6 Commits

  1. 2
      Info.plist
  2. 105
      src/AppDelegate.swift
  3. 100
      src/InputBoxManager.swift
  4. 2
      src/PingMonitor.swift
  5. 696
      src/SonioxClient.swift
  6. 47
      src/TextProcessor.swift
  7. 3
      src/main.swift

2
Info.plist

@ -19,7 +19,7 @@
<key>CFBundlePackageType</key> <key>CFBundlePackageType</key>
<string>APPL</string> <string>APPL</string>
<key>CFBundleShortVersionString</key> <key>CFBundleShortVersionString</key>
<string>2.10</string>
<string>2.16</string>
<key>CFBundleVersion</key> <key>CFBundleVersion</key>
<string>1</string> <string>1</string>
<key>LSMinimumSystemVersion</key> <key>LSMinimumSystemVersion</key>

105
src/AppDelegate.swift

@ -101,7 +101,8 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
self.stopRecording() self.stopRecording()
} }
self.pendingStopWorkItem = work self.pendingStopWorkItem = work
DispatchQueue.main.asyncAfter(deadline: .now() + 0.65, execute: work)
// 0.35s (350ms) acoustic tail buffer captures trailing syllables at key release without sluggish delay
DispatchQueue.main.asyncAfter(deadline: .now() + 0.35, execute: work)
} }
} }
} }
@ -158,6 +159,9 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
PingMonitor.shared.onNetworkRestored = { [weak self] in PingMonitor.shared.onNetworkRestored = { [weak self] in
guard let self = self else { return } guard let self = self else { return }
if self.audioRecorder.isRecording, let active = self.activeSession, !active.isReady {
active.triggerMidRecordingReconnect(cleanArchive: false, reason: "Network restored while recording")
}
SonioxSessionPool.shared.reconnectAll() SonioxSessionPool.shared.reconnectAll()
let gen = self.currentGeneration() let gen = self.currentGeneration()
OfflineAudioQueueManager.shared.processPendingQueueIfNeeded(isCancelled: { [weak self] in OfflineAudioQueueManager.shared.processPendingQueueIfNeeded(isCancelled: { [weak self] in
@ -166,7 +170,10 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
}) })
} }
PingMonitor.shared.onInterfaceChanged = {
PingMonitor.shared.onInterfaceChanged = { [weak self] in
if let self = self, self.audioRecorder.isRecording {
self.activeSession?.triggerMidRecordingReconnect(cleanArchive: false, reason: "Network interface changed while recording")
}
SonioxSessionPool.shared.reconnectAll() SonioxSessionPool.shared.reconnectAll()
} }
@ -294,6 +301,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
session.onEndTriggerDetected = { [weak self] in session.onEndTriggerDetected = { [weak self] in
guard let self = self, self.currentGeneration() == myGen else { return } guard let self = self, self.currentGeneration() == myGen else { return }
guard HotkeyManager.shared.currentMode != .pushToTalk else { return }
print("AppDelegate: In-flight end trigger ('and'/'تمام') detected! Stopping recording immediately...") print("AppDelegate: In-flight end trigger ('and'/'تمام') detected! Stopping recording immediately...")
self.stopRecording() self.stopRecording()
} }
@ -305,14 +313,16 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
self.hasSpokenInCurrentSession = true self.hasSpokenInCurrentSession = true
self.lastVoiceDetectedTime = Date() self.lastVoiceDetectedTime = Date()
let deltaWords = delta.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if let lastWord = deltaWords.last.map({ InputBoxManager.stripPunctuation($0).lowercased() }) {
if lastWord == "and" || lastWord == "تمام" || lastWord == "end" {
print("AppDelegate: In-speech end trigger detected ('\(lastWord)'). Finalizing speech...")
DispatchQueue.main.async {
self.stopRecording()
if HotkeyManager.shared.currentMode != .pushToTalk {
let deltaWords = delta.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if let lastWord = deltaWords.last.map({ InputBoxManager.stripPunctuation($0).lowercased() }) {
if lastWord == "and" || lastWord == "تمام" || lastWord == "end" {
print("AppDelegate: In-speech end trigger detected ('\(lastWord)'). Finalizing speech...")
DispatchQueue.main.async {
self.stopRecording()
}
return
} }
return
} }
} }
@ -404,15 +414,24 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
session.setRecordingActive(false) session.setRecordingActive(false)
let audioSeconds = Double(pcmData.count) / 32000.0 let audioSeconds = Double(pcmData.count) / 32000.0
// FAST PATH FOR SILENT STALL / OFFLINE NETWORK:
// If user spoke (>0.8s) and socket is dead, disconnected, or had a silent TCP stall with 0 tokens,
// FAST PATH FOR SILENT STALL / OFFLINE NETWORK / UNTRANSCRIBED TAIL AUDIO:
// If user spoke (>0.8s) and socket is dead, disconnected, or has untranscribed tail audio from a mid-speech drop,
// do NOT wait 4.0s-10s for finalize timeout — jump straight into Smart Audio Recovery in 0ms! // do NOT wait 4.0s-10s for finalize timeout — jump straight into Smart Audio Recovery in 0ms!
if hasSpokenInCurrentSession && audioSeconds >= 0.8 { if hasSpokenInCurrentSession && audioSeconds >= 0.8 {
if !PingMonitor.shared.isPathSatisfied || !session.isAlive || (session.hasSilentStall && session.totalReceivedTokensCount == 0) {
print("AppDelegate: ⚡️ Fast-path recovery triggered immediately (Silent stall / dead socket detected during speech).")
if !PingMonitor.shared.isPathSatisfied || !session.isAlive || session.hasUntranscribedTailAudio || (session.hasSilentStall && session.totalReceivedTokensCount == 0) {
let confirmedPrefix = session.confirmedTranscriptPrefix()
let tailAudio = session.untranscribedTailAudioData()
let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines)
print("AppDelegate: ⚡️ Fast-path recovery triggered immediately (confirmedPrefix: \(confirmedPrefix.count) chars, tailAudio: \(tailAudio.count) bytes).")
session.abort() session.abort()
self.activeSession = nil self.activeSession = nil
attemptFullAudioRecovery(reason: "Fast-path stall detection", generation: myGen)
attemptFullAudioRecovery(
reason: "Fast-path stall/disconnect detection",
partialFallback: bestEffort,
confirmedPrefix: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? confirmedPrefix : "",
customAudioData: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? tailAudio : nil,
generation: myGen
)
return return
} }
} }
@ -440,9 +459,17 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
self.finishFinalization(text: text, error: nil, generation: myGen) self.finishFinalization(text: text, error: nil, generation: myGen)
} }
case .failure(let err): case .failure(let err):
let confirmedPrefix = session.confirmedTranscriptPrefix()
let tailAudio = session.untranscribedTailAudioData()
let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines) let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines)
print("AppDelegate: Live session error (\(err.localizedDescription)). Launching full buffered audio recovery...")
self.attemptFullAudioRecovery(reason: err.localizedDescription, partialFallback: bestEffort, generation: myGen)
print("AppDelegate: Live session error (\(err.localizedDescription)). Launching buffered audio recovery (prefix: \(confirmedPrefix.count) chars, tail: \(tailAudio.count) bytes)...")
self.attemptFullAudioRecovery(
reason: err.localizedDescription,
partialFallback: bestEffort,
confirmedPrefix: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? confirmedPrefix : "",
customAudioData: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? tailAudio : nil,
generation: myGen
)
} }
} }
@ -458,12 +485,21 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
completed = true completed = true
lock.unlock() lock.unlock()
let confirmedPrefix = session.confirmedTranscriptPrefix()
let tailAudio = session.untranscribedTailAudioData()
let hadTail = session.hasUntranscribedTailAudio
let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines) let bestEffort = session.currentBestTranscript().trimmingCharacters(in: .whitespacesAndNewlines)
session.abort() session.abort()
if bestEffort.isEmpty && !InputBoxManager.shared.hasStreamedText && (self.hasSpokenInCurrentSession || pcmData.count > 16000) {
print("AppDelegate: Finalize timeout with empty text. Triggering Smart Audio Recovery...")
self.attemptFullAudioRecovery(reason: "Finalize timeout", generation: myGen)
if hadTail || (bestEffort.isEmpty && !InputBoxManager.shared.hasStreamedText && (self.hasSpokenInCurrentSession || pcmData.count > 16000)) {
print("AppDelegate: Finalize timeout with untranscribed tail/empty text. Triggering Smart Audio Recovery...")
self.attemptFullAudioRecovery(
reason: "Finalize timeout",
partialFallback: bestEffort,
confirmedPrefix: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? confirmedPrefix : "",
customAudioData: !confirmedPrefix.isEmpty && tailAudio.count > 6400 ? tailAudio : nil,
generation: myGen
)
} else { } else {
self.finishFinalization(text: bestEffort, error: nil, generation: myGen) self.finishFinalization(text: bestEffort, error: nil, generation: myGen)
} }
@ -472,13 +508,20 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
DispatchQueue.main.asyncAfter(deadline: .now() + finalizeTimeout, execute: watchdog) DispatchQueue.main.asyncAfter(deadline: .now() + finalizeTimeout, execute: watchdog)
} }
private func attemptFullAudioRecovery(reason: String, partialFallback: String = "", generation: UInt64) {
private func attemptFullAudioRecovery(
reason: String,
partialFallback: String = "",
confirmedPrefix: String = "",
customAudioData: Data? = nil,
generation: UInt64
) {
guard currentGeneration() == generation else { return } guard currentGeneration() == generation else { return }
let audioToRecover = self.lastRecordedAudio
let audioToRecover = customAudioData ?? self.lastRecordedAudio
let audioSec = Double(audioToRecover.count) / 32000.0 let audioSec = Double(audioToRecover.count) / 32000.0
guard audioToRecover.count > 6400 else { guard audioToRecover.count > 6400 else {
finishFinalization(text: partialFallback.isEmpty ? nil : partialFallback, error: reason, generation: generation)
let fallbackText = !partialFallback.isEmpty ? partialFallback : confirmedPrefix
finishFinalization(text: fallbackText.isEmpty ? nil : fallbackText, error: reason, generation: generation)
return return
} }
@ -501,31 +544,33 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
guard let self = self, self.currentGeneration() == generation else { return } guard let self = self, self.currentGeneration() == generation else { return }
switch result { switch result {
case .success(let recovered): case .success(let recovered):
let trimmed = recovered.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmed.isEmpty {
let trimmedTail = recovered.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmedTail.isEmpty {
let combinedText = confirmedPrefix.isEmpty ? trimmedTail : (confirmedPrefix + " " + trimmedTail)
VoiceCommandLogger.shared.logRecovery( VoiceCommandLogger.shared.logRecovery(
audioBytes: audioToRecover.count, audioBytes: audioToRecover.count,
durationSec: audioSec, durationSec: audioSec,
reason: reason, reason: reason,
outcome: "SUCCESS (Full Buffered Audio Recovered)",
recoveredText: trimmed
outcome: confirmedPrefix.isEmpty ? "SUCCESS (Full Buffered Audio Recovered)" : "SUCCESS (Confirmed Prefix + Tail Audio Recovered)",
recoveredText: combinedText
) )
self.finishFinalization(text: trimmed, error: nil, generation: generation)
self.finishFinalization(text: combinedText, error: nil, generation: generation)
return return
} }
case .failure: case .failure:
break break
} }
if !partialFallback.isEmpty {
let fallbackToUse = !partialFallback.isEmpty ? partialFallback : confirmedPrefix
if !fallbackToUse.isEmpty {
VoiceCommandLogger.shared.logRecovery( VoiceCommandLogger.shared.logRecovery(
audioBytes: audioToRecover.count, audioBytes: audioToRecover.count,
durationSec: audioSec, durationSec: audioSec,
reason: reason, reason: reason,
outcome: "PARTIAL FALLBACK USED", outcome: "PARTIAL FALLBACK USED",
recoveredText: partialFallback
recoveredText: fallbackToUse
) )
self.finishFinalization(text: partialFallback, error: nil, generation: generation)
self.finishFinalization(text: fallbackToUse, error: nil, generation: generation)
} else { } else {
// Save to persistent offline queue so when internet comes back in a few minutes, it is automatically transcribed! // Save to persistent offline queue so when internet comes back in a few minutes, it is automatically transcribed!
OfflineAudioQueueManager.shared.enqueueFailedAudio(pcmData: audioToRecover) OfflineAudioQueueManager.shared.enqueueFailedAudio(pcmData: audioToRecover)

100
src/InputBoxManager.swift

@ -63,22 +63,24 @@ public final class InputBoxManager {
public func streamDelta(text: String) { public func streamDelta(text: String) {
guard !text.isEmpty else { return } guard !text.isEmpty else { return }
// Suppress trailing end triggers ("and", "تمام", "end") from live typing
let triggerWords = text.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if let lastRaw = triggerWords.last {
let last = InputBoxManager.stripPunctuation(lastRaw).lowercased()
if last == "and" || last == "تمام" || last == "end" {
if triggerWords.count <= 1 {
return // Drop completely if it was just the end word
// Suppress trailing end triggers ("and", "تمام", "end") ONLY in hands-free/toggle mode, never in Push-To-Talk
if HotkeyManager.shared.currentMode != .pushToTalk {
let triggerWords = text.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if let lastRaw = triggerWords.last {
let last = InputBoxManager.stripPunctuation(lastRaw).lowercased()
if last == "and" || last == "تمام" || last == "end" {
if triggerWords.count <= 1 {
return
}
let textWithoutEnd = triggerWords.dropLast().joined(separator: " ") + " "
streamDelta(text: textWithoutEnd)
return
} }
let textWithoutEnd = triggerWords.dropLast().joined(separator: " ") + " "
streamDelta(text: textWithoutEnd)
return
} }
} }
// Apply lightweight inline domain vocabulary so streamed words match final text
let normalizedDelta = SonioxSettings.shared.domainVocabulary ? TextProcessor.shared.applyDomainVocabularyInline(text) : text
// Apply lightweight inline formatting (domain vocabulary, voice punctuation, digits) so streamed words match final text
let normalizedDelta = TextProcessor.shared.processLiveChunk(text)
lock.lock() lock.lock()
let expectedGen = sessionGeneration let expectedGen = sessionGeneration
@ -224,6 +226,7 @@ public final class InputBoxManager {
// This guarantees that numbers (۱ -> 1), ellipses (... -> . . .), and punctuation tokens have identical word counts & tokens! // This guarantees that numbers (۱ -> 1), ellipses (... -> . . .), and punctuation tokens have identical word counts & tokens!
let processedStreamed = rawStreamed.isEmpty ? "" : TextProcessor.shared.process(text: rawStreamed) let processedStreamed = rawStreamed.isEmpty ? "" : TextProcessor.shared.process(text: rawStreamed)
let cleanStreamed = processedStreamed.trimmingCharacters(in: .whitespacesAndNewlines) let cleanStreamed = processedStreamed.trimmingCharacters(in: .whitespacesAndNewlines)
let streamedEndedWithSpace = rawStreamed.unicodeScalars.last.map { CharacterSet.whitespacesAndNewlines.contains($0) } ?? false
pasteQueue.async { [weak self] in pasteQueue.async { [weak self] in
guard let self = self else { return } guard let self = self else { return }
@ -253,17 +256,14 @@ public final class InputBoxManager {
// Case 1: Nothing was streamed live -> Paste entire text // Case 1: Nothing was streamed live -> Paste entire text
if cleanStreamed.isEmpty { if cleanStreamed.isEmpty {
let pb = NSPasteboard.general
pb.clearContents()
pb.setString(textWithTrailing, forType: .string)
self.postCleanCmdV()
self.pasteSynchronously(textWithTrailing)
restoreClipboardFullText() restoreClipboardFullText()
return return
} }
// Case 2: Exactly matches streamed text // Case 2: Exactly matches streamed text
if cleanStreamed == trimmed { if cleanStreamed == trimmed {
if addTrailingSpace {
if addTrailingSpace && !streamedEndedWithSpace {
self.pasteSynchronously(" ") self.pasteSynchronously(" ")
} }
restoreClipboardFullText() restoreClipboardFullText()
@ -272,9 +272,15 @@ public final class InputBoxManager {
// Case 3: Streamed prefix is exact prefix of trimmed final text // Case 3: Streamed prefix is exact prefix of trimmed final text
if trimmed.count > cleanStreamed.count && trimmed.hasPrefix(cleanStreamed) { if trimmed.count > cleanStreamed.count && trimmed.hasPrefix(cleanStreamed) {
let suffix = String(trimmed.dropFirst(cleanStreamed.count))
let suffixToPaste = addTrailingSpace ? (suffix + " ") : suffix
self.pasteSynchronously(suffixToPaste)
let rawSuffix = String(trimmed.dropFirst(cleanStreamed.count))
let cleanSuffix = rawSuffix.trimmingCharacters(in: .whitespacesAndNewlines)
if !cleanSuffix.isEmpty {
let needsLeadingSpace = !streamedEndedWithSpace && (rawSuffix.first?.isWhitespace == true)
let suffixToPaste = (needsLeadingSpace ? " " : "") + cleanSuffix + (addTrailingSpace ? " " : "")
self.pasteSynchronously(suffixToPaste)
} else if addTrailingSpace && !streamedEndedWithSpace {
self.pasteSynchronously(" ")
}
restoreClipboardFullText() restoreClipboardFullText()
return return
} }
@ -374,12 +380,14 @@ public final class InputBoxManager {
if startIndex < finalPairs.count { if startIndex < finalPairs.count {
let remainingWords = finalPairs.dropFirst(startIndex).map { $0.raw } let remainingWords = finalPairs.dropFirst(startIndex).map { $0.raw }
let suffix = remainingWords.joined(separator: " ") let suffix = remainingWords.joined(separator: " ")
let suffixToPaste = addTrailingSpace ? (" " + suffix + " ") : (" " + suffix)
let leadingSpace = streamedEndedWithSpace ? "" : " "
let trailingSpace = addTrailingSpace ? " " : ""
let suffixToPaste = leadingSpace + suffix + trailingSpace
self.pasteSynchronously(suffixToPaste) self.pasteSynchronously(suffixToPaste)
restoreClipboardFullText() restoreClipboardFullText()
return return
} else { } else {
if addTrailingSpace {
if addTrailingSpace && !streamedEndedWithSpace {
self.pasteSynchronously(" ") self.pasteSynchronously(" ")
} }
restoreClipboardFullText() restoreClipboardFullText()
@ -390,13 +398,39 @@ public final class InputBoxManager {
} }
} }
/// Places text on NSPasteboard, posts Cmd+V, and waits 45ms so target app consumes this chunk BEFORE next clipboard write!
/// Places core word(s) on NSPasteboard via Cmd+V, while injecting leading/trailing spaces
/// via native Space keystrokes so apps that `.trim()` pasted clipboard text (like Hermes Desktop)
/// never glue words together or double-space!
private func pasteSynchronously(_ text: String) { private func pasteSynchronously(_ text: String) {
guard !text.isEmpty else { return }
let horizontalSpaceSet = CharacterSet(charactersIn: " \t")
let hasLeadingSpace = text.unicodeScalars.first.map { horizontalSpaceSet.contains($0) } ?? false
let hasTrailingSpace = text.unicodeScalars.last.map { horizontalSpaceSet.contains($0) } ?? false
let core = text.trimmingCharacters(in: horizontalSpaceSet)
if core.isEmpty {
if hasLeadingSpace || hasTrailingSpace {
postCleanSpace()
usleep(12000)
}
return
}
if hasLeadingSpace {
postCleanSpace()
usleep(12000)
}
let pb = NSPasteboard.general let pb = NSPasteboard.general
pb.clearContents() pb.clearContents()
pb.setString(text, forType: .string)
pb.setString(core, forType: .string)
postCleanCmdV() postCleanCmdV()
usleep(45000) // 45ms guard ensures browser/app reads NSPasteboard before next chunk overwrites it
usleep(22000) // 22ms guard ensures browser/app reads NSPasteboard before next chunk overwrites it (~45 chunks/sec capacity)
if hasTrailingSpace {
postCleanSpace()
usleep(12000)
}
} }
private func postCleanCmdV() { private func postCleanCmdV() {
@ -411,4 +445,20 @@ public final class InputBoxManager {
up.post(tap: .cgAnnotatedSessionEventTap) up.post(tap: .cgAnnotatedSessionEventTap)
} }
} }
private func postCleanSpace() {
let src = CGEventSource(stateID: .privateState)
let spaceKeyCode: CGKeyCode = 0x31 // ' ' (kVK_Space)
var spaceChar: UniChar = 0x0020
if let down = CGEvent(keyboardEventSource: src, virtualKey: spaceKeyCode, keyDown: true),
let up = CGEvent(keyboardEventSource: src, virtualKey: spaceKeyCode, keyDown: false) {
down.flags = []
up.flags = []
down.keyboardSetUnicodeString(stringLength: 1, unicodeString: &spaceChar)
up.keyboardSetUnicodeString(stringLength: 1, unicodeString: &spaceChar)
down.post(tap: .cgAnnotatedSessionEventTap)
up.post(tap: .cgAnnotatedSessionEventTap)
}
}
} }

2
src/PingMonitor.swift

@ -31,7 +31,7 @@ public enum PingQuality {
public final class PingMonitor { public final class PingMonitor {
public static let shared = PingMonitor() public static let shared = PingMonitor()
private let targetURL = URL(string: "https://translate.compare.soniox.com/")!
private let targetURL = URL(string: "https://stt.newhorizonco.uk/health")!
private var session: URLSession! private var session: URLSession!
private var timer: Timer? private var timer: Timer?
private var isRunning = false private var isRunning = false

696
src/SonioxClient.swift

@ -2,11 +2,22 @@ import Foundation
import AppKit import AppKit
public final class SonioxLiveSession { public final class SonioxLiveSession {
public static let primaryWsBase = "wss://translate.compare.soniox.com/compare/api/compare-websocket"
public static let fallbackWsBase = "wss://stt.compare.soniox.com/compare/api/compare-websocket"
public static let primaryWsBase = "wss://stt.newhorizonco.uk/ws/live-stt"
public static let fallbackWsBase = "wss://stt.newhorizonco.uk/ws/live-translate"
private static let apiKey = "nwh_live_Y54Zfil6MdYZEOu4IHlBzTWhGputnVbJ"
private static let userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" private static let userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
public static func origin(for endpoint: String) -> String { public static func origin(for endpoint: String) -> String {
return endpoint.contains("translate") ? "https://translate.compare.soniox.com" : "https://stt.compare.soniox.com"
return "https://stt.newhorizonco.uk"
}
private static func baseQuery(withHints hints: [String], targetLanguage: String?) -> String {
var parts: [String] = []
let hintStr = hints.map { "language_hints=\($0)" }.joined(separator: "&")
if !hintStr.isEmpty { parts.append(hintStr) }
parts.append("key=\(apiKey)")
if let target = targetLanguage, !target.isEmpty, target != "off" {
parts.append("target_language=\(target)")
}
return parts.joined(separator: "&")
} }
// Pre-compiled static regexes for zero-allocation hot path // Pre-compiled static regexes for zero-allocation hot path
@ -19,6 +30,9 @@ public final class SonioxLiveSession {
private static let punctCharSet = CharacterSet(charactersIn: ".,!?:;،؛؟\"'()[]{}«»-–—") private static let punctCharSet = CharacterSet(charactersIn: ".,!?:;،؛؟\"'()[]{}«»-–—")
public let targetLanguage: String? public let targetLanguage: String?
public let languageHints: [String]
private var isFallbackSwitched = false
private var initialAudioBuffer: [Data] = []
public var isTranslationMode: Bool { public var isTranslationMode: Bool {
guard let target = targetLanguage, !target.isEmpty, target != "off" else { return false } guard let target = targetLanguage, !target.isEmpty, target != "off" else { return false }
return true return true
@ -32,10 +46,16 @@ public final class SonioxLiveSession {
private var isAborted = false private var isAborted = false
private let lock = NSLock() private let lock = NSLock()
// Transcripts & Stall tracking
// Transcripts & Stall tracking (Multi-segment continuous speech + checkpointed audio buffer)
private var archivedCommittedText: String = ""
private var committedFinalTokens: [String] = [] private var committedFinalTokens: [String] = []
private var currentNonFinalTokens: [String] = [] private var currentNonFinalTokens: [String] = []
private var streamedPrefix: String = "" private var streamedPrefix: String = ""
private var currentSegmentAudioBuffer = Data()
private var lastConfirmedWordAudioByteOffset: Int = 0
private var isReconnectingMidRecording = false
private var isFlushingReplayBuffer = false
private var segmentConnectTime: Date = Date()
private var sentAudioBytes: Int = 0 private var sentAudioBytes: Int = 0
public private(set) var totalReceivedTokensCount: Int = 0 public private(set) var totalReceivedTokensCount: Int = 0
public private(set) var hasSilentStall: Bool = false public private(set) var hasSilentStall: Bool = false
@ -58,8 +78,44 @@ public final class SonioxLiveSession {
isRecordingActive = active isRecordingActive = active
if active { if active {
lastDataTime = Date() lastDataTime = Date()
segmentConnectTime = Date()
}
lock.unlock()
}
/// True when a mid-recording disconnect/stall occurred and there is >0.35s of unconfirmed tail audio waiting to be transcribed.
public var hasUntranscribedTailAudio: Bool {
lock.lock()
defer { lock.unlock() }
let tailCount = max(0, currentSegmentAudioBuffer.count - lastConfirmedWordAudioByteOffset)
let hadDrop = hasNetworkFailure || hasSilentStall || isReconnectingMidRecording || !isConnected
return hadDrop && tailCount > 11200
}
/// Returns the already-confirmed transcript prefix across all completed segments and finalized tokens.
public func confirmedTranscriptPrefix() -> String {
lock.lock()
let segCommitted = committedFinalTokens.joined().trimmingCharacters(in: .whitespacesAndNewlines)
let rawPrefix: String
if archivedCommittedText.isEmpty {
rawPrefix = segCommitted
} else if segCommitted.isEmpty {
rawPrefix = archivedCommittedText
} else {
rawPrefix = archivedCommittedText + " " + segCommitted
} }
lock.unlock() lock.unlock()
return SonioxLiveSession.sanitizeText(rawPrefix)
}
/// Returns the slice of raw PCM audio from `lastConfirmedWordAudioByteOffset` to the end of `currentSegmentAudioBuffer`.
public func untranscribedTailAudioData() -> Data {
lock.lock()
defer { lock.unlock() }
guard currentSegmentAudioBuffer.count > lastConfirmedWordAudioByteOffset else {
return Data()
}
return currentSegmentAudioBuffer.subdata(in: lastConfirmedWordAudioByteOffset..<currentSegmentAudioBuffer.count)
} }
public var isReady: Bool { public var isReady: Bool {
@ -124,22 +180,12 @@ public final class SonioxLiveSession {
targetLanguage: String? = nil, targetLanguage: String? = nil,
endpoint: String = SonioxLiveSession.primaryWsBase endpoint: String = SonioxLiveSession.primaryWsBase
) { ) {
self.languageHints = languageHints
let effectiveTarget = targetLanguage ?? (SonioxSettings.shared.targetLanguage.isEnabled ? SonioxSettings.shared.targetLanguage.rawValue : nil) let effectiveTarget = targetLanguage ?? (SonioxSettings.shared.targetLanguage.isEnabled ? SonioxSettings.shared.targetLanguage.rawValue : nil)
self.targetLanguage = effectiveTarget self.targetLanguage = effectiveTarget
let hints = languageHints.map { "language_hints=\($0)" }.joined(separator: "&")
var queryParams: [String] = []
if !hints.isEmpty {
queryParams.append(hints)
}
queryParams.append("enable_speaker_diarization=false")
queryParams.append("enable_language_identification=true")
queryParams.append("enable_endpoint_detection=false")
queryParams.append("providers=soniox")
if let target = effectiveTarget, !target.isEmpty, target != "off" {
queryParams.append("target_language=\(target)")
}
let urlStr = "\(endpoint)?" + queryParams.joined(separator: "&")
let queryStr = SonioxLiveSession.baseQuery(withHints: languageHints, targetLanguage: effectiveTarget)
let urlStr = "\(endpoint)?\(queryStr)"
guard let url = URL(string: urlStr) else { return } guard let url = URL(string: urlStr) else { return }
var request = URLRequest(url: url) var request = URLRequest(url: url)
@ -164,49 +210,310 @@ public final class SonioxLiveSession {
if error == nil && !self.isClosed && !self.isAborted { if error == nil && !self.isClosed && !self.isAborted {
self.isConnected = true self.isConnected = true
self.lastDataTime = Date() self.lastDataTime = Date()
self.initialAudioBuffer.removeAll()
self.lock.unlock() self.lock.unlock()
self.onConnectionStateChanged?(true) self.onConnectionStateChanged?(true)
print("SonioxLiveSession: Connected successfully.")
print("SonioxLiveSession: Connected successfully (\(endpoint)).")
} else { } else {
if let error = error {
print("SonioxLiveSession: Ping failed on \(endpoint):", error)
}
if endpoint == SonioxLiveSession.primaryWsBase && !self.isClosed && !self.isAborted && !self.isFallbackSwitched {
self.lock.unlock()
self.switchToFallback()
return
}
self.hasNetworkFailure = true self.hasNetworkFailure = true
self.lock.unlock() self.lock.unlock()
self.onNetworkDegraded?(true) self.onNetworkDegraded?(true)
if let error = error {
print("SonioxLiveSession: Ping failed:", error)
}
} }
} }
startReceiving() startReceiving()
} }
/// Called while user is actively speaking to detect half-open TCP silent stalls
private func switchToFallback() {
lock.lock()
guard !isFallbackSwitched, !isClosed, !isAborted else {
lock.unlock()
return
}
isFallbackSwitched = true
print("SonioxLiveSession: ⚡️ Primary connection failed. Seamlessly switching to fallback endpoint: \(SonioxLiveSession.fallbackWsBase)")
webSocketTask?.cancel(with: .goingAway, reason: nil)
let endpoint = SonioxLiveSession.fallbackWsBase
let queryStr = SonioxLiveSession.baseQuery(withHints: languageHints, targetLanguage: targetLanguage)
let urlStr = "\(endpoint)?\(queryStr)"
guard let url = URL(string: urlStr), let session = urlSession else {
hasNetworkFailure = true
lock.unlock()
onNetworkDegraded?(true)
return
}
var request = URLRequest(url: url)
request.setValue(SonioxLiveSession.userAgent, forHTTPHeaderField: "User-Agent")
request.setValue(SonioxLiveSession.origin(for: endpoint), forHTTPHeaderField: "Origin")
request.timeoutInterval = 10.0
let newTask = session.webSocketTask(with: request)
self.webSocketTask = newTask
lock.unlock()
newTask.resume()
newTask.sendPing { [weak self] err in
guard let self = self else { return }
self.lock.lock()
if err == nil && !self.isClosed && !self.isAborted {
self.isConnected = true
self.hasNetworkFailure = false
self.lastDataTime = Date()
let bufferedChunks = self.initialAudioBuffer
self.initialAudioBuffer.removeAll()
self.lock.unlock()
self.onConnectionStateChanged?(true)
print("SonioxLiveSession: ✅ Fallback endpoint connected! Flushing \(bufferedChunks.count) buffered chunks...")
for chunk in bufferedChunks {
newTask.send(.data(chunk)) { _ in }
}
} else {
self.hasNetworkFailure = true
self.lock.unlock()
self.onNetworkDegraded?(true)
if let err = err {
print("SonioxLiveSession: Fallback ping failed:", err)
}
}
}
startReceiving()
}
/// Called while user is actively speaking to detect half-open TCP silent stalls and auto-reconnect seamlessly
public func checkSilentStallWhileSpeaking() { public func checkSilentStallWhileSpeaking() {
lock.lock() lock.lock()
let connected = isConnected && !isClosed && !isAborted let connected = isConnected && !isClosed && !isAborted
let activeRec = isRecordingActive && !isFinalizing && !isReconnectingMidRecording
let sent = sentAudioBytes let sent = sentAudioBytes
let elapsedSinceData = Date().timeIntervalSince(lastDataTime) let elapsedSinceData = Date().timeIntervalSince(lastDataTime)
lock.unlock() lock.unlock()
// If >1.5s of audio (48KB) was sent while speaking, but 0 frames arrived for >3.0s:
if connected && sent > 48000 && elapsedSinceData > 3.0 {
// If >1.0s of audio (32KB) was sent while speaking, but 0 frames arrived for >3.5s:
if connected && sent > 32000 && elapsedSinceData > 3.5 {
lock.lock() lock.lock()
hasSilentStall = true hasSilentStall = true
lock.unlock() lock.unlock()
DispatchQueue.main.async { [weak self] in DispatchQueue.main.async { [weak self] in
self?.onNetworkDegraded?(true) self?.onNetworkDegraded?(true)
} }
if activeRec {
triggerMidRecordingReconnect(cleanArchive: false, reason: "Silent TCP/STT stall (\(String(format: "%.1f", elapsedSinceData))s without frames while speaking)")
}
}
}
/// Seamlessly archives confirmed transcript tokens and reconnects a fresh WebSocket in the background
/// while the user continues holding Push-To-Talk / speaking for 5–10+ minutes!
public func triggerMidRecordingReconnect(cleanArchive: Bool, reason: String) {
lock.lock()
guard isRecordingActive, !isFinalizing, !isClosed, !isAborted, !isReconnectingMidRecording else {
lock.unlock()
return
}
isReconnectingMidRecording = true
isConnected = false
let segCommitted = committedFinalTokens.joined()
let segNonFinal = currentNonFinalTokens.joined()
let toArchive = cleanArchive ? (segCommitted + segNonFinal) : segCommitted
let cleanToArchive = toArchive.trimmingCharacters(in: .whitespacesAndNewlines)
if !cleanToArchive.isEmpty {
if archivedCommittedText.isEmpty {
archivedCommittedText = cleanToArchive
} else {
archivedCommittedText += " " + cleanToArchive
}
}
committedFinalTokens.removeAll()
currentNonFinalTokens.removeAll()
if cleanArchive {
currentSegmentAudioBuffer.removeAll(keepingCapacity: true)
lastConfirmedWordAudioByteOffset = 0
} else if lastConfirmedWordAudioByteOffset > 0 && lastConfirmedWordAudioByteOffset <= currentSegmentAudioBuffer.count {
currentSegmentAudioBuffer.removeSubrange(0..<lastConfirmedWordAudioByteOffset)
lastConfirmedWordAudioByteOffset = 0
}
if !archivedCommittedText.isEmpty {
streamedPrefix = archivedCommittedText + " "
}
let bufferedBytes = currentSegmentAudioBuffer.count
let archivedCount = archivedCommittedText.count
let oldTask = webSocketTask
let oldSession = urlSession
webSocketTask = nil
urlSession = nil
lock.unlock()
print("SonioxLiveSession: 🔄 Mid-recording auto-reconnect triggered (\(reason)). Archived \(archivedCount) chars, buffering \(bufferedBytes) bytes of unconfirmed audio...")
oldTask?.cancel(with: .goingAway, reason: nil)
oldSession?.invalidateAndCancel()
attemptMidRecordingConnect(attempt: 1)
}
private func attemptMidRecordingConnect(attempt: Int) {
lock.lock()
guard isRecordingActive, !isFinalizing, !isClosed, !isAborted else {
isReconnectingMidRecording = false
lock.unlock()
return
}
let endpoint = (attempt % 2 == 1) ? SonioxLiveSession.primaryWsBase : SonioxLiveSession.fallbackWsBase
let queryStr = SonioxLiveSession.baseQuery(withHints: languageHints, targetLanguage: targetLanguage)
let urlStr = "\(endpoint)?\(queryStr)"
guard let url = URL(string: urlStr) else {
isReconnectingMidRecording = false
lock.unlock()
return
}
var request = URLRequest(url: url)
request.setValue(SonioxLiveSession.userAgent, forHTTPHeaderField: "User-Agent")
request.setValue(SonioxLiveSession.origin(for: endpoint), forHTTPHeaderField: "Origin")
request.timeoutInterval = 6.0
let config = URLSessionConfiguration.default
config.waitsForConnectivity = false
config.requestCachePolicy = .reloadIgnoringLocalCacheData
let newSession = URLSession(configuration: config)
let newTask = newSession.webSocketTask(with: request)
self.urlSession = newSession
self.webSocketTask = newTask
lock.unlock()
newTask.resume()
newTask.sendPing { [weak self] err in
guard let self = self else { return }
self.lock.lock()
if err == nil && !self.isClosed && !self.isAborted && !self.isFinalizing {
self.isConnected = true
self.hasNetworkFailure = false
self.hasSilentStall = false
self.isReconnectingMidRecording = false
self.isFlushingReplayBuffer = true
self.lastDataTime = Date()
self.segmentConnectTime = Date()
self.sentAudioBytes = 0
let replaySnapshot = self.currentSegmentAudioBuffer
self.lock.unlock()
self.onNetworkDegraded?(false)
self.onConnectionStateChanged?(true)
print("SonioxLiveSession: ✅ Mid-recording reconnect succeeded on attempt \(attempt) (\(endpoint))! Replaying \(replaySnapshot.count) bytes of buffered speech...")
self.startReceiving()
self.flushReplayBufferToSocket(task: newTask, offset: 0)
} else {
let shouldRetry = self.isRecordingActive && !self.isFinalizing && !self.isClosed && !self.isAborted
self.webSocketTask = nil
self.urlSession = nil
self.lock.unlock()
newTask.cancel()
newSession.invalidateAndCancel()
if shouldRetry {
let delay = min(1.5, 0.5 + Double(min(attempt, 4)) * 0.25)
DispatchQueue.global(qos: .userInitiated).asyncAfter(deadline: .now() + delay) { [weak self] in
self?.attemptMidRecordingConnect(attempt: attempt + 1)
}
}
}
}
}
private func flushReplayBufferToSocket(task: URLSessionWebSocketTask, offset: Int) {
lock.lock()
guard isConnected, !isClosed, !isAborted, !isFinalizing, webSocketTask === task else {
isFlushingReplayBuffer = false
lock.unlock()
return
}
let total = currentSegmentAudioBuffer.count
if offset >= total {
isFlushingReplayBuffer = false
lock.unlock()
return
}
let chunkSize = 32000 // 1.0s chunk
let end = min(offset + chunkSize, total)
let slice = currentSegmentAudioBuffer.subdata(in: offset..<end)
lock.unlock()
task.send(.data(slice)) { [weak self] err in
guard let self = self else { return }
if let err = err {
print("SonioxLiveSession: Error flushing replay buffer:", err)
self.lock.lock()
self.isFlushingReplayBuffer = false
self.hasNetworkFailure = true
self.isConnected = false
let activeRec = self.isRecordingActive && !self.isFinalizing && !self.isClosed && !self.isAborted
self.lock.unlock()
if activeRec {
self.triggerMidRecordingReconnect(cleanArchive: false, reason: "Replay flush send error")
}
return
}
self.lock.lock()
self.sentAudioBytes += slice.count
self.lock.unlock()
self.flushReplayBufferToSocket(task: task, offset: end)
} }
} }
public func sendAudioChunk(_ data: Data) { public func sendAudioChunk(_ data: Data) {
lock.lock() lock.lock()
guard !isFinalizing, !isClosed, !isAborted, let task = webSocketTask else {
guard !isFinalizing, !isClosed, !isAborted else {
lock.unlock()
return
}
currentSegmentAudioBuffer.append(data)
if !isConnected {
if initialAudioBuffer.count < 120 {
initialAudioBuffer.append(data)
}
let shouldReconnect = isRecordingActive && hasNetworkFailure && !isReconnectingMidRecording
lock.unlock()
if shouldReconnect {
triggerMidRecordingReconnect(cleanArchive: false, reason: "Audio chunk arrived while disconnected")
}
return
}
if isReconnectingMidRecording || isFlushingReplayBuffer {
lock.unlock()
return
}
guard let task = webSocketTask else {
lock.unlock() lock.unlock()
return return
} }
// Proactive segment rotation at >150s when at a clean committed pause boundary so server never kills long 5–10m streams
let segAge = Date().timeIntervalSince(segmentConnectTime)
let canRotateProactively = segAge > 150.0 && currentNonFinalTokens.isEmpty && !committedFinalTokens.isEmpty
lock.unlock() lock.unlock()
if canRotateProactively {
triggerMidRecordingReconnect(cleanArchive: true, reason: "Proactive 150s continuous speech segment rollover")
return
}
let t0 = Date() let t0 = Date()
let message = URLSessionWebSocketTask.Message.data(data) let message = URLSessionWebSocketTask.Message.data(data)
task.send(message) { [weak self] error in task.send(message) { [weak self] error in
@ -214,10 +521,19 @@ public final class SonioxLiveSession {
if let error = error { if let error = error {
print("SonioxLiveSession: Error streaming audio chunk:", error) print("SonioxLiveSession: Error streaming audio chunk:", error)
self.lock.lock() self.lock.lock()
if !self.isFallbackSwitched && !self.isFinalizing && !self.isClosed && !self.isAborted && self.archivedCommittedText.isEmpty && self.committedFinalTokens.isEmpty {
self.lock.unlock()
self.switchToFallback()
return
}
self.hasNetworkFailure = true self.hasNetworkFailure = true
self.isConnected = false self.isConnected = false
let activeRec = self.isRecordingActive && !self.isFinalizing && !self.isClosed && !self.isAborted
self.lock.unlock() self.lock.unlock()
self.onNetworkDegraded?(true) self.onNetworkDegraded?(true)
if activeRec {
self.triggerMidRecordingReconnect(cleanArchive: false, reason: "Send audio chunk error: \(error.localizedDescription)")
}
} else { } else {
self.lock.lock() self.lock.lock()
self.sentAudioBytes += data.count self.sentAudioBytes += data.count
@ -232,15 +548,24 @@ public final class SonioxLiveSession {
public func finalizeStream() { public func finalizeStream() {
lock.lock() lock.lock()
guard !isFinalizing, !isClosed, !isAborted, let task = webSocketTask else {
guard !isFinalizing, !isClosed, !isAborted else {
lock.unlock() lock.unlock()
return return
} }
isFinalizing = true isFinalizing = true
let connected = isConnected && !isReconnectingMidRecording
let task = webSocketTask
lock.unlock() lock.unlock()
guard connected, let task = task else {
print("SonioxLiveSession: finalizeStream called while socket disconnected/reconnecting — completing immediately for tail recovery.")
completeWithCurrentText()
return
}
print("SonioxLiveSession: Sending finalize packet...") print("SonioxLiveSession: Sending finalize packet...")
// 1. Send trailing 300ms (9600 bytes at 16kHz 16-bit PCM) silence frame // 1. Send trailing 300ms (9600 bytes at 16kHz 16-bit PCM) silence frame
// so acoustic CTC model cleanly flushes and resolves trailing fast speech syllables
let silenceChunk = Data(count: 9600) let silenceChunk = Data(count: 9600)
let silenceMsg = URLSessionWebSocketTask.Message.data(silenceChunk) let silenceMsg = URLSessionWebSocketTask.Message.data(silenceChunk)
task.send(silenceMsg) { _ in } task.send(silenceMsg) { _ in }
@ -314,14 +639,15 @@ public final class SonioxLiveSession {
self.lock.lock() self.lock.lock()
self.hasNetworkFailure = true self.hasNetworkFailure = true
self.isConnected = false self.isConnected = false
let activeRecording = self.isRecordingActive
let activeRecording = self.isRecordingActive && !self.isFinalizing && !self.isClosed && !self.isAborted
self.lock.unlock() self.lock.unlock()
self.onNetworkDegraded?(true) self.onNetworkDegraded?(true)
if !activeRecording { if !activeRecording {
self.completeWithCurrentText() self.completeWithCurrentText()
} else { } else {
print("SonioxLiveSession: Network dropped mid-recording. Audio is buffered for Smart Recovery upon key release.")
print("SonioxLiveSession: Network dropped mid-recording. Triggering automatic live reconnect...")
self.triggerMidRecordingReconnect(cleanArchive: false, reason: "WebSocket receive error: \(error.localizedDescription)")
} }
} }
} }
@ -376,37 +702,77 @@ public final class SonioxLiveSession {
totalReceivedTokensCount += tokenCountDelta totalReceivedTokensCount += tokenCountDelta
if !newFinals.isEmpty { if !newFinals.isEmpty {
committedFinalTokens.append(contentsOf: newFinals) committedFinalTokens.append(contentsOf: newFinals)
// Advance confirmed audio checkpoint in `currentSegmentAudioBuffer`:
// Keep a safety acoustic lookback proportional to unfinalized words (plus 0.8s = 25600 bytes)
let unfinalizedWords = newNonFinals.joined().components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }.count
let safetyLookbackBytes = max(25600, (unfinalizedWords + 1) * 16000)
if currentSegmentAudioBuffer.count > safetyLookbackBytes {
lastConfirmedWordAudioByteOffset = max(lastConfirmedWordAudioByteOffset, currentSegmentAudioBuffer.count - safetyLookbackBytes)
}
} }
currentNonFinalTokens = newNonFinals currentNonFinalTokens = newNonFinals
let fullCommitted = committedFinalTokens.joined() let fullCommitted = committedFinalTokens.joined()
let fullNonFinal = currentNonFinalTokens.joined() let fullNonFinal = currentNonFinalTokens.joined()
let combined = fullCommitted + fullNonFinal
let segmentCombined = fullCommitted + fullNonFinal
let combined: String
if archivedCommittedText.isEmpty {
combined = segmentCombined
} else if segmentCombined.isEmpty {
combined = archivedCommittedText
} else {
let cleanSeg = segmentCombined.trimmingCharacters(in: .whitespaces)
combined = archivedCommittedText + (cleanSeg.isEmpty ? "" : (" " + cleanSeg))
}
// Check for in-flight end trigger ("and", "تمام", "end") in combined text
// Check for in-flight end trigger ("and", "تمام", "end") ONLY in hands-free/toggle mode and ONLY on finalized tokens!
// Never trigger auto-stop during Push-To-Talk while user is physically holding the hotkey!
var triggerDetected = false var triggerDetected = false
let trimmedCombined = combined.trimmingCharacters(in: .whitespacesAndNewlines)
let currentWords = trimmedCombined.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if let lastRaw = currentWords.last {
let cleanLast = InputBoxManager.stripPunctuation(lastRaw).lowercased()
if cleanLast == "and" || cleanLast == "تمام" || cleanLast == "end" {
triggerDetected = true
if HotkeyManager.shared.currentMode != .pushToTalk && currentNonFinalTokens.isEmpty && !newFinals.isEmpty {
let trimmedCombined = combined.trimmingCharacters(in: .whitespacesAndNewlines)
let currentWords = trimmedCombined.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if let lastRaw = currentWords.last {
let cleanLast = InputBoxManager.stripPunctuation(lastRaw).lowercased()
if cleanLast == "and" || cleanLast == "تمام" || cleanLast == "end" {
triggerDetected = true
}
} }
} }
// Emit live streaming deltas STRICTLY from Soniox-finalized tokens (is_final == true):
// Check if this message indicates end of session or final delivery
let sessionEnded = json["session_ended"] as? Bool ?? false
let sessionDone = (json["type"] as? String) == "session_done"
let isFinished = json["finished"] as? Bool ?? false
let hasRawFinished: Bool = {
guard let raw = json["raw"] as? String else { return false }
return raw.contains("\"finished\":true") || raw.contains("\"finished\": true")
}()
let isTerminatingMessage = gotFin || sessionEnded || sessionDone || isFinished || hasRawFinished
let stillActivelyRecording = self.isRecordingActive && !self.isFinalizing
// Emit live streaming deltas in real time as the user speaks!
// We stream completed words in `combined` with a 1-word acoustic lookahead holdback so every emitted chunk ends with a trailing space.
var deltaToEmit: String? = nil var deltaToEmit: String? = nil
if !triggerDetected && fullCommitted.count > streamedPrefix.count {
if fullCommitted.hasPrefix(streamedPrefix) {
let delta = String(fullCommitted.dropFirst(streamedPrefix.count))
streamedPrefix = fullCommitted
deltaToEmit = delta
} else {
let common = fullCommitted.commonPrefix(with: streamedPrefix)
if fullCommitted.count > common.count {
let delta = String(fullCommitted.dropFirst(common.count))
streamedPrefix = fullCommitted
if !triggerDetected && !isTerminatingMessage && !self.isFinalizing {
let rTrimmedCombined = combined.replacingOccurrences(of: #"\s+$"#, with: "", options: .regularExpression)
var stableCandidate = ""
if let lastSpaceRange = rTrimmedCombined.rangeOfCharacter(from: .whitespacesAndNewlines, options: .backwards) {
stableCandidate = String(rTrimmedCombined[..<lastSpaceRange.upperBound])
}
if stableCandidate.count > streamedPrefix.count {
if stableCandidate.hasPrefix(streamedPrefix) {
let delta = String(stableCandidate.dropFirst(streamedPrefix.count))
streamedPrefix = stableCandidate
deltaToEmit = delta deltaToEmit = delta
} else {
let streamedWordsCount = streamedPrefix.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }.count
let candidateWords = stableCandidate.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty }
if candidateWords.count > streamedWordsCount {
let delta = candidateWords.dropFirst(streamedWordsCount).joined(separator: " ") + " "
streamedPrefix = stableCandidate
deltaToEmit = delta
}
} }
} }
} }
@ -427,18 +793,33 @@ public final class SonioxLiveSession {
self?.onPartialText?(combined) self?.onPartialText?(combined)
} }
} }
}
let sessionEnded = json["session_ended"] as? Bool ?? false
let sessionDone = (json["type"] as? String) == "session_done"
let isFinished = json["finished"] as? Bool ?? false
let hasRawFinished: Bool = {
guard let raw = json["raw"] as? String else { return false }
return raw.contains("\"finished\":true") || raw.contains("\"finished\": true")
}()
if gotFin || sessionEnded || sessionDone || isFinished || hasRawFinished {
completeWithCurrentText()
if isTerminatingMessage {
if stillActivelyRecording {
print("SonioxLiveSession: Server ended WebSocket segment while user is still speaking — seamlessly rotating to fresh segment...")
triggerMidRecordingReconnect(cleanArchive: true, reason: "Server ended WebSocket segment while recording is still active")
} else {
completeWithCurrentText()
}
}
} else {
let sessionEnded = json["session_ended"] as? Bool ?? false
let sessionDone = (json["type"] as? String) == "session_done"
let isFinished = json["finished"] as? Bool ?? false
let hasRawFinished: Bool = {
guard let raw = json["raw"] as? String else { return false }
return raw.contains("\"finished\":true") || raw.contains("\"finished\": true")
}()
if sessionEnded || sessionDone || isFinished || hasRawFinished {
lock.lock()
let stillActivelyRecording = self.isRecordingActive && !self.isFinalizing
lock.unlock()
if stillActivelyRecording {
triggerMidRecordingReconnect(cleanArchive: true, reason: "Server session_done while recording is still active")
} else {
completeWithCurrentText()
}
}
} }
} }
@ -446,7 +827,15 @@ public final class SonioxLiveSession {
lock.lock() lock.lock()
let fullCommitted = committedFinalTokens.joined() let fullCommitted = committedFinalTokens.joined()
let fullNonFinal = currentNonFinalTokens.joined() let fullNonFinal = currentNonFinalTokens.joined()
let rawCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal)
let segCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal)
let rawCombined: String
if archivedCommittedText.isEmpty {
rawCombined = segCombined
} else if segCombined.isEmpty {
rawCombined = archivedCommittedText
} else {
rawCombined = archivedCommittedText + " " + segCombined.trimmingCharacters(in: .whitespaces)
}
lock.unlock() lock.unlock()
return SonioxLiveSession.sanitizeText(rawCombined) return SonioxLiveSession.sanitizeText(rawCombined)
} }
@ -457,10 +846,23 @@ public final class SonioxLiveSession {
lock.unlock() lock.unlock()
return return
} }
let tailCount = max(0, currentSegmentAudioBuffer.count - lastConfirmedWordAudioByteOffset)
let hadDrop = hasNetworkFailure || hasSilentStall || isReconnectingMidRecording || !isConnected
let needsTailRecovery = hadDrop && tailCount > 11200
isClosed = true isClosed = true
isReconnectingMidRecording = false
let fullCommitted = committedFinalTokens.joined() let fullCommitted = committedFinalTokens.joined()
let fullNonFinal = currentNonFinalTokens.joined() let fullNonFinal = currentNonFinalTokens.joined()
let rawCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal)
let segCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal)
let rawCombined: String
if archivedCommittedText.isEmpty {
rawCombined = segCombined
} else if segCombined.isEmpty {
rawCombined = archivedCommittedText
} else {
rawCombined = archivedCommittedText + " " + segCombined.trimmingCharacters(in: .whitespaces)
}
let cleaned = SonioxLiveSession.sanitizeText(rawCombined) let cleaned = SonioxLiveSession.sanitizeText(rawCombined)
let failureOccurred = hasNetworkFailure let failureOccurred = hasNetworkFailure
@ -474,7 +876,9 @@ public final class SonioxLiveSession {
stopKeepAliveHeartbeat() stopKeepAliveHeartbeat()
DispatchQueue.main.async { DispatchQueue.main.async {
if cleaned.isEmpty && failureOccurred {
if needsTailRecovery {
cb?(.failure(NSError(domain: "SonioxLiveSession", code: -103, userInfo: [NSLocalizedDescriptionKey: "Partial transcript with \(tailCount) bytes of untranscribed tail audio"])))
} else if cleaned.isEmpty && failureOccurred {
cb?(.failure(NSError(domain: "SonioxLiveSession", code: -101, userInfo: [NSLocalizedDescriptionKey: "WebSocket network failure with empty transcript"]))) cb?(.failure(NSError(domain: "SonioxLiveSession", code: -101, userInfo: [NSLocalizedDescriptionKey: "WebSocket network failure with empty transcript"])))
} else { } else {
cb?(.success(cleaned)) cb?(.success(cleaned))
@ -490,6 +894,10 @@ public final class SonioxLiveSession {
isConnected = false isConnected = false
isRecordingActive = false isRecordingActive = false
isFinalizing = false isFinalizing = false
isReconnectingMidRecording = false
isFlushingReplayBuffer = false
initialAudioBuffer.removeAll()
currentSegmentAudioBuffer.removeAll()
onPartialText = nil onPartialText = nil
onCommittedDelta = nil onCommittedDelta = nil
onEndTriggerDetected = nil onEndTriggerDetected = nil
@ -532,6 +940,128 @@ public final class SonioxLiveSession {
let durationSec = Double(pcmData.count) / 32000.0 let durationSec = Double(pcmData.count) / 32000.0
print("SonioxLiveSession: Initiating Smart Burst Recovery: \(pcmData.count) bytes (\(String(format: "%.1f", durationSec))s audio, target: \(effectiveTarget ?? "orig"))...") print("SonioxLiveSession: Initiating Smart Burst Recovery: \(pcmData.count) bytes (\(String(format: "%.1f", durationSec))s audio, target: \(effectiveTarget ?? "orig"))...")
// For multi-minute recordings (>60s = 1,920,000 bytes), split into ~60s silence-aligned chunks
// so individual WebSocket burst connections never hit server buffer limits or timeouts!
let maxSingleBurstBytes = 1_920_000
if pcmData.count > maxSingleBurstBytes {
let chunks = splitIntoSilenceAlignedChunks(pcmData: pcmData, targetChunkBytes: maxSingleBurstBytes)
print("SonioxLiveSession: Splitting long recovery audio (\(String(format: "%.1f", durationSec))s) into \(chunks.count) sequential chunks...")
var recoveredParts: [String] = []
func processChunk(at index: Int) {
if isCancelled() {
completion(.failure(NSError(domain: "SonioxLiveSession", code: -999, userInfo: [NSLocalizedDescriptionKey: "Cancelled by Escape"])))
return
}
if index >= chunks.count {
let combinedText = recoveredParts.joined(separator: " ").trimmingCharacters(in: .whitespacesAndNewlines)
if !combinedText.isEmpty {
completion(.success(combinedText))
} else {
completion(.failure(NSError(domain: "SonioxLiveSession", code: -104, userInfo: [NSLocalizedDescriptionKey: "Empty chunked recovery result"])))
}
return
}
DispatchQueue.main.async {
onStatusUpdate?("PART \(index + 1)/\(chunks.count)")
}
retrySingleChunk(
pcmData: chunks[index],
languageHints: languageHints,
effectiveTarget: effectiveTarget,
isCancelled: isCancelled,
onProgress: onProgress,
onStatusUpdate: onStatusUpdate
) { result in
switch result {
case .success(let partText):
let cleanPart = partText.trimmingCharacters(in: .whitespacesAndNewlines)
if !cleanPart.isEmpty {
recoveredParts.append(cleanPart)
}
processChunk(at: index + 1)
case .failure(let err):
if !recoveredParts.isEmpty {
// Salvage already-recovered chunks if a later chunk fails
let partial = recoveredParts.joined(separator: " ").trimmingCharacters(in: .whitespacesAndNewlines)
completion(.success(partial))
} else {
completion(.failure(err))
}
}
}
}
processChunk(at: 0)
return
}
retrySingleChunk(
pcmData: pcmData,
languageHints: languageHints,
effectiveTarget: effectiveTarget,
isCancelled: isCancelled,
onProgress: onProgress,
onStatusUpdate: onStatusUpdate,
completion: completion
)
}
/// Splits a multi-minute 16kHz 16-bit mono PCM buffer at low-energy silence boundaries near `targetChunkBytes`.
private static func splitIntoSilenceAlignedChunks(pcmData: Data, targetChunkBytes: Int) -> [Data] {
var chunks: [Data] = []
var offset = 0
let total = pcmData.count
let searchWindowBytes = 160_000 // Search within last 5.0s of each 60s window for lowest energy pause
let windowStepBytes = 3200 // 100ms window
while offset < total {
let remaining = total - offset
if remaining <= targetChunkBytes + 160_000 {
chunks.append(pcmData.subdata(in: offset..<total))
break
}
var bestCut = offset + targetChunkBytes
let searchStart = max(offset + targetChunkBytes - searchWindowBytes, offset + 64000)
let searchEnd = min(offset + targetChunkBytes + 32000, total - windowStepBytes)
if searchEnd > searchStart {
var minEnergy: Float = .greatestFiniteMagnitude
pcmData.withUnsafeBytes { rawBuf in
guard let basePtr = rawBuf.baseAddress else { return }
var pos = searchStart & ~1 // align to 2-byte Int16 sample
while pos < searchEnd {
let sampleCount = windowStepBytes / 2
let ptr = basePtr.advanced(by: pos).assumingMemoryBound(to: Int16.self)
var sumSq: Float = 0
for i in 0..<sampleCount {
let s = Float(ptr[i])
sumSq += s * s
}
if sumSq < minEnergy {
minEnergy = sumSq
bestCut = pos + (windowStepBytes / 2)
}
pos += windowStepBytes
}
}
}
bestCut = bestCut & ~1
chunks.append(pcmData.subdata(in: offset..<bestCut))
offset = bestCut
}
return chunks
}
private static func retrySingleChunk(
pcmData: Data,
languageHints: [String],
effectiveTarget: String?,
isCancelled: @escaping () -> Bool,
onProgress: ((String) -> Void)?,
onStatusUpdate: ((String) -> Void)?,
completion: @escaping (Result<String, Error>) -> Void
) {
// Multi-stage resilient retry pipeline across dual endpoints with automatic backoff: // Multi-stage resilient retry pipeline across dual endpoints with automatic backoff:
// Attempt 1: Primary WS // Attempt 1: Primary WS
// Attempt 2: Fallback WS // Attempt 2: Fallback WS
@ -614,19 +1144,8 @@ public final class SonioxLiveSession {
completion: @escaping (Result<String, Error>) -> Void completion: @escaping (Result<String, Error>) -> Void
) { ) {
let isTranslationMode = targetLanguage != nil && !targetLanguage!.isEmpty && targetLanguage != "off" let isTranslationMode = targetLanguage != nil && !targetLanguage!.isEmpty && targetLanguage != "off"
let hints = languageHints.map { "language_hints=\($0)" }.joined(separator: "&")
var queryParams: [String] = []
if !hints.isEmpty {
queryParams.append(hints)
}
queryParams.append("enable_speaker_diarization=false")
queryParams.append("enable_language_identification=true")
queryParams.append("enable_endpoint_detection=false")
queryParams.append("providers=soniox")
if let target = targetLanguage, !target.isEmpty, target != "off" {
queryParams.append("target_language=\(target)")
}
let urlStr = "\(endpoint)?" + queryParams.joined(separator: "&")
let queryStr = SonioxLiveSession.baseQuery(withHints: languageHints, targetLanguage: targetLanguage)
let urlStr = "\(endpoint)?\(queryStr)"
guard let url = URL(string: urlStr) else { guard let url = URL(string: urlStr) else {
completion(.failure(NSError(domain: "Soniox", code: -1, userInfo: [NSLocalizedDescriptionKey: "Invalid URL"]))) completion(.failure(NSError(domain: "Soniox", code: -1, userInfo: [NSLocalizedDescriptionKey: "Invalid URL"])))
return return
@ -637,7 +1156,7 @@ public final class SonioxLiveSession {
request.setValue(origin(for: endpoint), forHTTPHeaderField: "Origin") request.setValue(origin(for: endpoint), forHTTPHeaderField: "Origin")
let durationSec = Double(pcmData.count) / 32000.0 let durationSec = Double(pcmData.count) / 32000.0
let watchdogTimeout = max(6.5, min(25.0, durationSec * 0.60 + 5.0))
let watchdogTimeout = max(10.0, min(65.0, durationSec * 0.90 + 10.0))
request.timeoutInterval = 5.0 // Fast handshake timeout so dead endpoint fails over in <5s request.timeoutInterval = 5.0 // Fast handshake timeout so dead endpoint fails over in <5s
let config = URLSessionConfiguration.ephemeral let config = URLSessionConfiguration.ephemeral
@ -796,7 +1315,7 @@ public final class SonioxLiveSession {
if done || isCancelled() { return } if done || isCancelled() { return }
if offset >= pcmData.count { if offset >= pcmData.count {
let silence = Data(count: 6400)
let silence = Data(count: 19200)
task.send(.data(silence)) { _ in task.send(.data(silence)) { _ in
task.send(.string("{\"type\": \"finalize\"}")) { _ in task.send(.string("{\"type\": \"finalize\"}")) { _ in
task.send(.string("END")) { _ in } task.send(.string("END")) { _ in }
@ -863,19 +1382,13 @@ public final class SonioxLiveSession {
var cleaned: [String] = [] var cleaned: [String] = []
let stopWords: Set<String> = ["sex", "from", "no", "oh", "you", "all", "know", "that", "god", "loves", "men", "for", "day", "night", "mankind", "wanted", "moments"] let stopWords: Set<String> = ["sex", "from", "no", "oh", "you", "all", "know", "that", "god", "loves", "men", "for", "day", "night", "mankind", "wanted", "moments"]
let preserveWords: Set<String> = [
"one", "two", "three", "four", "five", "six", "seven", "eight", "nine",
"tab", "slash", "account", "telegram", "ide", "code", "chatgpt", "claude",
"github", "git", "firefox", "chrome", "safari", "figma", "terminal",
"notes", "music", "finder", "settings", "status", "cancel", "new", "close", "reopen", "search"
]
if faRatio >= 0.25 { if faRatio >= 0.25 {
for w in words { for w in words {
if matches(enRegex, in: w) && !matches(faRegex, in: w) { if matches(enRegex, in: w) && !matches(faRegex, in: w) {
let cleanW = w.lowercased().trimmingCharacters(in: punctCharSet) let cleanW = w.lowercased().trimmingCharacters(in: punctCharSet)
// Strip ONLY known acoustic noise hallucinations; preserve legitimate English technical terms (Soniox, Voice, Sync, Mac, Push, Talk, API, etc.)
if stopWords.contains(cleanW) { continue } if stopWords.contains(cleanW) { continue }
if faRatio >= 0.70 && !preserveWords.contains(cleanW) { continue }
} }
cleaned.append(w) cleaned.append(w)
} }
@ -994,12 +1507,25 @@ public final class SonioxSessionPool {
} }
let age = Date().timeIntervalSince(createdAt) let age = Date().timeIntervalSince(createdAt)
if let session = session, session.isReady && age < 55.0 {
if let session = session, session.isReady && age < 60.0 {
session.stopKeepAliveHeartbeat() session.stopKeepAliveHeartbeat()
print("SonioxSessionPool: Acquired HOT prewarmed session (age: \(String(format: "%.1f", age))s, 0ms connect delay!)") print("SonioxSessionPool: Acquired HOT prewarmed session (age: \(String(format: "%.1f", age))s, 0ms connect delay!)")
return session return session
} }
// If prewarmed session is connecting or slightly aged, wait up to 0.4s for it before aborting
if let session = session, !session.isReady && age < 15.0 {
let tStart = Date()
while Date().timeIntervalSince(tStart) < 0.40 {
if session.isReady {
session.stopKeepAliveHeartbeat()
print("SonioxSessionPool: Acquired session after \(Int(Date().timeIntervalSince(tStart)*1000))ms connection wait.")
return session
}
usleep(25000) // 25ms
}
}
session?.stopKeepAliveHeartbeat() session?.stopKeepAliveHeartbeat()
session?.abort() session?.abort()

47
src/TextProcessor.swift

@ -48,15 +48,15 @@ public final class TextProcessor {
if SonioxSettings.shared.voicePunctuation { if SonioxSettings.shared.voicePunctuation {
result = applyVoicePunctuation(result) result = applyVoicePunctuation(result)
} }
// 2. Domain Vocabulary Enhancement // 2. Domain Vocabulary Enhancement
if SonioxSettings.shared.domainVocabulary { if SonioxSettings.shared.domainVocabulary {
result = applyDomainVocabulary(result) result = applyDomainVocabulary(result)
} }
// 3. Number Formatting // 3. Number Formatting
result = applyNumberFormatting(result, format: SonioxSettings.shared.numberFormat) result = applyNumberFormatting(result, format: SonioxSettings.shared.numberFormat)
// 3.5. Sanitize foreign hallucinations & word loops // 3.5. Sanitize foreign hallucinations & word loops
result = SonioxLiveSession.sanitizeText(result) result = SonioxLiveSession.sanitizeText(result)
@ -64,14 +64,39 @@ public final class TextProcessor {
if SonioxSettings.shared.normalizeText { if SonioxSettings.shared.normalizeText {
result = normalizeText(result) result = normalizeText(result)
} }
// 5. Wake Word & End Trigger sanitization // 5. Wake Word & End Trigger sanitization
result = stripTrailingEndTrigger(result) result = stripTrailingEndTrigger(result)
return result.trimmingCharacters(in: .whitespacesAndNewlines) return result.trimmingCharacters(in: .whitespacesAndNewlines)
}
public func applyDomainVocabularyInline(_ text: String) -> String {
}
/// Lightweight inline formatter applied to live streaming chunks so words, domain terms,
/// and numbers match the final post-processed text in real time.
public func processLiveChunk(_ text: String) -> String {
if text.isEmpty { return "" }
let hadLeadingSpace = text.first?.isWhitespace == true
let hadTrailingSpace = text.last?.isWhitespace == true
var result = text
if SonioxSettings.shared.voicePunctuation {
result = applyVoicePunctuation(result)
}
if SonioxSettings.shared.domainVocabulary {
result = applyDomainVocabulary(result)
}
result = applyNumberFormatting(result, format: SonioxSettings.shared.numberFormat)
let resultHasLeadingSpace = result.first?.isWhitespace == true
let resultHasTrailingSpace = result.last?.isWhitespace == true
let trimmed = result.trimmingCharacters(in: .whitespaces)
if trimmed.isEmpty { return text }
let keepLeading = hadLeadingSpace || resultHasLeadingSpace
let keepTrailing = hadTrailingSpace || resultHasTrailingSpace
return (keepLeading ? " " : "") + trimmed + (keepTrailing ? " " : "")
}
public func applyDomainVocabularyInline(_ text: String) -> String {
return applyDomainVocabulary(text) return applyDomainVocabulary(text)
} }
@ -220,7 +245,6 @@ public final class TextProcessor {
(" شصت ", " 60 "), (" هفتاد ", " 70 "), (" هشتاد ", " 80 "), (" نود ", " 90 "), (" شصت ", " 60 "), (" هفتاد ", " 70 "), (" هشتاد ", " 80 "), (" نود ", " 90 "),
(" صد ", " 100 "), (" دویست ", " 200 "), (" سیصد ", " 300 "), (" چهارصد ", " 400 "), (" صد ", " 100 "), (" دویست ", " 200 "), (" سیصد ", " 300 "), (" چهارصد ", " 400 "),
(" پانصد ", " 500 "), (" ششصد ", " 600 "), (" هفتصد ", " 700 "), (" هشتصد ", " 800 "), (" نهصد ", " 900 "), (" پانصد ", " 500 "), (" ششصد ", " 600 "), (" هفتصد ", " 700 "), (" هشتصد ", " 800 "), (" نهصد ", " 900 "),
(" هزار ", " 1000 "), (" میلیون ", " 1000000 "),
(" صفر ", " 0 "), (" یک ", " 1 "), (" دو ", " 2 "), (" سه ", " 3 "), (" چهار ", " 4 "), (" صفر ", " 0 "), (" یک ", " 1 "), (" دو ", " 2 "), (" سه ", " 3 "), (" چهار ", " 4 "),
(" پنج ", " 5 "), (" شش ", " 6 "), (" هفت ", " 7 "), (" هشت ", " 8 "), (" نه ", " 9 ") (" پنج ", " 5 "), (" شش ", " 6 "), (" هفت ", " 7 "), (" هشت ", " 8 "), (" نه ", " 9 ")
] ]
@ -262,9 +286,12 @@ public final class TextProcessor {
t = t.replacingOccurrences(of: " " + p, with: p) t = t.replacingOccurrences(of: " " + p, with: p)
} }
t = t.replacingOccurrences(of: "...", with: "…") t = t.replacingOccurrences(of: "...", with: "…")
for p in [".", "،", "؛", "؟", "!", "…"] {
for p in ["،", "؛", "؟", "!", "…"] {
t = t.replacingOccurrences(of: p, with: p + " ") t = t.replacingOccurrences(of: p, with: p + " ")
} }
// Only add space after "." when it is NOT between two digits (preserve 4.3, 3.8, IP addresses)
let dotNotDecimal = try! NSRegularExpression(pattern: #"(\.)(?![0-9۰-۹\s\.])"#, options: [])
t = dotNotDecimal.stringByReplacingMatches(in: t, options: [], range: NSRange(location: 0, length: t.utf16.count), withTemplate: ". ")
t = t.replacingOccurrences(of: "…", with: "...") t = t.replacingOccurrences(of: "…", with: "...")
let speechFixes: [(String, String)] = [ let speechFixes: [(String, String)] = [
(" ماا ", " ما "), (" ماا ", " ما "),

3
src/main.swift

@ -1,5 +1,8 @@
import Cocoa import Cocoa
setbuf(stdout, nil)
setbuf(stderr, nil)
func checkSingleInstance() { func checkSingleInstance() {
let myPid = ProcessInfo.processInfo.processIdentifier let myPid = ProcessInfo.processInfo.processIdentifier

Loading…
Cancel
Save