From a4e1eef0a6761ebf572f4a677e36992eafe1e04f Mon Sep 17 00:00:00 2001 From: Ali Alavi Date: Mon, 26 Oct 2026 18:08:43 +0000 Subject: [PATCH] fix(streaming): restore real-time word sync with 1-word lookahead holdback, 350ms tail grace & 22ms paste guard (v2.13) --- Info.plist | 2 +- src/AppDelegate.swift | 3 ++- src/InputBoxManager.swift | 8 +++--- src/SonioxClient.swift | 55 ++++++++++++++++++++++++++------------- src/TextProcessor.swift | 43 +++++++++++++++++++++++------- 5 files changed, 77 insertions(+), 34 deletions(-) diff --git a/Info.plist b/Info.plist index 83001df..d6fdb03 100644 --- a/Info.plist +++ b/Info.plist @@ -19,7 +19,7 @@ CFBundlePackageType APPL CFBundleShortVersionString - 2.12 + 2.13 CFBundleVersion 1 LSMinimumSystemVersion diff --git a/src/AppDelegate.swift b/src/AppDelegate.swift index 4779c24..06ff40c 100644 --- a/src/AppDelegate.swift +++ b/src/AppDelegate.swift @@ -101,7 +101,8 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { self.stopRecording() } self.pendingStopWorkItem = work - DispatchQueue.main.asyncAfter(deadline: .now() + 0.65, execute: work) + // 0.35s (350ms) acoustic tail buffer captures trailing syllables at key release without sluggish delay + DispatchQueue.main.asyncAfter(deadline: .now() + 0.35, execute: work) } } } diff --git a/src/InputBoxManager.swift b/src/InputBoxManager.swift index 5d419b9..b426023 100644 --- a/src/InputBoxManager.swift +++ b/src/InputBoxManager.swift @@ -77,8 +77,8 @@ public final class InputBoxManager { } } - // Apply lightweight inline domain vocabulary so streamed words match final text - let normalizedDelta = SonioxSettings.shared.domainVocabulary ? TextProcessor.shared.applyDomainVocabularyInline(text) : text + // Apply lightweight inline formatting (domain vocabulary, voice punctuation, digits) so streamed words match final text + let normalizedDelta = TextProcessor.shared.processLiveChunk(text) lock.lock() let expectedGen = sessionGeneration @@ -390,13 +390,13 @@ public final class InputBoxManager { } } - /// Places text on NSPasteboard, posts Cmd+V, and waits 45ms so target app consumes this chunk BEFORE next clipboard write! + /// Places text on NSPasteboard, posts Cmd+V, and waits 22ms so target app consumes this chunk BEFORE next clipboard write! private func pasteSynchronously(_ text: String) { let pb = NSPasteboard.general pb.clearContents() pb.setString(text, forType: .string) postCleanCmdV() - usleep(45000) // 45ms guard ensures browser/app reads NSPasteboard before next chunk overwrites it + usleep(22000) // 22ms guard ensures browser/app reads NSPasteboard before next chunk overwrites it (~45 chunks/sec capacity) } private func postCleanCmdV() { diff --git a/src/SonioxClient.swift b/src/SonioxClient.swift index beab560..b83d0c0 100644 --- a/src/SonioxClient.swift +++ b/src/SonioxClient.swift @@ -328,9 +328,9 @@ public final class SonioxLiveSession { lock.unlock() print("SonioxLiveSession: Sending finalize packet...") - // 1. Send trailing 600ms (19200 bytes at 16kHz 16-bit PCM) silence frame + // 1. Send trailing 300ms (9600 bytes at 16kHz 16-bit PCM) silence frame // so acoustic CTC model cleanly flushes and resolves trailing fast speech syllables - let silenceChunk = Data(count: 19200) + let silenceChunk = Data(count: 9600) let silenceMsg = URLSessionWebSocketTask.Message.data(silenceChunk) task.send(silenceMsg) { _ in } @@ -493,16 +493,41 @@ public final class SonioxLiveSession { }() let isTerminatingMessage = gotFin || sessionEnded || sessionDone || isFinished || hasRawFinished - // Emit live streaming deltas STRICTLY from Soniox-finalized tokens (is_final == true) - // and ONLY during active speech, NEVER on terminal finalize packets! - // When terminating, the final text is safely delivered to completeWithCurrentText() for full normalization - // and atomic insertion, completely eliminating dual-paste race conditions! + // Emit live streaming deltas in real time as the user speaks! + // Soniox batches `is_final: true` tokens only every 6-8 seconds, which would make live typing feel lagging and out of sync. + // However, completed words in `combined` (with a 1-word acoustic lookahead holdback) are 100% stable and appear within ~150ms! + // We stream `fullCommitted` PLUS any completed words in `fullNonFinal` except the trailing word currently being uttered. var deltaToEmit: String? = nil - if !triggerDetected && !isTerminatingMessage && !self.isFinalizing && fullCommitted.count > streamedPrefix.count { - if fullCommitted.hasPrefix(streamedPrefix) { - let delta = String(fullCommitted.dropFirst(streamedPrefix.count)) - streamedPrefix = fullCommitted - deltaToEmit = delta + if !triggerDetected && !isTerminatingMessage && !self.isFinalizing { + var stableCandidate = fullCommitted + if let lastSpaceRange = fullNonFinal.rangeOfCharacter(from: .whitespacesAndNewlines, options: .backwards) { + let completedNonFinal = String(fullNonFinal[.. streamedPrefix.count { + if stableCandidate.hasPrefix(streamedPrefix) { + let delta = String(stableCandidate.dropFirst(streamedPrefix.count)) + streamedPrefix = stableCandidate + deltaToEmit = delta + } else { + let common = stableCandidate.commonPrefix(with: streamedPrefix) + if stableCandidate.count > common.count { + let delta = String(stableCandidate.dropFirst(common.count)) + streamedPrefix = stableCandidate + deltaToEmit = delta + } + } } } lock.unlock() @@ -962,19 +987,13 @@ public final class SonioxLiveSession { var cleaned: [String] = [] let stopWords: Set = ["sex", "from", "no", "oh", "you", "all", "know", "that", "god", "loves", "men", "for", "day", "night", "mankind", "wanted", "moments"] - let preserveWords: Set = [ - "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", - "tab", "slash", "account", "telegram", "ide", "code", "chatgpt", "claude", - "github", "git", "firefox", "chrome", "safari", "figma", "terminal", - "notes", "music", "finder", "settings", "status", "cancel", "new", "close", "reopen", "search" - ] if faRatio >= 0.25 { for w in words { if matches(enRegex, in: w) && !matches(faRegex, in: w) { let cleanW = w.lowercased().trimmingCharacters(in: punctCharSet) + // Strip ONLY known acoustic noise hallucinations; preserve legitimate English technical terms (Soniox, Voice, Sync, Mac, Push, Talk, API, etc.) if stopWords.contains(cleanW) { continue } - if faRatio >= 0.70 && !preserveWords.contains(cleanW) { continue } } cleaned.append(w) } diff --git a/src/TextProcessor.swift b/src/TextProcessor.swift index 876f9f0..05d50ad 100644 --- a/src/TextProcessor.swift +++ b/src/TextProcessor.swift @@ -48,15 +48,15 @@ public final class TextProcessor { if SonioxSettings.shared.voicePunctuation { result = applyVoicePunctuation(result) } - + // 2. Domain Vocabulary Enhancement if SonioxSettings.shared.domainVocabulary { result = applyDomainVocabulary(result) } - + // 3. Number Formatting result = applyNumberFormatting(result, format: SonioxSettings.shared.numberFormat) - + // 3.5. Sanitize foreign hallucinations & word loops result = SonioxLiveSession.sanitizeText(result) @@ -64,14 +64,35 @@ public final class TextProcessor { if SonioxSettings.shared.normalizeText { result = normalizeText(result) } - + // 5. Wake Word & End Trigger sanitization result = stripTrailingEndTrigger(result) - + return result.trimmingCharacters(in: .whitespacesAndNewlines) - } - - public func applyDomainVocabularyInline(_ text: String) -> String { + } + + /// Lightweight inline formatter applied to live streaming chunks so words, domain terms, + /// and numbers match the final post-processed text in real time. + public func processLiveChunk(_ text: String) -> String { + if text.isEmpty { return "" } + let hasLeadingSpace = text.first?.isWhitespace == true + let hasTrailingSpace = text.last?.isWhitespace == true + + var result = text + if SonioxSettings.shared.voicePunctuation { + result = applyVoicePunctuation(result) + } + if SonioxSettings.shared.domainVocabulary { + result = applyDomainVocabulary(result) + } + result = applyNumberFormatting(result, format: SonioxSettings.shared.numberFormat) + + let trimmed = result.trimmingCharacters(in: .whitespaces) + if trimmed.isEmpty { return text } + return (hasLeadingSpace ? " " : "") + trimmed + (hasTrailingSpace ? " " : "") + } + + public func applyDomainVocabularyInline(_ text: String) -> String { return applyDomainVocabulary(text) } @@ -220,7 +241,6 @@ public final class TextProcessor { (" شصت ", " 60 "), (" هفتاد ", " 70 "), (" هشتاد ", " 80 "), (" نود ", " 90 "), (" صد ", " 100 "), (" دویست ", " 200 "), (" سیصد ", " 300 "), (" چهارصد ", " 400 "), (" پانصد ", " 500 "), (" ششصد ", " 600 "), (" هفتصد ", " 700 "), (" هشتصد ", " 800 "), (" نهصد ", " 900 "), - (" هزار ", " 1000 "), (" میلیون ", " 1000000 "), (" صفر ", " 0 "), (" یک ", " 1 "), (" دو ", " 2 "), (" سه ", " 3 "), (" چهار ", " 4 "), (" پنج ", " 5 "), (" شش ", " 6 "), (" هفت ", " 7 "), (" هشت ", " 8 "), (" نه ", " 9 ") ] @@ -262,9 +282,12 @@ public final class TextProcessor { t = t.replacingOccurrences(of: " " + p, with: p) } t = t.replacingOccurrences(of: "...", with: "…") - for p in [".", "،", "؛", "؟", "!", "…"] { + for p in ["،", "؛", "؟", "!", "…"] { t = t.replacingOccurrences(of: p, with: p + " ") } + // Only add space after "." when it is NOT between two digits (preserve 4.3, 3.8, IP addresses) + let dotNotDecimal = try! NSRegularExpression(pattern: #"(\.)(?![0-9۰-۹\s\.])"#, options: []) + t = dotNotDecimal.stringByReplacingMatches(in: t, options: [], range: NSRange(location: 0, length: t.utf16.count), withTemplate: ". ") t = t.replacingOccurrences(of: "…", with: "...") let speechFixes: [(String, String)] = [ (" ماا ", " ما "),