Browse Source

fix(streaming): restore real-time word sync with 1-word lookahead holdback, 350ms tail grace & 22ms paste guard (v2.13)

main
Ali Alavi 3 weeks ago
parent
commit
a4e1eef0a6
  1. 2
      Info.plist
  2. 3
      src/AppDelegate.swift
  3. 8
      src/InputBoxManager.swift
  4. 55
      src/SonioxClient.swift
  5. 43
      src/TextProcessor.swift

2
Info.plist

@ -19,7 +19,7 @@
<key>CFBundlePackageType</key> <key>CFBundlePackageType</key>
<string>APPL</string> <string>APPL</string>
<key>CFBundleShortVersionString</key> <key>CFBundleShortVersionString</key>
<string>2.12</string>
<string>2.13</string>
<key>CFBundleVersion</key> <key>CFBundleVersion</key>
<string>1</string> <string>1</string>
<key>LSMinimumSystemVersion</key> <key>LSMinimumSystemVersion</key>

3
src/AppDelegate.swift

@ -101,7 +101,8 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
self.stopRecording() self.stopRecording()
} }
self.pendingStopWorkItem = work self.pendingStopWorkItem = work
DispatchQueue.main.asyncAfter(deadline: .now() + 0.65, execute: work)
// 0.35s (350ms) acoustic tail buffer captures trailing syllables at key release without sluggish delay
DispatchQueue.main.asyncAfter(deadline: .now() + 0.35, execute: work)
} }
} }
} }

8
src/InputBoxManager.swift

@ -77,8 +77,8 @@ public final class InputBoxManager {
} }
} }
// Apply lightweight inline domain vocabulary so streamed words match final text
let normalizedDelta = SonioxSettings.shared.domainVocabulary ? TextProcessor.shared.applyDomainVocabularyInline(text) : text
// Apply lightweight inline formatting (domain vocabulary, voice punctuation, digits) so streamed words match final text
let normalizedDelta = TextProcessor.shared.processLiveChunk(text)
lock.lock() lock.lock()
let expectedGen = sessionGeneration let expectedGen = sessionGeneration
@ -390,13 +390,13 @@ public final class InputBoxManager {
} }
} }
/// Places text on NSPasteboard, posts Cmd+V, and waits 45ms so target app consumes this chunk BEFORE next clipboard write!
/// Places text on NSPasteboard, posts Cmd+V, and waits 22ms so target app consumes this chunk BEFORE next clipboard write!
private func pasteSynchronously(_ text: String) { private func pasteSynchronously(_ text: String) {
let pb = NSPasteboard.general let pb = NSPasteboard.general
pb.clearContents() pb.clearContents()
pb.setString(text, forType: .string) pb.setString(text, forType: .string)
postCleanCmdV() postCleanCmdV()
usleep(45000) // 45ms guard ensures browser/app reads NSPasteboard before next chunk overwrites it
usleep(22000) // 22ms guard ensures browser/app reads NSPasteboard before next chunk overwrites it (~45 chunks/sec capacity)
} }
private func postCleanCmdV() { private func postCleanCmdV() {

55
src/SonioxClient.swift

@ -328,9 +328,9 @@ public final class SonioxLiveSession {
lock.unlock() lock.unlock()
print("SonioxLiveSession: Sending finalize packet...") print("SonioxLiveSession: Sending finalize packet...")
// 1. Send trailing 600ms (19200 bytes at 16kHz 16-bit PCM) silence frame
// 1. Send trailing 300ms (9600 bytes at 16kHz 16-bit PCM) silence frame
// so acoustic CTC model cleanly flushes and resolves trailing fast speech syllables // so acoustic CTC model cleanly flushes and resolves trailing fast speech syllables
let silenceChunk = Data(count: 19200)
let silenceChunk = Data(count: 9600)
let silenceMsg = URLSessionWebSocketTask.Message.data(silenceChunk) let silenceMsg = URLSessionWebSocketTask.Message.data(silenceChunk)
task.send(silenceMsg) { _ in } task.send(silenceMsg) { _ in }
@ -493,16 +493,41 @@ public final class SonioxLiveSession {
}() }()
let isTerminatingMessage = gotFin || sessionEnded || sessionDone || isFinished || hasRawFinished let isTerminatingMessage = gotFin || sessionEnded || sessionDone || isFinished || hasRawFinished
// Emit live streaming deltas STRICTLY from Soniox-finalized tokens (is_final == true)
// and ONLY during active speech, NEVER on terminal finalize packets!
// When terminating, the final text is safely delivered to completeWithCurrentText() for full normalization
// and atomic insertion, completely eliminating dual-paste race conditions!
// Emit live streaming deltas in real time as the user speaks!
// Soniox batches `is_final: true` tokens only every 6-8 seconds, which would make live typing feel lagging and out of sync.
// However, completed words in `combined` (with a 1-word acoustic lookahead holdback) are 100% stable and appear within ~150ms!
// We stream `fullCommitted` PLUS any completed words in `fullNonFinal` except the trailing word currently being uttered.
var deltaToEmit: String? = nil var deltaToEmit: String? = nil
if !triggerDetected && !isTerminatingMessage && !self.isFinalizing && fullCommitted.count > streamedPrefix.count {
if fullCommitted.hasPrefix(streamedPrefix) {
let delta = String(fullCommitted.dropFirst(streamedPrefix.count))
streamedPrefix = fullCommitted
deltaToEmit = delta
if !triggerDetected && !isTerminatingMessage && !self.isFinalizing {
var stableCandidate = fullCommitted
if let lastSpaceRange = fullNonFinal.rangeOfCharacter(from: .whitespacesAndNewlines, options: .backwards) {
let completedNonFinal = String(fullNonFinal[..<lastSpaceRange.upperBound])
let trailingToken = String(fullNonFinal[lastSpaceRange.upperBound...]).trimmingCharacters(in: .whitespacesAndNewlines)
if !trailingToken.isEmpty {
// At least one subsequent word has started, so `completedNonFinal` has full acoustic lookahead
stableCandidate = fullCommitted + completedNonFinal
} else {
// Trailing space at the very end of `fullNonFinal`: hold back the last word until the next word begins
let trimmedNF = completedNonFinal.trimmingCharacters(in: .whitespacesAndNewlines)
if let prevSpaceRange = trimmedNF.rangeOfCharacter(from: .whitespacesAndNewlines, options: .backwards) {
stableCandidate = fullCommitted + String(trimmedNF[..<prevSpaceRange.upperBound])
}
}
}
if stableCandidate.count > streamedPrefix.count {
if stableCandidate.hasPrefix(streamedPrefix) {
let delta = String(stableCandidate.dropFirst(streamedPrefix.count))
streamedPrefix = stableCandidate
deltaToEmit = delta
} else {
let common = stableCandidate.commonPrefix(with: streamedPrefix)
if stableCandidate.count > common.count {
let delta = String(stableCandidate.dropFirst(common.count))
streamedPrefix = stableCandidate
deltaToEmit = delta
}
}
} }
} }
lock.unlock() lock.unlock()
@ -962,19 +987,13 @@ public final class SonioxLiveSession {
var cleaned: [String] = [] var cleaned: [String] = []
let stopWords: Set<String> = ["sex", "from", "no", "oh", "you", "all", "know", "that", "god", "loves", "men", "for", "day", "night", "mankind", "wanted", "moments"] let stopWords: Set<String> = ["sex", "from", "no", "oh", "you", "all", "know", "that", "god", "loves", "men", "for", "day", "night", "mankind", "wanted", "moments"]
let preserveWords: Set<String> = [
"one", "two", "three", "four", "five", "six", "seven", "eight", "nine",
"tab", "slash", "account", "telegram", "ide", "code", "chatgpt", "claude",
"github", "git", "firefox", "chrome", "safari", "figma", "terminal",
"notes", "music", "finder", "settings", "status", "cancel", "new", "close", "reopen", "search"
]
if faRatio >= 0.25 { if faRatio >= 0.25 {
for w in words { for w in words {
if matches(enRegex, in: w) && !matches(faRegex, in: w) { if matches(enRegex, in: w) && !matches(faRegex, in: w) {
let cleanW = w.lowercased().trimmingCharacters(in: punctCharSet) let cleanW = w.lowercased().trimmingCharacters(in: punctCharSet)
// Strip ONLY known acoustic noise hallucinations; preserve legitimate English technical terms (Soniox, Voice, Sync, Mac, Push, Talk, API, etc.)
if stopWords.contains(cleanW) { continue } if stopWords.contains(cleanW) { continue }
if faRatio >= 0.70 && !preserveWords.contains(cleanW) { continue }
} }
cleaned.append(w) cleaned.append(w)
} }

43
src/TextProcessor.swift

@ -48,15 +48,15 @@ public final class TextProcessor {
if SonioxSettings.shared.voicePunctuation { if SonioxSettings.shared.voicePunctuation {
result = applyVoicePunctuation(result) result = applyVoicePunctuation(result)
} }
// 2. Domain Vocabulary Enhancement // 2. Domain Vocabulary Enhancement
if SonioxSettings.shared.domainVocabulary { if SonioxSettings.shared.domainVocabulary {
result = applyDomainVocabulary(result) result = applyDomainVocabulary(result)
} }
// 3. Number Formatting // 3. Number Formatting
result = applyNumberFormatting(result, format: SonioxSettings.shared.numberFormat) result = applyNumberFormatting(result, format: SonioxSettings.shared.numberFormat)
// 3.5. Sanitize foreign hallucinations & word loops // 3.5. Sanitize foreign hallucinations & word loops
result = SonioxLiveSession.sanitizeText(result) result = SonioxLiveSession.sanitizeText(result)
@ -64,14 +64,35 @@ public final class TextProcessor {
if SonioxSettings.shared.normalizeText { if SonioxSettings.shared.normalizeText {
result = normalizeText(result) result = normalizeText(result)
} }
// 5. Wake Word & End Trigger sanitization // 5. Wake Word & End Trigger sanitization
result = stripTrailingEndTrigger(result) result = stripTrailingEndTrigger(result)
return result.trimmingCharacters(in: .whitespacesAndNewlines) return result.trimmingCharacters(in: .whitespacesAndNewlines)
}
public func applyDomainVocabularyInline(_ text: String) -> String {
}
/// Lightweight inline formatter applied to live streaming chunks so words, domain terms,
/// and numbers match the final post-processed text in real time.
public func processLiveChunk(_ text: String) -> String {
if text.isEmpty { return "" }
let hasLeadingSpace = text.first?.isWhitespace == true
let hasTrailingSpace = text.last?.isWhitespace == true
var result = text
if SonioxSettings.shared.voicePunctuation {
result = applyVoicePunctuation(result)
}
if SonioxSettings.shared.domainVocabulary {
result = applyDomainVocabulary(result)
}
result = applyNumberFormatting(result, format: SonioxSettings.shared.numberFormat)
let trimmed = result.trimmingCharacters(in: .whitespaces)
if trimmed.isEmpty { return text }
return (hasLeadingSpace ? " " : "") + trimmed + (hasTrailingSpace ? " " : "")
}
public func applyDomainVocabularyInline(_ text: String) -> String {
return applyDomainVocabulary(text) return applyDomainVocabulary(text)
} }
@ -220,7 +241,6 @@ public final class TextProcessor {
(" شصت ", " 60 "), (" هفتاد ", " 70 "), (" هشتاد ", " 80 "), (" نود ", " 90 "), (" شصت ", " 60 "), (" هفتاد ", " 70 "), (" هشتاد ", " 80 "), (" نود ", " 90 "),
(" صد ", " 100 "), (" دویست ", " 200 "), (" سیصد ", " 300 "), (" چهارصد ", " 400 "), (" صد ", " 100 "), (" دویست ", " 200 "), (" سیصد ", " 300 "), (" چهارصد ", " 400 "),
(" پانصد ", " 500 "), (" ششصد ", " 600 "), (" هفتصد ", " 700 "), (" هشتصد ", " 800 "), (" نهصد ", " 900 "), (" پانصد ", " 500 "), (" ششصد ", " 600 "), (" هفتصد ", " 700 "), (" هشتصد ", " 800 "), (" نهصد ", " 900 "),
(" هزار ", " 1000 "), (" میلیون ", " 1000000 "),
(" صفر ", " 0 "), (" یک ", " 1 "), (" دو ", " 2 "), (" سه ", " 3 "), (" چهار ", " 4 "), (" صفر ", " 0 "), (" یک ", " 1 "), (" دو ", " 2 "), (" سه ", " 3 "), (" چهار ", " 4 "),
(" پنج ", " 5 "), (" شش ", " 6 "), (" هفت ", " 7 "), (" هشت ", " 8 "), (" نه ", " 9 ") (" پنج ", " 5 "), (" شش ", " 6 "), (" هفت ", " 7 "), (" هشت ", " 8 "), (" نه ", " 9 ")
] ]
@ -262,9 +282,12 @@ public final class TextProcessor {
t = t.replacingOccurrences(of: " " + p, with: p) t = t.replacingOccurrences(of: " " + p, with: p)
} }
t = t.replacingOccurrences(of: "...", with: "…") t = t.replacingOccurrences(of: "...", with: "…")
for p in [".", "،", "؛", "؟", "!", "…"] {
for p in ["،", "؛", "؟", "!", "…"] {
t = t.replacingOccurrences(of: p, with: p + " ") t = t.replacingOccurrences(of: p, with: p + " ")
} }
// Only add space after "." when it is NOT between two digits (preserve 4.3, 3.8, IP addresses)
let dotNotDecimal = try! NSRegularExpression(pattern: #"(\.)(?![0-9۰-۹\s\.])"#, options: [])
t = dotNotDecimal.stringByReplacingMatches(in: t, options: [], range: NSRange(location: 0, length: t.utf16.count), withTemplate: ". ")
t = t.replacingOccurrences(of: "…", with: "...") t = t.replacingOccurrences(of: "…", with: "...")
let speechFixes: [(String, String)] = [ let speechFixes: [(String, String)] = [
(" ماا ", " ما "), (" ماا ", " ما "),

Loading…
Cancel
Save