diff --git a/src/AppDelegate.swift b/src/AppDelegate.swift index 9a8fe16..e04f5c3 100644 --- a/src/AppDelegate.swift +++ b/src/AppDelegate.swift @@ -10,6 +10,8 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { private var currentAudioLevel: Float = 0.0 private var latestPartialText: String? = nil private var pendingStopWorkItem: DispatchWorkItem? = nil + private var isNetworkDegraded: Bool = false + private var lastRecordedAudio: Data = Data() public func applicationDidFinishLaunching(_ notification: Notification) { if UserDefaults.standard.object(forKey: "SonioxPlaySounds") == nil { @@ -60,7 +62,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { guard let self = self else { return } self.currentAudioLevel = level if self.audioRecorder.isRecording { - HUDOverlayController.shared.show(state: .recording(level: level, liveText: self.latestPartialText)) + HUDOverlayController.shared.show(state: .recording(level: level, liveText: self.latestPartialText, isLagging: self.isNetworkDegraded)) } } @@ -109,47 +111,47 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { if audioRecorder.isRecording || isBusyFinalizing { return } latestPartialText = nil + isNetworkDegraded = false + lastRecordedAudio.removeAll() - // Acquire pre-warmed / fast-connected WebSocket session + // 1. Find or focus an input box in the active window before user starts speaking + InputBoxManager.shared.ensureFocusedInputBox() + + // 2. Acquire pre-warmed / fast-connected WebSocket session let session = SonioxSessionPool.shared.acquireSession() self.activeSession = session + // Handle network degradation / packet loss warning + session.onNetworkDegraded = { [weak self] degraded in + guard let self = self else { return } + self.isNetworkDegraded = degraded + if self.audioRecorder.isRecording { + HUDOverlayController.shared.show(state: .recording(level: self.currentAudioLevel, liveText: self.latestPartialText, isLagging: degraded)) + } + } + + // Handle streaming delta text (incremental committed words) + session.onCommittedDelta = { [weak self] delta in + guard let self = self else { return } + // If it's an executive slash command, do NOT type it into the input box + if SonioxSettings.shared.voiceCommands && VoiceCommandExecutor.shared.isSlashCommand(text: delta) { + return + } + InputBoxManager.shared.streamDelta(text: delta) + } + session.onPartialText = { [weak self] partial in guard let self = self else { return } self.latestPartialText = partial if self.audioRecorder.isRecording { - HUDOverlayController.shared.show(state: .recording(level: self.currentAudioLevel, liveText: partial)) + HUDOverlayController.shared.show(state: .recording(level: self.currentAudioLevel, liveText: partial, isLagging: self.isNetworkDegraded)) } } session.onFinalResult = { [weak self] result in DispatchQueue.main.async { guard let self = self else { return } - self.isBusyFinalizing = false - self.activeSession = nil - self.statusBarController.updateIcon(state: .idle) - - switch result { - case .success(let text): - let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { return } - - // 1. Check for Executive Voice Slash Commands - if SonioxSettings.shared.voiceCommands && VoiceCommandExecutor.shared.isSlashCommand(text: trimmed) { - VoiceCommandExecutor.shared.execute(rawText: trimmed) - return // STRICT: If speech starts with slash, NEVER paste into any active input box! - } - - // 2. Normal dictation processing & paste - let processed = TextProcessor.shared.process(text: trimmed) - if !processed.isEmpty { - VoiceCommandLogger.shared.logDictation(rawSpeech: trimmed, processed: processed) - HistoryManager.shared.addEntry(processed) - self.pasteTextToFrontmostApp(text: processed) - } - case .failure(let error): - print("Soniox finalize error:", error) - } + self.handleFinalResult(result) } } @@ -158,7 +160,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { playSystemSound(name: "Tink") statusBarController.updateIcon(state: .recording) statusBarController.buildMenu(isRecording: true) - HUDOverlayController.shared.show(state: .recording(level: 0.1, liveText: nil)) + HUDOverlayController.shared.show(state: .recording(level: 0.1, liveText: nil, isLagging: false)) } catch { print("Failed to start audio recording:", error) activeSession?.cancel() @@ -170,7 +172,9 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { public func stopRecording() { guard audioRecorder.isRecording else { return } - _ = audioRecorder.stopRecording() + let pcmData = audioRecorder.stopRecording() + self.lastRecordedAudio = pcmData + playSystemSound(name: "Pop") statusBarController.updateIcon(state: .transcribing) statusBarController.buildMenu(isRecording: false) @@ -178,8 +182,8 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { isBusyFinalizing = true - // Safety timeout: If internet drops or server halts during finalization, guarantee completion within 3.5s - DispatchQueue.main.asyncAfter(deadline: .now() + 3.5) { [weak self] in + // Safety timeout watchdog: finalize must complete within 4.0s + DispatchQueue.main.asyncAfter(deadline: .now() + 4.0) { [weak self] in guard let self = self, self.isBusyFinalizing else { return } print("AppDelegate: Finalize safety watchdog triggered — recovering transcript.") self.activeSession?.completeWithCurrentText() @@ -188,35 +192,73 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { activeSession?.finalizeStream() } - private func pasteTextToFrontmostApp(text: String) { - guard !text.isEmpty else { return } - - // 1. Put into system pasteboard - let textToPaste = SonioxSettings.shared.addTrailingSpace ? (text + " ") : text - let pb = NSPasteboard.general - pb.clearContents() - pb.setString(textToPaste, forType: .string) + private func handleFinalResult(_ result: Result) { + self.activeSession = nil + self.statusBarController.updateIcon(state: .idle) - // 2. Check Accessibility permission - if !AXIsProcessTrusted() { - print("Accessibility permission NOT granted for auto-paste!") - let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary - AXIsProcessTrustedWithOptions(options) - return + var rawText = "" + switch result { + case .success(let text): + rawText = text.trimmingCharacters(in: .whitespacesAndNewlines) + case .failure(let error): + print("Soniox primary finalize error:", error) } - // 3. Instant single paste via CGEvent - DispatchQueue.main.asyncAfter(deadline: .now() + 0.03) { - let source = CGEventSource(stateID: .combinedSessionState) - let vKeyCode: CGKeyCode = 0x09 // 'v' key + // Check if smart audio recovery is needed: + // If text is empty or we experienced network failure, and we captured substantial audio (> 0.8s) + let needsRetry = (rawText.isEmpty || self.isNetworkDegraded) && self.lastRecordedAudio.count > 25600 + + if needsRetry { + print("AppDelegate: Network drop or truncated output detected. Triggering Smart Audio Recovery with \(self.lastRecordedAudio.count) bytes...") + HUDOverlayController.shared.show(state: .retrying(message: "اینترنت ضعیف... در حال بازیابی صوت")) - if let keyDown = CGEvent(keyboardEventSource: source, virtualKey: vKeyCode, keyDown: true), - let keyUp = CGEvent(keyboardEventSource: source, virtualKey: vKeyCode, keyDown: false) { - keyDown.flags = .maskCommand - keyUp.flags = .maskCommand - keyDown.post(tap: .cghidEventTap) - keyUp.post(tap: .cghidEventTap) + let audioCopy = self.lastRecordedAudio + let hints = SonioxSettings.shared.language.hints + + SonioxLiveSession.retryFullBufferedAudio(pcmData: audioCopy, languageHints: hints, onProgress: { progress in + HUDOverlayController.shared.show(state: .recording(level: 0.2, liveText: progress, isLagging: true)) + }) { [weak self] retryResult in + guard let self = self else { return } + self.isBusyFinalizing = false + + switch retryResult { + case .success(let recoveredText): + let trimmed = recoveredText.trimmingCharacters(in: .whitespacesAndNewlines) + let finalText = trimmed.isEmpty ? rawText : trimmed + self.deliverOutputText(finalText) + case .failure: + self.deliverOutputText(rawText) + } } + return + } + + self.isBusyFinalizing = false + deliverOutputText(rawText) + } + + private func deliverOutputText(_ text: String) { + guard !text.isEmpty else { + HUDOverlayController.shared.hide() + return + } + + // 1. Check for Executive Voice Slash Commands + if SonioxSettings.shared.voiceCommands && VoiceCommandExecutor.shared.isSlashCommand(text: text) { + HUDOverlayController.shared.show(state: .success(text: "دستور صوتی")) + VoiceCommandExecutor.shared.execute(rawText: text) + return // STRICT: Slash commands are executed, never pasted! + } + + // 2. Normal dictation processing & paste + let processed = TextProcessor.shared.process(text: text) + if !processed.isEmpty { + VoiceCommandLogger.shared.logDictation(rawSpeech: text, processed: processed) + HistoryManager.shared.addEntry(processed) + InputBoxManager.shared.commitFinalText(processedText: processed, addTrailingSpace: SonioxSettings.shared.addTrailingSpace) + HUDOverlayController.shared.show(state: .success(text: processed)) + } else { + HUDOverlayController.shared.hide() } } diff --git a/src/HUDOverlay.swift b/src/HUDOverlay.swift index eb38a09..8495fbf 100644 --- a/src/HUDOverlay.swift +++ b/src/HUDOverlay.swift @@ -1,8 +1,10 @@ import Cocoa +import QuartzCore public enum HUDState { case hidden - case recording(level: Float, liveText: String? = nil) + case recording(level: Float, liveText: String? = nil, isLagging: Bool = false) + case retrying(message: String) case transcribing case success(text: String) case error(message: String) @@ -10,7 +12,306 @@ public enum HUDState { public final class HUDOverlayController { public static let shared = HUDOverlayController() - private init() {} - public func show(state: HUDState) {} - public func hide() {} + + private var window: NSPanel? + private var orbView: SiriOrbView? + private var statusLabel: NSTextField? + private var isVisible = false + + private init() { + DispatchQueue.main.async { + self.setupWindow() + } + } + + private func setupWindow() { + let panel = NSPanel( + contentRect: NSRect(x: 0, y: 0, width: 220, height: 96), + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: false + ) + panel.isOpaque = false + panel.backgroundColor = .clear + panel.level = .floating + panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary] + panel.hasShadow = false + panel.ignoresMouseEvents = true + panel.alphaValue = 0.0 + + let container = NSView(frame: panel.contentView!.bounds) + container.wantsLayer = true + + // Siri-like Circular Orb (Center) + let orbSize: CGFloat = 58 + let orb = SiriOrbView(frame: NSRect(x: (220 - orbSize) / 2, y: 30, width: orbSize, height: orbSize)) + container.addSubview(orb) + self.orbView = orb + + // Caption / Status Pill under the Orb + let label = NSTextField(frame: NSRect(x: 10, y: 4, width: 200, height: 22)) + label.isBezeled = false + label.drawsBackground = false + label.isEditable = false + label.isSelectable = false + label.alignment = .center + label.textColor = NSColor.white.withAlphaComponent(0.9) + label.font = NSFont.systemFont(ofSize: 11, weight: .medium) + label.stringValue = "" + container.addSubview(label) + self.statusLabel = label + + panel.contentView = container + self.window = panel + reposition() + } + + private func reposition() { + guard let window = self.window, let screen = NSScreen.main else { return } + let screenRect = screen.visibleFrame + let x = screenRect.midX - (window.frame.width / 2) + let y = screenRect.minY + 36 + window.setFrameOrigin(NSPoint(x: x, y: y)) + } + + public func show(state: HUDState) { + DispatchQueue.main.async { [weak self] in + guard let self = self else { return } + if self.window == nil { + self.setupWindow() + } + guard let window = self.window, let orb = self.orbView else { return } + self.reposition() + + switch state { + case .hidden: + self.hide() + return + + case .recording(let level, let liveText, let isLagging): + if !self.isVisible { + window.orderFrontRegardless() + window.animator().alphaValue = 1.0 + self.isVisible = true + } + orb.updateAudioLevel(level, isLagging: isLagging) + if isLagging { + self.statusLabel?.stringValue = "اینترنت ضعیف... در حال دریافت" + self.statusLabel?.textColor = NSColor(red: 1.0, green: 0.75, blue: 0.2, alpha: 1.0) + } else if let txt = liveText, !txt.isEmpty { + // Show compact preview of last few words + let words = txt.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } + let tail = words.suffix(4).joined(separator: " ") + self.statusLabel?.stringValue = tail + self.statusLabel?.textColor = NSColor.white.withAlphaComponent(0.85) + } else { + self.statusLabel?.stringValue = "گوش می‌دهم..." + self.statusLabel?.textColor = NSColor.white.withAlphaComponent(0.7) + } + + case .retrying(let message): + if !self.isVisible { + window.orderFrontRegardless() + window.animator().alphaValue = 1.0 + self.isVisible = true + } + orb.showRetryState() + self.statusLabel?.stringValue = message + self.statusLabel?.textColor = NSColor(red: 1.0, green: 0.65, blue: 0.1, alpha: 1.0) + + case .transcribing: + orb.showTranscribingState() + self.statusLabel?.stringValue = "تبدیل به متن..." + self.statusLabel?.textColor = NSColor.white.withAlphaComponent(0.8) + + case .success(let text): + orb.showSuccessState() + let words = text.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } + let preview = words.suffix(3).joined(separator: " ") + self.statusLabel?.stringValue = preview.isEmpty ? "انجام شد" : preview + self.statusLabel?.textColor = NSColor(red: 0.3, green: 0.9, blue: 0.4, alpha: 1.0) + DispatchQueue.main.asyncAfter(deadline: .now() + 0.6) { + self.hide() + } + + case .error(let msg): + orb.showErrorState() + self.statusLabel?.stringValue = msg + self.statusLabel?.textColor = NSColor(red: 1.0, green: 0.3, blue: 0.3, alpha: 1.0) + DispatchQueue.main.asyncAfter(deadline: .now() + 1.2) { + self.hide() + } + } + } + } + + public func hide() { + DispatchQueue.main.async { [weak self] in + guard let self = self, let window = self.window, self.isVisible else { return } + NSAnimationContext.runAnimationGroup({ context in + context.duration = 0.25 + window.animator().alphaValue = 0.0 + }, completionHandler: { + window.orderOut(nil) + self.isVisible = false + self.orbView?.reset() + }) + } + } +} + +// MARK: - Apple Siri-like Gradient Orb View + +final class SiriOrbView: NSView { + private let gradientLayer = CAGradientLayer() + private let glowLayer = CALayer() + private let backdropLayer = CALayer() + private var baseRadius: CGFloat = 29 + + override init(frame frameRect: NSRect) { + super.init(frame: frameRect) + wantsLayer = true + setupLayers() + } + + required init?(coder: NSCoder) { + super.init(coder: coder) + wantsLayer = true + setupLayers() + } + + private func setupLayers() { + guard let layer = self.layer else { return } + layer.masksToBounds = false + + let bounds = self.bounds + baseRadius = bounds.width / 2.0 + + // 1. Soft Outer Glow + glowLayer.frame = bounds.insetBy(dx: -10, dy: -10) + glowLayer.cornerRadius = (bounds.width + 20) / 2.0 + glowLayer.backgroundColor = NSColor(red: 0.0, green: 0.6, blue: 1.0, alpha: 0.35).cgColor + glowLayer.shadowColor = NSColor(red: 0.4, green: 0.0, blue: 1.0, alpha: 0.6).cgColor + glowLayer.shadowRadius = 14 + glowLayer.shadowOpacity = 0.8 + glowLayer.shadowOffset = .zero + layer.addSublayer(glowLayer) + + // 2. Dark Translucent Backdrop + backdropLayer.frame = bounds + backdropLayer.cornerRadius = baseRadius + backdropLayer.backgroundColor = NSColor.black.withAlphaComponent(0.65).cgColor + backdropLayer.borderColor = NSColor.white.withAlphaComponent(0.2).cgColor + backdropLayer.borderWidth = 1.0 + layer.addSublayer(backdropLayer) + + // 3. Siri Multicolored Gradient Core + gradientLayer.frame = bounds.insetBy(dx: 4, dy: 4) + gradientLayer.cornerRadius = (bounds.width - 8) / 2.0 + gradientLayer.colors = [ + NSColor(red: 0.0, green: 0.85, blue: 1.0, alpha: 0.95).cgColor, // Cyan + NSColor(red: 0.45, green: 0.15, blue: 0.95, alpha: 0.95).cgColor, // Violet + NSColor(red: 0.95, green: 0.05, blue: 0.65, alpha: 0.9).cgColor // Magenta + ] + gradientLayer.startPoint = CGPoint(x: 0.0, y: 0.0) + gradientLayer.endPoint = CGPoint(x: 1.0, y: 1.0) + layer.addSublayer(gradientLayer) + + // Subtle constant breathing idle animation + startOrbRotation() + } + + private func startOrbRotation() { + let rotate = CABasicAnimation(keyPath: "transform.rotation.z") + rotate.fromValue = 0.0 + rotate.toValue = 2 * Double.pi + rotate.duration = 4.0 + rotate.repeatCount = .infinity + rotate.isRemovedOnCompletion = false + gradientLayer.add(rotate, forKey: "siriRotate") + } + + func updateAudioLevel(_ level: Float, isLagging: Bool) { + CATransaction.begin() + CATransaction.setDisableActions(true) + + if isLagging { + // Amber warm warning colors + gradientLayer.colors = [ + NSColor(red: 1.0, green: 0.6, blue: 0.0, alpha: 0.95).cgColor, + NSColor(red: 1.0, green: 0.3, blue: 0.0, alpha: 0.95).cgColor, + NSColor(red: 0.9, green: 0.8, blue: 0.1, alpha: 0.9).cgColor + ] + glowLayer.backgroundColor = NSColor(red: 1.0, green: 0.5, blue: 0.0, alpha: 0.4).cgColor + glowLayer.shadowColor = NSColor(red: 1.0, green: 0.4, blue: 0.0, alpha: 0.7).cgColor + } else { + // Standard vibrant Siri gradient + gradientLayer.colors = [ + NSColor(red: 0.0, green: 0.85, blue: 1.0, alpha: 0.95).cgColor, + NSColor(red: 0.45, green: 0.15, blue: 0.95, alpha: 0.95).cgColor, + NSColor(red: 0.95, green: 0.05, blue: 0.65, alpha: 0.9).cgColor + ] + glowLayer.backgroundColor = NSColor(red: 0.0, green: 0.6, blue: 1.0, alpha: 0.35).cgColor + glowLayer.shadowColor = NSColor(red: 0.4, green: 0.0, blue: 1.0, alpha: 0.6).cgColor + } + + // Audio reactive scale: expand slightly with microphone volume + let clamped = CGFloat(min(max(level, 0.0), 1.0)) + let scale = 1.0 + (clamped * 0.35) + self.layer?.sublayerTransform = CATransform3DMakeScale(scale, scale, 1.0) + glowLayer.shadowRadius = 14 + (clamped * 12) + + CATransaction.commit() + } + + func showRetryState() { + CATransaction.begin() + CATransaction.setDisableActions(false) + CATransaction.setAnimationDuration(0.3) + gradientLayer.colors = [ + NSColor(red: 1.0, green: 0.55, blue: 0.0, alpha: 0.95).cgColor, + NSColor(red: 1.0, green: 0.25, blue: 0.0, alpha: 0.95).cgColor, + NSColor(red: 0.95, green: 0.85, blue: 0.2, alpha: 0.9).cgColor + ] + glowLayer.shadowColor = NSColor.orange.cgColor + CATransaction.commit() + } + + func showTranscribingState() { + CATransaction.begin() + CATransaction.setDisableActions(true) + self.layer?.sublayerTransform = CATransform3DMakeScale(0.95, 0.95, 1.0) + CATransaction.commit() + } + + func showSuccessState() { + CATransaction.begin() + CATransaction.setDisableActions(false) + CATransaction.setAnimationDuration(0.2) + gradientLayer.colors = [ + NSColor(red: 0.2, green: 0.9, blue: 0.4, alpha: 0.95).cgColor, + NSColor(red: 0.0, green: 0.7, blue: 0.8, alpha: 0.95).cgColor + ] + glowLayer.shadowColor = NSColor.green.cgColor + CATransaction.commit() + } + + func showErrorState() { + CATransaction.begin() + CATransaction.setDisableActions(false) + CATransaction.setAnimationDuration(0.2) + gradientLayer.colors = [ + NSColor(red: 1.0, green: 0.2, blue: 0.2, alpha: 0.95).cgColor, + NSColor(red: 0.8, green: 0.0, blue: 0.3, alpha: 0.95).cgColor + ] + glowLayer.shadowColor = NSColor.red.cgColor + CATransaction.commit() + } + + func reset() { + CATransaction.begin() + CATransaction.setDisableActions(true) + self.layer?.sublayerTransform = CATransform3DIdentity + CATransaction.commit() + } } diff --git a/src/InputBoxManager.swift b/src/InputBoxManager.swift new file mode 100644 index 0000000..2a67a9c --- /dev/null +++ b/src/InputBoxManager.swift @@ -0,0 +1,218 @@ +import Cocoa +import ApplicationServices + +public final class InputBoxManager { + public static let shared = InputBoxManager() + + private var targetedElement: AXUIElement? + private var streamedCharacterCount: Int = 0 + private var didStreamAnyText: Bool = false + private let lock = NSLock() + + private init() {} + + /// Prepares and guarantees that an input box is focused before or at the start of recording + @discardableResult + public func ensureFocusedInputBox() -> AXUIElement? { + lock.lock() + defer { lock.unlock() } + + streamedCharacterCount = 0 + didStreamAnyText = false + targetedElement = nil + + guard AXIsProcessTrusted() else { + print("InputBoxManager: AX not trusted, falling back to general keyboard events") + return nil + } + + guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil } + let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) + + // 1. Check if an element is already focused in the frontmost application + var focusedObj: CFTypeRef? + if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success, + let focused = focusedObj { + let elem = (focused as! AXUIElement) + if isEditableElement(elem) { + targetedElement = elem + return elem + } + } + + // 2. Also check systemWide focus + let sysWide = AXUIElementCreateSystemWide() + if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success, + let focused = focusedObj { + let elem = (focused as! AXUIElement) + if isEditableElement(elem) { + targetedElement = elem + return elem + } + } + + // 3. If no input is focused, search the active window of the frontmost app for an input field + var windowObj: CFTypeRef? + if AXUIElementCopyAttributeValue(appElem, kAXFocusedWindowAttribute as CFString, &windowObj) == .success, + let win = windowObj { + let winElem = (win as! AXUIElement) + if let bestInput = findBestInputElement(in: winElem) { + // Focus it via Accessibility + AXUIElementSetAttributeValue(bestInput, kAXFocusedAttribute as CFString, kCFBooleanTrue) + + // If it has a clickable position, click its center to ensure real input focus + if let center = getElementCenter(bestInput) { + clickAtPoint(center) + } + + targetedElement = bestInput + return bestInput + } + } + + return nil + } + + /// Stream delta text (incremental words) live into the focused input box + public func streamDelta(text: String) { + guard !text.isEmpty else { return } + + lock.lock() + didStreamAnyText = true + streamedCharacterCount += text.count + let target = targetedElement + lock.unlock() + + // 1. Try direct Accessibility insertion if element supports it + if let elem = target { + var selectedTextObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXSelectedTextAttribute as CFString, &selectedTextObj) == .success { + let err = AXUIElementSetAttributeValue(elem, kAXSelectedTextAttribute as CFString, text as CFString) + if err == .success { + return + } + } + } + + // 2. Native Unicode CGEvent typing (universal, works in Telegram, browsers, Electron, native apps) + typeUnicodeString(text) + } + + /// Commits final text to the input box and system clipboard + public func commitFinalText(processedText: String, addTrailingSpace: Bool) { + guard !processedText.isEmpty else { return } + + let textToSave = addTrailingSpace ? (processedText + " ") : processedText + + // 1. Always put full processed text into clipboard + let pb = NSPasteboard.general + pb.clearContents() + pb.setString(textToSave, forType: .string) + + lock.lock() + let hadStreamed = didStreamAnyText + let streamedCount = streamedCharacterCount + targetedElement = nil + lock.unlock() + + // If we didn't stream live (e.g. command or burst retry), paste the full text now + if !hadStreamed || streamedCount == 0 { + pasteClipboardToFrontmostApp() + } + } + + /// Simulates Cmd+V to paste clipboard contents + public func pasteClipboardToFrontmostApp() { + DispatchQueue.main.asyncAfter(deadline: .now() + 0.02) { + let source = CGEventSource(stateID: .combinedSessionState) + let vKeyCode: CGKeyCode = 0x09 // 'v' + if let down = CGEvent(keyboardEventSource: source, virtualKey: vKeyCode, keyDown: true), + let up = CGEvent(keyboardEventSource: source, virtualKey: vKeyCode, keyDown: false) { + down.flags = .maskCommand + up.flags = .maskCommand + down.post(tap: .cghidEventTap) + up.post(tap: .cghidEventTap) + } + } + } + + // MARK: - Helpers + + private func isEditableElement(_ elem: AXUIElement) -> Bool { + var roleObj: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) + let role = (roleObj as? String) ?? "" + if role == "AXTextField" || role == "AXTextArea" || role == "AXComboBox" || role == "AXWebArea" { + return true + } + var valObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success { + return true + } + return false + } + + private func findBestInputElement(in parent: AXUIElement, maxDepth: Int = 8, depth: Int = 0) -> AXUIElement? { + if depth > maxDepth { return nil } + + var roleObj: CFTypeRef? + AXUIElementCopyAttributeValue(parent, kAXRoleAttribute as CFString, &roleObj) + let role = (roleObj as? String) ?? "" + + if role == "AXTextField" || role == "AXTextArea" || role == "AXComboBox" { + return parent + } + + var childrenObj: CFTypeRef? + if AXUIElementCopyAttributeValue(parent, kAXChildrenAttribute as CFString, &childrenObj) == .success, + let children = childrenObj as? [AXUIElement] { + for child in children { + if let found = findBestInputElement(in: child, maxDepth: maxDepth, depth: depth + 1) { + return found + } + } + } + return nil + } + + private func getElementCenter(_ elem: AXUIElement) -> CGPoint? { + var posVal: CFTypeRef? + var sizeVal: CFTypeRef? + guard AXUIElementCopyAttributeValue(elem, kAXPositionAttribute as CFString, &posVal) == .success, + AXUIElementCopyAttributeValue(elem, kAXSizeAttribute as CFString, &sizeVal) == .success else { + return nil + } + var pt = CGPoint.zero + var sz = CGSize.zero + guard AXValueGetValue(posVal as! AXValue, .cgPoint, &pt), + AXValueGetValue(sizeVal as! AXValue, .cgSize, &sz) else { + return nil + } + return CGPoint(x: pt.x + sz.width / 2.0, y: pt.y + sz.height / 2.0) + } + + private func clickAtPoint(_ point: CGPoint) { + let source = CGEventSource(stateID: .hidSystemState) + if let clickDown = CGEvent(mouseEventSource: source, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left), + let clickUp = CGEvent(mouseEventSource: source, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) { + clickDown.post(tap: .cghidEventTap) + clickUp.post(tap: .cghidEventTap) + } + } + + private func typeUnicodeString(_ string: String) { + let source = CGEventSource(stateID: .hidSystemState) + let utf16 = Array(string.utf16) + + guard let keyDown = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: true), + let keyUp = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: false) else { + return + } + + keyDown.keyboardSetUnicodeString(stringLength: utf16.count, unicodeString: utf16) + keyUp.keyboardSetUnicodeString(stringLength: utf16.count, unicodeString: utf16) + + keyDown.post(tap: .cghidEventTap) + keyUp.post(tap: .cghidEventTap) + } +} diff --git a/src/SonioxClient.swift b/src/SonioxClient.swift index ebde563..a0facd3 100644 --- a/src/SonioxClient.swift +++ b/src/SonioxClient.swift @@ -1,10 +1,10 @@ import Foundation public final class SonioxLiveSession { - private let primaryWsBase = "wss://translate.compare.soniox.com/compare/api/compare-websocket" - private let fallbackWsBase = "wss://stt.compare.soniox.com/compare/api/compare-websocket" - private let userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" - private let origin = "https://translate.compare.soniox.com" + public static let primaryWsBase = "wss://translate.compare.soniox.com/compare/api/compare-websocket" + public static let fallbackWsBase = "wss://stt.compare.soniox.com/compare/api/compare-websocket" + private static let userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + private static let origin = "https://translate.compare.soniox.com" private var webSocketTask: URLSessionWebSocketTask? private var urlSession: URLSession? @@ -17,9 +17,15 @@ public final class SonioxLiveSession { private var committedFinalTokens: [String] = [] private var currentNonFinalTokens: [String] = [] + // Callbacks public var onPartialText: ((String) -> Void)? + public var onCommittedDelta: ((String) -> Void)? public var onFinalResult: ((Result) -> Void)? public var onConnectionStateChanged: ((Bool) -> Void)? + public var onNetworkDegraded: ((Bool) -> Void)? + + public private(set) var hasNetworkFailure: Bool = false + private var lastDataTime: Date = Date() public var isReady: Bool { lock.lock() @@ -27,15 +33,15 @@ public final class SonioxLiveSession { return isConnected && !isClosed && !isFinalizing } - public init(languageHints: [String] = ["fa", "en", "ar"]) { + public init(languageHints: [String] = ["fa", "en", "ar"], endpoint: String = SonioxLiveSession.primaryWsBase) { let hints = languageHints.map { "language_hints=\($0)" }.joined(separator: "&") - let urlStr = "\(primaryWsBase)?\(hints)&enable_speaker_diarization=false&enable_language_identification=true&enable_endpoint_detection=false&providers=soniox" + let urlStr = "\(endpoint)?\(hints)&enable_speaker_diarization=false&enable_language_identification=true&enable_endpoint_detection=false&providers=soniox" guard let url = URL(string: urlStr) else { return } var request = URLRequest(url: url) - request.setValue(userAgent, forHTTPHeaderField: "User-Agent") - request.setValue(origin, forHTTPHeaderField: "Origin") - request.timeoutInterval = 20.0 + request.setValue(SonioxLiveSession.userAgent, forHTTPHeaderField: "User-Agent") + request.setValue(SonioxLiveSession.origin, forHTTPHeaderField: "Origin") + request.timeoutInterval = 25.0 let config = URLSessionConfiguration.default config.waitsForConnectivity = true @@ -47,7 +53,7 @@ public final class SonioxLiveSession { self.webSocketTask = task task.resume() - // Fast ping to verify connection + // Ping to verify connection & latency task.sendPing { [weak self] error in guard let self = self else { return } self.lock.lock() @@ -55,9 +61,11 @@ public final class SonioxLiveSession { self.isConnected = true self.lock.unlock() self.onConnectionStateChanged?(true) - print("SonioxLiveSession: WebSocket connected successfully.") + print("SonioxLiveSession: Connected successfully.") } else { + self.hasNetworkFailure = true self.lock.unlock() + self.onNetworkDegraded?(true) if let error = error { print("SonioxLiveSession: Ping failed:", error) } @@ -73,9 +81,13 @@ public final class SonioxLiveSession { guard !isFinalizing, !isClosed, let task = webSocketTask else { return } let message = URLSessionWebSocketTask.Message.data(data) - task.send(message) { error in + task.send(message) { [weak self] error in if let error = error { - print("Error streaming audio chunk:", error) + print("SonioxLiveSession: Error streaming audio chunk:", error) + self?.lock.lock() + self?.hasNetworkFailure = true + self?.lock.unlock() + self?.onNetworkDegraded?(true) } } } @@ -99,14 +111,24 @@ public final class SonioxLiveSession { let finalizeMsg = URLSessionWebSocketTask.Message.string("{\"type\": \"finalize\"}") task.send(finalizeMsg) { [weak self] error in if let error = error { - print("Error sending finalize:", error) + print("SonioxLiveSession: Error sending finalize:", error) + self?.lock.lock() + self?.hasNetworkFailure = true + self?.lock.unlock() self?.completeWithCurrentText() } } - // Safety timeout fallback: finalize must complete within 3.0s (accommodates high network TTFB) - DispatchQueue.global().asyncAfter(deadline: .now() + 3.0) { [weak self] in - self?.completeWithCurrentText() + // Safety timeout fallback: finalize must complete within 3.5s + DispatchQueue.global().asyncAfter(deadline: .now() + 3.5) { [weak self] in + guard let self = self else { return } + self.lock.lock() + let stillOpen = !self.isClosed + self.lock.unlock() + if stillOpen { + print("SonioxLiveSession: Finalize timeout reached — completing.") + self.completeWithCurrentText() + } } } @@ -124,6 +146,7 @@ public final class SonioxLiveSession { switch result { case .success(let message): + self.lastDataTime = Date() var textReceived: String? switch message { case .string(let str): @@ -140,7 +163,11 @@ public final class SonioxLiveSession { self.startReceiving() case .failure(let error): - print("WebSocket receive status:", error) + print("SonioxLiveSession: WebSocket receive error:", error) + self.lock.lock() + self.hasNetworkFailure = true + self.lock.unlock() + self.onNetworkDegraded?(true) self.completeWithCurrentText() } } @@ -192,6 +219,14 @@ public final class SonioxLiveSession { let combined = fullCommitted + fullNonFinal lock.unlock() + // Emit streaming delta for newly committed words + if !newFinals.isEmpty { + let delta = newFinals.joined() + DispatchQueue.main.async { [weak self] in + self?.onCommittedDelta?(delta) + } + } + if !combined.isEmpty { DispatchQueue.main.async { [weak self] in self?.onPartialText?(combined) @@ -217,7 +252,7 @@ public final class SonioxLiveSession { let fullCommitted = committedFinalTokens.joined() let fullNonFinal = currentNonFinalTokens.joined() let rawCombined = fullCommitted.isEmpty ? fullNonFinal : (fullCommitted + fullNonFinal) - let cleaned = sanitizeText(rawCombined) + let cleaned = SonioxLiveSession.sanitizeText(rawCombined) let cb = onFinalResult webSocketTask?.cancel(with: .normalClosure, reason: nil) @@ -239,7 +274,173 @@ public final class SonioxLiveSession { lock.unlock() } - private func sanitizeText(_ text: String) -> String { + // MARK: - Smart Audio Recovery / Fast Burst Retry + + /// Re-transcribes the full audio buffer when the live stream had packet drops, disconnects, or missing trailing words. + public static func retryFullBufferedAudio( + pcmData: Data, + languageHints: [String], + onProgress: ((String) -> Void)? = nil, + completion: @escaping (Result) -> Void + ) { + guard !pcmData.isEmpty else { + completion(.success("")) + return + } + + print("SonioxLiveSession: Starting Smart Burst Retry with \(pcmData.count) bytes (\(Double(pcmData.count)/32000.0)s audio)...") + + // Try Primary endpoint, failover to Fallback if needed + executeBurstSession(endpoint: primaryWsBase, pcmData: pcmData, languageHints: languageHints, onProgress: onProgress) { result in + switch result { + case .success(let text): + if !text.isEmpty { + completion(.success(text)) + } else { + // Try fallback endpoint + print("SonioxLiveSession: Primary retry returned empty, attempting fallback endpoint...") + executeBurstSession(endpoint: fallbackWsBase, pcmData: pcmData, languageHints: languageHints, onProgress: onProgress, completion: completion) + } + case .failure(let error): + print("SonioxLiveSession: Primary retry failed (\(error)), attempting fallback endpoint...") + executeBurstSession(endpoint: fallbackWsBase, pcmData: pcmData, languageHints: languageHints, onProgress: onProgress, completion: completion) + } + } + } + + private static func executeBurstSession( + endpoint: String, + pcmData: Data, + languageHints: [String], + onProgress: ((String) -> Void)?, + completion: @escaping (Result) -> Void + ) { + let hints = languageHints.map { "language_hints=\($0)" }.joined(separator: "&") + let urlStr = "\(endpoint)?\(hints)&enable_speaker_diarization=false&enable_language_identification=true&enable_endpoint_detection=false&providers=soniox" + guard let url = URL(string: urlStr) else { + completion(.failure(NSError(domain: "Soniox", code: -1, userInfo: [NSLocalizedDescriptionKey: "Invalid URL"]))) + return + } + + var request = URLRequest(url: url) + request.setValue(userAgent, forHTTPHeaderField: "User-Agent") + request.setValue(origin, forHTTPHeaderField: "Origin") + request.timeoutInterval = 25.0 + + let config = URLSessionConfiguration.default + let session = URLSession(configuration: config) + let task = session.webSocketTask(with: request) + task.resume() + + var fullCommitted: [String] = [] + var fullNonFinal: [String] = [] + var isDone = false + let lock = NSLock() + + func finishOnce(with text: String) { + lock.lock() + if isDone { lock.unlock(); return } + isDone = true + task.cancel(with: .normalClosure, reason: nil) + lock.unlock() + + let sanitized = sanitizeText(text) + DispatchQueue.main.async { + completion(.success(sanitized)) + } + } + + // Receive handler + func receiveNext() { + task.receive { result in + lock.lock() + if isDone { lock.unlock(); return } + lock.unlock() + + switch result { + case .success(let msg): + var textStr: String? + switch msg { + case .string(let s): textStr = s + case .data(let d): textStr = String(data: d, encoding: .utf8) + @unknown default: break + } + + if let raw = textStr, let jData = raw.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: jData) as? [String: Any] { + + var gotFin = false + if let parts = json["parts"] as? [[String: Any]] { + for p in parts { + if (p["translation_status"] as? String) == "translation" { continue } + let t = p["text"] as? String ?? "" + let isFinal = p["is_final"] as? Bool ?? false + if t.contains("") { + gotFin = true + let clean = t.replacingOccurrences(of: "", with: "") + if !clean.isEmpty { fullCommitted.append(clean) } + } else if isFinal { + if !t.isEmpty { fullCommitted.append(t) } + } else { + if !t.isEmpty { fullNonFinal.append(t) } + } + } + } + + let combined = fullCommitted.joined() + fullNonFinal.joined() + if !combined.isEmpty { + DispatchQueue.main.async { + onProgress?(combined) + } + } + + let ended = (json["session_ended"] as? Bool ?? false) || ((json["type"] as? String) == "session_done") + if gotFin || ended { + finishOnce(with: combined) + return + } + } + receiveNext() + + case .failure(let err): + print("SonioxLiveSession: Burst receive error:", err) + let combined = fullCommitted.joined() + fullNonFinal.joined() + finishOnce(with: combined) + } + } + } + + receiveNext() + + // Pump audio in high-speed bursts (16KB chunks) + DispatchQueue.global(qos: .userInitiated).async { + let chunkSize = 16000 // 0.5s audio + var offset = 0 + while offset < pcmData.count { + let length = min(chunkSize, pcmData.count - offset) + let chunk = pcmData.subdata(in: offset..<(offset + length)) + task.send(URLSessionWebSocketTask.Message.data(chunk)) { _ in } + offset += length + usleep(8000) // 8ms pace + } + + // Send trailing silence + finalize + let silence = Data(count: 3840) + task.send(URLSessionWebSocketTask.Message.data(silence)) { _ in } + let finalizeMsg = URLSessionWebSocketTask.Message.string("{\"type\": \"finalize\"}") + task.send(finalizeMsg) { _ in } + } + + // Watchdog timeout for burst retry: max 5 seconds + DispatchQueue.global().asyncAfter(deadline: .now() + 5.0) { + let combined = fullCommitted.joined() + fullNonFinal.joined() + finishOnce(with: combined) + } + } + + // MARK: - Sanitization + + public static func sanitizeText(_ text: String) -> String { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) if trimmed.isEmpty { return "" } @@ -291,7 +492,8 @@ public final class SonioxLiveSession { } } -/// Pre-warms and pools active WebSocket sessions for 0ms start latency +// MARK: - Pre-warmed Session Pool + public final class SonioxSessionPool { public static let shared = SonioxSessionPool()