Browse Source

fix(sync): universal synthetic paste engine, ghost sync suppression for web/electron apps and robust bidirectional state machine

main
Ali Alavi 20 hours ago
parent
commit
be80173395
  1. 36
      android/app/src/main/java/com/soniox/remotemic/MainActivity.kt
  2. 32
      android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt
  3. 201
      mac/src/FocusedInputSync.swift

36
android/app/src/main/java/com/soniox/remotemic/MainActivity.kt

@ -88,7 +88,7 @@ class MainActivity : AppCompatActivity() {
setupUI()
checkPermissions()
AppLogger.log("Main", "اپلیکیشن راه‌اندازی شد (طراحی ریسپانسیو و فوکوس کامل اینپوت‌باکس روی کیبورد)")
AppLogger.log("Main", "اپلیکیشن همگام‌سازی صوتی سانی‌اوکس راه‌اندازی شد (نسخه بهینه‌شده فوق‌سریع v5.2)")
// Initialize Collaborative WebSocket Client
streamDictationClient = StreamDictationClient(
@ -100,7 +100,7 @@ class MainActivity : AppCompatActivity() {
)
},
onSyncStateReceived = { state ->
// Drop echoes and do not interrupt active user editing on phone
// Drop echoes originated from phone itself
if (state.source != "android") {
val timeSinceLocalEdit = System.currentTimeMillis() - lastLocalUserEditTime
@ -109,6 +109,19 @@ class MainActivity : AppCompatActivity() {
return@StreamDictationClient
}
// Monotonic revision check
if (state.revision > 0 && state.revision < currentRevision) {
return@StreamDictationClient
}
// Ghost empty sync protection for opaque/Electron apps (e.g. Antigravity)
if (state.text.isEmpty() && lastLocalText.isNotEmpty() && state.source == "mac") {
if (state.app.isNotEmpty() && state.app != "Desktop" && state.app != "App") {
binding.tvMacStatus.text = "متصل به ${state.app} 🖥️"
}
return@StreamDictationClient
}
if (state.text != lastLocalText) {
isApplyingRemoteUpdate = true
lastLocalText = state.text
@ -212,10 +225,19 @@ class MainActivity : AppCompatActivity() {
val prefix = if (start > 0) current.substring(0, start) else ""
val suffix = if (end < current.length) current.substring(end) else ""
val formattedSpeech = if (prefix.isNotEmpty() && !prefix.endsWith(" ") && !speechText.startsWith(" ")) {
" $speechText"
} else {
speechText
val trimmedSpeech = speechText.trim()
if (trimmedSpeech.isEmpty()) return
// Smart spacing for Persian / English word boundaries
val needsPreSpace = prefix.isNotEmpty() && !prefix.endsWith(" ") && !prefix.endsWith("\n")
val needsPostSpace = suffix.isNotEmpty() && !suffix.startsWith(" ") && !suffix.startsWith("\n") &&
!suffix.startsWith(",") && !suffix.startsWith("،") && !suffix.startsWith(".") &&
!suffix.startsWith("؟") && !suffix.startsWith("!") && !suffix.startsWith(":")
val formattedSpeech = buildString {
if (needsPreSpace) append(" ")
append(trimmedSpeech)
if (needsPostSpace) append(" ")
}
val mergedText = "$prefix$formattedSpeech$suffix"
@ -228,7 +250,7 @@ class MainActivity : AppCompatActivity() {
binding.etTranscript.setSelection(newCursor)
isApplyingRemoteUpdate = false
AppLogger.log("Main", "تزریق گفتار در نشانگر: '$speechText' (موقعیت جدید: $newCursor)")
AppLogger.log("Main", "تزریق گفتار در نشانگر: '$trimmedSpeech' (موقعیت جدید: $newCursor)")
streamDictationClient?.sendLocalSyncState(mergedText, newCursor)
}

32
android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt

@ -52,7 +52,7 @@ class StreamDictationClient(
.retryOnConnectionFailure(true)
.build()
private var webSocket: WebSocket? = null
private var activeWebSocket: WebSocket? = null
private val isConnected = AtomicBoolean(false)
private var currentSessionId: String = ""
private var isSessionActive = AtomicBoolean(false)
@ -63,21 +63,21 @@ class StreamDictationClient(
@Synchronized
fun connectWebSocket() {
if (isConnected.get() && webSocket != null) return
if (isConnected.get() && activeWebSocket != null) return
val cleanHost = host.removePrefix("http://").removePrefix("https://").removePrefix("ws://").removePrefix("wss://")
val wsUrl = "ws://$cleanHost/ws/stream"
AppLogger.log(tag, "اتصال به سوکت همگام‌سازی بلادرنگ: $wsUrl")
val req = Request.Builder().url(wsUrl).build()
webSocket = okHttpClient.newWebSocket(req, object : WebSocketListener() {
override fun onOpen(ws: WebSocket, response: Response) {
activeWebSocket = okHttpClient.newWebSocket(req, object : WebSocketListener() {
override fun onOpen(webSocket: WebSocket, response: Response) {
AppLogger.log(tag, "🟢 سوکت همگام‌سازی بلادرنگ متصل شد")
isConnected.set(true)
mainHandler.post { onConnectionStateChanged(true) }
}
override fun onMessage(ws: WebSocket, text: String) {
override fun onMessage(webSocket: WebSocket, text: String) {
try {
val json = JSONObject(text)
val type = json.optString("type")
@ -121,18 +121,18 @@ class StreamDictationClient(
}
}
override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) {
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
AppLogger.log(tag, "🔴 قطع سوکت: ${t.message}. تلاش مجدد...")
isConnected.set(false)
webSocket = null
activeWebSocket = null
mainHandler.post { onConnectionStateChanged(false) }
mainHandler.postDelayed({ connectWebSocket() }, 2000)
}
override fun onClosed(ws: WebSocket, code: Int, reason: String) {
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
AppLogger.log(tag, "سوکت بسته شد ($code). تلاش مجدد...")
isConnected.set(false)
webSocket = null
activeWebSocket = null
mainHandler.post { onConnectionStateChanged(false) }
mainHandler.postDelayed({ connectWebSocket() }, 2000)
}
@ -143,7 +143,7 @@ class StreamDictationClient(
* Broadcasts phone's updated text & cursor to Mac in sub-15ms
*/
fun sendLocalSyncState(text: String, cursor: Int, selection: Int = 0) {
if (!isConnected.get() || webSocket == null) {
if (!isConnected.get() || activeWebSocket == null) {
connectWebSocket()
}
val payload = JSONObject().apply {
@ -154,7 +154,7 @@ class StreamDictationClient(
put("selection", selection)
put("timestamp", System.currentTimeMillis() / 1000.0)
}.toString()
webSocket?.send(payload)
activeWebSocket?.send(payload)
}
@SuppressLint("MissingPermission")
@ -165,7 +165,7 @@ class StreamDictationClient(
isRecording.set(true)
isSessionActive.set(true)
if (!isConnected.get() || webSocket == null) {
if (!isConnected.get() || activeWebSocket == null) {
connectWebSocket()
}
@ -174,7 +174,7 @@ class StreamDictationClient(
put("session_id", currentSessionId)
put("cursor_pos", cursorPos)
}.toString()
webSocket?.send(startFrame)
activeWebSocket?.send(startFrame)
AppLogger.log(tag, "🎙️ شروع استریم گفتار در موقعیت نشانگر $cursorPos...")
val minBufferSize = AudioRecord.getMinBufferSize(sampleRate, channelConfig, audioFormat)
@ -207,7 +207,7 @@ class StreamDictationClient(
val bytesRead = audioRecord?.read(chunk, 0, chunk.size) ?: -1
if (bytesRead > 0) {
val slice = if (bytesRead == chunk.size) chunk.clone() else chunk.copyOf(bytesRead)
webSocket?.send(slice.toByteString())
activeWebSocket?.send(slice.toByteString())
var sum = 0.0
val samplesCount = bytesRead / 2
@ -255,7 +255,7 @@ class StreamDictationClient(
put("session_id", currentSessionId)
put("cursor_pos", cursorPos)
}.toString()
webSocket?.send(stopFrame)
activeWebSocket?.send(stopFrame)
AppLogger.log(tag, "⏹️ پایان ضبط. دریافت متن نهایی...")
}
@ -263,7 +263,7 @@ class StreamDictationClient(
isRecording.set(false)
try {
audioRecord?.release()
webSocket?.close(1000, "Client Shutdown")
activeWebSocket?.close(1000, "Client Shutdown")
} catch (e: Exception) {}
}
}

201
mac/src/FocusedInputSync.swift

@ -32,6 +32,16 @@ public final class FocusedInputSync {
private var localRevision: Int64 = 0
private var remoteChangeExpiryTime: Double = 0
// Known Web, Electron, Terminal, and Custom UI apps that do NOT accept direct AXValue writes
// but require fast, reliable native keystroke paste (Cmd+V / Cmd+A + Cmd+V)
private let pastePreferredApps: Set<String> = [
"antigravity", "antigravity helper", "google chrome", "chromium", "brave browser",
"arc", "microsoft edge", "firefox", "safari", "code", "cursor", "visual studio code",
"slack", "discord", "telegram", "whatsapp", "signal", "ghostty", "iterm2", "iterm",
"terminal", "alacritty", "kitty", "jetbrains", "idea", "webstorm", "pycharm",
"datagrip", "sublime text", "notion", "obsidian", "linear", "warp"
]
private init() {
self.systemWideElement = AXUIElementCreateSystemWide()
enableGlobalAccessibility()
@ -42,7 +52,19 @@ public final class FocusedInputSync {
AXUIElementSetAttributeValue(systemWideElement, "AXManualAccessibility" as CFString, kCFBooleanTrue)
}
private func findFocusedDescendant(_ elem: AXUIElement) -> AXUIElement? {
private func isAppPastePreferred(_ appName: String) -> Bool {
let lower = appName.lowercased()
for pref in pastePreferredApps {
if lower.contains(pref) {
return true
}
}
return false
}
private func findFocusedDescendant(_ elem: AXUIElement, depth: Int = 0) -> AXUIElement? {
if depth > 12 { return nil }
var isFocusedObj: CFTypeRef?
if AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &isFocusedObj) == .success,
let isFocused = isFocusedObj as? Bool, isFocused {
@ -58,7 +80,7 @@ public final class FocusedInputSync {
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success,
let children = childrenObj as? [AXUIElement] {
for child in children {
if let found = findFocusedDescendant(child) {
if let found = findFocusedDescendant(child, depth: depth + 1) {
return found
}
}
@ -66,8 +88,8 @@ public final class FocusedInputSync {
return nil
}
public func getFocusedElement() -> (AXUIElement, String)? {
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil }
public func getFocusedElement() -> (AXUIElement?, String) {
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return (nil, "App") }
let appName = frontApp.localizedName ?? "App"
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier)
@ -76,7 +98,7 @@ public final class FocusedInputSync {
var targetElem: AXUIElement?
// 1. Try system wide focused element
// 1. Try system-wide focused element
var focusedObj: CFTypeRef?
if AXUIElementCopyAttributeValue(systemWideElement, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success,
let obj = focusedObj {
@ -103,25 +125,59 @@ public final class FocusedInputSync {
}
}
// 3. Recursive search in tree
// 3. Try App focused window's focused element
if targetElem == nil {
targetElem = findFocusedDescendant(appElem)
var focusedWinObj: CFTypeRef?
if AXUIElementCopyAttributeValue(appElem, kAXFocusedWindowAttribute as CFString, &focusedWinObj) == .success,
let win = focusedWinObj {
var winFocObj: CFTypeRef?
if AXUIElementCopyAttributeValue((win as! AXUIElement), kAXFocusedUIElementAttribute as CFString, &winFocObj) == .success,
let obj = winFocObj {
targetElem = (obj as! AXUIElement)
}
}
}
if let elem = targetElem {
return (elem, appName)
// 4. Recursive search in tree
if targetElem == nil {
targetElem = findFocusedDescendant(appElem)
}
return nil
return (targetElem, appName)
}
/// Inspects the current focused element and returns a state snapshot if changed
/// Inspects the current focused element and returns a state snapshot if changed on Mac
public func inspectCurrentState() -> MacInputState? {
let now = Date().timeIntervalSince1970
if isApplyingRemoteChange || now < remoteChangeExpiryTime {
return nil
}
guard let (elem, appName) = getFocusedElement() else { return nil }
let (elemOpt, appName) = getFocusedElement()
// If app changed
let appChanged = (appName != lastObservedApp)
if appChanged {
lastObservedApp = appName
}
guard let elem = elemOpt else {
// When no native AX element is available (e.g. Electron web canvas or Antigravity),
// DO NOT emit empty text which would wipe out the mobile client transcript!
if appChanged && !lastObservedText.isEmpty {
localRevision += 1
return MacInputState(app: appName, text: lastObservedText, cursor: lastObservedCursor, selection: 0, revision: localRevision)
}
return nil
}
// Check role
var roleObj: CFTypeRef?
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj)
let role = roleObj as? String ?? ""
// Only inspect text if element is a recognized text-capable input
let isTextRole = (role == "AXTextField" || role == "AXTextArea" || role == "AXComboBox" || role == "AXSearchField")
// Extract text
var text = ""
@ -135,7 +191,7 @@ public final class FocusedInputSync {
}
}
if text.isEmpty {
if text.isEmpty && isTextRole {
var countObj: CFTypeRef?
if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countObj) == .success,
let count = countObj as? Int, count > 0 {
@ -150,6 +206,12 @@ public final class FocusedInputSync {
}
}
// Ghost empty suppression: If an opaque web app reports empty string, but we had existing text,
// and element is NOT an explicit empty native text field, ignore the empty read.
if text.isEmpty && !lastObservedText.isEmpty && !isTextRole && isAppPastePreferred(appName) {
return nil
}
// Extract cursor & selection
var cursor = text.count
var selLen = 0
@ -164,7 +226,7 @@ public final class FocusedInputSync {
}
// Check if text or cursor actually changed on Mac
if text == lastObservedText && cursor == lastObservedCursor && appName == lastObservedApp {
if text == lastObservedText && cursor == lastObservedCursor && !appChanged {
return nil
}
@ -180,10 +242,11 @@ public final class FocusedInputSync {
@discardableResult
public func applyRemoteUpdate(text: String, cursor: Int? = nil, isFullReplace: Bool = true) -> Bool {
isApplyingRemoteChange = true
remoteChangeExpiryTime = Date().timeIntervalSince1970 + 0.45 // 450ms quiet window
// 750ms quiet window to prevent self-echo and race conditions
remoteChangeExpiryTime = Date().timeIntervalSince1970 + 0.75
defer {
DispatchQueue.main.asyncAfter(deadline: .now() + 0.45) {
DispatchQueue.main.asyncAfter(deadline: .now() + 0.75) {
self.isApplyingRemoteChange = false
}
}
@ -198,62 +261,98 @@ public final class FocusedInputSync {
lastObservedText = cleanText
lastObservedCursor = targetCursor
guard let (elem, appName) = getFocusedElement() else {
// Fallback: clipboard paste
lastObservedApp = NSWorkspace.shared.frontmostApplication?.localizedName ?? "App"
let (elemOpt, appName) = getFocusedElement()
lastObservedApp = appName
// If the target app is a Web/Electron/IDE application (e.g. Antigravity, Chrome, VS Code, Discord, Slack)
// or no native AX element was resolved, bypass AXValue write and execute rock-solid synthetic paste!
if isAppPastePreferred(appName) || elemOpt == nil {
print("FocusedInputSync: 🎯 Target app '\(appName)' is Web/Electron/Paste-preferred. Executing Universal Synthetic Paste.")
return pasteViaKeystroke(cleanText, isFullReplace: isFullReplace)
}
lastObservedApp = appName
if isFullReplace {
// Try setting AXValue directly
let setErr = AXUIElementSetAttributeValue(elem, kAXValueAttribute as CFString, cleanText as CFTypeRef)
if setErr == .success {
if let cursor = cursor {
var range = CFRange(location: min(cursor, cleanText.count), length: 0)
if let axRange = AXValueCreate(.cfRange, &range) {
AXUIElementSetAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, axRange)
// For Native AppKit/Cocoa apps (e.g. TextEdit, Notes, Finder):
if let elem = elemOpt {
if isFullReplace {
let setErr = AXUIElementSetAttributeValue(elem, kAXValueAttribute as CFString, cleanText as CFTypeRef)
if setErr == .success {
if let cursor = cursor {
var range = CFRange(location: min(cursor, cleanText.count), length: 0)
if let axRange = AXValueCreate(.cfRange, &range) {
AXUIElementSetAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, axRange)
}
}
print("FocusedInputSync: ✅ Updated native AXValue for \(appName)")
return true
}
} else {
let setSelErr = AXUIElementSetAttributeValue(elem, kAXSelectedTextAttribute as CFString, cleanText as CFTypeRef)
if setSelErr == .success {
print("FocusedInputSync: ✅ Inserted native text via AXSelectedText for \(appName)")
return true
}
print("FocusedInputSync: ✅ Updated AXValue for \(appName)")
return true
}
} else {
// Try setting selected text
let setSelErr = AXUIElementSetAttributeValue(elem, kAXSelectedTextAttribute as CFString, cleanText as CFTypeRef)
if setSelErr == .success {
print("FocusedInputSync: ✅ Inserted text via AXSelectedText for \(appName)")
return true
}
}
// Reliable fallback for any situation
return pasteViaKeystroke(cleanText, isFullReplace: isFullReplace)
}
/**
* Ultra-reliable synthetic keystroke injection (Cmd+A -> Cmd+V)
* Includes proper key-dwell times, modifier flag persistence, and clipboard preservation.
*/
private func pasteViaKeystroke(_ text: String, isFullReplace: Bool) -> Bool {
let pasteboard = NSPasteboard.general
// 1. Snapshot previous clipboard to restore after paste
let oldString = pasteboard.string(forType: .string)
// 2. Set new text to pasteboard
pasteboard.clearContents()
pasteboard.setString(text, forType: .string)
let src = CGEventSource(stateID: .hidSystemState)
let src = CGEventSource(stateID: .combinedSessionState)
if isFullReplace {
// Select all: Cmd + A
let aDown = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: true)
aDown?.flags = .maskCommand
let aUp = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: false)
aDown?.post(tap: .cghidEventTap)
aUp?.post(tap: .cghidEventTap)
// Select all: Cmd + A (virtualKey 0 = 'a')
if let aDown = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: true) {
aDown.flags = .maskCommand
aDown.post(tap: .cghidEventTap)
}
usleep(12000) // 12ms key-down dwell time
if let aUp = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: false) {
aUp.flags = .maskCommand
aUp.post(tap: .cghidEventTap)
}
usleep(20000)
// 35ms settling delay for Electron / DOM selection update
usleep(35000)
}
// Paste: Cmd + V (virtualKey 9 = 'v')
if let vDown = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: true) {
vDown.flags = .maskCommand
vDown.post(tap: .cghidEventTap)
}
usleep(12000) // 12ms key-down dwell time
// Paste: Cmd + V
let vDown = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: true)
vDown?.flags = .maskCommand
let vUp = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: false)
vDown?.post(tap: .cghidEventTap)
vUp?.post(tap: .cghidEventTap)
if let vUp = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: false) {
vUp.flags = .maskCommand
vUp.post(tap: .cghidEventTap)
}
// 3. Asynchronously restore previous clipboard content after 250ms
if let previousText = oldString, previousText != text {
DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + 0.25) {
// If pasteboard hasn't been changed by user in the meantime, restore
if pasteboard.string(forType: .string) == text {
pasteboard.clearContents()
pasteboard.setString(previousText, forType: .string)
}
}
}
return true
}

Loading…
Cancel
Save