Browse Source

fix: smooth character-by-character deletion and echo suppression without backspace fighting

main
Ali Alavi 1 day ago
parent
commit
85096007e9
  1. 44
      android/app/src/main/java/com/soniox/remotemic/MainActivity.kt
  2. 39
      mac/src/FocusedInputSync.swift

44
android/app/src/main/java/com/soniox/remotemic/MainActivity.kt

@ -10,6 +10,8 @@ import android.content.Context
import android.content.pm.PackageManager import android.content.pm.PackageManager
import android.os.Build import android.os.Build
import android.os.Bundle import android.os.Bundle
import android.os.Handler
import android.os.Looper
import android.os.VibrationEffect import android.os.VibrationEffect
import android.os.Vibrator import android.os.Vibrator
import android.os.VibratorManager import android.os.VibratorManager
@ -49,10 +51,14 @@ class MainActivity : AppCompatActivity() {
// Authoritative Permanent Gateway Server on Linux (116.16.16.19:8999) // Authoritative Permanent Gateway Server on Linux (116.16.16.19:8999)
private val gatewayHost = "116.16.16.19:8999" private val gatewayHost = "116.16.16.19:8999"
// Live Synchronized State (Google Docs / Figma style)
// Live Synchronized State (Collaborative Engine)
private var currentRevision: Long = 0L private var currentRevision: Long = 0L
private var isApplyingRemoteUpdate = false private var isApplyingRemoteUpdate = false
private var lastLocalText = "" private var lastLocalText = ""
private var lastLocalUserEditTime = 0L
private val debounceHandler = Handler(Looper.getMainLooper())
private var pendingSyncRunnable: Runnable? = null
// Voice Insertion Anchor // Voice Insertion Anchor
private var voiceInsertionCursorStart = 0 private var voiceInsertionCursorStart = 0
@ -79,7 +85,7 @@ class MainActivity : AppCompatActivity() {
setupUI() setupUI()
checkPermissions() checkPermissions()
AppLogger.log("Main", "اپلیکیشن راه‌اندازی شد (همگام‌سازی بلادرنگ Google Docs/Figma Style با مک)")
AppLogger.log("Main", "اپلیکیشن راه‌اندازی شد (همگام‌سازی بلادرنگ بدون باگ پاک‌کردن)")
// Initialize Collaborative WebSocket Client // Initialize Collaborative WebSocket Client
streamDictationClient = StreamDictationClient( streamDictationClient = StreamDictationClient(
@ -91,16 +97,23 @@ class MainActivity : AppCompatActivity() {
) )
}, },
onSyncStateReceived = { state -> onSyncStateReceived = { state ->
// Apply update from Mac if source != android and revision is newer
// Drop echoes and do not interrupt active user editing on phone
if (state.source != "android") { if (state.source != "android") {
val timeSinceLocalEdit = System.currentTimeMillis() - lastLocalUserEditTime
// If user is actively typing or backspacing right now on phone, do not override with remote echo
if (timeSinceLocalEdit < 1000L && binding.etTranscript.hasFocus()) {
return@StreamDictationClient
}
if (state.text != lastLocalText) { if (state.text != lastLocalText) {
isApplyingRemoteUpdate = true isApplyingRemoteUpdate = true
lastLocalText = state.text lastLocalText = state.text
currentRevision = state.revision currentRevision = state.revision
// Preserve cursor safely
val currentCursor = binding.etTranscript.selectionStart val currentCursor = binding.etTranscript.selectionStart
binding.etTranscript.setText(state.text) binding.etTranscript.setText(state.text)
val targetCursor = if (binding.etTranscript.hasFocus() && currentCursor >= 0) { val targetCursor = if (binding.etTranscript.hasFocus() && currentCursor >= 0) {
currentCursor.coerceIn(0, state.text.length) currentCursor.coerceIn(0, state.text.length)
} else { } else {
@ -143,20 +156,15 @@ class MainActivity : AppCompatActivity() {
) )
} }
/**
* Dynamically adjusts the layout when virtual keyboard opens or closes
*/
private fun setupKeyboardInsetHandling() { private fun setupKeyboardInsetHandling() {
ViewCompat.setOnApplyWindowInsetsListener(binding.rootLayout) { _, insets -> ViewCompat.setOnApplyWindowInsetsListener(binding.rootLayout) { _, insets ->
val imeVisible = insets.isVisible(WindowInsetsCompat.Type.ime()) val imeVisible = insets.isVisible(WindowInsetsCompat.Type.ime())
if (imeVisible) { if (imeVisible) {
// Keyboard OPEN: Hide huge bottom circle, expand editor box to full height directly above keyboard
binding.bottomVoiceSection.visibility = View.GONE binding.bottomVoiceSection.visibility = View.GONE
binding.tvSubtitle.visibility = View.GONE binding.tvSubtitle.visibility = View.GONE
binding.btnMiniMic.visibility = View.VISIBLE binding.btnMiniMic.visibility = View.VISIBLE
} else { } else {
// Keyboard CLOSED: Restore spacious layout with large glowing mic button
binding.bottomVoiceSection.visibility = View.VISIBLE binding.bottomVoiceSection.visibility = View.VISIBLE
binding.tvSubtitle.visibility = View.VISIBLE binding.tvSubtitle.visibility = View.VISIBLE
binding.btnMiniMic.visibility = View.GONE binding.btnMiniMic.visibility = View.GONE
@ -166,9 +174,6 @@ class MainActivity : AppCompatActivity() {
} }
} }
/**
* Slices speech text right at the exact cursor/selection location
*/
private fun insertSpeechAtCursor(speechText: String) { private fun insertSpeechAtCursor(speechText: String) {
val current = binding.etTranscript.text?.toString() ?: "" val current = binding.etTranscript.text?.toString() ?: ""
val start = voiceInsertionCursorStart.coerceIn(0, current.length) val start = voiceInsertionCursorStart.coerceIn(0, current.length)
@ -188,18 +193,18 @@ class MainActivity : AppCompatActivity() {
isApplyingRemoteUpdate = true isApplyingRemoteUpdate = true
lastLocalText = mergedText lastLocalText = mergedText
lastLocalUserEditTime = System.currentTimeMillis()
binding.etTranscript.setText(mergedText) binding.etTranscript.setText(mergedText)
binding.etTranscript.setSelection(newCursor) binding.etTranscript.setSelection(newCursor)
isApplyingRemoteUpdate = false isApplyingRemoteUpdate = false
AppLogger.log("Main", "تزریق گفتار در نشانگر: '$speechText' (موقعیت جدید: $newCursor)") AppLogger.log("Main", "تزریق گفتار در نشانگر: '$speechText' (موقعیت جدید: $newCursor)")
// Broadcast updated state to Mac immediately!
streamDictationClient?.sendLocalSyncState(mergedText, newCursor) streamDictationClient?.sendLocalSyncState(mergedText, newCursor)
} }
private fun setupUI() { private fun setupUI() {
// Real-time TextWatcher for local keyboard typing
// Real-time TextWatcher for local keyboard typing & backspacing
binding.etTranscript.addTextChangedListener(object : TextWatcher { binding.etTranscript.addTextChangedListener(object : TextWatcher {
override fun beforeTextChanged(s: CharSequence?, start: Int, count: Int, after: Int) {} override fun beforeTextChanged(s: CharSequence?, start: Int, count: Int, after: Int) {}
override fun onTextChanged(s: CharSequence?, start: Int, before: Int, count: Int) { override fun onTextChanged(s: CharSequence?, start: Int, before: Int, count: Int) {
@ -207,11 +212,17 @@ class MainActivity : AppCompatActivity() {
val wordCount = if (text.trim().isEmpty()) 0 else text.trim().split("\\s+".toRegex()).size val wordCount = if (text.trim().isEmpty()) 0 else text.trim().split("\\s+".toRegex()).size
binding.tvCharCount.text = "$wordCount کلمه" binding.tvCharCount.text = "$wordCount کلمه"
// If typed locally by user, broadcast to Mac immediately!
if (!isApplyingRemoteUpdate && !isCurrentlyRecording) { if (!isApplyingRemoteUpdate && !isCurrentlyRecording) {
lastLocalText = text lastLocalText = text
lastLocalUserEditTime = System.currentTimeMillis()
val cur = binding.etTranscript.selectionStart.coerceIn(0, text.length) val cur = binding.etTranscript.selectionStart.coerceIn(0, text.length)
streamDictationClient?.sendLocalSyncState(text, cur)
// Debounce sending to Mac by 60ms to allow 120Hz lag-free backspacing and typing
pendingSyncRunnable?.let { debounceHandler.removeCallbacks(it) }
pendingSyncRunnable = Runnable {
streamDictationClient?.sendLocalSyncState(text, cur)
}
debounceHandler.postDelayed(pendingSyncRunnable!!, 60)
} }
} }
override fun afterTextChanged(s: Editable?) {} override fun afterTextChanged(s: Editable?) {}
@ -222,6 +233,7 @@ class MainActivity : AppCompatActivity() {
isApplyingRemoteUpdate = true isApplyingRemoteUpdate = true
binding.etTranscript.setText("") binding.etTranscript.setText("")
lastLocalText = "" lastLocalText = ""
lastLocalUserEditTime = System.currentTimeMillis()
isApplyingRemoteUpdate = false isApplyingRemoteUpdate = false
streamDictationClient?.sendLocalSyncState("", 0) streamDictationClient?.sendLocalSyncState("", 0)
binding.tvInstruction.text = getString(R.string.hold_to_speak) binding.tvInstruction.text = getString(R.string.hold_to_speak)

39
mac/src/FocusedInputSync.swift

@ -26,8 +26,11 @@ public final class FocusedInputSync {
private let systemWideElement: AXUIElement private let systemWideElement: AXUIElement
private var isApplyingRemoteChange: Bool = false private var isApplyingRemoteChange: Bool = false
private var lastObservedHash: Int = 0
private var lastObservedText: String = ""
private var lastObservedCursor: Int = -1
private var lastObservedApp: String = ""
private var localRevision: Int64 = 0 private var localRevision: Int64 = 0
private var remoteChangeExpiryTime: Double = 0
private init() { private init() {
self.systemWideElement = AXUIElementCreateSystemWide() self.systemWideElement = AXUIElementCreateSystemWide()
@ -113,7 +116,11 @@ public final class FocusedInputSync {
/// Inspects the current focused element and returns a state snapshot if changed /// Inspects the current focused element and returns a state snapshot if changed
public func inspectCurrentState() -> MacInputState? { public func inspectCurrentState() -> MacInputState? {
guard !isApplyingRemoteChange else { return nil }
let now = Date().timeIntervalSince1970
if isApplyingRemoteChange || now < remoteChangeExpiryTime {
return nil
}
guard let (elem, appName) = getFocusedElement() else { return nil } guard let (elem, appName) = getFocusedElement() else { return nil }
// Extract text // Extract text
@ -156,10 +163,14 @@ public final class FocusedInputSync {
} }
} }
// Check hash to prevent echo loops
let currentHash = "\(appName)_\(text)_\(cursor)_\(selLen)".hashValue
guard currentHash != lastObservedHash else { return nil }
lastObservedHash = currentHash
// Check if text or cursor actually changed on Mac
if text == lastObservedText && cursor == lastObservedCursor && appName == lastObservedApp {
return nil
}
lastObservedText = text
lastObservedCursor = cursor
lastObservedApp = appName
localRevision += 1 localRevision += 1
return MacInputState(app: appName, text: text, cursor: cursor, selection: selLen, revision: localRevision) return MacInputState(app: appName, text: text, cursor: cursor, selection: selLen, revision: localRevision)
@ -169,22 +180,30 @@ public final class FocusedInputSync {
@discardableResult @discardableResult
public func applyRemoteUpdate(text: String, cursor: Int? = nil, isFullReplace: Bool = true) -> Bool { public func applyRemoteUpdate(text: String, cursor: Int? = nil, isFullReplace: Bool = true) -> Bool {
isApplyingRemoteChange = true isApplyingRemoteChange = true
remoteChangeExpiryTime = Date().timeIntervalSince1970 + 0.45 // 450ms quiet window
defer { defer {
DispatchQueue.main.asyncAfter(deadline: .now() + 0.12) {
DispatchQueue.main.asyncAfter(deadline: .now() + 0.45) {
self.isApplyingRemoteChange = false self.isApplyingRemoteChange = false
} }
} }
// 1. Flatten all newlines and carriage returns
// 1. Clean and normalize text
var cleanText = text.components(separatedBy: .newlines).joined(separator: " ") var cleanText = text.components(separatedBy: .newlines).joined(separator: " ")
cleanText = cleanText.replacingOccurrences(of: "\t", with: " ") cleanText = cleanText.replacingOccurrences(of: "\t", with: " ")
cleanText = cleanText.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) cleanText = cleanText.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression)
cleanText = cleanText.trimmingCharacters(in: .whitespacesAndNewlines) cleanText = cleanText.trimmingCharacters(in: .whitespacesAndNewlines)
let targetCursor = cursor ?? cleanText.count
lastObservedText = cleanText
lastObservedCursor = targetCursor
guard let (elem, appName) = getFocusedElement() else { guard let (elem, appName) = getFocusedElement() else {
// Fallback: clipboard paste // Fallback: clipboard paste
lastObservedApp = NSWorkspace.shared.frontmostApplication?.localizedName ?? "App"
return pasteViaKeystroke(cleanText, isFullReplace: isFullReplace) return pasteViaKeystroke(cleanText, isFullReplace: isFullReplace)
} }
lastObservedApp = appName
if isFullReplace { if isFullReplace {
// Try setting AXValue directly // Try setting AXValue directly
@ -196,7 +215,6 @@ public final class FocusedInputSync {
AXUIElementSetAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, axRange) AXUIElementSetAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, axRange)
} }
} }
lastObservedHash = "\(appName)_\(cleanText)_\(cursor ?? cleanText.count)_0".hashValue
print("FocusedInputSync: ✅ Updated AXValue for \(appName)") print("FocusedInputSync: ✅ Updated AXValue for \(appName)")
return true return true
} }
@ -204,7 +222,6 @@ public final class FocusedInputSync {
// Try setting selected text // Try setting selected text
let setSelErr = AXUIElementSetAttributeValue(elem, kAXSelectedTextAttribute as CFString, cleanText as CFTypeRef) let setSelErr = AXUIElementSetAttributeValue(elem, kAXSelectedTextAttribute as CFString, cleanText as CFTypeRef)
if setSelErr == .success { if setSelErr == .success {
lastObservedHash = "\(appName)_\(cleanText)_\(cursor ?? cleanText.count)_0".hashValue
print("FocusedInputSync: ✅ Inserted text via AXSelectedText for \(appName)") print("FocusedInputSync: ✅ Inserted text via AXSelectedText for \(appName)")
return true return true
} }
@ -228,7 +245,7 @@ public final class FocusedInputSync {
aDown?.post(tap: .cghidEventTap) aDown?.post(tap: .cghidEventTap)
aUp?.post(tap: .cghidEventTap) aUp?.post(tap: .cghidEventTap)
usleep(25000)
usleep(20000)
} }
// Paste: Cmd + V // Paste: Cmd + V

Loading…
Cancel
Save