Browse Source

fix(mac): process-targeted AppleScript injection and non-activating floating HUD window

main
Ali Alavi 24 hours ago
parent
commit
20bcb17c68
  1. 17
      mac/src/AppDelegate.swift
  2. 169
      mac/src/FocusedInputSync.swift
  3. 11
      mac/src/HUDOverlay.swift

17
mac/src/AppDelegate.swift

@ -60,11 +60,16 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
startRemotePasteServer()
// Connect to Linux Persistent Gateway (via local tunnel 18999 -> 8999)
RelayClient.shared.onRemoteUpdateReceived = { [weak self] text, cursor, isFullReplace in
guard let self = self else { return }
print("AppDelegate: 📥 Clean Remote Input Update (Silent): '\(text.prefix(30))...' (replace: \(isFullReplace))")
HUDOverlayController.shared.show(state: .success(text: text))
RelayClient.shared.onRemoteUpdateReceived = { text, cursor, isFullReplace in
print("AppDelegate: 📥 Clean Remote Input Update: '\(text.prefix(30))...' (replace: \(isFullReplace))")
// 1. FIRST apply the remote update directly into the active input while window is in pristine focus!
FocusedInputSync.shared.applyRemoteUpdate(text: text, cursor: cursor, isFullReplace: isFullReplace)
// 2. THEN show the floating success HUD notification
DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) {
HUDOverlayController.shared.show(state: .success(text: text))
}
}
RelayClient.shared.start()
@ -91,8 +96,8 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
}
private func handleRemoteConnection(_ connection: NWConnection) {
connection.receive(minimumIncompleteLength: 1, maximumLength: 65536) { [weak self] data, _, _, _ in
guard let self = self, let data = data, let reqStr = String(data: data, encoding: .utf8) else {
connection.receive(minimumIncompleteLength: 1, maximumLength: 65536) { data, _, _, _ in
guard let data = data, let reqStr = String(data: data, encoding: .utf8) else {
connection.cancel()
return
}

169
mac/src/FocusedInputSync.swift

@ -1,5 +1,6 @@
import Cocoa
import ApplicationServices
import CoreGraphics
public struct MacInputState: Codable {
public let source: String
@ -52,7 +53,7 @@ public final class FocusedInputSync {
AXUIElementSetAttributeValue(systemWideElement, "AXManualAccessibility" as CFString, kCFBooleanTrue)
}
private func isAppPastePreferred(_ appName: String) -> Bool {
public func isAppPastePreferred(_ appName: String) -> Bool {
let lower = appName.lowercased()
for pref in pastePreferredApps {
if lower.contains(pref) {
@ -62,6 +63,46 @@ public final class FocusedInputSync {
return false
}
/// Resolves the actual user-facing application, bypassing system overlays like UserNotificationCenter
public func getRealFrontmostApp() -> NSRunningApplication? {
// 1. If standard frontmost app is a real regular user app, return it
if let front = NSWorkspace.shared.frontmostApplication,
front.activationPolicy == .regular,
let bundleId = front.bundleIdentifier,
!bundleId.contains("notificationcenter"),
!bundleId.contains("controlcenter"),
!bundleId.contains("WindowManager"),
!bundleId.contains("Soniox") {
return front
}
// 2. Otherwise, find top on-screen Layer 0 window from CGWindowList
let windowList = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? []
for win in windowList {
let layer = win[kCGWindowLayer as String] as? Int ?? -1
let pid = win[kCGWindowOwnerPID as String] as? pid_t ?? 0
let bounds = win[kCGWindowBounds as String] as? [String: Any] ?? [:]
let width = bounds["Width"] as? CGFloat ?? 0
let height = bounds["Height"] as? CGFloat ?? 0
// Only consider standard app windows (layer 0, reasonable size)
if layer == 0 && width > 100 && height > 100 {
if let app = NSRunningApplication(processIdentifier: pid),
app.activationPolicy == .regular,
let bundleId = app.bundleIdentifier,
!bundleId.contains("notificationcenter"),
!bundleId.contains("controlcenter"),
!bundleId.contains("WindowManager"),
!bundleId.contains("Soniox") {
return app
}
}
}
return NSWorkspace.shared.frontmostApplication
}
private func findFocusedDescendant(_ elem: AXUIElement, depth: Int = 0) -> AXUIElement? {
if depth > 12 { return nil }
@ -89,7 +130,7 @@ public final class FocusedInputSync {
}
public func getFocusedElement() -> (AXUIElement?, String) {
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return (nil, "App") }
guard let frontApp = getRealFrontmostApp() else { return (nil, "App") }
let appName = frontApp.localizedName ?? "App"
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier)
@ -162,8 +203,7 @@ public final class FocusedInputSync {
}
guard let elem = elemOpt else {
// When no native AX element is available (e.g. Electron web canvas or Antigravity),
// DO NOT emit empty text which would wipe out the mobile client transcript!
// When in opaque Electron/Antigravity containers, do not emit ghost empty text
if appChanged && !lastObservedText.isEmpty {
localRevision += 1
return MacInputState(app: appName, text: lastObservedText, cursor: lastObservedCursor, selection: 0, revision: localRevision)
@ -175,8 +215,6 @@ public final class FocusedInputSync {
var roleObj: CFTypeRef?
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj)
let role = roleObj as? String ?? ""
// Only inspect text if element is a recognized text-capable input
let isTextRole = (role == "AXTextField" || role == "AXTextArea" || role == "AXComboBox" || role == "AXSearchField")
// Extract text
@ -265,10 +303,10 @@ public final class FocusedInputSync {
lastObservedApp = appName
// If the target app is a Web/Electron/IDE application (e.g. Antigravity, Chrome, VS Code, Discord, Slack)
// or no native AX element was resolved, bypass AXValue write and execute rock-solid synthetic paste!
// or no native AX element was resolved, execute process-targeted synthetic paste!
if isAppPastePreferred(appName) || elemOpt == nil {
print("FocusedInputSync: 🎯 Target app '\(appName)' is Web/Electron/Paste-preferred. Executing Universal Synthetic Paste.")
return pasteViaKeystroke(cleanText, isFullReplace: isFullReplace)
print("FocusedInputSync: 🎯 Target app '\(appName)' is Web/Electron. Executing Process-Targeted Paste.")
return pasteViaKeystroke(cleanText, appName: appName, isFullReplace: isFullReplace)
}
// For Native AppKit/Cocoa apps (e.g. TextEdit, Notes, Finder):
@ -295,58 +333,107 @@ public final class FocusedInputSync {
}
// Reliable fallback for any situation
return pasteViaKeystroke(cleanText, isFullReplace: isFullReplace)
return pasteViaKeystroke(cleanText, appName: appName, isFullReplace: isFullReplace)
}
/**
* Ultra-reliable synthetic keystroke injection (Cmd+A -> Cmd+V)
* Includes proper key-dwell times, modifier flag persistence, and clipboard preservation.
* Ultra-reliable Process-Targeted synthetic keystroke injection (AppleScript System Events + Quartz HID)
* Forces frontmost focus on the target app process and injects Cmd+V directly into its input element.
*/
private func pasteViaKeystroke(_ text: String, isFullReplace: Bool) -> Bool {
private func pasteViaKeystroke(_ text: String, appName: String, isFullReplace: Bool) -> Bool {
let pasteboard = NSPasteboard.general
// 1. Snapshot previous clipboard to restore after paste
let oldString = pasteboard.string(forType: .string)
// 2. Set new text to pasteboard
// 2. Set new speech text to pasteboard
pasteboard.clearContents()
pasteboard.setString(text, forType: .string)
let src = CGEventSource(stateID: .combinedSessionState)
// 3. Ensure target app is activated
if let realApp = getRealFrontmostApp() {
realApp.activate(options: [])
usleep(25000) // 25ms focus settling
}
if isFullReplace {
// Select all: Cmd + A (virtualKey 0 = 'a')
if let aDown = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: true) {
aDown.flags = .maskCommand
aDown.post(tap: .cghidEventTap)
// 4. Engine 1: Process-Targeted AppleScript System Events
var appleScriptSuccess = false
let safeAppName = appName.replacingOccurrences(of: "\"", with: "\\\"")
let scriptSource = isFullReplace ? """
tell application "\(safeAppName)" to activate
tell application "System Events"
tell process "\(safeAppName)"
set frontmost to true
keystroke "a" using command down
delay 0.03
keystroke "v" using command down
end tell
end tell
""" : """
tell application "\(safeAppName)" to activate
tell application "System Events"
tell process "\(safeAppName)"
set frontmost to true
keystroke "v" using command down
end tell
end tell
"""
if let script = NSAppleScript(source: scriptSource) {
var errorDict: NSDictionary?
script.executeAndReturnError(&errorDict)
if errorDict == nil {
appleScriptSuccess = true
print("FocusedInputSync: ✅ Process-Targeted AppleScript Injected into '\(safeAppName)' (replace: \(isFullReplace))")
} else {
print("FocusedInputSync: ⚠️ Targeted AppleScript warning: \(errorDict ?? [:]), trying general System Events")
// General fallback without process qualification
let generalScriptSource = isFullReplace ? "tell application \"System Events\" to {keystroke \"a\" using command down, delay 0.03, keystroke \"v\" using command down}" : "tell application \"System Events\" to keystroke \"v\" using command down"
if let genScript = NSAppleScript(source: generalScriptSource) {
var genErr: NSDictionary?
genScript.executeAndReturnError(&genErr)
if genErr == nil {
appleScriptSuccess = true
}
}
}
usleep(12000) // 12ms key-down dwell time
}
// 5. Engine 2: Quartz HID Event Tap Fallback (if AppleScript didn't run)
if !appleScriptSuccess {
let src = CGEventSource(stateID: .hidSystemState)
if let aUp = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: false) {
aUp.flags = .maskCommand
aUp.post(tap: .cghidEventTap)
if isFullReplace {
// Select all: Cmd + A (virtualKey 0 = 'a')
if let aDown = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: true) {
aDown.flags = .maskCommand
aDown.post(tap: .cghidEventTap)
}
usleep(15000)
if let aUp = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: false) {
aUp.flags = .maskCommand
aUp.post(tap: .cghidEventTap)
}
usleep(40000)
}
// 35ms settling delay for Electron / DOM selection update
usleep(35000)
}
// Paste: Cmd + V (virtualKey 9 = 'v')
if let vDown = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: true) {
vDown.flags = .maskCommand
vDown.post(tap: .cghidEventTap)
}
usleep(12000) // 12ms key-down dwell time
if let vUp = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: false) {
vUp.flags = .maskCommand
vUp.post(tap: .cghidEventTap)
// Paste: Cmd + V (virtualKey 9 = 'v')
if let vDown = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: true) {
vDown.flags = .maskCommand
vDown.post(tap: .cghidEventTap)
}
usleep(15000)
if let vUp = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: false) {
vUp.flags = .maskCommand
vUp.post(tap: .cghidEventTap)
}
print("FocusedInputSync: ✅ Injected via Quartz HID Event Tap (replace: \(isFullReplace))")
}
// 3. Asynchronously restore previous clipboard content after 250ms
// 6. Asynchronously restore previous clipboard content after 600ms
if let previousText = oldString, previousText != text {
DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + 0.25) {
// If pasteboard hasn't been changed by user in the meantime, restore
DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + 0.60) {
if pasteboard.string(forType: .string) == text {
pasteboard.clearContents()
pasteboard.setString(previousText, forType: .string)

11
mac/src/HUDOverlay.swift

@ -8,10 +8,16 @@ public enum HUDState {
case error(message: String)
}
final class NonActivatingFloatingPanel: NSPanel {
override var canBecomeKey: Bool { return false }
override var canBecomeMain: Bool { return false }
override var acceptsFirstResponder: Bool { return false }
}
public final class HUDOverlayController {
public static let shared = HUDOverlayController()
private var window: NSPanel?
private var window: NonActivatingFloatingPanel?
private var visualEffectView: NSVisualEffectView?
private var iconImageView: NSImageView?
private var titleLabel: NSTextField?
@ -27,7 +33,7 @@ public final class HUDOverlayController {
let width: CGFloat = 460
let height: CGFloat = 80
let panel = NSPanel(
let panel = NonActivatingFloatingPanel(
contentRect: NSRect(x: 0, y: 0, width: width, height: height),
styleMask: [.borderless, .nonactivatingPanel],
backing: .buffered,
@ -39,6 +45,7 @@ public final class HUDOverlayController {
panel.backgroundColor = .clear
panel.hasShadow = true
panel.ignoresMouseEvents = true
panel.becomesKeyOnlyIfNeeded = false
panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary]
let visualEffect = NSVisualEffectView(frame: NSRect(x: 0, y: 0, width: width, height: height))

Loading…
Cancel
Save