Browse Source
feat: Google Docs & Figma style real-time bi-directional input synchronization (v5.0)
main
feat: Google Docs & Figma style real-time bi-directional input synchronization (v5.0)
main
17 changed files with 1053 additions and 513 deletions
-
112android/app/src/main/java/com/soniox/remotemic/MainActivity.kt
-
77android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt
-
6android/app/src/main/java/com/soniox/remotemic/VoiceRecognitionActivity.kt
-
237android/server/relay_server.py
-
206mac/src/FocusedInputSync.swift
-
92mac/src/RelayClient.swift
-
42mac/test_apps_ax.swift
-
107mac/test_focus_detector.swift
-
110mac/test_full_ax.swift
-
45mac/test_inspect.swift
-
45mac/test_inspect2.swift
-
76mac/test_inspect3.swift
-
81mac/test_live_ax.swift
-
21mac/test_timer.swift
-
39mac/test_tree.swift
-
33mac/test_win_ax.swift
-
237server/relay_server.py
@ -0,0 +1,42 @@ |
|||||
|
import Cocoa |
||||
|
import ApplicationServices |
||||
|
|
||||
|
func inspectApp(named targetName: String) { |
||||
|
guard let app = NSWorkspace.shared.runningApplications.first(where: { $0.localizedName == targetName }) else { |
||||
|
print("App \(targetName) not running") |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
print("\n--- Inspecting \(targetName) (PID: \(app.processIdentifier)) ---") |
||||
|
let appElem = AXUIElementCreateApplication(app.processIdentifier) |
||||
|
AXUIElementSetAttributeValue(appElem, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
||||
|
AXUIElementSetAttributeValue(appElem, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
||||
|
|
||||
|
var focusedElemObj: CFTypeRef? |
||||
|
let err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) |
||||
|
print("kAXFocusedUIElementAttribute error:", err.rawValue) |
||||
|
|
||||
|
if err == .success, let elem = focusedElemObj { |
||||
|
let axElem = elem as! AXUIElement |
||||
|
var roleObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleObj) |
||||
|
print("Role:", roleObj ?? "none") |
||||
|
|
||||
|
var valObj: CFTypeRef? |
||||
|
let valErr = AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) |
||||
|
print("Value err:", valErr.rawValue, "Value:", valObj ?? "none") |
||||
|
|
||||
|
var rangeObj: CFTypeRef? |
||||
|
let rangeErr = AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &rangeObj) |
||||
|
if rangeErr == .success, let axRange = rangeObj { |
||||
|
var range = CFRange() |
||||
|
if AXValueGetValue(axRange as! AXValue, .cfRange, &range) { |
||||
|
print("Cursor: loc=\(range.location), len=\(range.length)") |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
inspectApp(named: "Telegram") |
||||
|
inspectApp(named: "Obsidian") |
||||
|
inspectApp(named: "firefox") |
||||
@ -0,0 +1,107 @@ |
|||||
|
import Cocoa |
||||
|
import ApplicationServices |
||||
|
|
||||
|
// Comprehensive recursive search for any element with AXFocused == true |
||||
|
func findActiveFocusedElement(_ elem: AXUIElement) -> AXUIElement? { |
||||
|
var isFocusedObj: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &isFocusedObj) == .success, |
||||
|
let isFocused = isFocusedObj as? Bool, isFocused { |
||||
|
var roleObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) |
||||
|
let role = (roleObj as? String) ?? "" |
||||
|
if role != "AXWindow" && role != "AXApplication" && role != "AXGroup" && role != "AXSplitGroup" && role != "AXScrollArea" { |
||||
|
return elem |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
var childrenObj: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, |
||||
|
let children = childrenObj as? [AXUIElement] { |
||||
|
for child in children { |
||||
|
if let found = findActiveFocusedElement(child) { |
||||
|
return found |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func getFocusedTextInfo() -> (String, String, Int, Int)? { |
||||
|
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil } |
||||
|
let appName = frontApp.localizedName ?? "App" |
||||
|
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
||||
|
|
||||
|
// 1. Try systemWide |
||||
|
let sysWide = AXUIElementCreateSystemWide() |
||||
|
var focusedElemObj: CFTypeRef? |
||||
|
var targetElem: AXUIElement? |
||||
|
|
||||
|
if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) == .success, |
||||
|
let obj = focusedElemObj { |
||||
|
targetElem = (obj as! AXUIElement) |
||||
|
} |
||||
|
|
||||
|
// 2. Try App focused element |
||||
|
if targetElem == nil { |
||||
|
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) == .success, |
||||
|
let obj = focusedElemObj { |
||||
|
targetElem = (obj as! AXUIElement) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// 3. Try Recursive search in app tree |
||||
|
if targetElem == nil { |
||||
|
targetElem = findActiveFocusedElement(appElem) |
||||
|
} |
||||
|
|
||||
|
guard let elem = targetElem else { return nil } |
||||
|
|
||||
|
// Extract text |
||||
|
var text = "" |
||||
|
var valObj: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success, |
||||
|
let val = valObj { |
||||
|
if let str = val as? String { |
||||
|
text = str |
||||
|
} else if let attrStr = val as? NSAttributedString { |
||||
|
text = attrStr.string |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Try parameterized string for range |
||||
|
if text.isEmpty { |
||||
|
var countObj: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countObj) == .success, |
||||
|
let count = countObj as? Int, count > 0 { |
||||
|
var range = CFRange(location: 0, length: count) |
||||
|
if let axRange = AXValueCreate(.cfRange, &range) { |
||||
|
var strObj: CFTypeRef? |
||||
|
if AXUIElementCopyParameterizedAttributeValue(elem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &strObj) == .success, |
||||
|
let str = strObj as? String { |
||||
|
text = str |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Cursor & Selection |
||||
|
var cursor = text.count |
||||
|
var selLen = 0 |
||||
|
var rangeObj: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, &rangeObj) == .success, |
||||
|
let axRange = rangeObj { |
||||
|
var range = CFRange() |
||||
|
if AXValueGetValue(axRange as! AXValue, .cfRange, &range) { |
||||
|
cursor = range.location |
||||
|
selLen = range.length |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return (appName, text, cursor, selLen) |
||||
|
} |
||||
|
|
||||
|
if let (app, text, cursor, selLen) = getFocusedTextInfo() { |
||||
|
print("Found! App: \(app), Text: '\(text)', Cursor: \(cursor), SelLen: \(selLen)") |
||||
|
} else { |
||||
|
print("No focused text element found") |
||||
|
} |
||||
@ -0,0 +1,110 @@ |
|||||
|
import Cocoa |
||||
|
import ApplicationServices |
||||
|
|
||||
|
func extractTextAndCursor(from elem: AXUIElement) -> (String, Int, Int)? { |
||||
|
var roleRef: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleRef) |
||||
|
let role = (roleRef as? String) ?? "" |
||||
|
|
||||
|
var currentText = "" |
||||
|
var valueRef: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valueRef) == .success, |
||||
|
let val = valueRef { |
||||
|
if let str = val as? String { |
||||
|
currentText = str |
||||
|
} else if let attrStr = val as? NSAttributedString { |
||||
|
currentText = attrStr.string |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if currentText.isEmpty { |
||||
|
var countRef: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countRef) == .success, |
||||
|
let count = countRef as? Int, count > 0 { |
||||
|
var range = CFRange(location: 0, length: count) |
||||
|
if let axRange = AXValueCreate(.cfRange, &range) { |
||||
|
var stringRef: CFTypeRef? |
||||
|
if AXUIElementCopyParameterizedAttributeValue(elem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &stringRef) == .success, |
||||
|
let str = stringRef as? String { |
||||
|
currentText = str |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
var cursor = currentText.count |
||||
|
var selLen = 0 |
||||
|
var rangeRef: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, &rangeRef) == .success, |
||||
|
let val = rangeRef { |
||||
|
var cfRange = CFRange() |
||||
|
if AXValueGetValue(val as! AXValue, .cfRange, &cfRange) { |
||||
|
cursor = cfRange.location |
||||
|
selLen = cfRange.length |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if !currentText.isEmpty || role == "AXTextField" || role == "AXTextArea" || role == "AXSearchField" || rangeRef != nil { |
||||
|
return (currentText, cursor, selLen) |
||||
|
} |
||||
|
|
||||
|
// Check focused child |
||||
|
var focusedChildRef: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXFocusedUIElementAttribute as CFString, &focusedChildRef) == .success, |
||||
|
let child = focusedChildRef { |
||||
|
if let res = extractTextAndCursor(from: child as! AXUIElement) { |
||||
|
return res |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Check children |
||||
|
var childrenRef: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenRef) == .success, |
||||
|
let children = childrenRef as? [AXUIElement] { |
||||
|
for child in children { |
||||
|
var isFocusedRef: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(child, kAXFocusedAttribute as CFString, &isFocusedRef) == .success, |
||||
|
let isFoc = isFocusedRef as? Bool, isFoc { |
||||
|
if let res = extractTextAndCursor(from: child) { |
||||
|
return res |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func testFullAX() { |
||||
|
let sysWide = AXUIElementCreateSystemWide() |
||||
|
AXUIElementSetAttributeValue(sysWide, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
||||
|
AXUIElementSetAttributeValue(sysWide, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
||||
|
|
||||
|
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return } |
||||
|
print("Front App:", frontApp.localizedName ?? "") |
||||
|
|
||||
|
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
||||
|
AXUIElementSetAttributeValue(appElem, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
||||
|
AXUIElementSetAttributeValue(appElem, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
||||
|
|
||||
|
var focusedElem: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focusedElem) == .success, |
||||
|
let elem = focusedElem { |
||||
|
if let (text, cursor, sel) = extractTextAndCursor(from: elem as! AXUIElement) { |
||||
|
print("Extracted from sysWide -> Text: '\(text)', Cursor: \(cursor), Sel: \(sel)") |
||||
|
return |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElem) == .success, |
||||
|
let elem = focusedElem { |
||||
|
if let (text, cursor, sel) = extractTextAndCursor(from: elem as! AXUIElement) { |
||||
|
print("Extracted from appElem -> Text: '\(text)', Cursor: \(cursor), Sel: \(sel)") |
||||
|
return |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
print("Could not extract text") |
||||
|
} |
||||
|
|
||||
|
testFullAX() |
||||
@ -0,0 +1,45 @@ |
|||||
|
import Cocoa |
||||
|
import ApplicationServices |
||||
|
|
||||
|
func inspect() { |
||||
|
guard let frontApp = NSWorkspace.shared.frontmostApplication else { |
||||
|
print("No front app") |
||||
|
return |
||||
|
} |
||||
|
print("Front App:", frontApp.localizedName ?? "", "PID:", frontApp.processIdentifier) |
||||
|
|
||||
|
let trusted = AXIsProcessTrusted() |
||||
|
print("AXIsProcessTrusted:", trusted) |
||||
|
|
||||
|
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
||||
|
var focusedElemObj: CFTypeRef? |
||||
|
let err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) |
||||
|
print("kAXFocusedUIElement error:", err.rawValue) |
||||
|
|
||||
|
if err == .success, let elem = focusedElemObj { |
||||
|
let axElem = elem as! AXUIElement |
||||
|
var roleObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleObj) |
||||
|
print("Role:", roleObj ?? "none") |
||||
|
|
||||
|
var valObj: CFTypeRef? |
||||
|
let valErr = AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) |
||||
|
print("Value err:", valErr.rawValue, "Value:", valObj ?? "nil") |
||||
|
|
||||
|
var selectedTextObj: CFTypeRef? |
||||
|
let selTxtErr = AXUIElementCopyAttributeValue(axElem, kAXSelectedTextAttribute as CFString, &selectedTextObj) |
||||
|
print("SelectedText err:", selTxtErr.rawValue, "SelectedText:", selectedTextObj ?? "nil") |
||||
|
|
||||
|
var rangeObj: CFTypeRef? |
||||
|
let rangeErr = AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &rangeObj) |
||||
|
print("Range err:", rangeErr.rawValue) |
||||
|
if rangeErr == .success, let axRange = rangeObj { |
||||
|
var range = CFRange() |
||||
|
if AXValueGetValue(axRange as! AXValue, .cfRange, &range) { |
||||
|
print("CFRange: loc=\(range.location), len=\(range.length)") |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
inspect() |
||||
@ -0,0 +1,45 @@ |
|||||
|
import Cocoa |
||||
|
import ApplicationServices |
||||
|
|
||||
|
func getFocusedElement() -> AXUIElement? { |
||||
|
// 1. System wide |
||||
|
let sysWide = AXUIElementCreateSystemWide() |
||||
|
var sysFocusedObj: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &sysFocusedObj) == .success, |
||||
|
let obj = sysFocusedObj { |
||||
|
return (obj as! AXUIElement) |
||||
|
} |
||||
|
|
||||
|
// 2. Frontmost App |
||||
|
if let frontApp = NSWorkspace.shared.frontmostApplication { |
||||
|
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
||||
|
var appFocusedObj: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &appFocusedObj) == .success, |
||||
|
let obj = appFocusedObj { |
||||
|
return (obj as! AXUIElement) |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func inspectElement(_ elem: AXUIElement) { |
||||
|
var attrNamesObj: CFArray? |
||||
|
AXUIElementCopyAttributeNames(elem, &attrNamesObj) |
||||
|
if let names = attrNamesObj as? [String] { |
||||
|
print("Attribute Names:", names) |
||||
|
for name in names { |
||||
|
var val: CFTypeRef? |
||||
|
let err = AXUIElementCopyAttributeValue(elem, name as CFString, &val) |
||||
|
if err == .success, let val = val { |
||||
|
print(" \(name): \(val)") |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if let focused = getFocusedElement() { |
||||
|
print("Found Focused Element:") |
||||
|
inspectElement(focused) |
||||
|
} else { |
||||
|
print("No focused element found") |
||||
|
} |
||||
@ -0,0 +1,76 @@ |
|||||
|
import Cocoa |
||||
|
import ApplicationServices |
||||
|
|
||||
|
func findFocusedDescendant(_ elem: AXUIElement) -> AXUIElement? { |
||||
|
// Check if this element is a text field/text area or has value |
||||
|
var roleObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) |
||||
|
let role = (roleObj as? String) ?? "" |
||||
|
|
||||
|
if role == "AXTextField" || role == "AXTextArea" || role == "AXComboBox" || role == "AXSearchField" { |
||||
|
return elem |
||||
|
} |
||||
|
|
||||
|
var focusedObj: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &focusedObj) == .success, |
||||
|
let isFocused = focusedObj as? Bool, isFocused { |
||||
|
// If it has value attribute, return it |
||||
|
var valObj: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success { |
||||
|
return elem |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Check children |
||||
|
var childrenObj: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, |
||||
|
let children = childrenObj as? [AXUIElement] { |
||||
|
for child in children { |
||||
|
if let found = findFocusedDescendant(child) { |
||||
|
return found |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func getDeepFocusedElement() -> (AXUIElement, String, String)? { |
||||
|
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil } |
||||
|
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
||||
|
|
||||
|
// First try standard focused element |
||||
|
var focusedObj: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success, |
||||
|
let elem = focusedObj as! AXUIElement? { |
||||
|
var roleObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) |
||||
|
let role = (roleObj as? String) ?? "" |
||||
|
|
||||
|
var valObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) |
||||
|
let val = (valObj as? String) ?? "" |
||||
|
|
||||
|
if !val.isEmpty || role == "AXTextField" || role == "AXTextArea" { |
||||
|
return (elem, role, val) |
||||
|
} |
||||
|
|
||||
|
// If it's a window or web area, search descendants |
||||
|
if let deep = findFocusedDescendant(elem) { |
||||
|
var deepRoleObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(deep, kAXRoleAttribute as CFString, &deepRoleObj) |
||||
|
let deepRole = (deepRoleObj as? String) ?? "" |
||||
|
|
||||
|
var deepValObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(deep, kAXValueAttribute as CFString, &deepValObj) |
||||
|
let deepVal = (deepValObj as? String) ?? "" |
||||
|
return (deep, deepRole, deepVal) |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
if let (elem, role, val) = getDeepFocusedElement() { |
||||
|
print("Found deep focused element! Role: \(role), Value: '\(val)'") |
||||
|
} else { |
||||
|
print("Deep focused element not found") |
||||
|
} |
||||
@ -0,0 +1,81 @@ |
|||||
|
import Cocoa |
||||
|
import ApplicationServices |
||||
|
|
||||
|
func testLiveAX() { |
||||
|
let sysWide = AXUIElementCreateSystemWide() |
||||
|
|
||||
|
// Enable Chromium/Electron accessibility |
||||
|
AXUIElementSetAttributeValue(sysWide, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
||||
|
AXUIElementSetAttributeValue(sysWide, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
||||
|
|
||||
|
guard let frontApp = NSWorkspace.shared.frontmostApplication else { |
||||
|
print("No front app") |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
print("Front App:", frontApp.localizedName ?? "", "PID:", frontApp.processIdentifier) |
||||
|
|
||||
|
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
||||
|
AXUIElementSetAttributeValue(appElem, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
||||
|
AXUIElementSetAttributeValue(appElem, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
||||
|
|
||||
|
// Try system wide focused element |
||||
|
var focusedUIElement: CFTypeRef? |
||||
|
var err = AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focusedUIElement) |
||||
|
if err != .success || focusedUIElement == nil { |
||||
|
err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedUIElement) |
||||
|
} |
||||
|
|
||||
|
guard err == .success, let elem = focusedUIElement else { |
||||
|
print("No focused element. Error:", err.rawValue) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
let axElem = elem as! AXUIElement |
||||
|
|
||||
|
var roleRef: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleRef) |
||||
|
let role = (roleRef as? String) ?? "AXUnknown" |
||||
|
print("Role:", role) |
||||
|
|
||||
|
// Extract text |
||||
|
var currentText = "" |
||||
|
var valueRef: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valueRef) == .success, |
||||
|
let val = valueRef { |
||||
|
if let str = val as? String { |
||||
|
currentText = str |
||||
|
} else if let attrStr = val as? NSAttributedString { |
||||
|
currentText = attrStr.string |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if currentText.isEmpty { |
||||
|
var countRef: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(axElem, kAXNumberOfCharactersAttribute as CFString, &countRef) == .success, |
||||
|
let count = countRef as? Int { |
||||
|
var range = CFRange(location: 0, length: count) |
||||
|
if let axRange = AXValueCreate(.cfRange, &range) { |
||||
|
var stringRef: CFTypeRef? |
||||
|
if AXUIElementCopyParameterizedAttributeValue(axElem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &stringRef) == .success, |
||||
|
let str = stringRef as? String { |
||||
|
currentText = str |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
print("Extracted Text: '\(currentText)'") |
||||
|
|
||||
|
// Selection / Cursor |
||||
|
var rangeRef: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &rangeRef) == .success, |
||||
|
let val = rangeRef { |
||||
|
var cfRange = CFRange() |
||||
|
if AXValueGetValue(val as! AXValue, .cfRange, &cfRange) { |
||||
|
print("Cursor Location: \(cfRange.location), Selection Length: \(cfRange.length)") |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
testLiveAX() |
||||
@ -0,0 +1,21 @@ |
|||||
|
import Foundation |
||||
|
import Cocoa |
||||
|
|
||||
|
class TestTimer { |
||||
|
private var timerSource: DispatchSourceTimer? |
||||
|
|
||||
|
func start() { |
||||
|
let queue = DispatchQueue(label: "com.test.timer", qos: .userInteractive) |
||||
|
let timer = DispatchSource.makeTimerSource(queue: queue) |
||||
|
timer.schedule(deadline: .now(), repeating: .milliseconds(50)) |
||||
|
timer.setEventHandler { |
||||
|
if let (elem, app) = FocusedInputSync.shared.getFocusedElement() { |
||||
|
if let state = FocusedInputSync.shared.inspectCurrentState() { |
||||
|
print("Live State detected: '\(state.text)' in \(state.app)") |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
timer.resume() |
||||
|
self.timerSource = timer |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,39 @@ |
|||||
|
import Cocoa |
||||
|
import ApplicationServices |
||||
|
|
||||
|
func printAXTree(_ elem: AXUIElement, depth: Int = 0) { |
||||
|
if depth > 7 { return } |
||||
|
let indent = String(repeating: " ", count: depth) |
||||
|
|
||||
|
var roleObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) |
||||
|
let role = (roleObj as? String) ?? "unknown" |
||||
|
|
||||
|
var valObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) |
||||
|
let val = (valObj as? String) ?? "" |
||||
|
|
||||
|
var titleObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(elem, kAXTitleAttribute as CFString, &titleObj) |
||||
|
let title = (titleObj as? String) ?? "" |
||||
|
|
||||
|
var focusedObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &focusedObj) |
||||
|
let isFocused = (focusedObj as? Bool) ?? false |
||||
|
|
||||
|
print("\(indent)[\(role)] title='\(title)' val='\(val)' focused=\(isFocused)") |
||||
|
|
||||
|
var childrenObj: CFTypeRef? |
||||
|
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, |
||||
|
let children = childrenObj as? [AXUIElement] { |
||||
|
for child in children { |
||||
|
printAXTree(child, depth: depth + 1) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if let frontApp = NSWorkspace.shared.frontmostApplication { |
||||
|
print("Front App:", frontApp.localizedName ?? "") |
||||
|
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
||||
|
printAXTree(appElem) |
||||
|
} |
||||
@ -0,0 +1,33 @@ |
|||||
|
import Cocoa |
||||
|
import ApplicationServices |
||||
|
|
||||
|
func inspectFrontApp() { |
||||
|
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return } |
||||
|
print("Frontmost App:", frontApp.localizedName ?? "") |
||||
|
|
||||
|
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
||||
|
|
||||
|
// 1. Try focused window |
||||
|
var windowObj: CFTypeRef? |
||||
|
var err = AXUIElementCopyAttributeValue(appElem, kAXFocusedWindowAttribute as CFString, &windowObj) |
||||
|
print("kAXFocusedWindowAttribute err:", err.rawValue) |
||||
|
|
||||
|
if err == .success, let win = windowObj { |
||||
|
let winElem = win as! AXUIElement |
||||
|
var focusedObj: CFTypeRef? |
||||
|
let winErr = AXUIElementCopyAttributeValue(winElem, kAXFocusedUIElementAttribute as CFString, &focusedObj) |
||||
|
print("Window focused element err:", winErr.rawValue) |
||||
|
if winErr == .success, let elem = focusedObj { |
||||
|
let axElem = elem as! AXUIElement |
||||
|
var roleObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleObj) |
||||
|
print("Role from window:", roleObj ?? "none") |
||||
|
|
||||
|
var valObj: CFTypeRef? |
||||
|
AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) |
||||
|
print("Value from window:", valObj ?? "none") |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
inspectFrontApp() |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue