Browse Source
feat: Google Docs & Figma style real-time bi-directional input synchronization (v5.0)
main
feat: Google Docs & Figma style real-time bi-directional input synchronization (v5.0)
main
17 changed files with 1053 additions and 513 deletions
-
112android/app/src/main/java/com/soniox/remotemic/MainActivity.kt
-
67android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt
-
6android/app/src/main/java/com/soniox/remotemic/VoiceRecognitionActivity.kt
-
227android/server/relay_server.py
-
180mac/src/FocusedInputSync.swift
-
90mac/src/RelayClient.swift
-
42mac/test_apps_ax.swift
-
107mac/test_focus_detector.swift
-
110mac/test_full_ax.swift
-
45mac/test_inspect.swift
-
45mac/test_inspect2.swift
-
76mac/test_inspect3.swift
-
81mac/test_live_ax.swift
-
21mac/test_timer.swift
-
39mac/test_tree.swift
-
33mac/test_win_ax.swift
-
227server/relay_server.py
@ -0,0 +1,42 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func inspectApp(named targetName: String) { |
|||
guard let app = NSWorkspace.shared.runningApplications.first(where: { $0.localizedName == targetName }) else { |
|||
print("App \(targetName) not running") |
|||
return |
|||
} |
|||
|
|||
print("\n--- Inspecting \(targetName) (PID: \(app.processIdentifier)) ---") |
|||
let appElem = AXUIElementCreateApplication(app.processIdentifier) |
|||
AXUIElementSetAttributeValue(appElem, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
|||
AXUIElementSetAttributeValue(appElem, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
|||
|
|||
var focusedElemObj: CFTypeRef? |
|||
let err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) |
|||
print("kAXFocusedUIElementAttribute error:", err.rawValue) |
|||
|
|||
if err == .success, let elem = focusedElemObj { |
|||
let axElem = elem as! AXUIElement |
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleObj) |
|||
print("Role:", roleObj ?? "none") |
|||
|
|||
var valObj: CFTypeRef? |
|||
let valErr = AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) |
|||
print("Value err:", valErr.rawValue, "Value:", valObj ?? "none") |
|||
|
|||
var rangeObj: CFTypeRef? |
|||
let rangeErr = AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &rangeObj) |
|||
if rangeErr == .success, let axRange = rangeObj { |
|||
var range = CFRange() |
|||
if AXValueGetValue(axRange as! AXValue, .cfRange, &range) { |
|||
print("Cursor: loc=\(range.location), len=\(range.length)") |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
inspectApp(named: "Telegram") |
|||
inspectApp(named: "Obsidian") |
|||
inspectApp(named: "firefox") |
|||
@ -0,0 +1,107 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
// Comprehensive recursive search for any element with AXFocused == true |
|||
func findActiveFocusedElement(_ elem: AXUIElement) -> AXUIElement? { |
|||
var isFocusedObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &isFocusedObj) == .success, |
|||
let isFocused = isFocusedObj as? Bool, isFocused { |
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) |
|||
let role = (roleObj as? String) ?? "" |
|||
if role != "AXWindow" && role != "AXApplication" && role != "AXGroup" && role != "AXSplitGroup" && role != "AXScrollArea" { |
|||
return elem |
|||
} |
|||
} |
|||
|
|||
var childrenObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, |
|||
let children = childrenObj as? [AXUIElement] { |
|||
for child in children { |
|||
if let found = findActiveFocusedElement(child) { |
|||
return found |
|||
} |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func getFocusedTextInfo() -> (String, String, Int, Int)? { |
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil } |
|||
let appName = frontApp.localizedName ?? "App" |
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
|
|||
// 1. Try systemWide |
|||
let sysWide = AXUIElementCreateSystemWide() |
|||
var focusedElemObj: CFTypeRef? |
|||
var targetElem: AXUIElement? |
|||
|
|||
if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) == .success, |
|||
let obj = focusedElemObj { |
|||
targetElem = (obj as! AXUIElement) |
|||
} |
|||
|
|||
// 2. Try App focused element |
|||
if targetElem == nil { |
|||
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) == .success, |
|||
let obj = focusedElemObj { |
|||
targetElem = (obj as! AXUIElement) |
|||
} |
|||
} |
|||
|
|||
// 3. Try Recursive search in app tree |
|||
if targetElem == nil { |
|||
targetElem = findActiveFocusedElement(appElem) |
|||
} |
|||
|
|||
guard let elem = targetElem else { return nil } |
|||
|
|||
// Extract text |
|||
var text = "" |
|||
var valObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success, |
|||
let val = valObj { |
|||
if let str = val as? String { |
|||
text = str |
|||
} else if let attrStr = val as? NSAttributedString { |
|||
text = attrStr.string |
|||
} |
|||
} |
|||
|
|||
// Try parameterized string for range |
|||
if text.isEmpty { |
|||
var countObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countObj) == .success, |
|||
let count = countObj as? Int, count > 0 { |
|||
var range = CFRange(location: 0, length: count) |
|||
if let axRange = AXValueCreate(.cfRange, &range) { |
|||
var strObj: CFTypeRef? |
|||
if AXUIElementCopyParameterizedAttributeValue(elem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &strObj) == .success, |
|||
let str = strObj as? String { |
|||
text = str |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
// Cursor & Selection |
|||
var cursor = text.count |
|||
var selLen = 0 |
|||
var rangeObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, &rangeObj) == .success, |
|||
let axRange = rangeObj { |
|||
var range = CFRange() |
|||
if AXValueGetValue(axRange as! AXValue, .cfRange, &range) { |
|||
cursor = range.location |
|||
selLen = range.length |
|||
} |
|||
} |
|||
|
|||
return (appName, text, cursor, selLen) |
|||
} |
|||
|
|||
if let (app, text, cursor, selLen) = getFocusedTextInfo() { |
|||
print("Found! App: \(app), Text: '\(text)', Cursor: \(cursor), SelLen: \(selLen)") |
|||
} else { |
|||
print("No focused text element found") |
|||
} |
|||
@ -0,0 +1,110 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func extractTextAndCursor(from elem: AXUIElement) -> (String, Int, Int)? { |
|||
var roleRef: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleRef) |
|||
let role = (roleRef as? String) ?? "" |
|||
|
|||
var currentText = "" |
|||
var valueRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valueRef) == .success, |
|||
let val = valueRef { |
|||
if let str = val as? String { |
|||
currentText = str |
|||
} else if let attrStr = val as? NSAttributedString { |
|||
currentText = attrStr.string |
|||
} |
|||
} |
|||
|
|||
if currentText.isEmpty { |
|||
var countRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countRef) == .success, |
|||
let count = countRef as? Int, count > 0 { |
|||
var range = CFRange(location: 0, length: count) |
|||
if let axRange = AXValueCreate(.cfRange, &range) { |
|||
var stringRef: CFTypeRef? |
|||
if AXUIElementCopyParameterizedAttributeValue(elem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &stringRef) == .success, |
|||
let str = stringRef as? String { |
|||
currentText = str |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
var cursor = currentText.count |
|||
var selLen = 0 |
|||
var rangeRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, &rangeRef) == .success, |
|||
let val = rangeRef { |
|||
var cfRange = CFRange() |
|||
if AXValueGetValue(val as! AXValue, .cfRange, &cfRange) { |
|||
cursor = cfRange.location |
|||
selLen = cfRange.length |
|||
} |
|||
} |
|||
|
|||
if !currentText.isEmpty || role == "AXTextField" || role == "AXTextArea" || role == "AXSearchField" || rangeRef != nil { |
|||
return (currentText, cursor, selLen) |
|||
} |
|||
|
|||
// Check focused child |
|||
var focusedChildRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXFocusedUIElementAttribute as CFString, &focusedChildRef) == .success, |
|||
let child = focusedChildRef { |
|||
if let res = extractTextAndCursor(from: child as! AXUIElement) { |
|||
return res |
|||
} |
|||
} |
|||
|
|||
// Check children |
|||
var childrenRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenRef) == .success, |
|||
let children = childrenRef as? [AXUIElement] { |
|||
for child in children { |
|||
var isFocusedRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(child, kAXFocusedAttribute as CFString, &isFocusedRef) == .success, |
|||
let isFoc = isFocusedRef as? Bool, isFoc { |
|||
if let res = extractTextAndCursor(from: child) { |
|||
return res |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func testFullAX() { |
|||
let sysWide = AXUIElementCreateSystemWide() |
|||
AXUIElementSetAttributeValue(sysWide, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
|||
AXUIElementSetAttributeValue(sysWide, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
|||
|
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return } |
|||
print("Front App:", frontApp.localizedName ?? "") |
|||
|
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
AXUIElementSetAttributeValue(appElem, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
|||
AXUIElementSetAttributeValue(appElem, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
|||
|
|||
var focusedElem: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focusedElem) == .success, |
|||
let elem = focusedElem { |
|||
if let (text, cursor, sel) = extractTextAndCursor(from: elem as! AXUIElement) { |
|||
print("Extracted from sysWide -> Text: '\(text)', Cursor: \(cursor), Sel: \(sel)") |
|||
return |
|||
} |
|||
} |
|||
|
|||
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElem) == .success, |
|||
let elem = focusedElem { |
|||
if let (text, cursor, sel) = extractTextAndCursor(from: elem as! AXUIElement) { |
|||
print("Extracted from appElem -> Text: '\(text)', Cursor: \(cursor), Sel: \(sel)") |
|||
return |
|||
} |
|||
} |
|||
|
|||
print("Could not extract text") |
|||
} |
|||
|
|||
testFullAX() |
|||
@ -0,0 +1,45 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func inspect() { |
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { |
|||
print("No front app") |
|||
return |
|||
} |
|||
print("Front App:", frontApp.localizedName ?? "", "PID:", frontApp.processIdentifier) |
|||
|
|||
let trusted = AXIsProcessTrusted() |
|||
print("AXIsProcessTrusted:", trusted) |
|||
|
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
var focusedElemObj: CFTypeRef? |
|||
let err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) |
|||
print("kAXFocusedUIElement error:", err.rawValue) |
|||
|
|||
if err == .success, let elem = focusedElemObj { |
|||
let axElem = elem as! AXUIElement |
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleObj) |
|||
print("Role:", roleObj ?? "none") |
|||
|
|||
var valObj: CFTypeRef? |
|||
let valErr = AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) |
|||
print("Value err:", valErr.rawValue, "Value:", valObj ?? "nil") |
|||
|
|||
var selectedTextObj: CFTypeRef? |
|||
let selTxtErr = AXUIElementCopyAttributeValue(axElem, kAXSelectedTextAttribute as CFString, &selectedTextObj) |
|||
print("SelectedText err:", selTxtErr.rawValue, "SelectedText:", selectedTextObj ?? "nil") |
|||
|
|||
var rangeObj: CFTypeRef? |
|||
let rangeErr = AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &rangeObj) |
|||
print("Range err:", rangeErr.rawValue) |
|||
if rangeErr == .success, let axRange = rangeObj { |
|||
var range = CFRange() |
|||
if AXValueGetValue(axRange as! AXValue, .cfRange, &range) { |
|||
print("CFRange: loc=\(range.location), len=\(range.length)") |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
inspect() |
|||
@ -0,0 +1,45 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func getFocusedElement() -> AXUIElement? { |
|||
// 1. System wide |
|||
let sysWide = AXUIElementCreateSystemWide() |
|||
var sysFocusedObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &sysFocusedObj) == .success, |
|||
let obj = sysFocusedObj { |
|||
return (obj as! AXUIElement) |
|||
} |
|||
|
|||
// 2. Frontmost App |
|||
if let frontApp = NSWorkspace.shared.frontmostApplication { |
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
var appFocusedObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &appFocusedObj) == .success, |
|||
let obj = appFocusedObj { |
|||
return (obj as! AXUIElement) |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func inspectElement(_ elem: AXUIElement) { |
|||
var attrNamesObj: CFArray? |
|||
AXUIElementCopyAttributeNames(elem, &attrNamesObj) |
|||
if let names = attrNamesObj as? [String] { |
|||
print("Attribute Names:", names) |
|||
for name in names { |
|||
var val: CFTypeRef? |
|||
let err = AXUIElementCopyAttributeValue(elem, name as CFString, &val) |
|||
if err == .success, let val = val { |
|||
print(" \(name): \(val)") |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
if let focused = getFocusedElement() { |
|||
print("Found Focused Element:") |
|||
inspectElement(focused) |
|||
} else { |
|||
print("No focused element found") |
|||
} |
|||
@ -0,0 +1,76 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func findFocusedDescendant(_ elem: AXUIElement) -> AXUIElement? { |
|||
// Check if this element is a text field/text area or has value |
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) |
|||
let role = (roleObj as? String) ?? "" |
|||
|
|||
if role == "AXTextField" || role == "AXTextArea" || role == "AXComboBox" || role == "AXSearchField" { |
|||
return elem |
|||
} |
|||
|
|||
var focusedObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &focusedObj) == .success, |
|||
let isFocused = focusedObj as? Bool, isFocused { |
|||
// If it has value attribute, return it |
|||
var valObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success { |
|||
return elem |
|||
} |
|||
} |
|||
|
|||
// Check children |
|||
var childrenObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, |
|||
let children = childrenObj as? [AXUIElement] { |
|||
for child in children { |
|||
if let found = findFocusedDescendant(child) { |
|||
return found |
|||
} |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func getDeepFocusedElement() -> (AXUIElement, String, String)? { |
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil } |
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
|
|||
// First try standard focused element |
|||
var focusedObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success, |
|||
let elem = focusedObj as! AXUIElement? { |
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) |
|||
let role = (roleObj as? String) ?? "" |
|||
|
|||
var valObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) |
|||
let val = (valObj as? String) ?? "" |
|||
|
|||
if !val.isEmpty || role == "AXTextField" || role == "AXTextArea" { |
|||
return (elem, role, val) |
|||
} |
|||
|
|||
// If it's a window or web area, search descendants |
|||
if let deep = findFocusedDescendant(elem) { |
|||
var deepRoleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(deep, kAXRoleAttribute as CFString, &deepRoleObj) |
|||
let deepRole = (deepRoleObj as? String) ?? "" |
|||
|
|||
var deepValObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(deep, kAXValueAttribute as CFString, &deepValObj) |
|||
let deepVal = (deepValObj as? String) ?? "" |
|||
return (deep, deepRole, deepVal) |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
if let (elem, role, val) = getDeepFocusedElement() { |
|||
print("Found deep focused element! Role: \(role), Value: '\(val)'") |
|||
} else { |
|||
print("Deep focused element not found") |
|||
} |
|||
@ -0,0 +1,81 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func testLiveAX() { |
|||
let sysWide = AXUIElementCreateSystemWide() |
|||
|
|||
// Enable Chromium/Electron accessibility |
|||
AXUIElementSetAttributeValue(sysWide, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
|||
AXUIElementSetAttributeValue(sysWide, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
|||
|
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { |
|||
print("No front app") |
|||
return |
|||
} |
|||
|
|||
print("Front App:", frontApp.localizedName ?? "", "PID:", frontApp.processIdentifier) |
|||
|
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
AXUIElementSetAttributeValue(appElem, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
|||
AXUIElementSetAttributeValue(appElem, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
|||
|
|||
// Try system wide focused element |
|||
var focusedUIElement: CFTypeRef? |
|||
var err = AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focusedUIElement) |
|||
if err != .success || focusedUIElement == nil { |
|||
err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedUIElement) |
|||
} |
|||
|
|||
guard err == .success, let elem = focusedUIElement else { |
|||
print("No focused element. Error:", err.rawValue) |
|||
return |
|||
} |
|||
|
|||
let axElem = elem as! AXUIElement |
|||
|
|||
var roleRef: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleRef) |
|||
let role = (roleRef as? String) ?? "AXUnknown" |
|||
print("Role:", role) |
|||
|
|||
// Extract text |
|||
var currentText = "" |
|||
var valueRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valueRef) == .success, |
|||
let val = valueRef { |
|||
if let str = val as? String { |
|||
currentText = str |
|||
} else if let attrStr = val as? NSAttributedString { |
|||
currentText = attrStr.string |
|||
} |
|||
} |
|||
|
|||
if currentText.isEmpty { |
|||
var countRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(axElem, kAXNumberOfCharactersAttribute as CFString, &countRef) == .success, |
|||
let count = countRef as? Int { |
|||
var range = CFRange(location: 0, length: count) |
|||
if let axRange = AXValueCreate(.cfRange, &range) { |
|||
var stringRef: CFTypeRef? |
|||
if AXUIElementCopyParameterizedAttributeValue(axElem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &stringRef) == .success, |
|||
let str = stringRef as? String { |
|||
currentText = str |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
print("Extracted Text: '\(currentText)'") |
|||
|
|||
// Selection / Cursor |
|||
var rangeRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &rangeRef) == .success, |
|||
let val = rangeRef { |
|||
var cfRange = CFRange() |
|||
if AXValueGetValue(val as! AXValue, .cfRange, &cfRange) { |
|||
print("Cursor Location: \(cfRange.location), Selection Length: \(cfRange.length)") |
|||
} |
|||
} |
|||
} |
|||
|
|||
testLiveAX() |
|||
@ -0,0 +1,21 @@ |
|||
import Foundation |
|||
import Cocoa |
|||
|
|||
class TestTimer { |
|||
private var timerSource: DispatchSourceTimer? |
|||
|
|||
func start() { |
|||
let queue = DispatchQueue(label: "com.test.timer", qos: .userInteractive) |
|||
let timer = DispatchSource.makeTimerSource(queue: queue) |
|||
timer.schedule(deadline: .now(), repeating: .milliseconds(50)) |
|||
timer.setEventHandler { |
|||
if let (elem, app) = FocusedInputSync.shared.getFocusedElement() { |
|||
if let state = FocusedInputSync.shared.inspectCurrentState() { |
|||
print("Live State detected: '\(state.text)' in \(state.app)") |
|||
} |
|||
} |
|||
} |
|||
timer.resume() |
|||
self.timerSource = timer |
|||
} |
|||
} |
|||
@ -0,0 +1,39 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func printAXTree(_ elem: AXUIElement, depth: Int = 0) { |
|||
if depth > 7 { return } |
|||
let indent = String(repeating: " ", count: depth) |
|||
|
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) |
|||
let role = (roleObj as? String) ?? "unknown" |
|||
|
|||
var valObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) |
|||
let val = (valObj as? String) ?? "" |
|||
|
|||
var titleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXTitleAttribute as CFString, &titleObj) |
|||
let title = (titleObj as? String) ?? "" |
|||
|
|||
var focusedObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &focusedObj) |
|||
let isFocused = (focusedObj as? Bool) ?? false |
|||
|
|||
print("\(indent)[\(role)] title='\(title)' val='\(val)' focused=\(isFocused)") |
|||
|
|||
var childrenObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, |
|||
let children = childrenObj as? [AXUIElement] { |
|||
for child in children { |
|||
printAXTree(child, depth: depth + 1) |
|||
} |
|||
} |
|||
} |
|||
|
|||
if let frontApp = NSWorkspace.shared.frontmostApplication { |
|||
print("Front App:", frontApp.localizedName ?? "") |
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
printAXTree(appElem) |
|||
} |
|||
@ -0,0 +1,33 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func inspectFrontApp() { |
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return } |
|||
print("Frontmost App:", frontApp.localizedName ?? "") |
|||
|
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
|
|||
// 1. Try focused window |
|||
var windowObj: CFTypeRef? |
|||
var err = AXUIElementCopyAttributeValue(appElem, kAXFocusedWindowAttribute as CFString, &windowObj) |
|||
print("kAXFocusedWindowAttribute err:", err.rawValue) |
|||
|
|||
if err == .success, let win = windowObj { |
|||
let winElem = win as! AXUIElement |
|||
var focusedObj: CFTypeRef? |
|||
let winErr = AXUIElementCopyAttributeValue(winElem, kAXFocusedUIElementAttribute as CFString, &focusedObj) |
|||
print("Window focused element err:", winErr.rawValue) |
|||
if winErr == .success, let elem = focusedObj { |
|||
let axElem = elem as! AXUIElement |
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleObj) |
|||
print("Role from window:", roleObj ?? "none") |
|||
|
|||
var valObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) |
|||
print("Value from window:", valObj ?? "none") |
|||
} |
|||
} |
|||
} |
|||
|
|||
inspectFrontApp() |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue