Browse Source
feat: real-time bi-directional input synchronization & cursor-aware speech insertion
main
feat: real-time bi-directional input synchronization & cursor-aware speech insertion
main
8 changed files with 1016 additions and 254 deletions
-
164android/app/src/main/java/com/soniox/remotemic/MainActivity.kt
-
63android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt
-
101android/app/src/main/java/com/soniox/remotemic/VoiceRecognitionActivity.kt
-
471android/server/relay_server.py
-
46mac/src/AppDelegate.swift
-
137mac/src/FocusedInputSync.swift
-
91mac/src/RelayClient.swift
-
197server/relay_server.py
@ -1,102 +1,65 @@ |
|||
package com.soniox.remotemic |
|||
|
|||
import android.Manifest |
|||
import android.app.Activity |
|||
import android.content.Intent |
|||
import android.content.pm.PackageManager |
|||
import android.os.Bundle |
|||
import android.speech.RecognizerIntent |
|||
import android.view.View |
|||
import android.widget.Toast |
|||
import androidx.activity.result.contract.ActivityResultContracts |
|||
import android.widget.TextView |
|||
import androidx.appcompat.app.AppCompatActivity |
|||
import androidx.core.content.ContextCompat |
|||
import com.soniox.remotemic.databinding.ActivityVoiceRecognitionBinding |
|||
|
|||
class VoiceRecognitionActivity : AppCompatActivity() { |
|||
|
|||
private lateinit var binding: ActivityVoiceRecognitionBinding |
|||
private var streamDictationClient: StreamDictationClient? = null |
|||
private var lastTranscript: String = "" |
|||
|
|||
private val requestPermissionLauncher = registerForActivityResult( |
|||
ActivityResultContracts.RequestPermission() |
|||
) { isGranted: Boolean -> |
|||
if (isGranted) { |
|||
startListening() |
|||
} else { |
|||
Toast.makeText(this, "دسترسی میکروفون برای تایپ صوتی الزامی است", Toast.LENGTH_SHORT).show() |
|||
setResult(Activity.RESULT_CANCELED) |
|||
finish() |
|||
} |
|||
} |
|||
private var streamClient: StreamDictationClient? = null |
|||
private lateinit var tvStatus: TextView |
|||
private lateinit var tvTranscript: TextView |
|||
|
|||
override fun onCreate(savedInstanceState: Bundle?) { |
|||
super.onCreate(savedInstanceState) |
|||
binding = ActivityVoiceRecognitionBinding.inflate(layoutInflater) |
|||
setContentView(binding.root) |
|||
|
|||
binding.btnDialogCancel.setOnClickListener { |
|||
streamDictationClient?.stopRecording() |
|||
setResult(Activity.RESULT_CANCELED) |
|||
finish() |
|||
} |
|||
setContentView(R.layout.activity_voice_recognition) |
|||
|
|||
binding.btnDialogDone.setOnClickListener { |
|||
finishWithResult(lastTranscript) |
|||
} |
|||
tvStatus = findViewById(R.id.tvDialogStatus) |
|||
tvTranscript = findViewById(R.id.tvDialogTranscript) |
|||
|
|||
if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) { |
|||
startListening() |
|||
} else { |
|||
requestPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) |
|||
} |
|||
} |
|||
val host = "116.16.16.19:8999" |
|||
|
|||
private fun startListening() { |
|||
val host = PreferencesManager.getServerHost(this) |
|||
streamDictationClient = StreamDictationClient( |
|||
streamClient = StreamDictationClient( |
|||
host = host, |
|||
onConnectionStateChanged = { connected -> |
|||
if (!connected) { |
|||
binding.tvDialogStatus.text = "در حال اتصال به سرور..." |
|||
} else { |
|||
binding.tvDialogStatus.text = "در حال گوش دادن..." |
|||
if (connected) { |
|||
tvStatus.text = "در حال گوش دادن..." |
|||
streamClient?.startRecording(0) |
|||
} |
|||
}, |
|||
onPartialText = { liveText -> |
|||
lastTranscript = liveText |
|||
binding.tvDialogTranscript.text = liveText |
|||
onMacInputStateReceived = {}, |
|||
onPartialText = { live -> |
|||
tvTranscript.text = live |
|||
}, |
|||
onAudioLevel = { level -> |
|||
val scale = 1.0f + (level * 0.4f) |
|||
binding.dialogGlow.scaleX = scale |
|||
binding.dialogGlow.scaleY = scale |
|||
}, |
|||
onCompleted = { finalText, _ -> |
|||
lastTranscript = finalText |
|||
binding.tvDialogTranscript.text = finalText |
|||
finishWithResult(finalText) |
|||
onAudioLevel = {}, |
|||
onCompleted = { text, _ -> |
|||
val resultIntent = Intent().apply { |
|||
val list = ArrayList<String>() |
|||
list.add(text) |
|||
putStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS, list) |
|||
} |
|||
setResult(Activity.RESULT_OK, resultIntent) |
|||
finish() |
|||
}, |
|||
onError = { errMsg -> |
|||
binding.tvDialogStatus.text = "خطا: $errMsg" |
|||
onError = { |
|||
setResult(Activity.RESULT_CANCELED) |
|||
finish() |
|||
} |
|||
) |
|||
|
|||
streamDictationClient?.startRecording() |
|||
} |
|||
|
|||
private fun finishWithResult(text: String) { |
|||
streamDictationClient?.stopRecording() |
|||
val resultIntent = Intent().apply { |
|||
putStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS, arrayListOf(text)) |
|||
findViewById<View>(R.id.btnDialogCancel).setOnClickListener { |
|||
streamClient?.stopRecording(0) |
|||
setResult(Activity.RESULT_CANCELED) |
|||
finish() |
|||
} |
|||
setResult(Activity.RESULT_OK, resultIntent) |
|||
finish() |
|||
} |
|||
|
|||
override fun onDestroy() { |
|||
super.onDestroy() |
|||
streamDictationClient?.release() |
|||
streamClient?.release() |
|||
} |
|||
} |
|||
@ -0,0 +1,471 @@ |
|||
#!/usr/bin/env python3 |
|||
""" |
|||
Soniox Bi-Directional Input Synchronization Gateway (v4.0) |
|||
- Mirrors Mac focused input box <--> Android phone in real-time. |
|||
- Supports inserting voice text at exact cursor location. |
|||
- Instant <15ms WebSocket push to Mac for editing and pasting. |
|||
""" |
|||
|
|||
import asyncio |
|||
import json |
|||
import logging |
|||
import re |
|||
import subprocess |
|||
import time |
|||
from collections import OrderedDict |
|||
import aiohttp |
|||
from aiohttp import web |
|||
import websockets |
|||
from websockets.protocol import State |
|||
|
|||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") |
|||
logger = logging.getLogger("SonioxRelay") |
|||
|
|||
connected_mac_websockets = set() |
|||
connected_phone_websockets = set() |
|||
recent_pasted_sessions = OrderedDict() # session_id -> (timestamp, text) |
|||
|
|||
latest_mac_input_state = { |
|||
"type": "mac_input_state", |
|||
"app": "Desktop", |
|||
"text": "", |
|||
"cursor": 0, |
|||
"selection": 0, |
|||
"timestamp": time.time() |
|||
} |
|||
|
|||
MAC_DIRECT_HTTP_URL = "http://116.16.16.20:8999/paste" |
|||
SONIOX_WS_URL = ( |
|||
"wss://translate.compare.soniox.com/compare/api/compare-websocket" |
|||
"?language_hints=fa&language_hints=en&language_hints=ar" |
|||
"&enable_speaker_diarization=false&enable_language_identification=true" |
|||
"&enable_endpoint_detection=false&providers=soniox" |
|||
) |
|||
SONIOX_HEADERS = { |
|||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36", |
|||
"Origin": "https://translate.compare.soniox.com", |
|||
} |
|||
|
|||
def is_ws_open(ws) -> bool: |
|||
if ws is None: |
|||
return False |
|||
if hasattr(ws, 'closed'): |
|||
return not ws.closed |
|||
if hasattr(ws, 'state'): |
|||
return ws.state == State.OPEN |
|||
return True |
|||
|
|||
class SonioxPool: |
|||
"""Pre-warms upstream WebSockets to Soniox for 0ms speech start delay.""" |
|||
def __init__(self, size=3): |
|||
self._pool = asyncio.Queue(maxsize=size) |
|||
self._refilling = False |
|||
|
|||
async def get_session(self): |
|||
while not self._pool.empty(): |
|||
try: |
|||
ws = self._pool.get_nowait() |
|||
if is_ws_open(ws): |
|||
asyncio.create_task(self.refill()) |
|||
return ws |
|||
except asyncio.QueueEmpty: |
|||
break |
|||
|
|||
logger.info("Pool empty, connecting fresh Soniox session...") |
|||
asyncio.create_task(self.refill()) |
|||
return await self._create_ws() |
|||
|
|||
async def _create_ws(self): |
|||
try: |
|||
return await websockets.connect( |
|||
SONIOX_WS_URL, |
|||
additional_headers=SONIOX_HEADERS, |
|||
open_timeout=3.5, |
|||
ping_interval=20, |
|||
) |
|||
except Exception as e: |
|||
logger.error("Failed to connect to upstream Soniox: %s", e) |
|||
return None |
|||
|
|||
async def refill(self): |
|||
if self._refilling or self._pool.full(): |
|||
return |
|||
self._refilling = True |
|||
try: |
|||
while not self._pool.full(): |
|||
ws = await self._create_ws() |
|||
if ws and is_ws_open(ws): |
|||
await self._pool.put(ws) |
|||
else: |
|||
break |
|||
finally: |
|||
self._refilling = False |
|||
|
|||
soniox_pool = SonioxPool() |
|||
|
|||
def sanitize_and_flatten_text(text: str) -> str: |
|||
""" |
|||
1. Removes all line breaks (\\r, \\n) and collapses whitespace into single spaces. |
|||
2. Strips English hallucination stop words during Persian speech. |
|||
3. Guarantees zero trailing/leading enters or spaces. |
|||
""" |
|||
if not text: |
|||
return "" |
|||
|
|||
flattened = re.sub(r"[\r\n\t]+", " ", text) |
|||
flattened = re.sub(r"\s+", " ", flattened).strip() |
|||
|
|||
if not flattened: |
|||
return "" |
|||
|
|||
words = flattened.split() |
|||
fa_pattern = re.compile(r"[\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]") |
|||
en_pattern = re.compile(r"[a-zA-Z]") |
|||
|
|||
fa_count = sum(1 for w in words if fa_pattern.search(w)) |
|||
en_count = sum(1 for w in words if en_pattern.search(w)) |
|||
total = fa_count + en_count |
|||
|
|||
if total == 0: |
|||
return flattened |
|||
|
|||
fa_ratio = fa_count / total |
|||
stop_words = {"sex", "from", "no", "oh", "you", "all", "know", "that", "god", "loves", "men", "for", "day", "night", "mankind", "wanted", "moments"} |
|||
|
|||
cleaned = [] |
|||
if fa_ratio >= 0.25: |
|||
for w in words: |
|||
if en_pattern.search(w) and not fa_pattern.search(w): |
|||
clean_w = re.sub(r"[.,!?:;،؛؟\"'()\[\]{}«»–—-]", "", w.lower()) |
|||
if clean_w in stop_words or fa_ratio >= 0.70: |
|||
continue |
|||
cleaned.append(w) |
|||
else: |
|||
cleaned = words |
|||
|
|||
result = " ".join(cleaned) |
|||
return re.sub(r"\s+", " ", result).strip() |
|||
|
|||
async def broadcast_to_macs(payload_dict: dict) -> bool: |
|||
"""Pushes command payload directly to Mac via persistent WebSocket in <15ms.""" |
|||
delivered = False |
|||
payload_str = json.dumps(payload_dict, ensure_ascii=False) |
|||
|
|||
dead_sockets = set() |
|||
for ws in list(connected_mac_websockets): |
|||
try: |
|||
if is_ws_open(ws): |
|||
await ws.send_str(payload_str) |
|||
delivered = True |
|||
logger.info("⚡ Pushed to Mac WS: %s", payload_dict.get("action") or payload_dict.get("type")) |
|||
else: |
|||
dead_sockets.add(ws) |
|||
except Exception: |
|||
dead_sockets.add(ws) |
|||
|
|||
for dead in dead_sockets: |
|||
connected_mac_websockets.discard(dead) |
|||
|
|||
if delivered: |
|||
return True |
|||
|
|||
# Fallback to SSH script if WebSocket temporarily disconnected |
|||
try: |
|||
text = payload_dict.get("text", "") |
|||
if text: |
|||
clean_text = sanitize_and_flatten_text(text) |
|||
escaped_text = clean_text.replace("'", "'\\''") |
|||
remote_cmd = ( |
|||
f"printf '%s' '{escaped_text}' | pbcopy && " |
|||
f"/usr/bin/osascript -e 'tell application \"System Events\" to keystroke \"v\" using command down'" |
|||
) |
|||
proc = await asyncio.create_subprocess_exec( |
|||
"ssh", "-p", "2222", "-o", "BatchMode=yes", "-o", "ConnectTimeout=2", "alig@127.0.0.1", |
|||
remote_cmd, |
|||
stdout=subprocess.PIPE, |
|||
stderr=subprocess.PIPE |
|||
) |
|||
stdout, stderr = await proc.communicate() |
|||
if proc.returncode == 0: |
|||
logger.info("✅ Pasted to Mac via SSH Tunnel fallback: '%s'", clean_text[:30]) |
|||
return True |
|||
except Exception as e: |
|||
logger.warning("SSH fallback error: %s", e) |
|||
|
|||
return False |
|||
|
|||
async def broadcast_to_phones(payload_dict: dict): |
|||
"""Pushes Mac input state changes to all connected Android clients.""" |
|||
payload_str = json.dumps(payload_dict, ensure_ascii=False) |
|||
dead_phones = set() |
|||
for ws in list(connected_phone_websockets): |
|||
try: |
|||
if is_ws_open(ws): |
|||
await ws.send_str(payload_str) |
|||
else: |
|||
dead_phones.add(ws) |
|||
except Exception: |
|||
dead_phones.add(ws) |
|||
|
|||
for dead in dead_phones: |
|||
connected_phone_websockets.discard(dead) |
|||
|
|||
async def handle_phone_stream_ws(request): |
|||
""" |
|||
⚡ Persistent Duplex Channel for Android Client: |
|||
- Receives live Mac input state on connect and continuously. |
|||
- Handles live audio streaming and returns live STT tokens. |
|||
- Receives user manual text edits and pushes them instantly to Mac. |
|||
""" |
|||
ws = web.WebSocketResponse(heartbeat=15.0) |
|||
await ws.prepare(request) |
|||
client_ip = request.remote |
|||
logger.info("📱 Android Client connected to Duplex Stream: %s", client_ip) |
|||
connected_phone_websockets.add(ws) |
|||
|
|||
# Immediately send the latest Mac input state to phone upon connection! |
|||
if latest_mac_input_state: |
|||
await ws.send_str(json.dumps(latest_mac_input_state, ensure_ascii=False)) |
|||
|
|||
active_soniox_ws = None |
|||
reader_task = None |
|||
stop_event = asyncio.Event() |
|||
current_session_id = "" |
|||
|
|||
full_final_tokens = [] |
|||
current_non_final = "" |
|||
|
|||
async def soniox_reader(soniox_ws, sid): |
|||
nonlocal current_non_final |
|||
try: |
|||
async for s_msg in soniox_ws: |
|||
if not is_ws_open(ws): |
|||
break |
|||
try: |
|||
data = json.loads(s_msg) |
|||
if data.get("type") == "data" and "parts" in data: |
|||
new_finals = [] |
|||
new_non_finals = [] |
|||
got_fin = False |
|||
for p in data["parts"]: |
|||
if p.get("translation_status") == "translation": |
|||
continue |
|||
txt = p.get("text", "") |
|||
is_final = p.get("is_final", False) |
|||
if "<fin>" in txt: |
|||
got_fin = True |
|||
clean = txt.replace("<fin>", "") |
|||
if clean: new_finals.append(clean) |
|||
elif is_final: |
|||
if txt: new_finals.append(txt) |
|||
else: |
|||
if txt: new_non_finals.append(txt) |
|||
|
|||
if new_finals: |
|||
full_final_tokens.extend(new_finals) |
|||
current_non_final = "".join(new_non_finals) |
|||
|
|||
live_text = sanitize_and_flatten_text("".join(full_final_tokens) + current_non_final) |
|||
if live_text and is_ws_open(ws): |
|||
await ws.send_str(json.dumps({ |
|||
"type": "live", |
|||
"session_id": sid, |
|||
"text": live_text |
|||
})) |
|||
|
|||
if got_fin or data.get("session_ended") or data.get("type") == "session_done": |
|||
stop_event.set() |
|||
break |
|||
except Exception as e: |
|||
logger.warning("Soniox parse error: %s", e) |
|||
except Exception as e: |
|||
logger.warning("Soniox reader error: %s", e) |
|||
finally: |
|||
stop_event.set() |
|||
|
|||
try: |
|||
async for msg in ws: |
|||
if msg.type == aiohttp.WSMsgType.BINARY: |
|||
# Live PCM audio chunk (2048 bytes / 64ms) |
|||
if active_soniox_ws and is_ws_open(active_soniox_ws): |
|||
await active_soniox_ws.send(msg.data) |
|||
|
|||
elif msg.type == aiohttp.WSMsgType.TEXT: |
|||
try: |
|||
data = json.loads(msg.data) |
|||
except Exception: |
|||
continue |
|||
|
|||
msg_type = data.get("type") or data.get("action") |
|||
sid = data.get("session_id", f"sess_{int(time.time()*1000)}") |
|||
|
|||
if msg_type == "start": |
|||
current_session_id = sid |
|||
full_final_tokens.clear() |
|||
current_non_final = "" |
|||
stop_event.clear() |
|||
|
|||
active_soniox_ws = await soniox_pool.get_session() |
|||
if not active_soniox_ws or not is_ws_open(active_soniox_ws): |
|||
await ws.send_str(json.dumps({"type": "error", "message": "Upstream Soniox unavailable"})) |
|||
continue |
|||
|
|||
if reader_task and not reader_task.done(): |
|||
reader_task.cancel() |
|||
reader_task = asyncio.create_task(soniox_reader(active_soniox_ws, current_session_id)) |
|||
logger.info("🎙️ Started Live Speech Session: %s", current_session_id) |
|||
|
|||
elif msg_type == "stop": |
|||
if active_soniox_ws and is_ws_open(active_soniox_ws): |
|||
await active_soniox_ws.send(json.dumps({"type": "finalize"})) |
|||
try: |
|||
await asyncio.wait_for(stop_event.wait(), timeout=0.55) |
|||
except asyncio.TimeoutError: |
|||
pass |
|||
|
|||
raw_final = "".join(full_final_tokens) + current_non_final |
|||
clean_final = sanitize_and_flatten_text(raw_final) |
|||
logger.info("⚡ Session %s final text: '%s'", sid, clean_final) |
|||
|
|||
# Return final voice transcription to Android |
|||
if is_ws_open(ws): |
|||
await ws.send_str(json.dumps({ |
|||
"type": "final", |
|||
"session_id": sid, |
|||
"text": clean_final, |
|||
"cursor_pos": data.get("cursor_pos", -1) |
|||
})) |
|||
|
|||
if reader_task and not reader_task.done(): |
|||
reader_task.cancel() |
|||
if active_soniox_ws: |
|||
try: |
|||
await active_soniox_ws.close() |
|||
except Exception: |
|||
pass |
|||
active_soniox_ws = None |
|||
|
|||
asyncio.create_task(soniox_pool.refill()) |
|||
|
|||
elif msg_type == "phone_input_edit" or msg_type == "update_mac_input": |
|||
# User manually edited text on phone or pressed "Insert in Mac" |
|||
edit_text = data.get("text", "") |
|||
cursor = data.get("cursor_pos") or data.get("cursor") |
|||
is_full_replace = data.get("is_full_replace", True) |
|||
logger.info("📱 Received phone edit to push to Mac: '%s' (cursor: %s)", edit_text[:30], cursor) |
|||
|
|||
mac_ok = await broadcast_to_macs({ |
|||
"action": "update_input", |
|||
"text": edit_text, |
|||
"cursor": cursor, |
|||
"is_full_replace": is_full_replace |
|||
}) |
|||
|
|||
# Update our cached latest state |
|||
latest_mac_input_state["text"] = edit_text |
|||
latest_mac_input_state["cursor"] = cursor if cursor is not None else len(edit_text) |
|||
latest_mac_input_state["timestamp"] = time.time() |
|||
|
|||
if is_ws_open(ws): |
|||
await ws.send_str(json.dumps({ |
|||
"type": "edit_ack", |
|||
"session_id": sid, |
|||
"mac_delivered": mac_ok |
|||
})) |
|||
|
|||
elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.ERROR): |
|||
break |
|||
|
|||
finally: |
|||
connected_phone_websockets.discard(ws) |
|||
if reader_task and not reader_task.done(): |
|||
reader_task.cancel() |
|||
if active_soniox_ws: |
|||
try: |
|||
await active_soniox_ws.close() |
|||
except Exception: |
|||
pass |
|||
asyncio.create_task(soniox_pool.refill()) |
|||
logger.info("📱 Android Client disconnected: %s", client_ip) |
|||
|
|||
return ws |
|||
|
|||
async def handle_mac_ws(request): |
|||
"""Persistent WebSocket for Mac Bridge (receives input state & pushes edits).""" |
|||
global latest_mac_input_state |
|||
ws = web.WebSocketResponse(heartbeat=10.0) |
|||
await ws.prepare(request) |
|||
client_ip = request.remote |
|||
logger.info("🖥️ Mac client connected to persistent WebSocket: %s", client_ip) |
|||
connected_mac_websockets.add(ws) |
|||
|
|||
try: |
|||
await ws.send_str(json.dumps({"type": "welcome", "message": "Connected to Soniox Duplex Gateway"})) |
|||
async for msg in ws: |
|||
if msg.type == aiohttp.WSMsgType.TEXT: |
|||
try: |
|||
data = json.loads(msg.data) |
|||
msg_type = data.get("type") |
|||
|
|||
if msg_type == "mac_input_state": |
|||
# Mac reports focused input box text and cursor position |
|||
latest_mac_input_state = data |
|||
# Broadcast immediately to phone! |
|||
await broadcast_to_phones(data) |
|||
|
|||
elif msg_type == "ping": |
|||
await ws.send_str(json.dumps({"type": "pong"})) |
|||
except Exception as e: |
|||
logger.warning("Mac message error: %s", e) |
|||
elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.ERROR): |
|||
break |
|||
finally: |
|||
connected_mac_websockets.discard(ws) |
|||
if is_ws_open(ws): |
|||
await ws.close() |
|||
logger.info("🖥️ Mac client disconnected: %s", client_ip) |
|||
|
|||
return ws |
|||
|
|||
async def handle_health(request): |
|||
return web.json_response({ |
|||
"status": "ok", |
|||
"service": "Soniox Bi-Directional Duplex Gateway v4.0", |
|||
"connected_macs": len(connected_mac_websockets), |
|||
"connected_phones": len(connected_phone_websockets), |
|||
"latest_mac_input_app": latest_mac_input_state.get("app", ""), |
|||
"latest_mac_input_text_len": len(latest_mac_input_state.get("text", "")) |
|||
}) |
|||
|
|||
async def handle_paste(request): |
|||
try: |
|||
data = await request.json() |
|||
text = data.get("text", "") |
|||
session_id = data.get("session_id", "") |
|||
cursor = data.get("cursor_pos") or data.get("cursor") |
|||
success = await broadcast_to_macs({ |
|||
"action": "update_input", |
|||
"text": text, |
|||
"cursor": cursor, |
|||
"is_full_replace": True |
|||
}) |
|||
return web.json_response({"status": "pasted" if success else "failed", "mac_delivered": success}) |
|||
except Exception as e: |
|||
return web.json_response({"error": str(e)}, status=400) |
|||
|
|||
async def start_background_tasks(app): |
|||
asyncio.create_task(soniox_pool.refill()) |
|||
|
|||
def create_app(): |
|||
app = web.Application(client_max_size=25 * 1024 * 1024) |
|||
app.on_startup.append(start_background_tasks) |
|||
app.router.add_get("/health", handle_health) |
|||
app.router.add_get("/status", handle_health) |
|||
app.router.add_post("/paste", handle_paste) |
|||
app.router.add_get("/ws/stream", handle_phone_stream_ws) |
|||
app.router.add_get("/ws/mac", handle_mac_ws) |
|||
return app |
|||
|
|||
if __name__ == "__main__": |
|||
app = create_app() |
|||
web.run_app(app, host="0.0.0.0", port=8999) |
|||
@ -0,0 +1,137 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
public struct MacInputState: Codable { |
|||
public let app: String |
|||
public let text: String |
|||
public let cursor: Int |
|||
public let selection: Int |
|||
public let timestamp: Double |
|||
|
|||
public init(app: String, text: String, cursor: Int, selection: Int) { |
|||
self.app = app |
|||
self.text = text |
|||
self.cursor = cursor |
|||
self.selection = selection |
|||
self.timestamp = Date().timeIntervalSince1970 |
|||
} |
|||
} |
|||
|
|||
public final class FocusedInputSync { |
|||
public static let shared = FocusedInputSync() |
|||
|
|||
private var lastState: MacInputState? |
|||
private var isUpdatingLocally = false |
|||
|
|||
public var onInputStateChanged: ((MacInputState) -> Void)? |
|||
|
|||
private init() {} |
|||
|
|||
/// Reads current focused element state (app name, text, cursor position, selection) |
|||
public func getCurrentState() -> MacInputState? { |
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil } |
|||
let appName = frontApp.localizedName ?? "App" |
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
|
|||
var focusedElemObj: CFTypeRef? |
|||
let err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) |
|||
guard err == .success, let elem = focusedElemObj else { |
|||
return MacInputState(app: appName, text: "", cursor: 0, selection: 0) |
|||
} |
|||
|
|||
let axElem = elem as! AXUIElement |
|||
var valObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) |
|||
let text = (valObj as? String) ?? "" |
|||
|
|||
var cursor = text.count |
|||
var selLen = 0 |
|||
var selectedRangeObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &selectedRangeObj) == .success, |
|||
let axVal = selectedRangeObj { |
|||
var range = CFRange() |
|||
if AXValueGetValue(axVal as! AXValue, .cfRange, &range) { |
|||
cursor = range.location |
|||
selLen = range.length |
|||
} |
|||
} |
|||
|
|||
return MacInputState(app: appName, text: text, cursor: cursor, selection: selLen) |
|||
} |
|||
|
|||
/// Applies updated full text or inserts at cursor position directly into active Mac input |
|||
@discardableResult |
|||
public func applyRemoteUpdate(text: String, cursor: Int? = nil, isFullReplace: Bool = true) -> Bool { |
|||
isUpdatingLocally = true |
|||
defer { |
|||
DispatchQueue.main.asyncAfter(deadline: .now() + 0.2) { |
|||
self.isUpdatingLocally = false |
|||
} |
|||
} |
|||
|
|||
// 1. Flatten all newlines and carriage returns |
|||
var cleanText = text.components(separatedBy: .newlines).joined(separator: " ") |
|||
cleanText = cleanText.replacingOccurrences(of: "\t", with: " ") |
|||
cleanText = cleanText.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) |
|||
cleanText = cleanText.trimmingCharacters(in: .whitespacesAndNewlines) |
|||
|
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return false } |
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
var focusedElemObj: CFTypeRef? |
|||
|
|||
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) == .success, |
|||
let elem = focusedElemObj { |
|||
let axElem = elem as! AXUIElement |
|||
|
|||
if isFullReplace { |
|||
// Try setting AXValue directly |
|||
let setErr = AXUIElementSetAttributeValue(axElem, kAXValueAttribute as CFString, cleanText as CFTypeRef) |
|||
if setErr == .success { |
|||
if let cursor = cursor { |
|||
var range = CFRange(location: min(cursor, cleanText.count), length: 0) |
|||
if let axRange = AXValueCreate(.cfRange, &range) { |
|||
AXUIElementSetAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, axRange) |
|||
} |
|||
} |
|||
print("FocusedInputSync: ✅ Updated AXValue directly for \(frontApp.localizedName ?? "")") |
|||
return true |
|||
} |
|||
} else { |
|||
// Try setting selected text |
|||
let setSelErr = AXUIElementSetAttributeValue(axElem, kAXSelectedTextAttribute as CFString, cleanText as CFTypeRef) |
|||
if setSelErr == .success { |
|||
print("FocusedInputSync: ✅ Inserted text via AXSelectedText for \(frontApp.localizedName ?? "")") |
|||
return true |
|||
} |
|||
} |
|||
} |
|||
|
|||
// Fallback: Clipboard Cmd+A + Cmd+V or pure Cmd+V |
|||
let pasteboard = NSPasteboard.general |
|||
pasteboard.clearContents() |
|||
pasteboard.setString(cleanText, forType: .string) |
|||
|
|||
let src = CGEventSource(stateID: .hidSystemState) |
|||
|
|||
if isFullReplace { |
|||
// Select all: Cmd + A |
|||
let aDown = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: true) |
|||
aDown?.flags = .maskCommand |
|||
let aUp = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: false) |
|||
aDown?.post(tap: .cghidEventTap) |
|||
aUp?.post(tap: .cghidEventTap) |
|||
|
|||
usleep(25000) |
|||
} |
|||
|
|||
// Paste: Cmd + V |
|||
let vDown = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: true) |
|||
vDown?.flags = .maskCommand |
|||
let vUp = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: false) |
|||
vDown?.post(tap: .cghidEventTap) |
|||
vUp?.post(tap: .cghidEventTap) |
|||
|
|||
print("FocusedInputSync: ✅ Injected clipboard keystroke to \(frontApp.localizedName ?? "")") |
|||
return true |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue