Browse Source

feat(v3.1): demand-driven mic lifecycle (0% idle CPU, orange dot off), live device switch & exact hardware-ready chime

main
Ali Alavi 20 hours ago
parent
commit
8e14b3bdce
  1. 2
      Info.plist
  2. 12
      src/AppDelegate.swift
  3. 171
      src/AudioRecorder.swift
  4. 25
      src/SonioxSettings.swift
  5. 23
      src/StatusBarController.swift

2
Info.plist

@ -19,7 +19,7 @@
<key>CFBundlePackageType</key> <key>CFBundlePackageType</key>
<string>APPL</string> <string>APPL</string>
<key>CFBundleShortVersionString</key> <key>CFBundleShortVersionString</key>
<string>3.0</string>
<string>3.1</string>
<key>CFBundleVersion</key> <key>CFBundleVersion</key>
<string>1</string> <string>1</string>
<key>LSMinimumSystemVersion</key> <key>LSMinimumSystemVersion</key>

12
src/AppDelegate.swift

@ -143,6 +143,12 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
self?.activeSession?.sendAudioChunk(chunk) self?.activeSession?.sendAudioChunk(chunk)
} }
// Play start chime the instant microphone hardware actually begins delivering audio
audioRecorder.onHardwareCapturingReady = { [weak self] in
guard let self = self, self.audioRecorder.isRecording else { return }
self.playStartSound()
}
// Start live ping monitor & wire network restoration hooks // Start live ping monitor & wire network restoration hooks
PingMonitor.shared.onPingUpdate = { [weak self] quality in PingMonitor.shared.onPingUpdate = { [weak self] quality in
guard let self = self else { return } guard let self = self else { return }
@ -241,6 +247,11 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
SonioxSessionPool.shared.prewarmNextSession() SonioxSessionPool.shared.prewarmNextSession()
} }
public func reloadAudioDevice() {
audioRecorder.resetSession()
try? audioRecorder.ensureSessionConfigured()
}
@objc public func toggleRecording() { @objc public func toggleRecording() {
if audioRecorder.isRecording { if audioRecorder.isRecording {
stopRecording() stopRecording()
@ -338,7 +349,6 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
pingQuality: PingMonitor.shared.currentQuality pingQuality: PingMonitor.shared.currentQuality
) )
) )
playStartSound()
} }
private func cancelShortTapRecording() { private func cancelShortTapRecording() {

171
src/AudioRecorder.swift

@ -3,6 +3,12 @@ import AppKit
import AVFoundation import AVFoundation
import CoreMedia import CoreMedia
/// Professional Demand-Driven Audio Recorder (v3.1):
/// - Pre-configures the AVCaptureSession graph WITHOUT activating mic hardware while idle
/// (macOS orange microphone indicator stays OFF, 0% idle CPU, Bluetooth AirPods stay in stereo A2DP).
/// - Starts capture immediately on `.userInteractive` queue when recording begins (~25–45ms attack).
/// - Shuts down mic hardware cleanly immediately after dictation (or after a 15s burst cooldown if selected).
/// - Uses `CMBlockBufferCopyDataBytes` for safe multi-channel (stereo/mono) & segmented buffer capture.
public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBufferDelegate { public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBufferDelegate {
private var captureSession: AVCaptureSession? private var captureSession: AVCaptureSession?
private var audioOutput: AVCaptureAudioDataOutput? private var audioOutput: AVCaptureAudioDataOutput?
@ -11,7 +17,10 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
private var pcmBuffer = Data() private var pcmBuffer = Data()
private var preRollBuffer = Data() private var preRollBuffer = Data()
private let preRollByteLimit = 9600 // 300ms rolling pre-roll at 16kHz 16-bit mono (32,000 bytes/sec)
private let preRollByteLimit = 9600 // 300ms rolling buffer used only during optional 15s warm cooldown
private var idleStopWorkItem: DispatchWorkItem?
private var didFireHardwareReadyForSession = false
private let lock = NSLock() private let lock = NSLock()
private let captureQueue = DispatchQueue(label: "com.soniox.audiocapture", qos: .userInteractive) private let captureQueue = DispatchQueue(label: "com.soniox.audiocapture", qos: .userInteractive)
private let sessionQueue = DispatchQueue(label: "com.soniox.audiosession", qos: .userInteractive) private let sessionQueue = DispatchQueue(label: "com.soniox.audiosession", qos: .userInteractive)
@ -23,6 +32,7 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
public private(set) var isRecording = false public private(set) var isRecording = false
public var onAudioLevelUpdate: ((Float) -> Void)? public var onAudioLevelUpdate: ((Float) -> Void)?
public var onAudioChunkAvailable: ((Data) -> Void)? public var onAudioChunkAvailable: ((Data) -> Void)?
public var onHardwareCapturingReady: (() -> Void)?
public override init() { public override init() {
self.targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 16000, channels: 1, interleaved: false)! self.targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 16000, channels: 1, interleaved: false)!
@ -37,10 +47,10 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
} }
NSWorkspace.shared.notificationCenter.addObserver(forName: NSWorkspace.didWakeNotification, object: nil, queue: .main) { [weak self] _ in NSWorkspace.shared.notificationCenter.addObserver(forName: NSWorkspace.didWakeNotification, object: nil, queue: .main) { [weak self] _ in
print("AudioRecorder: Mac woke from sleep. Resetting and priming audio session in 1.0s...")
print("AudioRecorder: Mac woke from sleep. Pre-configuring idle audio graph (mic stays OFF)...")
self?.resetSession() self?.resetSession()
DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) {
try? self?.ensureSessionRunning()
try? self?.ensureSessionConfigured()
} }
} }
@ -58,6 +68,9 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
public func resetSession() { public func resetSession() {
lock.lock() lock.lock()
isRecording = false isRecording = false
didFireHardwareReadyForSession = false
idleStopWorkItem?.cancel()
idleStopWorkItem = nil
let session = captureSession let session = captureSession
captureSession = nil captureSession = nil
audioOutput = nil audioOutput = nil
@ -75,7 +88,7 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
} }
} }
} }
print("AudioRecorder: Hardware capture session reset.")
print("AudioRecorder: Hardware capture session reset (mic OFF).")
} }
public func requestMicrophonePermission(completion: @escaping (Bool) -> Void) { public func requestMicrophonePermission(completion: @escaping (Bool) -> Void) {
@ -114,14 +127,12 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
} }
return AVCaptureDevice.default(for: .audio) ?? all.first return AVCaptureDevice.default(for: .audio) ?? all.first
} }
public func ensureSessionConfigured() throws {
try ensureSessionRunning()
}
public func ensureSessionRunning() throws {
/// Pre-wires the AVCaptureSession graph WITHOUT activating the microphone hardware!
/// Keeps macOS orange mic indicator OFF and CPU at 0% while making `startRunning()` ultra-fast.
public func ensureSessionConfigured() throws {
lock.lock() lock.lock()
if let session = captureSession, session.isRunning {
if captureSession != nil {
lock.unlock() lock.unlock()
return return
} }
@ -131,29 +142,24 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
throw NSError(domain: "AudioRecorder", code: 1, userInfo: [NSLocalizedDescriptionKey: "No microphone device found"]) throw NSError(domain: "AudioRecorder", code: 1, userInfo: [NSLocalizedDescriptionKey: "No microphone device found"])
} }
let session = AVCaptureSession()
let input = try AVCaptureDeviceInput(device: device)
if session.canAddInput(input) {
session.addInput(input)
}
let output = AVCaptureAudioDataOutput()
output.setSampleBufferDelegate(self, queue: captureQueue)
if session.canAddOutput(output) {
session.addOutput(output)
}
lock.lock() lock.lock()
if captureSession == nil {
let session = AVCaptureSession()
let input = try AVCaptureDeviceInput(device: device)
if session.canAddInput(input) {
session.addInput(input)
}
let output = AVCaptureAudioDataOutput()
output.setSampleBufferDelegate(self, queue: captureQueue)
if session.canAddOutput(output) {
session.addOutput(output)
}
if self.captureSession == nil {
self.captureSession = session self.captureSession = session
self.audioOutput = output self.audioOutput = output
print("AudioRecorder: Capture session graph configured for device:", device.localizedName)
} }
let session = captureSession
lock.unlock() lock.unlock()
if let session = session, !session.isRunning {
session.startRunning()
print("AudioRecorder: Pre-warmed hot audio session running for device:", device.localizedName)
}
print("AudioRecorder: Idle capture graph pre-configured for '\(device.localizedName)' (mic hardware OFF).")
} }
public func startRecording() throws { public func startRecording() throws {
@ -162,35 +168,61 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
lock.unlock() lock.unlock()
return return
} }
isRecording = true
let preRoll = preRollBuffer
idleStopWorkItem?.cancel()
idleStopWorkItem = nil
lock.unlock()
try ensureSessionConfigured()
lock.lock()
let session = captureSession
let alreadyHot = session?.isRunning ?? false
let preRoll = alreadyHot ? preRollBuffer : Data()
preRollBuffer.removeAll(keepingCapacity: true) preRollBuffer.removeAll(keepingCapacity: true)
pcmBuffer.removeAll(keepingCapacity: true) pcmBuffer.removeAll(keepingCapacity: true)
pcmBuffer.append(preRoll) pcmBuffer.append(preRoll)
let session = captureSession
isRecording = true
didFireHardwareReadyForSession = alreadyHot
lock.unlock() lock.unlock()
if let session = session, !session.isRunning {
if let session = session, !alreadyHot {
sessionQueue.async { sessionQueue.async {
if !session.isRunning { if !session.isRunning {
session.startRunning() session.startRunning()
} }
} }
} else if session == nil {
sessionQueue.async { [weak self] in
try? self?.ensureSessionRunning()
}
} }
if !preRoll.isEmpty {
onAudioChunkAvailable?(preRoll)
if alreadyHot {
if !preRoll.isEmpty {
onAudioChunkAvailable?(preRoll)
}
DispatchQueue.main.async { [weak self] in
self?.onHardwareCapturingReady?()
}
print("AudioRecorder: Recording started on warm mic (pre-roll: \(preRoll.count) bytes).")
} else {
print("AudioRecorder: Starting mic hardware on demand (0% idle mode).")
} }
let preRollMs = Int(Double(preRoll.count) / 32.0)
print("AudioRecorder: Instant 0ms recording started (hot mic, pre-roll: \(preRoll.count) bytes / \(preRollMs)ms).")
} }
public func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { public func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
// Audio converter runs continuously to maintain 300ms rolling pre-roll ring buffer!
lock.lock()
let active = isRecording
let keepWarm = (SonioxSettings.shared.micStandbyMode == .warmCooldown15s) && (idleStopWorkItem != nil)
let shouldNotifyReady = active && !didFireHardwareReadyForSession
if shouldNotifyReady {
didFireHardwareReadyForSession = true
}
lock.unlock()
if shouldNotifyReady {
DispatchQueue.main.async { [weak self] in
self?.onHardwareCapturingReady?()
}
}
guard active || keepWarm else { return }
guard let formatDesc = CMSampleBufferGetFormatDescription(sampleBuffer) else { return } guard let formatDesc = CMSampleBufferGetFormatDescription(sampleBuffer) else { return }
let srcFormat = AVAudioFormat(cmAudioFormatDescription: formatDesc) let srcFormat = AVAudioFormat(cmAudioFormatDescription: formatDesc)
@ -272,36 +304,68 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
pcmBuffer.append(data) pcmBuffer.append(data)
lock.unlock() lock.unlock()
// Stream live audio chunk to WebSocket immediately
onAudioChunkAvailable?(data) onAudioChunkAvailable?(data)
// Calculate decibel loudness for perceptual, high-responsiveness audio meter
var sumSquare: Float = 0 var sumSquare: Float = 0
for i in 0..<frameLen { for i in 0..<frameLen {
let sample = Float(int16Ptr[i]) / 32768.0 let sample = Float(int16Ptr[i]) / 32768.0
sumSquare += sample * sample sumSquare += sample * sample
} }
let rms = sqrt(sumSquare / Float(frameLen)) let rms = sqrt(sumSquare / Float(frameLen))
// Convert to decibels (-50 dB noise floor to -12 dB peak speech)
let db = 20.0 * log10(max(rms, 0.00001)) let db = 20.0 * log10(max(rms, 0.00001))
let normalizedLevel = min(max((db + 48.0) / 36.0, 0.0), 1.0) let normalizedLevel = min(max((db + 48.0) / 36.0, 0.0), 1.0)
DispatchQueue.main.async { [weak self] in DispatchQueue.main.async { [weak self] in
self?.onAudioLevelUpdate?(normalizedLevel) self?.onAudioLevelUpdate?(normalizedLevel)
} }
} else {
// Continuous 300ms rolling pre-roll ring buffer (9,600 bytes at 16kHz Int16 mono)
} else if keepWarm {
preRollBuffer.append(data) preRollBuffer.append(data)
if preRollBuffer.count > preRollByteLimit { if preRollBuffer.count > preRollByteLimit {
preRollBuffer.removeFirst(preRollBuffer.count - preRollByteLimit) preRollBuffer.removeFirst(preRollBuffer.count - preRollByteLimit)
} }
lock.unlock() lock.unlock()
} else {
lock.unlock()
} }
} }
} }
} }
/// Non-blocking drain of the serial capture queue (keeps audio engine hot for 0ms next capture)
/// Schedules clean microphone hardware shutdown so the macOS orange mic indicator turns OFF when idle.
private func scheduleHardwareShutdown(delaySeconds: TimeInterval) {
lock.lock()
idleStopWorkItem?.cancel()
let work = DispatchWorkItem { [weak self] in
guard let self = self else { return }
self.lock.lock()
let stillIdle = !self.isRecording
let session = self.captureSession
if stillIdle {
self.idleStopWorkItem = nil
self.preRollBuffer.removeAll(keepingCapacity: true)
}
self.lock.unlock()
if stillIdle, let session = session {
self.sessionQueue.async {
if session.isRunning {
session.stopRunning()
print("AudioRecorder: Microphone hardware turned OFF (idle — orange dot cleared).")
}
}
}
}
idleStopWorkItem = work
lock.unlock()
if delaySeconds <= 0.0 {
work.perform()
} else {
DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + delaySeconds, execute: work)
}
}
/// Non-blocking drain of serial captureQueue + clean demand-driven mic hardware shutdown.
public func stopRecording() -> Data { public func stopRecording() -> Data {
lock.lock() lock.lock()
if !isRecording { if !isRecording {
@ -311,10 +375,10 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
} }
lock.unlock() lock.unlock()
// Synchronize with serial captureQueue (<1ms) so any in-flight sampleBuffer finishes cleanly
captureQueue.sync { captureQueue.sync {
self.lock.lock() self.lock.lock()
self.isRecording = false self.isRecording = false
self.didFireHardwareReadyForSession = false
self.preRollBuffer.removeAll(keepingCapacity: true) self.preRollBuffer.removeAll(keepingCapacity: true)
self.lock.unlock() self.lock.unlock()
} }
@ -323,17 +387,22 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
let data = pcmBuffer let data = pcmBuffer
lock.unlock() lock.unlock()
print("AudioRecorder: Stopped. Audio engine kept hot for 0ms delay. Total PCM captured: \(data.count) bytes.")
let cooldown: TimeInterval = (SonioxSettings.shared.micStandbyMode == .warmCooldown15s) ? 15.0 : 0.0
scheduleHardwareShutdown(delaySeconds: cooldown)
print("AudioRecorder: Stopped (cooldown: \(Int(cooldown))s). Total PCM captured: \(data.count) bytes.")
return data return data
} }
/// Immediate abort (e.g. Escape key pressed) — discards buffered audio immediately
/// Immediate abort (Escape key or short tap) — discards audio and turns mic hardware OFF immediately.
public func abortRecording() { public func abortRecording() {
lock.lock() lock.lock()
isRecording = false isRecording = false
didFireHardwareReadyForSession = false
pcmBuffer.removeAll(keepingCapacity: true) pcmBuffer.removeAll(keepingCapacity: true)
preRollBuffer.removeAll(keepingCapacity: true) preRollBuffer.removeAll(keepingCapacity: true)
lock.unlock() lock.unlock()
print("AudioRecorder: Aborted immediately.")
scheduleHardwareShutdown(delaySeconds: 0.0)
print("AudioRecorder: Aborted immediately (mic OFF).")
} }
} }

25
src/SonioxSettings.swift

@ -165,6 +165,20 @@ public enum ClipboardBehavior: String, CaseIterable {
} }
} }
public enum MicStandbyMode: String, CaseIterable {
case immediateOff = "immediateOff"
case warmCooldown15s = "warmCooldown15s"
public var displayName: String {
switch self {
case .immediateOff:
return "🔇 Turn Mic Off Immediately When Idle (Default)"
case .warmCooldown15s:
return "⏱️ Auto-Off After 15s Idle (Fast Back-to-Back)"
}
}
}
public final class SonioxSettings { public final class SonioxSettings {
public static let shared = SonioxSettings() public static let shared = SonioxSettings()
@ -298,6 +312,16 @@ public final class SonioxSettings {
} }
} }
public var micStandbyMode: MicStandbyMode {
get {
let val = defaults.string(forKey: "SonioxMicStandbyMode") ?? MicStandbyMode.immediateOff.rawValue
return MicStandbyMode(rawValue: val) ?? .immediateOff
}
set {
defaults.set(newValue.rawValue, forKey: "SonioxMicStandbyMode")
}
}
public var playSounds: Bool { public var playSounds: Bool {
get { get {
if defaults.object(forKey: "SonioxPlaySounds") == nil { return true } if defaults.object(forKey: "SonioxPlaySounds") == nil { return true }
@ -344,6 +368,7 @@ public final class SonioxSettings {
defaults.removeObject(forKey: "SonioxTypingDeliveryMode") defaults.removeObject(forKey: "SonioxTypingDeliveryMode")
defaults.removeObject(forKey: "SonioxLiveStreamEngineMode") defaults.removeObject(forKey: "SonioxLiveStreamEngineMode")
defaults.removeObject(forKey: "SonioxClipboardBehavior") defaults.removeObject(forKey: "SonioxClipboardBehavior")
defaults.removeObject(forKey: "SonioxMicStandbyMode")
defaults.removeObject(forKey: "SonioxPlaySounds") defaults.removeObject(forKey: "SonioxPlaySounds")
defaults.removeObject(forKey: "SonioxLaunchAtLogin") defaults.removeObject(forKey: "SonioxLaunchAtLogin")
defaults.removeObject(forKey: "SonioxPreferredAudioDevice") defaults.removeObject(forKey: "SonioxPreferredAudioDevice")

23
src/StatusBarController.swift

@ -228,6 +228,18 @@ public final class StatusBarController: NSObject, NSMenuDelegate {
streamEngineMenuItem.submenu = streamEngineMenu streamEngineMenuItem.submenu = streamEngineMenu
advancedMenu.addItem(streamEngineMenuItem) advancedMenu.addItem(streamEngineMenuItem)
let micStandbyMenu = NSMenu()
for mode in MicStandbyMode.allCases {
let item = NSMenuItem(title: mode.displayName, action: #selector(selectMicStandbyAction(_:)), keyEquivalent: "")
item.target = self
item.representedObject = mode
item.state = (SonioxSettings.shared.micStandbyMode == mode) ? .on : .off
micStandbyMenu.addItem(item)
}
let micStandbyMenuItem = NSMenuItem(title: "🎙️ Microphone Idle Standby", action: nil, keyEquivalent: "")
micStandbyMenuItem.submenu = micStandbyMenu
advancedMenu.addItem(micStandbyMenuItem)
advancedMenu.addItem(NSMenuItem.separator()) advancedMenu.addItem(NSMenuItem.separator())
let slashCmdItem = NSMenuItem(title: "⚡️ Executive Voice Slash Commands", action: #selector(toggleVoiceCommandsAction(_:)), keyEquivalent: "") let slashCmdItem = NSMenuItem(title: "⚡️ Executive Voice Slash Commands", action: #selector(toggleVoiceCommandsAction(_:)), keyEquivalent: "")
@ -426,6 +438,16 @@ public final class StatusBarController: NSObject, NSMenuDelegate {
} }
} }
@objc private func selectMicStandbyAction(_ sender: NSMenuItem) {
if let mode = sender.representedObject as? MicStandbyMode {
SonioxSettings.shared.micStandbyMode = mode
if mode == .immediateOff {
AppDelegate.shared?.reloadAudioDevice()
}
buildMenu()
}
}
@objc private func selectHotkeyAction(_ sender: NSMenuItem) { @objc private func selectHotkeyAction(_ sender: NSMenuItem) {
if let preset = sender.representedObject as? HotkeyPreset { if let preset = sender.representedObject as? HotkeyPreset {
HotkeyManager.shared.currentPreset = preset HotkeyManager.shared.currentPreset = preset
@ -450,6 +472,7 @@ public final class StatusBarController: NSObject, NSMenuDelegate {
@objc private func selectAudioDeviceAction(_ sender: NSMenuItem) { @objc private func selectAudioDeviceAction(_ sender: NSMenuItem) {
let deviceID = sender.representedObject as? String let deviceID = sender.representedObject as? String
SonioxSettings.shared.preferredAudioDeviceID = deviceID SonioxSettings.shared.preferredAudioDeviceID = deviceID
AppDelegate.shared?.reloadAudioDevice()
buildMenu() buildMenu()
} }

Loading…
Cancel
Save