feat: native multi-keyword wake word engine (Jarvis, Computer, Alexa) with Sherpa-ONNX Zipformer

.gitignore

@@ -0,0 +1,4 @@
+build/
+*.dmg
+*.zip
+.DS_Store

include/SherpaOnnx/module.modulemap

@@ -0,0 +1,4 @@
+module SherpaOnnx {
+    header "c-api.h"
+    export *
+}

package.sh

@@ -15,12 +15,21 @@ rm -rf "$APP_BUNDLE"
 # 1. Compile Swift sources
 swiftc -O -target arm64-apple-macosx13.0 \
   -framework Cocoa -framework AVFoundation -framework Carbon \
+  -I "$PROJECT_DIR/include/SherpaOnnx" \
+  -L "$PROJECT_DIR/Frameworks" -lsherpa-onnx-c-api \
+  -Xlinker -rpath -Xlinker "@executable_path/../Frameworks" \
   "$PROJECT_DIR"/src/*.swift \
   -o "$BUILD_DIR/SonioxVoice"
 
 # 2. Assemble .app bundle
 mkdir -p "$APP_BUNDLE/Contents/MacOS"
 mkdir -p "$APP_BUNDLE/Contents/Resources"
+mkdir -p "$APP_BUNDLE/Contents/Frameworks"
+
+cp -R "$PROJECT_DIR/Frameworks/"* "$APP_BUNDLE/Contents/Frameworks/"
+
+mkdir -p "$APP_BUNDLE/Contents/Resources/wakeword_models"
+cp -R "$PROJECT_DIR/resources/wakeword_models/"* "$APP_BUNDLE/Contents/Resources/wakeword_models/"
 
 if [ ! -f "$PROJECT_DIR/resources/AppIcon.icns" ] && [ -d "$PROJECT_DIR/resources/AppIcon.iconset" ]; then
     iconutil -c icns "$PROJECT_DIR/resources/AppIcon.iconset" -o "$PROJECT_DIR/resources/AppIcon.icns"

resources/wakeword_models/keywords.txt

@@ -0,0 +1,7 @@
+▁JA R VI S
+▁HE Y ▁JA R VI S
+▁COMP U TER
+▁HE Y ▁COMP U TER
+▁A LE X A
+▁G HO LA M
+▁GO LA M

resources/wakeword_models/tokens.txt

@@ -0,0 +1,500 @@
+<blk> 0
+<sos/eos> 1
+<unk> 2
+S 3
+T 4
+▁THE 5
+▁A 6
+▁TO 7
+▁AND 8
+N 9
+E 10
+D 11
+ING 12
+' 13
+▁OF 14
+▁I 15
+▁IN 16
+Y 17
+ED 18
+M 19
+▁ 20
+▁THAT 21
+O 22
+C 23
+R 24
+A 25
+P 26
+▁YOU 27
+U 28
+RE 29
+▁IT 30
+▁S 31
+G 32
+L 33
+B 34
+▁IS 35
+I 36
+AL 37
+ER 38
+▁RE 39
+▁C 40
+▁WE 41
+▁SO 42
+▁BE 43
+OR 44
+AR 45
+IN 46
+▁FOR 47
+F 48
+▁HE 49
+LE 50
+LY 51
+▁E 52
+▁WAS 53
+K 54
+▁F 55
+W 56
+VE 57
+RI 58
+CH 59
+LL 60
+▁THIS 61
+▁T 62
+EN 63
+▁ON 64
+IT 65
+CE 66
+▁WITH 67
+▁B 68
+▁W 69
+▁ME 70
+▁DO 71
+▁DE 72
+▁HAVE 73
+▁BUT 74
+ENT 75
+▁P 76
+UR 77
+ON 78
+▁AS 79
+IC 80
+TER 81
+TE 82
+▁THEY 83
+ES 84
+RA 85
+▁NOT 86
+TH 87
+▁ARE 88
+RO 89
+IL 90
+ATION 91
+AN 92
+▁ST 93
+▁WHAT 94
+SE 95
+LI 96
+V 97
+▁CAN 98
+▁G 99
+▁OR 100
+▁ALL 101
+NE 102
+▁AT 103
+▁CON 104
+H 105
+▁LIKE 106
+▁MO 107
+▁HIS 108
+▁AN 109
+▁MA 110
+▁THERE 111
+▁SU 112
+▁MY 113
+NG 114
+- 115
+▁ABOUT 116
+▁SE 117
+LA 118
+▁ONE 119
+LO 120
+▁SHE 121
+CK 122
+▁HAD 123
+AD 124
+MENT 125
+VER 126
+▁CA 127
+IR 128
+▁IF 129
+▁NO 130
+EL 131
+▁O 132
+▁FROM 133
+▁PA 134
+▁YOUR 135
+▁KNOW 136
+RY 137
+ION 138
+UN 139
+▁JUST 140
+▁UP 141
+▁US 142
+▁WHO 143
+▁GO 144
+ME 145
+▁EX 146
+▁OUT 147
+ND 148
+CI 149
+▁PO 150
+▁HER 151
+▁FA 152
+▁PRO 153
+▁WHEN 154
+ATE 155
+AT 156
+▁BY 157
+PE 158
+AS 159
+▁LA 160
+GE 161
+▁CO 162
+▁PEOPLE 163
+▁K 164
+▁SOME 165
+IGHT 166
+▁THINK 167
+VI 168
+DE 169
+▁LE 170
+▁WERE 171
+▁LI 172
+OM 173
+▁WOULD 174
+▁SEE 175
+CO 176
+▁BO 177
+▁OUR 178
+▁HOW 179
+ET 180
+Z 181
+UL 182
+▁HO 183
+IG 184
+▁GET 185
+▁DON 186
+IS 187
+URE 188
+▁WHICH 189
+▁MI 190
+IES 191
+US 192
+X 193
+▁MORE 194
+▁NOW 195
+▁TIME 196
+▁UN 197
+▁FI 198
+TY 199
+▁WILL 200
+RU 201
+IVE 202
+▁THEIR 203
+▁SAY 204
+ITY 205
+▁HA 206
+LU 207
+▁RO 208
+HO 209
+OL 210
+IST 211
+AGE 212
+MA 213
+KE 214
+▁RA 215
+ID 216
+ERS 217
+▁LO 218
+▁SP 219
+▁DA 220
+▁SAID 221
+TION 222
+THER 223
+▁EN 224
+▁BECAUSE 225
+▁HAS 226
+EST 227
+TI 228
+ACT 229
+▁BA 230
+▁VERY 231
+▁THEM 232
+▁BEEN 233
+▁WA 234
+ABLE 235
+▁SA 236
+▁DID 237
+▁HIM 238
+▁DIS 239
+▁WANT 240
+▁OTHER 241
+IM 242
+▁PRE 243
+▁LOOK 244
+▁GOING 245
+▁VI 246
+▁WORK 247
+▁DR 248
+ENCE 249
+▁DI 250
+FF 251
+TA 252
+▁REALLY 253
+▁RIGHT 254
+MO 255
+ALLY 256
+▁INTO 257
+OW 258
+▁WAY 259
+▁FE 260
+▁COULD 261
+▁THEN 262
+TED 263
+▁HERE 264
+▁THESE 265
+▁OVER 266
+▁WELL 267
+▁NA 268
+▁ANY 269
+HE 270
+▁COMP 271
+▁COM 272
+▁TH 273
+▁EVEN 274
+▁TWO 275
+▁JO 276
+▁MAKE 277
+▁BACK 278
+▁WHERE 279
+SH 280
+▁QU 281
+MAN 282
+▁YEAR 283
+▁TRA 284
+▁CHA 285
+QU 286
+CA 287
+▁ALSO 288
+CTION 289
+ANCE 290
+UM 291
+PER 292
+DI 293
+▁HU 294
+▁MU 295
+FUL 296
+▁NEW 297
+ARD 298
+J 299
+ICAL 300
+ANT 301
+END 302
+▁MAR 303
+▁NE 304
+IF 305
+DER 306
+ONE 307
+▁THOSE 308
+PP 309
+▁MAN 310
+▁GOOD 311
+▁TAKE 312
+SIDE 313
+PO 314
+HA 315
+▁LITTLE 316
+▁APP 317
+▁BI 318
+UGH 319
+▁COME 320
+ISH 321
+TIC 322
+▁PER 323
+▁THINGS 324
+▁MUCH 325
+▁FIRST 326
+▁THAN 327
+IA 328
+▁AGAIN 329
+▁STA 330
+▁PART 331
+▁GA 332
+▁MEAN 333
+▁LET 334
+▁DOWN 335
+▁NEED 336
+▁GU 337
+LES 338
+▁RU 339
+ATED 340
+▁SHA 341
+VO 342
+▁TALK 343
+▁DAY 344
+▁CHI 345
+▁SOMETHING 346
+▁AFTER 347
+▁THOUGH 348
+▁CALL 349
+▁GRA 350
+▁EVERY 351
+▁KIND 352
+ROW 353
+AIN 354
+▁BU 355
+▁THROUGH 356
+▁SHOULD 357
+▁ONLY 358
+▁TA 359
+▁MIN 360
+▁GOT 361
+IAN 362
+▁GREAT 363
+▁RI 364
+▁DOES 365
+▁THING 366
+DAY 367
+OUS 368
+▁START 369
+▁MOST 370
+▁COL 371
+PH 372
+▁LOT 373
+IZE 374
+BER 375
+IOUS 376
+▁LIFE 377
+▁MANY 378
+▁FEEL 379
+ITION 380
+▁VA 381
+NESS 382
+▁BEFORE 383
+▁THREE 384
+▁WORLD 385
+▁GIVE 386
+▁WHY 387
+▁PI 388
+▁LONG 389
+PORT 390
+▁SHOW 391
+▁ACTUALLY 392
+▁NEVER 393
+IV 394
+FORM 395
+▁THANK 396
+▁HI 397
+▁PUT 398
+▁LOVE 399
+▁QUESTION 400
+▁HELP 401
+▁HAPPEN 402
+▁MR 403
+▁OWN 404
+WARD 405
+▁MADE 406
+▁HAND 407
+▁STATE 408
+IBLE 409
+▁TELL 410
+▁PH 411
+▁CAME 412
+▁TEA 413
+▁STILL 414
+▁DIFFERENT 415
+▁PLACE 416
+▁MIGHT 417
+▁FIND 418
+▁DU 419
+▁IMP 420
+▁OLD 421
+▁PERSON 422
+▁COUNT 423
+▁JA 424
+▁POINT 425
+▁HIGH 426
+SHIP 427
+▁LAST 428
+▁SAME 429
+▁FOUR 430
+▁AROUND 431
+▁OH 432
+▁CHANGE 433
+▁UNDER 434
+▁HOME 435
+▁ANOTHER 436
+▁FOUND 437
+ATIVE 438
+▁TURN 439
+▁YES 440
+LAND 441
+▁ALWAYS 442
+▁YEAH 443
+▁SCHOOL 444
+▁JE 445
+▁NEXT 446
+▁WORD 447
+▁JU 448
+LINE 449
+▁SIX 450
+▁LEARN 451
+▁BOOK 452
+▁PLAY 453
+▁AWAY 454
+▁HOUSE 455
+▁PLAN 456
+▁WHILE 457
+▁COMMUNI 458
+▁FRIEND 459
+▁AMERICA 460
+▁INTEREST 461
+▁KEEP 462
+▁BETTER 463
+▁IDEA 464
+▁HARD 465
+▁TWENTY 466
+▁THOUSAND 467
+▁HUNDRED 468
+▁WEEK 469
+▁NUMBER 470
+▁COURSE 471
+▁OKAY 472
+PRESS 473
+▁SYSTEM 474
+▁NINE 475
+▁UNDERSTAND 476
+▁WITHOUT 477
+▁BELIEVE 478
+▁POWER 479
+▁MAYBE 480
+▁BUILD 481
+▁MONEY 482
+▁BETWEEN 483
+▁GONNA 484
+▁IMPORTANT 485
+▁ANYTHING 486
+▁LEFT 487
+▁HAVING 488
+▁WHITE 489
+▁PROBLEM 490
+▁SECOND 491
+▁SPEAK 492
+▁TOGETHER 493
+▁BLACK 494
+▁PUBLIC 495
+▁HEALTH 496
+▁NOTHING 497
+▁EVERYTHING 498
+Q 499

src/AppDelegate.swift

@@ -93,6 +93,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
         // Check permissions on start
         checkInitialPermissions()
         setupDarwinNotifications()
+        setupWakeWord()
     }
     
     private func checkInitialPermissions() {
@@ -249,6 +250,9 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
         statusBarController.buildMenu(isRecording: false)
         
         isBusyFinalizing = true
+        if SonioxSettings.shared.wakeWordEnabled {
+            self.audioRecorder.startPassiveListening()
+        }
         
         // Check if the live session suffered a network failure during recording
         let sessionDropped = (self.activeSession == nil) || (self.activeSession?.hasNetworkFailure == true) || self.isNetworkDegraded
@@ -438,4 +442,39 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
         CFNotificationCenterAddObserver(notifyCenter, selfPtr, toggleCb, "com.soniox.voice.toggle" as CFString, nil, .deliverImmediately)
     }
 
+
+    private func setupWakeWord() {
+        // Connect WakeWordEngine callbacks
+        WakeWordEngine.shared.onWakeWordDetected = { [weak self] keyword in
+            guard let self = self else { return }
+            print("AppDelegate: 🎯 WAKE WORD DETECTED: '\(keyword)'! Starting recording...")
+            DispatchQueue.main.async {
+                if !self.audioRecorder.isRecording {
+                    self.startRecording()
+                }
+            }
+        }
+        
+        audioRecorder.onPassiveAudioChunkAvailable = { chunk in
+            WakeWordEngine.shared.acceptAudioChunk(chunk)
+        }
+        
+        // Listen to wake word setting changes from Status Bar menu
+        NotificationCenter.default.addObserver(forName: NSNotification.Name("SonioxWakeWordSettingChanged"), object: nil, queue: .main) { [weak self] _ in
+            guard let self = self else { return }
+            if SonioxSettings.shared.wakeWordEnabled {
+                self.audioRecorder.startPassiveListening()
+            } else {
+                self.audioRecorder.pausePassiveListening()
+            }
+        }
+        
+        // Initial start if enabled
+        if SonioxSettings.shared.wakeWordEnabled {
+            DispatchQueue.main.asyncAfter(deadline: .now() + 1.2) { [weak self] in
+                self?.audioRecorder.startPassiveListening()
+            }
+        }
+    }
+
 }

src/AudioRecorder.swift

@@ -14,8 +14,10 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
     private let captureQueue = DispatchQueue(label: "com.soniox.audiocapture", qos: .userInteractive)
     
     public private(set) var isRecording = false
+    public private(set) var isPassiveListening = false
     public var onAudioLevelUpdate: ((Float) -> Void)?
     public var onAudioChunkAvailable: ((Data) -> Void)?
+    public var onPassiveAudioChunkAvailable: ((Data) -> Void)?
     
     public override init() {
         self.targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 16000, channels: 1, interleaved: false)!
@@ -227,4 +229,34 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
         print("AudioRecorder: Stopped. Microphone hardware OFF. Total PCM captured: \(pcmBuffer.count) bytes.")
         return pcmBuffer
     }
+
+    public func startPassiveListening() {
+        lock.lock()
+        defer { lock.unlock() }
+        guard !isRecording, !isPassiveListening else { return }
+        guard SonioxSettings.shared.wakeWordEnabled else { return }
+        
+        do {
+            try ensureSessionConfigured()
+            if let session = captureSession, !session.isRunning {
+                session.startRunning()
+            }
+            isPassiveListening = true
+            print("AudioRecorder: Passive wake-word listening active (Jarvis, Computer, Alexa).")
+        } catch {
+            print("AudioRecorder: Failed to start passive listening:", error)
+        }
+    }
+    
+    public func pausePassiveListening() {
+        lock.lock()
+        defer { lock.unlock() }
+        isPassiveListening = false
+        if !isRecording {
+            if let session = captureSession, session.isRunning {
+                session.stopRunning()
+            }
+        }
+    }
+
 }

src/SonioxSettings.swift

@@ -163,6 +163,16 @@ public final class SonioxSettings {
         }
     }
     
+    public var wakeWordEnabled: Bool {
+        get {
+            if defaults.object(forKey: "SonioxWakeWordEnabled") == nil { return true }
+            return defaults.bool(forKey: "SonioxWakeWordEnabled")
+        }
+        set {
+            defaults.set(newValue, forKey: "SonioxWakeWordEnabled")
+        }
+    }
+    
     public var voicePunctuation: Bool {
         get {
             if defaults.object(forKey: "SonioxVoicePunctuation") == nil { return true }

src/StatusBarController.swift

@@ -73,6 +73,13 @@ public final class StatusBarController: NSObject, NSMenuDelegate {
         recordItem.target = self
         menu.addItem(recordItem)
         
+        // Wake Word Toggle Item
+        let wakeWordTitle = "🗣️ Wake Word (Jarvis, Computer, Alexa)"
+        let wakeWordItem = NSMenuItem(title: wakeWordTitle, action: #selector(toggleWakeWordAction), keyEquivalent: "")
+        wakeWordItem.target = self
+        wakeWordItem.state = SonioxSettings.shared.wakeWordEnabled ? .on : .off
+        menu.addItem(wakeWordItem)
+        
         menu.addItem(NSMenuItem.separator())
         
         // 2. Dictation Mode Submenu
@@ -593,4 +600,11 @@ public final class StatusBarController: NSObject, NSMenuDelegate {
     @objc private func quitAction() {
         NSApplication.shared.terminate(nil)
     }
+
+    @objc private func toggleWakeWordAction() {
+        SonioxSettings.shared.wakeWordEnabled.toggle()
+        buildMenu()
+        NotificationCenter.default.post(name: NSNotification.Name("SonioxWakeWordSettingChanged"), object: nil)
+    }
+
 }

src/TextProcessor.swift

@@ -260,7 +260,7 @@ public final class TextProcessor {
 
     public func stripLeadingWakeWord(_ text: String) -> String {
         var t = text.trimmingCharacters(in: .whitespacesAndNewlines)
-        let pattern = #"^(غلام|gholam|gulam)[،,:\s]+"#
+        let pattern = #"^(غلام|gholam|gulam|jarvis|hey jarvis|computer|hey computer|alexa)[،,:\s]+"#
         if let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive) {
             let range = NSRange(location: 0, length: t.utf16.count)
             t = regex.stringByReplacingMatches(in: t, options: [], range: range, withTemplate: "")

src/WakeWordEngine.swift

@@ -0,0 +1,135 @@
+import Foundation
+import SherpaOnnx
+
+public final class WakeWordEngine {
+    public static let shared = WakeWordEngine()
+    
+    private var spotter: OpaquePointer?
+    private var stream: OpaquePointer?
+    private let lock = NSLock()
+    private var isInitialized = false
+    
+    public var onWakeWordDetected: ((String) -> Void)?
+    
+    private init() {
+        initializeEngine()
+    }
+    
+    private func cStr(_ s: String) -> UnsafePointer<CChar> {
+        return UnsafePointer(strdup(s)!)
+    }
+    
+    public func initializeEngine() {
+        lock.lock()
+        defer { lock.unlock() }
+        
+        if isInitialized { return }
+        
+        let fileManager = FileManager.default
+        let bundleResourcePath = Bundle.main.resourcePath.map { $0 + "/wakeword_models" } ?? ""
+        let fallbackPath = "/Users/alialavi/.wakeword-models/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"
+        
+        let modelDir: String
+        if fileManager.fileExists(atPath: bundleResourcePath + "/tokens.txt") {
+            modelDir = bundleResourcePath
+        } else if fileManager.fileExists(atPath: fallbackPath + "/tokens.txt") {
+            modelDir = fallbackPath
+        } else {
+            print("WakeWordEngine: Model files not found.")
+            return
+        }
+        
+        let keywordsPath: String
+        let customKeywords = Bundle.main.resourcePath.map { $0 + "/wakeword_models/keywords.txt" } ?? ""
+        if fileManager.fileExists(atPath: customKeywords) {
+            keywordsPath = customKeywords
+        } else {
+            keywordsPath = modelDir + "/keywords.txt"
+        }
+        
+        var featConfig = SherpaOnnxFeatureConfig(sample_rate: 16000, feature_dim: 80)
+        var modelConfig = SherpaOnnxOnlineModelConfig()
+        modelConfig.tokens = cStr(modelDir + "/tokens.txt")
+        modelConfig.transducer.encoder = cStr(modelDir + "/encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx")
+        modelConfig.transducer.decoder = cStr(modelDir + "/decoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx")
+        modelConfig.transducer.joiner = cStr(modelDir + "/joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx")
+        modelConfig.num_threads = 2
+        modelConfig.provider = cStr("cpu")
+        
+        var config = SherpaOnnxKeywordSpotterConfig(
+            feat_config: featConfig,
+            model_config: modelConfig,
+            max_active_paths: 4,
+            num_trailing_blanks: 1,
+            keywords_score: 1.0,
+            keywords_threshold: 0.25,
+            keywords_file: cStr(keywordsPath),
+            keywords_buf: nil,
+            keywords_buf_size: 0
+        )
+        
+        guard let spotter = SherpaOnnxCreateKeywordSpotter(&config) else {
+            print("WakeWordEngine: Failed to create SherpaOnnxKeywordSpotter")
+            return
+        }
+        guard let stream = SherpaOnnxCreateKeywordStream(spotter) else {
+            print("WakeWordEngine: Failed to create KeywordStream")
+            SherpaOnnxDestroyKeywordSpotter(spotter)
+            return
+        }
+        
+        self.spotter = spotter
+        self.stream = stream
+        self.isInitialized = true
+        print("WakeWordEngine: Initialized successfully with keywords at \(keywordsPath)")
+    }
+    
+    public func acceptAudioChunk(_ pcmData: Data) {
+        lock.lock()
+        defer { lock.unlock() }
+        
+        guard isInitialized, let spotter = self.spotter, let stream = self.stream else { return }
+        
+        let sampleCount = pcmData.count / 2
+        guard sampleCount > 0 else { return }
+        
+        var floatSamples = [Float](repeating: 0, count: sampleCount)
+        pcmData.withUnsafeBytes { raw in
+            let ptr = raw.bindMemory(to: Int16.self)
+            for i in 0..<sampleCount {
+                floatSamples[i] = Float(ptr[i]) / 32768.0
+            }
+        }
+        
+        floatSamples.withUnsafeBufferPointer { buf in
+            guard let ptr = buf.baseAddress else { return }
+            SherpaOnnxOnlineStreamAcceptWaveform(stream, 16000, ptr, Int32(sampleCount))
+        }
+        
+        while SherpaOnnxIsKeywordStreamReady(spotter, stream) == 1 {
+            SherpaOnnxDecodeKeywordStream(spotter, stream)
+            if let resultPtr = SherpaOnnxGetKeywordResult(spotter, stream) {
+                let kw = String(cString: resultPtr.pointee.keyword).trimmingCharacters(in: .whitespacesAndNewlines)
+                if !kw.isEmpty {
+                    print("WakeWordEngine: 🎯 WAKE WORD DETECTED: '\(kw)'!")
+                    SherpaOnnxResetKeywordStream(spotter, stream)
+                    SherpaOnnxDestroyKeywordResult(resultPtr)
+                    
+                    DispatchQueue.main.async { [weak self] in
+                        self?.onWakeWordDetected?(kw)
+                    }
+                    break
+                }
+                SherpaOnnxDestroyKeywordResult(resultPtr)
+            }
+        }
+    }
+    
+    public func reset() {
+        lock.lock()
+        defer { lock.unlock() }
+        if let spotter = self.spotter, let stream = self.stream {
+            SherpaOnnxResetKeywordStream(spotter, stream)
+        }
+    }
+}