Browse Source

feat: native multi-keyword wake word engine (Jarvis, Computer, Alexa) with Sherpa-ONNX Zipformer

main
Ali Alavi 1 week ago
parent
commit
07f8ff2b41
  1. 4
      .gitignore
  2. 4723
      include/SherpaOnnx/c-api.h
  3. 4
      include/SherpaOnnx/module.modulemap
  4. 9
      package.sh
  5. BIN
      resources/wakeword_models/decoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx
  6. BIN
      resources/wakeword_models/encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx
  7. BIN
      resources/wakeword_models/joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx
  8. 7
      resources/wakeword_models/keywords.txt
  9. 500
      resources/wakeword_models/tokens.txt
  10. 39
      src/AppDelegate.swift
  11. 32
      src/AudioRecorder.swift
  12. 10
      src/SonioxSettings.swift
  13. 14
      src/StatusBarController.swift
  14. 2
      src/TextProcessor.swift
  15. 135
      src/WakeWordEngine.swift

4
.gitignore

@ -0,0 +1,4 @@
build/
*.dmg
*.zip
.DS_Store

4723
include/SherpaOnnx/c-api.h
File diff suppressed because it is too large
View File

4
include/SherpaOnnx/module.modulemap

@ -0,0 +1,4 @@
module SherpaOnnx {
header "c-api.h"
export *
}

9
package.sh

@ -15,12 +15,21 @@ rm -rf "$APP_BUNDLE"
# 1. Compile Swift sources
swiftc -O -target arm64-apple-macosx13.0 \
-framework Cocoa -framework AVFoundation -framework Carbon \
-I "$PROJECT_DIR/include/SherpaOnnx" \
-L "$PROJECT_DIR/Frameworks" -lsherpa-onnx-c-api \
-Xlinker -rpath -Xlinker "@executable_path/../Frameworks" \
"$PROJECT_DIR"/src/*.swift \
-o "$BUILD_DIR/SonioxVoice"
# 2. Assemble .app bundle
mkdir -p "$APP_BUNDLE/Contents/MacOS"
mkdir -p "$APP_BUNDLE/Contents/Resources"
mkdir -p "$APP_BUNDLE/Contents/Frameworks"
cp -R "$PROJECT_DIR/Frameworks/"* "$APP_BUNDLE/Contents/Frameworks/"
mkdir -p "$APP_BUNDLE/Contents/Resources/wakeword_models"
cp -R "$PROJECT_DIR/resources/wakeword_models/"* "$APP_BUNDLE/Contents/Resources/wakeword_models/"
if [ ! -f "$PROJECT_DIR/resources/AppIcon.icns" ] && [ -d "$PROJECT_DIR/resources/AppIcon.iconset" ]; then
iconutil -c icns "$PROJECT_DIR/resources/AppIcon.iconset" -o "$PROJECT_DIR/resources/AppIcon.icns"

BIN
resources/wakeword_models/decoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx

BIN
resources/wakeword_models/encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx

BIN
resources/wakeword_models/joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx

7
resources/wakeword_models/keywords.txt

@ -0,0 +1,7 @@
▁JA R VI S
▁HE Y ▁JA R VI S
▁COMP U TER
▁HE Y ▁COMP U TER
▁A LE X A
▁G HO LA M
▁GO LA M

500
resources/wakeword_models/tokens.txt

@ -0,0 +1,500 @@
<blk> 0
<sos/eos> 1
<unk> 2
S 3
T 4
▁THE 5
▁A 6
▁TO 7
▁AND 8
N 9
E 10
D 11
ING 12
' 13
▁OF 14
▁I 15
▁IN 16
Y 17
ED 18
M 19
▁ 20
▁THAT 21
O 22
C 23
R 24
A 25
P 26
▁YOU 27
U 28
RE 29
▁IT 30
▁S 31
G 32
L 33
B 34
▁IS 35
I 36
AL 37
ER 38
▁RE 39
▁C 40
▁WE 41
▁SO 42
▁BE 43
OR 44
AR 45
IN 46
▁FOR 47
F 48
▁HE 49
LE 50
LY 51
▁E 52
▁WAS 53
K 54
▁F 55
W 56
VE 57
RI 58
CH 59
LL 60
▁THIS 61
▁T 62
EN 63
▁ON 64
IT 65
CE 66
▁WITH 67
▁B 68
▁W 69
▁ME 70
▁DO 71
▁DE 72
▁HAVE 73
▁BUT 74
ENT 75
▁P 76
UR 77
ON 78
▁AS 79
IC 80
TER 81
TE 82
▁THEY 83
ES 84
RA 85
▁NOT 86
TH 87
▁ARE 88
RO 89
IL 90
ATION 91
AN 92
▁ST 93
▁WHAT 94
SE 95
LI 96
V 97
▁CAN 98
▁G 99
▁OR 100
▁ALL 101
NE 102
▁AT 103
▁CON 104
H 105
▁LIKE 106
▁MO 107
▁HIS 108
▁AN 109
▁MA 110
▁THERE 111
▁SU 112
▁MY 113
NG 114
- 115
▁ABOUT 116
▁SE 117
LA 118
▁ONE 119
LO 120
▁SHE 121
CK 122
▁HAD 123
AD 124
MENT 125
VER 126
▁CA 127
IR 128
▁IF 129
▁NO 130
EL 131
▁O 132
▁FROM 133
▁PA 134
▁YOUR 135
▁KNOW 136
RY 137
ION 138
UN 139
▁JUST 140
▁UP 141
▁US 142
▁WHO 143
▁GO 144
ME 145
▁EX 146
▁OUT 147
ND 148
CI 149
▁PO 150
▁HER 151
▁FA 152
▁PRO 153
▁WHEN 154
ATE 155
AT 156
▁BY 157
PE 158
AS 159
▁LA 160
GE 161
▁CO 162
▁PEOPLE 163
▁K 164
▁SOME 165
IGHT 166
▁THINK 167
VI 168
DE 169
▁LE 170
▁WERE 171
▁LI 172
OM 173
▁WOULD 174
▁SEE 175
CO 176
▁BO 177
▁OUR 178
▁HOW 179
ET 180
Z 181
UL 182
▁HO 183
IG 184
▁GET 185
▁DON 186
IS 187
URE 188
▁WHICH 189
▁MI 190
IES 191
US 192
X 193
▁MORE 194
▁NOW 195
▁TIME 196
▁UN 197
▁FI 198
TY 199
▁WILL 200
RU 201
IVE 202
▁THEIR 203
▁SAY 204
ITY 205
▁HA 206
LU 207
▁RO 208
HO 209
OL 210
IST 211
AGE 212
MA 213
KE 214
▁RA 215
ID 216
ERS 217
▁LO 218
▁SP 219
▁DA 220
▁SAID 221
TION 222
THER 223
▁EN 224
▁BECAUSE 225
▁HAS 226
EST 227
TI 228
ACT 229
▁BA 230
▁VERY 231
▁THEM 232
▁BEEN 233
▁WA 234
ABLE 235
▁SA 236
▁DID 237
▁HIM 238
▁DIS 239
▁WANT 240
▁OTHER 241
IM 242
▁PRE 243
▁LOOK 244
▁GOING 245
▁VI 246
▁WORK 247
▁DR 248
ENCE 249
▁DI 250
FF 251
TA 252
▁REALLY 253
▁RIGHT 254
MO 255
ALLY 256
▁INTO 257
OW 258
▁WAY 259
▁FE 260
▁COULD 261
▁THEN 262
TED 263
▁HERE 264
▁THESE 265
▁OVER 266
▁WELL 267
▁NA 268
▁ANY 269
HE 270
▁COMP 271
▁COM 272
▁TH 273
▁EVEN 274
▁TWO 275
▁JO 276
▁MAKE 277
▁BACK 278
▁WHERE 279
SH 280
▁QU 281
MAN 282
▁YEAR 283
▁TRA 284
▁CHA 285
QU 286
CA 287
▁ALSO 288
CTION 289
ANCE 290
UM 291
PER 292
DI 293
▁HU 294
▁MU 295
FUL 296
▁NEW 297
ARD 298
J 299
ICAL 300
ANT 301
END 302
▁MAR 303
▁NE 304
IF 305
DER 306
ONE 307
▁THOSE 308
PP 309
▁MAN 310
▁GOOD 311
▁TAKE 312
SIDE 313
PO 314
HA 315
▁LITTLE 316
▁APP 317
▁BI 318
UGH 319
▁COME 320
ISH 321
TIC 322
▁PER 323
▁THINGS 324
▁MUCH 325
▁FIRST 326
▁THAN 327
IA 328
▁AGAIN 329
▁STA 330
▁PART 331
▁GA 332
▁MEAN 333
▁LET 334
▁DOWN 335
▁NEED 336
▁GU 337
LES 338
▁RU 339
ATED 340
▁SHA 341
VO 342
▁TALK 343
▁DAY 344
▁CHI 345
▁SOMETHING 346
▁AFTER 347
▁THOUGH 348
▁CALL 349
▁GRA 350
▁EVERY 351
▁KIND 352
ROW 353
AIN 354
▁BU 355
▁THROUGH 356
▁SHOULD 357
▁ONLY 358
▁TA 359
▁MIN 360
▁GOT 361
IAN 362
▁GREAT 363
▁RI 364
▁DOES 365
▁THING 366
DAY 367
OUS 368
▁START 369
▁MOST 370
▁COL 371
PH 372
▁LOT 373
IZE 374
BER 375
IOUS 376
▁LIFE 377
▁MANY 378
▁FEEL 379
ITION 380
▁VA 381
NESS 382
▁BEFORE 383
▁THREE 384
▁WORLD 385
▁GIVE 386
▁WHY 387
▁PI 388
▁LONG 389
PORT 390
▁SHOW 391
▁ACTUALLY 392
▁NEVER 393
IV 394
FORM 395
▁THANK 396
▁HI 397
▁PUT 398
▁LOVE 399
▁QUESTION 400
▁HELP 401
▁HAPPEN 402
▁MR 403
▁OWN 404
WARD 405
▁MADE 406
▁HAND 407
▁STATE 408
IBLE 409
▁TELL 410
▁PH 411
▁CAME 412
▁TEA 413
▁STILL 414
▁DIFFERENT 415
▁PLACE 416
▁MIGHT 417
▁FIND 418
▁DU 419
▁IMP 420
▁OLD 421
▁PERSON 422
▁COUNT 423
▁JA 424
▁POINT 425
▁HIGH 426
SHIP 427
▁LAST 428
▁SAME 429
▁FOUR 430
▁AROUND 431
▁OH 432
▁CHANGE 433
▁UNDER 434
▁HOME 435
▁ANOTHER 436
▁FOUND 437
ATIVE 438
▁TURN 439
▁YES 440
LAND 441
▁ALWAYS 442
▁YEAH 443
▁SCHOOL 444
▁JE 445
▁NEXT 446
▁WORD 447
▁JU 448
LINE 449
▁SIX 450
▁LEARN 451
▁BOOK 452
▁PLAY 453
▁AWAY 454
▁HOUSE 455
▁PLAN 456
▁WHILE 457
▁COMMUNI 458
▁FRIEND 459
▁AMERICA 460
▁INTEREST 461
▁KEEP 462
▁BETTER 463
▁IDEA 464
▁HARD 465
▁TWENTY 466
▁THOUSAND 467
▁HUNDRED 468
▁WEEK 469
▁NUMBER 470
▁COURSE 471
▁OKAY 472
PRESS 473
▁SYSTEM 474
▁NINE 475
▁UNDERSTAND 476
▁WITHOUT 477
▁BELIEVE 478
▁POWER 479
▁MAYBE 480
▁BUILD 481
▁MONEY 482
▁BETWEEN 483
▁GONNA 484
▁IMPORTANT 485
▁ANYTHING 486
▁LEFT 487
▁HAVING 488
▁WHITE 489
▁PROBLEM 490
▁SECOND 491
▁SPEAK 492
▁TOGETHER 493
▁BLACK 494
▁PUBLIC 495
▁HEALTH 496
▁NOTHING 497
▁EVERYTHING 498
Q 499

39
src/AppDelegate.swift

@ -93,6 +93,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
// Check permissions on start
checkInitialPermissions()
setupDarwinNotifications()
setupWakeWord()
}
private func checkInitialPermissions() {
@ -249,6 +250,9 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
statusBarController.buildMenu(isRecording: false)
isBusyFinalizing = true
if SonioxSettings.shared.wakeWordEnabled {
self.audioRecorder.startPassiveListening()
}
// Check if the live session suffered a network failure during recording
let sessionDropped = (self.activeSession == nil) || (self.activeSession?.hasNetworkFailure == true) || self.isNetworkDegraded
@ -438,4 +442,39 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
CFNotificationCenterAddObserver(notifyCenter, selfPtr, toggleCb, "com.soniox.voice.toggle" as CFString, nil, .deliverImmediately)
}
private func setupWakeWord() {
// Connect WakeWordEngine callbacks
WakeWordEngine.shared.onWakeWordDetected = { [weak self] keyword in
guard let self = self else { return }
print("AppDelegate: 🎯 WAKE WORD DETECTED: '\(keyword)'! Starting recording...")
DispatchQueue.main.async {
if !self.audioRecorder.isRecording {
self.startRecording()
}
}
}
audioRecorder.onPassiveAudioChunkAvailable = { chunk in
WakeWordEngine.shared.acceptAudioChunk(chunk)
}
// Listen to wake word setting changes from Status Bar menu
NotificationCenter.default.addObserver(forName: NSNotification.Name("SonioxWakeWordSettingChanged"), object: nil, queue: .main) { [weak self] _ in
guard let self = self else { return }
if SonioxSettings.shared.wakeWordEnabled {
self.audioRecorder.startPassiveListening()
} else {
self.audioRecorder.pausePassiveListening()
}
}
// Initial start if enabled
if SonioxSettings.shared.wakeWordEnabled {
DispatchQueue.main.asyncAfter(deadline: .now() + 1.2) { [weak self] in
self?.audioRecorder.startPassiveListening()
}
}
}
}

32
src/AudioRecorder.swift

@ -14,8 +14,10 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
private let captureQueue = DispatchQueue(label: "com.soniox.audiocapture", qos: .userInteractive)
public private(set) var isRecording = false
public private(set) var isPassiveListening = false
public var onAudioLevelUpdate: ((Float) -> Void)?
public var onAudioChunkAvailable: ((Data) -> Void)?
public var onPassiveAudioChunkAvailable: ((Data) -> Void)?
public override init() {
self.targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 16000, channels: 1, interleaved: false)!
@ -227,4 +229,34 @@ public final class AudioRecorder: NSObject, AVCaptureAudioDataOutputSampleBuffer
print("AudioRecorder: Stopped. Microphone hardware OFF. Total PCM captured: \(pcmBuffer.count) bytes.")
return pcmBuffer
}
public func startPassiveListening() {
lock.lock()
defer { lock.unlock() }
guard !isRecording, !isPassiveListening else { return }
guard SonioxSettings.shared.wakeWordEnabled else { return }
do {
try ensureSessionConfigured()
if let session = captureSession, !session.isRunning {
session.startRunning()
}
isPassiveListening = true
print("AudioRecorder: Passive wake-word listening active (Jarvis, Computer, Alexa).")
} catch {
print("AudioRecorder: Failed to start passive listening:", error)
}
}
public func pausePassiveListening() {
lock.lock()
defer { lock.unlock() }
isPassiveListening = false
if !isRecording {
if let session = captureSession, session.isRunning {
session.stopRunning()
}
}
}
}

10
src/SonioxSettings.swift

@ -163,6 +163,16 @@ public final class SonioxSettings {
}
}
public var wakeWordEnabled: Bool {
get {
if defaults.object(forKey: "SonioxWakeWordEnabled") == nil { return true }
return defaults.bool(forKey: "SonioxWakeWordEnabled")
}
set {
defaults.set(newValue, forKey: "SonioxWakeWordEnabled")
}
}
public var voicePunctuation: Bool {
get {
if defaults.object(forKey: "SonioxVoicePunctuation") == nil { return true }

14
src/StatusBarController.swift

@ -73,6 +73,13 @@ public final class StatusBarController: NSObject, NSMenuDelegate {
recordItem.target = self
menu.addItem(recordItem)
// Wake Word Toggle Item
let wakeWordTitle = "🗣️ Wake Word (Jarvis, Computer, Alexa)"
let wakeWordItem = NSMenuItem(title: wakeWordTitle, action: #selector(toggleWakeWordAction), keyEquivalent: "")
wakeWordItem.target = self
wakeWordItem.state = SonioxSettings.shared.wakeWordEnabled ? .on : .off
menu.addItem(wakeWordItem)
menu.addItem(NSMenuItem.separator())
// 2. Dictation Mode Submenu
@ -593,4 +600,11 @@ public final class StatusBarController: NSObject, NSMenuDelegate {
@objc private func quitAction() {
NSApplication.shared.terminate(nil)
}
@objc private func toggleWakeWordAction() {
SonioxSettings.shared.wakeWordEnabled.toggle()
buildMenu()
NotificationCenter.default.post(name: NSNotification.Name("SonioxWakeWordSettingChanged"), object: nil)
}
}

2
src/TextProcessor.swift

@ -260,7 +260,7 @@ public final class TextProcessor {
public func stripLeadingWakeWord(_ text: String) -> String {
var t = text.trimmingCharacters(in: .whitespacesAndNewlines)
let pattern = #"^(غلام|gholam|gulam)[،,:\s]+"#
let pattern = #"^(غلام|gholam|gulam|jarvis|hey jarvis|computer|hey computer|alexa)[،,:\s]+"#
if let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive) {
let range = NSRange(location: 0, length: t.utf16.count)
t = regex.stringByReplacingMatches(in: t, options: [], range: range, withTemplate: "")

135
src/WakeWordEngine.swift

@ -0,0 +1,135 @@
import Foundation
import SherpaOnnx
public final class WakeWordEngine {
public static let shared = WakeWordEngine()
private var spotter: OpaquePointer?
private var stream: OpaquePointer?
private let lock = NSLock()
private var isInitialized = false
public var onWakeWordDetected: ((String) -> Void)?
private init() {
initializeEngine()
}
private func cStr(_ s: String) -> UnsafePointer<CChar> {
return UnsafePointer(strdup(s)!)
}
public func initializeEngine() {
lock.lock()
defer { lock.unlock() }
if isInitialized { return }
let fileManager = FileManager.default
let bundleResourcePath = Bundle.main.resourcePath.map { $0 + "/wakeword_models" } ?? ""
let fallbackPath = "/Users/alialavi/.wakeword-models/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"
let modelDir: String
if fileManager.fileExists(atPath: bundleResourcePath + "/tokens.txt") {
modelDir = bundleResourcePath
} else if fileManager.fileExists(atPath: fallbackPath + "/tokens.txt") {
modelDir = fallbackPath
} else {
print("WakeWordEngine: Model files not found.")
return
}
let keywordsPath: String
let customKeywords = Bundle.main.resourcePath.map { $0 + "/wakeword_models/keywords.txt" } ?? ""
if fileManager.fileExists(atPath: customKeywords) {
keywordsPath = customKeywords
} else {
keywordsPath = modelDir + "/keywords.txt"
}
var featConfig = SherpaOnnxFeatureConfig(sample_rate: 16000, feature_dim: 80)
var modelConfig = SherpaOnnxOnlineModelConfig()
modelConfig.tokens = cStr(modelDir + "/tokens.txt")
modelConfig.transducer.encoder = cStr(modelDir + "/encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx")
modelConfig.transducer.decoder = cStr(modelDir + "/decoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx")
modelConfig.transducer.joiner = cStr(modelDir + "/joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx")
modelConfig.num_threads = 2
modelConfig.provider = cStr("cpu")
var config = SherpaOnnxKeywordSpotterConfig(
feat_config: featConfig,
model_config: modelConfig,
max_active_paths: 4,
num_trailing_blanks: 1,
keywords_score: 1.0,
keywords_threshold: 0.25,
keywords_file: cStr(keywordsPath),
keywords_buf: nil,
keywords_buf_size: 0
)
guard let spotter = SherpaOnnxCreateKeywordSpotter(&config) else {
print("WakeWordEngine: Failed to create SherpaOnnxKeywordSpotter")
return
}
guard let stream = SherpaOnnxCreateKeywordStream(spotter) else {
print("WakeWordEngine: Failed to create KeywordStream")
SherpaOnnxDestroyKeywordSpotter(spotter)
return
}
self.spotter = spotter
self.stream = stream
self.isInitialized = true
print("WakeWordEngine: Initialized successfully with keywords at \(keywordsPath)")
}
public func acceptAudioChunk(_ pcmData: Data) {
lock.lock()
defer { lock.unlock() }
guard isInitialized, let spotter = self.spotter, let stream = self.stream else { return }
let sampleCount = pcmData.count / 2
guard sampleCount > 0 else { return }
var floatSamples = [Float](repeating: 0, count: sampleCount)
pcmData.withUnsafeBytes { raw in
let ptr = raw.bindMemory(to: Int16.self)
for i in 0..<sampleCount {
floatSamples[i] = Float(ptr[i]) / 32768.0
}
}
floatSamples.withUnsafeBufferPointer { buf in
guard let ptr = buf.baseAddress else { return }
SherpaOnnxOnlineStreamAcceptWaveform(stream, 16000, ptr, Int32(sampleCount))
}
while SherpaOnnxIsKeywordStreamReady(spotter, stream) == 1 {
SherpaOnnxDecodeKeywordStream(spotter, stream)
if let resultPtr = SherpaOnnxGetKeywordResult(spotter, stream) {
let kw = String(cString: resultPtr.pointee.keyword).trimmingCharacters(in: .whitespacesAndNewlines)
if !kw.isEmpty {
print("WakeWordEngine: 🎯 WAKE WORD DETECTED: '\(kw)'!")
SherpaOnnxResetKeywordStream(spotter, stream)
SherpaOnnxDestroyKeywordResult(resultPtr)
DispatchQueue.main.async { [weak self] in
self?.onWakeWordDetected?(kw)
}
break
}
SherpaOnnxDestroyKeywordResult(resultPtr)
}
}
}
public func reset() {
lock.lock()
defer { lock.unlock() }
if let spotter = self.spotter, let stream = self.stream {
SherpaOnnxResetKeywordStream(spotter, stream)
}
}
}
Loading…
Cancel
Save