File size: 31,713 Bytes
d530cd8 54d2210 d530cd8 0bed40c 7ec132a 54d2210 f36b9c7 d530cd8 d2a78c4 0bed40c d2a78c4 0bed40c 7ec132a 11ee0ea d530cd8 0bed40c b925a97 7ec132a 0bed40c d530cd8 f36b9c7 6b1c6af f36b9c7 d530cd8 f36b9c7 d530cd8 bc9c65c d530cd8 bc9c65c d530cd8 bc9c65c d530cd8 bc9c65c d530cd8 bc9c65c d2a78c4 bc9c65c d2a78c4 0bed40c 7ec132a 0bed40c d2a78c4 0bed40c d2a78c4 0bed40c d2a78c4 0bed40c d2a78c4 0bed40c d2a78c4 0bed40c d2a78c4 0bed40c d2a78c4 2578c9e d2a78c4 0bed40c d2a78c4 bc9c65c da5882b bc9c65c da5882b bc9c65c d530cd8 a260d88 d530cd8 0bed40c d530cd8 0bed40c 11ee0ea 0bed40c d530cd8 d2a78c4 d530cd8 a260d88 d530cd8 a260d88 d530cd8 a260d88 d530cd8 11ee0ea 2578c9e 11ee0ea 2578c9e 11ee0ea 2578c9e 11ee0ea 2578c9e 54d2210 2578c9e 11ee0ea 2578c9e 54d2210 11ee0ea d530cd8 d12b0d9 11ee0ea 2578c9e d530cd8 da5882b d530cd8 bc9c65c d530cd8 bc9c65c d530cd8 a260d88 d530cd8 a260d88 d530cd8 a260d88 11ee0ea 0bed40c 7ec132a 0bed40c a260d88 11ee0ea 0bed40c a260d88 11ee0ea a260d88 d530cd8 a260d88 d530cd8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 | /**
* FriendService β in-process VRM companion brain service.
*
* Mirror of FeishuService pattern:
* - Singleton, runs in the main CLI process
* - subscribe()/subscribeToInbound() for React external store sync
* - enqueue() with origin tracking for message submission
* - In-process audio capture via cpal (src/services/voice.ts)
* - SSE broadcast for VRM display commands
*
* Eliminates the need for:
* - A separate background server subprocess (port 3456)
* - A CLI SDK subprocess (conversationService.startSession)
* - Server-side arecord/parecord audio capture
*/
import { broadcastToVrm, type VrmBroadcastPayload } from './sse.js';
import { getPrefs } from './prefs.js';
import { stripForTts } from './text-utils.js';
import { edgeTts, qwenTts, registerAudioFile, getAudioFile } from './tts.js';
import { splitSentences } from './text-utils.js';
import { SileroVad } from './voice/vad-service.js';
import { HighPassFilter } from './voice/highpass-filter.js';
import { readFileSync } from 'node:fs';
import { buildVrmSystemPrompt } from '../skills/bundled/friendPrompt.js';
// ββ Types ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
export type FriendServiceState = {
status: 'stopped' | 'starting' | 'running' | 'error';
lastError?: string;
/** Number of active SSE display clients */
displayClientCount?: number;
/** Current capture status (for voice call interim polling) */
captureStatus?: { capturing: boolean; interimText?: string };
};
type Listener = () => void;
/** Inbound event for bridge hook consumption */
export type FriendInboundEvent = {
text: string;
};
type InboundListener = (event: FriendInboundEvent) => void;
// ββ Audio capture types (cpal wrapper) βββββββββββββββββββββββββββββββββ
type AudioCaptureProvider = {
startRecording(
onData: (chunk: Buffer) => void,
onEnd: () => void,
): Promise<boolean>;
stopRecording(): Promise<void>;
isRecording(): boolean;
};
// ββ Service implementation βββββββββββββββββββββββββββββββββββββββββββββ
class FriendService {
private listeners = new Set<Listener>();
private inboundListeners = new Set<InboundListener>();
private state: FriendServiceState = { status: 'stopped' };
/** Audio capture in progress? */
private capturing = false;
/** Accumulated STT text chunks during capture */
private captureTranscripts: string[] = [];
/** Interim (non-final) text during active capture */
private captureInterimText = '';
/** Resolver for the current stopVoiceCapture() call */
private captureResolver: ((text: string) => void) | null = null;
/** Lazy-loaded cpal audio capture module */
private audioCapture: AudioCaptureProvider | null = null;
/** Active STT connection (Anthropic/Doubao/Whisper) during capture */
private sttConnection: { send: (chunk: Buffer) => void; finalize: () => Promise<void>; close: () => void } | null = null;
/** STT provider/language cached for connection re-creation during segmentation */
private captureProvider = '';
private captureLanguage = '';
/** Guard against concurrent flush calls */
private _flushing = false;
/** Silero VAD instance (real ML-based voice activity detection) */
private vadInstance: SileroVad | null = null;
/** High-pass filter to remove low-frequency noise (fan/AC hum) before VAD */
private highpassFilter = new HighPassFilter(80);
/** When muted, audio from arecord is not forwarded to STT or VAD (prevents echo) */
private muted = false;
/** Timer to automatically unmute after estimated TTS playback */
private muteTimer: ReturnType<typeof setTimeout> | null = null;
// ββ React sync external store interface ββββββββββββββββββββββββββββββ
subscribe(listener: Listener): () => void {
this.listeners.add(listener);
return () => this.listeners.delete(listener);
}
subscribeToInbound(listener: InboundListener): () => void {
this.inboundListeners.add(listener);
return () => this.inboundListeners.delete(listener);
}
getStateSnapshot(): FriendServiceState {
return this.state;
}
// ββ Lifecycle ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
async start(): Promise<void> {
if (this.state.status === 'running') return;
this.setState({ status: 'starting', lastError: undefined });
try {
// Pre-warm Silero VAD (loads ONNX model + onnxruntime-web WASM)
if (!this.vadInstance) {
const vad = new SileroVad({
onSpeechStart: () => {},
onSpeechEnd: (_audio) => {
this._flushVadSegment().catch((e) =>
console.error('[FriendService] VAD segment flush error:', e),
);
},
}, {
// Strict thresholds for near-field mic β rejects keyboard/ambient noise
positiveSpeechThreshold: 0.90, // need 90% confidence (was 80%)
negativeSpeechThreshold: 0.40, // drop below 40% to end (was 50%)
preSpeechTriggerFrames: 16, // require ~512ms sustained speech (was 480ms)
minSpeechFrames: 8, // ~256ms min speech (was 192ms)
redemptionFrames: 15, // ~480ms silence before segment end (was 640ms)
rmsThreshold: 0.015, // -36dBFS noise floor (was -40dBFS)
});
vad.init().then(() => {
this.vadInstance = vad;
}).catch((e) => {
console.warn('[FriendService] VAD init failed (non-fatal, voice capture falls back to F2-only):', e);
});
}
this.setState({ status: 'running' });
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
this.setState({ status: 'error', lastError: msg });
throw err;
}
}
async stop(): Promise<void> {
// Stop any active capture
if (this.capturing) {
await this.stopVoiceCapture().catch(() => {});
}
this.audioCapture = null;
this.sttConnection = null;
if (this.state.status !== 'stopped') {
this.setState({ status: 'stopped' });
}
}
// ββ Text relay via messageQueueManager.enqueue() βββββββββββββββββββββ
/**
* Send text through the main CLI conversation.
* Mirrors FeishuService's enqueue() pattern with origin tracking.
*/
sendText(text: string): void {
const trimmed = text.trim();
if (!trimmed) return;
// Notify inbound listeners (bridge hook uses this for turn tracking)
for (const listener of this.inboundListeners) {
listener({ text: trimmed });
}
// Build the override system prompt (persona-only, no VersperClaw CLI prompt)
const baseVrmPrompt = buildVrmSystemPrompt();
const overrideSystemPrompt = `${baseVrmPrompt}\n\nIMPORTANT: You are ONLY the character(s) defined above. Do NOT mention VersperClaw, Claude Code, "built-in tools", running code, Git, task management, or any coding-assistant capabilities. You may use available tools when appropriate, but your identity and behavior must follow your character persona strictly.`;
// Dynamically import enqueue to avoid circular deps
import('../utils/messageQueueManager.js').then(({ enqueue }) => {
enqueue({
value: trimmed,
mode: 'prompt',
skipSlashCommands: true,
bridgeOrigin: true,
origin: { kind: 'channel', server: 'friend' },
overrideSystemPrompt,
});
}).catch((err) => {
console.error('[FriendService] enqueue failed:', err);
});
}
// ββ Voice capture ββββββββββββββββββββββββββββββββββββββββββββββββββββ
/**
* Start in-process voice capture using cpal.
* Audio is forwarded to the configured STT provider.
* Wraps initialization in a timeout (12s) to prevent hanging
* when STT provider or audio device is unavailable.
*/
async startVoiceCapture(): Promise<void> {
if (this.capturing) return;
const prefs = getPrefs();
let provider = prefs.sttProvider;
const language = prefs.sttLanguage || 'zh';
// Auto-detect STT provider if not configured or set to 'browser' (not available in WebKitGTK)
if (!provider || provider === 'browser') {
provider = await this.detectAvailableSttProvider();
}
this.captureTranscripts = [];
this.captureInterimText = '';
this.capturing = true;
try {
// Wrap the whole initialization in a 12s timeout to avoid hanging
// when STT provider or audio device is unavailable.
await this.withTimeout(
this._initVoiceCapture(provider, language),
12000,
`Voice initialization timed out. Check that your microphone is accessible and STT provider "${provider}" is configured correctly.`,
);
} catch (err) {
this.capturing = false;
this.sttConnection?.close();
this.sttConnection = null;
throw err;
}
}
/**
* Internal voice capture initialization (STT connection + audio capture).
* Separated so startVoiceCapture() can wrap it with a timeout.
*/
private async _initVoiceCapture(
provider: string,
language: string,
): Promise<void> {
this.captureProvider = provider;
this.captureLanguage = language;
// 1. Start STT provider connection (with inner 8s timeout)
const conn = await this.startSttConnectionWithTimeout(provider, language);
this.sttConnection = conn;
// 2. Load audio capture module (cpal)
const audio = await this.loadAudioCapture();
// 3. Start cpal recording β chunks go to STT
const ok = await audio.startRecording(
(chunk: Buffer) => {
this.sttConnection?.send(chunk);
},
() => {
// Capture ended (user stop or silence detection)
},
);
if (!ok) {
throw new Error('Native audio capture unavailable');
}
// 4. Start Silero VAD for real-time speech-end detection
// audio chunks flow to VAD via the arecord data callback (see loadAudioCapture)
if (this.vadInstance) {
try {
this.highpassFilter.reset(); // reset filter state for new session
this.vadInstance.start();
} catch (e) {
console.warn('[FriendService] VAD start error (non-fatal):', e);
}
}
}
/**
* Flush the current STT segment and start a new one, triggered by
* Silero VAD's onSpeechEnd callback.
*
* The audio from the just-ended speech segment has already been sent
* to the STT connection. We swap to a fresh connection so the next
* speech segment starts clean.
*/
private async _flushVadSegment(): Promise<void> {
if (this._flushing || !this.capturing) return;
this._flushing = true;
try {
const oldConn = this.sttConnection;
if (!oldConn) return;
// 1. Create a new STT connection for ongoing audio
const newConn = await this.startSttConnectionWithTimeout(
this.captureProvider,
this.captureLanguage,
);
this.sttConnection = newConn;
// 2. Finalize old connection (sends buffered audio to STT provider)
await oldConn.finalize().catch(() => {});
oldConn.close();
// 3. Send accumulated transcript to the CLI conversation
const transcript = this.captureTranscripts.join('').trim();
if (transcript) {
this.captureTranscripts = [];
this.sendText(transcript);
// Mute the entire AI turn: from transcript submission β AI processing
// (tools, deep thinking) β response β TTS playback.
// This prevents both TTS echo AND capturing accidental speech during
// AI processing (e.g. "hmm", "ok").
this.startAiTurnMute();
}
} catch (err) {
console.error('[FriendService] _flushVadSegment error:', err);
} finally {
this._flushing = false;
}
}
/** Race a promise against a timeout */
private async withTimeout<T>(
promise: Promise<T>,
ms: number,
message: string,
): Promise<T> {
return Promise.race([
promise,
new Promise<never>((_, reject) =>
setTimeout(() => reject(new Error(message)), ms),
),
]);
}
/**
* Auto-detect the first available STT provider.
* Tries: Groq Whisper (cloud, API key) β local Whisper β Anthropic Voice Stream β Doubao ASR
*/
private async detectAvailableSttProvider(): Promise<string> {
console.log('[FriendService] detectAvailableSttProvider: checking available providers...');
// Check Groq API key first (fastest β no Python, just a REST call)
// Keys are resolved from: prefs β process.env β ~/.claude/settings.json
try {
const { isGroqAvailable } = await import('../services/voice/groqSTT.js');
if (isGroqAvailable()) {
console.log('[FriendService] detectAvailableSttProvider: Groq API key found');
return 'groq';
}
} catch { /* ignore */ }
// Check local Whisper (no external API keys needed)
try {
const { checkLocalWhisperAvailable } = await import(
'../services/voice/whisperSTT.js'
);
const avail = await checkLocalWhisperAvailable();
console.log('[FriendService] detectAvailableSttProvider: local Whisper available:', avail);
if (avail) {
return 'local';
}
} catch (e) {
console.warn('[FriendService] detectAvailableSttProvider: local whisper check failed:', e);
}
// Check Anthropic Voice Stream
try {
const { isVoiceStreamAvailable } = await import(
'../services/voiceStreamSTT.js'
);
if (isVoiceStreamAvailable()) {
return 'anthropic';
}
} catch { /* skip */ }
// Check Doubao credentials file
try {
const path = await import('node:path');
const fs = await import('node:fs');
const homeDir = process.env.HOME || process.env.USERPROFILE || '';
const credsPath = path.join(homeDir, '.claude', 'tts', 'doubao', 'credentials.json');
if (fs.existsSync(credsPath)) {
return 'doubao';
}
} catch { /* skip */ }
throw new Error(
'No STT provider available. Install local Whisper:\n' +
' pip install openai-whisper\n\n' +
'Or configure an STT provider in Friend settings (Settings β STT Provider).',
);
}
/**
* Start STT connection with a timeout to prevent hanging
* when the provider is unavailable (e.g. Python Whisper not installed).
*/
private async startSttConnectionWithTimeout(
provider: string,
language: string,
): Promise<{ send: (chunk: Buffer) => void; finalize: () => Promise<void>; close: () => void }> {
const timeoutMs = 8000;
const result = await Promise.race([
this.startSttConnection(provider, language),
new Promise<never>((_, reject) =>
setTimeout(
() =>
reject(
new Error(
`STT provider "${provider}" timed out after ${timeoutMs / 1000}s.` +
(provider === 'local'
? '\nInstall local Whisper: pip install openai-whisper'
: ''),
),
),
timeoutMs,
),
),
]);
return result;
}
/**
* Stop voice capture and return the accumulated transcript.
*/
async stopVoiceCapture(): Promise<string> {
return this._stopCapture();
}
/**
* Internal: stop audio capture + finalize STT, return transcript.
*/
private async _stopCapture(): Promise<string> {
console.log(`[FriendService] _stopCapture: capturing=${this.capturing} sttConnection=${this.sttConnection ? 'exists' : 'null'}`);
if (!this.capturing) return '';
// Stop audio recording first (no more audio β no more STT/VAD input)
if (this.audioCapture) {
await this.audioCapture.stopRecording().catch(() => {});
}
this.capturing = false;
// Clear mute state
this.clearMute();
// Reset VAD state silently (don't fire onSpeechEnd β we're finalizing below)
if (this.vadInstance) {
try {
this.vadInstance.reset();
} catch { /* ignore */ }
}
// Finalize STT connection
const conn = this.sttConnection;
this.sttConnection = null;
if (conn) {
try {
await conn.finalize();
conn.close();
} catch {
// ignore finalization errors
}
}
// Send any remaining transcript to CLI before returning
const remaining = this.captureTranscripts.join('').trim();
if (remaining) {
this.sendText(remaining);
}
const transcript = this.captureTranscripts.join('');
console.log(`[FriendService] _stopCapture: transcript="${transcript}" (len=${transcript.length})`);
this.captureTranscripts = [];
this.captureInterimText = '';
this.setState({ captureStatus: { capturing: false } });
return transcript;
}
/**
* Get current capture status (for push-to-talk mode).
*/
getCaptureStatus(): { capturing: boolean; interimText?: string } {
const status = this.state.captureStatus ?? { capturing: false };
return { ...status };
}
/**
* Transcribe an audio buffer (PCM/WAV) using the configured STT provider
* and send the resulting text to the CLI conversation.
*
* Called by the HTTP endpoint when the browser VAD detects a speech segment.
*/
async transcribeAudioSegment(audioBuffer: Buffer): Promise<string> {
const prefs = getPrefs();
let provider = prefs.sttProvider;
if (!provider || provider === 'browser') {
provider = await this.detectAvailableSttProvider();
}
// Create a temporary STT connection just for this segment
const conn = await this.startSttConnectionWithTimeout(
provider,
prefs.sttLanguage || 'zh',
);
try {
conn.send(audioBuffer);
await conn.finalize();
conn.close();
} catch (err) {
conn.close();
throw err;
}
// Get transcript from the finalization callback
const transcript = this.captureTranscripts.join('');
this.captureTranscripts = [];
if (transcript.trim()) {
this.sendText(transcript);
}
return transcript;
}
/**
* Start mute at the beginning of an AI turn (when user speech is submitted).
*
* Audio from arecord is blocked from STT/VAD during:
* AI processing (tools, deep thinking) β response generation β TTS playback
*
* A long timeout (30s) covers the AI processing window without estimation.
* The timer is later refined by extendMuteForTts() when TTS audio is ready.
*/
private startAiTurnMute(): void {
if (!this.capturing) return;
// Clear any existing timer (from a previous turn)
if (this.muteTimer) clearTimeout(this.muteTimer);
this.muted = true;
this.vadInstance?.pause();
// 30s covers almost all AI response cycles (tools, deep thinking, etc.).
// The timer is reset in extendMuteForTts() when TTS duration is known.
this.muteTimer = setTimeout(() => this.unmute(), 30_000);
}
/**
* Refine the mute timer to match actual TTS audio duration once it's ready.
* Called from broadcastResponse() after TTS generation succeeds.
* Resets the timer to exactly the audio playback length.
*/
private extendMuteForTts(audioId: string): void {
if (!this.capturing) return;
if (!this.muted) return; // turn already ended, don't re-mute
// Clear the generous timer from startAiTurnMute
if (this.muteTimer) clearTimeout(this.muteTimer);
// Parse exact audio duration from the MP3 file
let muteMs = this.getMp3DurationMs(audioId);
if (muteMs <= 0) muteMs = 3000; // safety fallback
this.muteTimer = setTimeout(() => this.unmute(), muteMs);
}
/** Unmute and resume VAD */
private unmute(): void {
this.muted = false;
this.muteTimer = null;
this.vadInstance?.start();
}
/** Parse MP3 file to get exact audio duration in milliseconds.
*
* Finds the first two frame syncs to determine the real frame size,
* then counts frames using that stride. Works for CBR output (Edge TTS)
* without relying on error-prone bitrate lookup tables.
*/
private getMp3DurationMs(audioId: string): number {
const filePath = getAudioFile(audioId);
if (!filePath) return 0;
let buf: Buffer;
try { buf = readFileSync(filePath); } catch { return 0; }
if (buf.length < 100) return 0;
const isSync = (p: number) =>
p + 1 < buf.length && buf[p] === 0xff && (buf[p + 1] & 0xe0) === 0xe0;
let offset = 0;
// Skip ID3v2 tag
if (buf[0] === 0x49 && buf[1] === 0x44 && buf[2] === 0x33) {
offset = 10 +
((buf[6] & 0x7f) << 21) |
((buf[7] & 0x7f) << 14) |
((buf[8] & 0x7f) << 7) |
(buf[9] & 0x7f);
}
// Find first two syncs to measure actual frame stride
let firstSync = -1;
let secondSync = -1;
for (let i = offset; i < buf.length - 3; i++) {
if (isSync(i)) {
if (firstSync === -1) firstSync = i;
else { secondSync = i; break; }
}
}
if (firstSync === -1 || secondSync === -1) return 0;
const frameSize = secondSync - firstSync; // real stride (CBR)
if (frameSize < 20) return 0;
// Parse frame header for sample rate and samples-per-frame
const h =
(buf[firstSync] << 24) |
(buf[firstSync + 1] << 16) |
(buf[firstSync + 2] << 8) |
buf[firstSync + 3];
const version = (h >> 19) & 0x3;
const sampleRateIdx = (h >> 10) & 0x3;
if (sampleRateIdx === 3) return 0;
const srTable: Record<number, number> = {
3: [44100, 48000, 32000][sampleRateIdx],
2: [22050, 24000, 16000][sampleRateIdx],
0: [11025, 12000, 8000][sampleRateIdx],
};
const sampleRate = srTable[version];
if (!sampleRate) return 0;
const isMpeg1 = version === 3;
const spf = isMpeg1 ? 1152 : 576;
// Count frames using stride
let frames = 0;
for (let pos = firstSync; pos + 3 < buf.length; pos += frameSize) {
// Sanity check: verify sync word
if (!isSync(pos)) {
// Frame may have been corrupted; scan forward to next sync
while (pos < buf.length - 3 && !isSync(pos)) pos++;
if (pos >= buf.length - 3) break;
}
frames++;
}
return Math.round((frames * spf) / sampleRate * 1000);
}
/** Clear mute state immediately */
private clearMute(): void {
if (this.muteTimer) {
clearTimeout(this.muteTimer);
this.muteTimer = null;
}
this.muted = false;
}
// ββ Response broadcast (called by useFriendBridge) βββββββββββββββββββ
/**
* Broadcast AI response to the VRM display layer via SSE.
* Generates TTS audio for completed sentences and sends
* emotion/action commands alongside text.
*/
async broadcastResponse(text: string): Promise<void> {
if (!text.trim()) return;
const prefs = getPrefs();
// Send the text β the frontend TextBubble splits and displays it
broadcastToVrm({ text });
// Generate TTS for the full response if enabled
if (prefs.ttsEnabled) {
try {
const audioId = await this.generateTts(text);
if (audioId) {
const fullUrl = `http://127.0.0.1:3456/plugins/friend/audio/${audioId}`;
broadcastToVrm({ audioUrl: fullUrl, sendFirstTts: true });
// Refine mute timer to exact TTS audio duration
this.extendMuteForTts(audioId);
}
} catch (err) {
console.warn('[FriendService] TTS generation failed:', err);
}
}
// Signal reply done
broadcastToVrm({ replyDone: true });
}
/**
* Broadcast VRM emotion/action command.
*/
broadcastVrm(payload: VrmBroadcastPayload): void {
broadcastToVrm(payload);
}
// ββ Private: STT connection factory ββββββββββββββββββββββββββββββββββ
private async startSttConnection(
provider: string,
language: string,
): Promise<{ send: (chunk: Buffer) => void; finalize: () => Promise<void>; close: () => void }> {
const callbacks = {
onTranscript: (text: string, isFinal: boolean) => {
if (isFinal) {
this.captureTranscripts.push(text);
this.captureInterimText = '';
} else {
this.captureInterimText = text;
}
// Update state for status polling
this.setState({
captureStatus: { capturing: true, interimText: this.captureInterimText },
});
},
onError: (_error: string) => {},
onClose: () => {},
onReady: (_conn: any) => {},
};
switch (provider) {
case 'anthropic': {
const { connectVoiceStream, isVoiceStreamAvailable } = await import(
'../services/voiceStreamSTT.js'
);
if (!isVoiceStreamAvailable()) {
throw new Error('Anthropic Voice Stream not available');
}
return await connectVoiceStream(callbacks, { language, keyterms: ['code', 'versperclaw'] });
}
case 'local': {
const { connectLocalWhisperStream, preloadWhisperModel } = await import(
'../services/voice/whisperSTT.js'
);
await preloadWhisperModel({ language });
return await connectLocalWhisperStream(callbacks, { language });
}
case 'doubao': {
const { connectDoubaoStream } = await import('../services/doubaoSTT.js');
return await connectDoubaoStream(callbacks, { language: language || 'zh' });
}
case 'groq': {
const { connectGroqStream } = await import(
'../services/voice/groqSTT.js'
);
return await connectGroqStream(callbacks, { language });
}
default:
throw new Error(`Unknown STT provider: ${provider}`);
}
}
// ββ Private: Audio capture (arecord/parecord subprocess) ββββββββββββββ
private async loadAudioCapture(): Promise<AudioCaptureProvider> {
if (this.audioCapture) return this.audioCapture;
// Use subprocess-based capture (arecord/parecord) on all platforms.
// Skipping the native cpal module because its synchronous NAPI call
// can block the event loop if ALSA initialization hangs, and there
// is no way to timeout a native binding call from JS.
const { spawn } = await import('node:child_process');
let captureProc: import('node:child_process').ChildProcess | null = null;
this.audioCapture = {
startRecording: async (onData, _onEnd) => {
for (const tool of ['arecord', 'parecord']) {
try {
const args = tool === 'arecord'
? ['-D', 'default', '-r', '16000', '-f', 'S16_LE', '-c', '1', '-t', 'raw', '-q']
: ['--raw', '--rate=16000', '--format=s16le', '--channels=1', '--latency-msec=20'];
const proc = spawn(tool, args, { stdio: ['pipe', 'pipe', 'pipe'] });
if (proc.pid === undefined) continue;
// Verify the tool actually produces audio data within 500ms.
// Some tools (e.g. parecord without PulseAudio) spawn successfully
// but exit immediately without any output β catch that here.
let dataArrived = false;
let verifyTimer: ReturnType<typeof setTimeout> | null = null;
const verified = await new Promise<boolean>((resolve) => {
const feedAudio = (c: Buffer) => {
// Skip when muted β prevents AI TTS echo from re-entering STT/VAD
if (this.muted) return;
// Forward to STT connection
onData(c);
// Forward to VAD for speech activity detection
if (this.vadInstance) {
const float32 = new Float32Array(c.length / 2);
for (let i = 0; i < float32.length; i++) {
float32[i] = c.readInt16LE(i * 2) / 32768;
}
// High-pass filter: remove sub-80Hz noise (fan/AC hum) before VAD
this.highpassFilter.process(float32);
this.vadInstance.processAudio(float32).catch(() => {});
}
};
const dataHandler = (chunk: Buffer) => {
dataArrived = true;
if (verifyTimer) { clearTimeout(verifyTimer); }
feedAudio(chunk);
// Swap to the permanent handler for subsequent chunks
proc.stdout?.removeListener('data', dataHandler);
proc.stdout?.on('data', feedAudio);
resolve(true);
};
proc.stdout?.on('data', dataHandler);
// Process exited before producing data β mark as failed
proc.on('exit', () => {
if (!dataArrived) {
if (verifyTimer) { clearTimeout(verifyTimer); }
resolve(false);
}
});
// No data within 500ms β assume failure
verifyTimer = setTimeout(() => {
if (!dataArrived) resolve(false);
}, 500);
});
if (verified) {
captureProc = proc;
proc.on('exit', () => { captureProc = null; });
return true;
}
// Verification failed β kill and try next tool
proc.kill('SIGTERM');
} catch {
continue;
}
}
return false;
},
stopRecording: async () => {
if (captureProc) {
captureProc.kill('SIGTERM');
setTimeout(() => {
try { captureProc?.kill('SIGKILL'); } catch {}
}, 2000);
captureProc = null;
}
},
isRecording: () => captureProc !== null,
};
return this.audioCapture;
}
// ββ Private: TTS generation ββββββββββββββββββββββββββββββββββββββββββ
private async generateTts(text: string): Promise<string | undefined> {
const prefs = getPrefs();
if (!prefs.ttsEnabled) return undefined;
const cleanText = stripForTts(text);
if (!cleanText) return undefined;
let result: { success: boolean; audioPath?: string; error?: string };
if (prefs.provider === 'qwen' && prefs.qwenKey) {
result = await qwenTts({
text: cleanText,
apiKey: prefs.qwenKey,
voice: prefs.voice,
model: prefs.qwenModel,
language: prefs.language,
});
} else {
result = await edgeTts({ text: cleanText, voice: prefs.voice });
}
if (result.success && result.audioPath) {
return registerAudioFile(result.audioPath);
}
return undefined;
}
// ββ Private: state management ββββββββββββββββββββββββββββββββββββββββ
private setState(next: Partial<FriendServiceState>): void {
this.state = { ...this.state, ...next };
for (const listener of this.listeners) listener();
}
}
// Singleton
export const friendService = new FriendService();
|