// Puck's voice. Two engines behind one seam: // - kokoro: local neural TTS (kokoro-js, Kokoro-82M ONNX) — runs in WebGPU/wasm // right here in the page; model is fetched once from the HF CDN and cached by // the browser. Genuinely characterful, fully local, works in the Tauri webview // and the HF Space alike. // - browser: OS SpeechSynthesis — instant, zero-download, the fallback when // Kokoro hasn't loaded yet or errors. // Gated by the interruption ladder (`speaks`) and shaped by mood + reaction. // This module is the ONLY audio producer — callers never touch an engine directly. import type { KokoroTTS } from "kokoro-js"; import type { Mood, ReactionKind } from "../engine"; export type VoiceMode = "off" | "whisper" | "full"; // browser = OS speechSynthesis · kokoro = local neural · say = macOS `say` via the // daemon (the only thing that works in the WKWebView overlay — it plays Mac-side). export type VoiceEngine = "browser" | "kokoro" | "say"; export type VoiceChannel = "interrupt" | "bubble" | "toast" | "reaction"; // ---- browser (OS) voices ---------------------------------------------------- let voices: SpeechSynthesisVoice[] = []; let chosen: SpeechSynthesisVoice | null = null; const PREFERRED = [ "Samantha", "Karen", "Moira", "Tessa", "Fiona", "Google UK English Female", "Google US English", ]; function refresh() { if (!("speechSynthesis" in window)) return; voices = window.speechSynthesis.getVoices() || []; chosen = PREFERRED.map((n) => voices.find((v) => v.name === n)).find(Boolean) ?? voices.find((v) => /en[-_]/i.test(v.lang) && /female/i.test(v.name)) ?? voices.find((v) => /en[-_]/i.test(v.lang)) ?? voices[0] ?? null; } if (typeof window !== "undefined" && "speechSynthesis" in window) { refresh(); window.speechSynthesis.onvoiceschanged = refresh; } /** OS voices for the Settings picker (english first). */ export function listVoices(): { name: string; lang: string }[] { return [...voices] .sort((a, b) => Number(/en/i.test(b.lang)) - Number(/en/i.test(a.lang))) .map((v) => ({ name: v.name, lang: v.lang })); } function voiceByName(name?: string): SpeechSynthesisVoice | null { if (!name) return chosen; return voices.find((v) => v.name === name) ?? chosen; } // ---- kokoro (neural) voices ------------------------------------------------- // Curated subset of Kokoro's voices that suit a small, bright familiar. export const KOKORO_VOICES = [ { id: "af_heart", label: "Heart · warm" }, { id: "af_bella", label: "Bella · bright" }, { id: "af_nicole", label: "Nicole · soft" }, { id: "af_aoede", label: "Aoede · gentle" }, { id: "af_sky", label: "Sky · light" }, { id: "af_jessica", label: "Jessica · airy" }, { id: "bf_emma", label: "Emma · british" }, { id: "am_puck", label: "Puck · playful" }, ] as const; export const DEFAULT_KOKORO_VOICE = "af_heart"; let kokoro: KokoroTTS | null = null; let kokoroPromise: Promise | null = null; export function kokoroReady(): boolean { return kokoro !== null; } // Only run neural TTS where WebGPU exists. Chromium-on-Mac has it; the Tauri // overlay's WKWebView does NOT — and loading transformers.js + the model there // crashed WebKit's content process. So gate Kokoro on WebGPU and let those // environments use the OS voice instead of a crash. export function kokoroSupported(): boolean { return typeof navigator !== "undefined" && "gpu" in navigator; } /** Lazy-load the model (code-split + fetched once, cached by the browser). */ export function loadKokoro( onProgress?: (p: { progress?: number; status?: string }) => void, ): Promise { if (kokoro) return Promise.resolve(kokoro); if (!kokoroPromise) { kokoroPromise = (async () => { const { KokoroTTS } = await import("kokoro-js"); const device = "gpu" in navigator ? "webgpu" : "wasm"; const tts = await KokoroTTS.from_pretrained("onnx-community/Kokoro-82M-v1.0-ONNX", { dtype: device === "webgpu" ? "fp32" : "q8", // fp32 only when the GPU can carry it device, progress_callback: onProgress as never, }); kokoro = tts; return tts; })().catch((e) => { kokoroPromise = null; // let a later attempt retry throw e; }); } return kokoroPromise; } // the single in-flight neural clip, so cancel/mute can stop it mid-sentence let currentAudio: HTMLAudioElement | null = null; function stopKokoro() { if (currentAudio) { currentAudio.pause(); currentAudio = null; } } async function speakKokoro( text: string, { voiceName, speed, onStart, onEnd, }: { voiceName?: string; speed: number; onStart?: () => void; onEnd?: () => void }, ): Promise { try { const tts = await loadKokoro(); const audio = await tts.generate(text, { voice: (voiceName || DEFAULT_KOKORO_VOICE) as never, speed }); const url = URL.createObjectURL(audio.toBlob()); stopKokoro(); const el = new Audio(url); currentAudio = el; el.onplay = () => onStart?.(); const done = () => { onEnd?.(); URL.revokeObjectURL(url); if (currentAudio === el) currentAudio = null; }; el.onended = done; el.onerror = done; await el.play(); return true; } catch (e) { // model still downloading, webgpu hiccup, autoplay block — fall back to the OS voice console.error("puck: kokoro tts failed → browser fallback", e); return false; } } // ---- native macOS voice (daemon `say`) -------------------------------------- // Plays on the Mac itself, so it works in the WKWebView overlay where browser // audio is dead. The endpoint blocks until speech ends (or is superseded), so // the fetch resolving == speech finished — good enough to drive the animation. let sayAbort: AbortController | null = null; async function sayViaDaemon( text: string, { voice, rate, pitch, onStart, onEnd, }: { voice?: string; rate: number; pitch?: number; onStart?: () => void; onEnd?: () => void }, ): Promise { try { sayAbort?.abort(); sayAbort = new AbortController(); onStart?.(); // macOS `say` has no pitch flag, but honors an inline [[pbas N]] (pitch baseline). // Map our ~0.6–1.8 pitch onto it so the overlay's native voice carries emotion too // (~50 ≈ neutral). The daemon passes text straight to `say`, so the command runs. const shaped = pitch && pitch !== 1 ? `[[pbas ${Math.round(Math.max(30, Math.min(92, 50 * pitch)))}]] ${text}` : text; const res = await fetch("/api/voice/say", { method: "POST", headers: { "Content-Type": "application/json" }, // rate as words-per-minute (say's -r); 175 ≈ default cadence body: JSON.stringify({ text: shaped, voice: voice || undefined, rate: Math.round(175 * rate) }), signal: sayAbort.signal, }); onEnd?.(); return res.ok; // 501 (non-mac) / offline → caller falls back to browser TTS } catch { onEnd?.(); // aborted (superseded) or daemon down return false; } } function stopSay() { sayAbort?.abort(); void fetch("/api/voice/stop", { method: "POST" }).catch(() => {}); } type SayVoice = { id: string; lang: string }; let sayVoiceCache: SayVoice[] | null = null; /** macOS `say` voices for the Settings picker (fetched once from the daemon). */ export async function sayVoices(): Promise { if (sayVoiceCache) return sayVoiceCache; let list: SayVoice[] = []; try { const res = await fetch("/api/voice/voices", { signal: AbortSignal.timeout(4000) }); if (res.ok) list = ((await res.json()) as { voices?: SayVoice[] }).voices ?? []; } catch { /* daemon down / non-mac — no native voices to list */ } sayVoiceCache = list; return list; } // ---- shared shaping --------------------------------------------------------- const clean = (t: string) => t .replace(/[\u{1F000}-\u{1FAFF}\u{2600}-\u{27BF}←-⇿✦✕▸◷◍❍☾⎇♪]/gu, "") .replace(/\s+/g, " ") .trim(); // A spoken notification is a glance, not a paragraph: the bubble keeps the full // (sometimes mythic, two-sentence) line, but speech takes just the first sentence, // capped, so `say` doesn't read a whole monologue. Shouts are already short → no-op. function concise(t: string, maxWords = 13): string { const first = t.split(/(?<=[.!?])\s+/)[0]?.trim() || t.trim(); const words = first.split(/\s+/); return words.length <= maxWords ? first : `${words.slice(0, maxWords).join(" ")}…`; } /** Should this channel speak under this mode? */ export function speaks(mode: VoiceMode, channel: VoiceChannel): boolean { if (mode === "off") return false; if (mode === "whisper") return channel === "interrupt"; return true; // full } // Mood sets the baseline delivery; a reaction layers a short emotional tilt on top. const MOOD_DELIVERY: Record = { curious: { rate: 1.0, pitch: 1.05 }, mischief: { rate: 1.12, pitch: 1.22 }, sleepy: { rate: 0.84, pitch: 0.9 }, proud: { rate: 0.98, pitch: 1.0 }, grumpy: { rate: 0.95, pitch: 0.92 }, }; const REACTION_TILT: Partial> = { nani: { rate: 0.18, pitch: 0.28 }, // startled, fast and high (confusion) celebrate: { rate: 0.06, pitch: 0.16 }, perk: { rate: 0.02, pitch: 0.12 }, dance: { rate: 0.1, pitch: 0.2 }, laugh: { rate: 0.12, pitch: 0.25 }, // bright, giddy (amusement) fret: { rate: 0.16, pitch: 0.16 }, // quick, tense (the human's upset) sad: { rate: -0.22, pitch: -0.24 }, // slow, low, heavy (wistful) shrug: { rate: -0.12, pitch: -0.12 }, sulk: { rate: -0.16, pitch: -0.16 }, }; export interface SpeakOpts { mood?: Mood; mischief?: number; mode?: VoiceMode; channel?: VoiceChannel; /** Layer a reaction's emotional tilt onto the mood baseline. */ reaction?: ReactionKind; engine?: VoiceEngine; /** Voice id/name for the active engine; falls back to that engine's default. */ voiceName?: string; onStart?: () => void; onEnd?: () => void; } function browserSpeak( text: string, o: { rate: number; pitch: number; mode: VoiceMode; channel: VoiceChannel; voiceName?: string; onStart?: () => void; onEnd?: () => void; }, ): boolean { if (!("speechSynthesis" in window)) { o.onEnd?.(); return false; } try { window.speechSynthesis.cancel(); const u = new SpeechSynthesisUtterance(clean(text)); const v = voiceByName(o.voiceName); if (v) u.voice = v; u.rate = o.rate; u.pitch = o.pitch; u.volume = o.channel === "interrupt" ? (o.mode === "whisper" ? 0.5 : 0.95) : o.channel === "reaction" ? 0.7 : 0.6; u.onstart = () => o.onStart?.(); u.onend = () => o.onEnd?.(); u.onerror = () => o.onEnd?.(); window.speechSynthesis.speak(u); return true; } catch { o.onEnd?.(); return false; } } export function speakAloud( text: string, { mood = "curious", mischief = 45, mode = "whisper", channel = "bubble", reaction, engine = "browser", voiceName, onStart, onEnd, }: SpeakOpts = {}, ): boolean { if (!speaks(mode, channel)) { onEnd?.(); return false; } const base = MOOD_DELIVERY[mood]; const tilt = (reaction && REACTION_TILT[reaction]) || { rate: 0, pitch: 0 }; const rate = Math.max(0.6, Math.min(1.6, base.rate + tilt.rate)); const pitch = Math.max(0.6, Math.min(1.8, base.pitch + tilt.pitch + (mischief - 45) / 300)); // the displayed bubble keeps the full line; speech gets the trimmed form const spoken = concise(clean(text)); if (engine === "say") { // Mac-side native voice (the overlay's only working path). Fall back to // browser TTS if the daemon is down / not macOS. void sayViaDaemon(spoken, { voice: voiceName, rate, pitch, onStart, onEnd }).then((ok) => { if (!ok) browserSpeak(spoken, { rate, pitch, mode, channel, onStart, onEnd }); }); return true; } if (engine === "kokoro" && kokoroSupported()) { const speed = Math.max(0.7, Math.min(1.4, rate)); // Kokoro has no pitch knob — voice + speed carry it void speakKokoro(spoken, { voiceName, speed, onStart, onEnd }).then((ok) => { if (!ok) browserSpeak(spoken, { rate, pitch, mode, channel, onStart, onEnd }); // graceful degrade }); return true; } // browser engine, or Kokoro unsupported here (e.g. the WKWebView overlay). // Only forward voiceName for the real browser engine — a Kokoro id ("af_heart") // means nothing to the OS, so let it auto-pick. return browserSpeak(spoken, { rate, pitch, mode, channel, voiceName: engine === "browser" ? voiceName : undefined, onStart, onEnd, }); } // Browsers block programmatic audio until the user has interacted with the page. // Neural speech `await`s model-load + inference, so by the time it calls play() // the gesture is long gone → NotAllowedError. Fix: on the FIRST real interaction, // play a silent clip — that grants the page sticky activation, so every later // (event-driven, gesture-less) utterance is allowed. Idempotent. let unlocked = false; export function unlockAudio(): void { if (unlocked) return; unlocked = true; try { const a = new Audio("data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA="); a.muted = true; void a.play().catch(() => {}); } catch { /* no Audio — neural path will just fall back to speechSynthesis */ } } function browserPreview(text: string, voiceName?: string): void { if (!("speechSynthesis" in window)) return; window.speechSynthesis.cancel(); const u = new SpeechSynthesisUtterance(text); const v = voiceByName(voiceName); if (v) u.voice = v; u.pitch = 1.15; u.rate = 1.02; window.speechSynthesis.speak(u); } /** Audition a voice in Settings — bypasses the mode gate (it's an explicit tap). */ export function previewVoice(engine: VoiceEngine, voiceName?: string): void { const text = "Hello — I'm Puck. I'll keep an eye on things."; if (engine === "say") { void sayViaDaemon(text, { voice: voiceName, rate: 1.0 }).then((ok) => { if (!ok) browserPreview(text); }); return; } if (engine === "kokoro" && kokoroSupported()) { // fall back to the OS voice if the model is still downloading or errors void speakKokoro(text, { voiceName, speed: 1.05 }).then((ok) => { if (!ok) browserPreview(text); }); return; } browserPreview(text, engine === "browser" ? voiceName : undefined); } export function cancelSpeech() { stopKokoro(); stopSay(); try { window.speechSynthesis?.cancel(); } catch { /* no speech engine — nothing to cancel */ } }