import type { ReachyMiniInstance } from "@pollen-robotics/reachy-mini-sdk"; import { RealtimeSession } from "./realtime"; import { getRobotMicTrack, getRobotSpeakerTrack, primeRemoteTrack, resumeAudio, setPhoneOutput, ttsLevel } from "./audio"; import { countVideoReceivers } from "./camera"; import type { ToolContext } from "./tools"; import type { RobotGestures } from "../robot/gestures"; import type { Lang } from "../i18n"; export type VoiceStatus = "idle" | "connecting" | "listening" | "speaking" | "error"; /** Robot-only voice loop: Reachy's mic in, Reachy's speaker out, and smooth * client-side head motion while it speaks or listens. There is deliberately * no phone-mic/phone-speaker mode — the interaction is with the robot. */ export class VoiceController { private session: RealtimeSession | null = null; status: VoiceStatus = "idle"; constructor( private opts: { getReachy: () => ReachyMiniInstance | null; gestures: RobotGestures | null; getApiKey: () => string | null; getLang: () => Lang; getName: () => string; getActivationMode: () => "always" | "name"; getAllowInterrupt: () => boolean; toolCtx: ToolContext; onStatus: (s: VoiceStatus) => void; onError: (message: string) => void; onTranscript: (role: "user" | "assistant", text: string) => void; }, ) {} get active(): boolean { return this.session !== null; } private setStatus(s: VoiceStatus): void { this.status = s; this.opts.onStatus(s); } async toggle(): Promise { if (this.active) { this.stop(); } else { await this.start(); } } async start(): Promise { const apiKey = this.opts.getApiKey(); if (!apiKey) { this.opts.onError("Add your OpenAI key in Settings to talk to Reachy."); return; } const reachy = this.opts.getReachy(); if (!reachy) { this.opts.onError("Connect to your Reachy first."); return; } this.setStatus("connecting"); await resumeAudio(); // Robot ears: the mic track the robot streams to us over WebRTC. const micTrack = getRobotMicTrack(reachy); if (!micTrack) { console.warn("[voice] no live robot audio receiver track found"); this.opts.onError("Reachy's microphone isn't available — try leaving and reconnecting."); this.setStatus("idle"); return; } // Chrome: remote tracks feed silence into WebAudio unless also consumed by // a media element. Prime it before the mic pump reads it. primeRemoteTrack(micTrack); // Robot voice: replace the SDK's silent placeholder on the WebRTC audio // sender with our TTS bus (SDK 1.8 contract — it never calls getUserMedia). if (!this.injectSpeakerTrack(reachy)) { this.opts.onError("Couldn't reach Reachy's speaker — try leaving and reconnecting."); this.setStatus("idle"); return; } setPhoneOutput(false); // the robot is the only voice console.log(`[voice] video receivers: ${countVideoReceivers(reachy)}`); this.opts.gestures?.cancel(); try { reachy.setMicMuted(false); reachy.sendRaw({ type: "set_wobbling", enabled: false }); // we own motion } catch { /* older daemon */ } this.startMotion(); this.session = new RealtimeSession({ apiKey, lang: this.opts.getLang(), robotName: this.opts.getName(), activationMode: this.opts.getActivationMode(), allowInterrupt: this.opts.getAllowInterrupt(), micTrack, toolCtx: this.opts.toolCtx, callbacks: { onError: (m) => { this.opts.onError(m); this.setStatus("error"); }, onUserText: (t) => this.opts.onTranscript("user", t), onAssistantText: (t) => this.opts.onTranscript("assistant", t), onSpeakingChange: (speaking) => { this.speaking = speaking; this.setStatus(speaking ? "speaking" : "listening"); }, onClosed: () => { if (this.session) this.stop(); }, }, }); this.session.start(); this.setStatus("listening"); } stop(): void { this.session?.close(); this.session = null; this.speaking = false; this.stopMotion(); const reachy = this.opts.getReachy(); try { this.restoreSpeakerTrack(); reachy?.setMicMuted(true); } catch { /* channel closed */ } this.setStatus("idle"); } // ── Robot-speaker audio injection (replaceTrack on the WebRTC audio sender) ── private audioSender: RTCRtpSender | null = null; private placeholderTrack: MediaStreamTrack | null = null; /** Returns true if our TTS track was attached to the robot's audio sender. */ private injectSpeakerTrack(reachy: ReachyMiniInstance | null): boolean { const pc = reachy?._pc ?? null; const ttsTrack = getRobotSpeakerTrack(); if (!pc || !ttsTrack) return false; const sender = pc.getSenders().find((s) => s.track && s.track.kind === "audio") ?? null; if (!sender) return false; this.audioSender = sender; this.placeholderTrack = sender.track; // the SDK's silent placeholder void sender.replaceTrack(ttsTrack); return true; } private restoreSpeakerTrack(): void { if (this.audioSender) { void this.audioSender.replaceTrack(this.placeholderTrack); this.audioSender = null; this.placeholderTrack = null; } } // ── Client-side motion (~20 Hz, reachy-stories style) ────────────────────── // Speaking: head wobble scaled by a smoothed speech envelope (with a floor so // speech is always visibly animated) + antennas eased up slowly. // Listening: slow attentive sway so the robot feels alive. // Antennas are never oscillated at audio rate — that was the buzz. private speaking = false; private motionRaf: number | null = null; private env = 0; private ant = 0; private lastCmd = 0; private lastLog = 0; private startMotion(): void { if (this.motionRaf !== null) return; this.env = 0; this.ant = 0; const loop = (now: number) => { this.motionRaf = requestAnimationFrame(loop); if (now - this.lastCmd < 50) return; // ~20 Hz this.lastCmd = now; const reachy = this.opts.getReachy(); if (!reachy) return; const t = now / 1000; const level = Math.min(ttsLevel(), 1); // Envelope: while speaking, never below 0.35 so motion is clearly visible // even in quiet passages; decays to 0 when not speaking. const target = this.speaking ? Math.max(level, 0.35) : 0; this.env += (target - this.env) * 0.15; const e = this.env; // Antennas ease toward a raised, slowly swaying pose while speaking. const antennaTarget = this.speaking ? 18 + 7 * Math.sin(t * 1.4) : 0; this.ant += (antennaTarget - this.ant) * 0.08; let ok = true; try { if (e > 0.03) { ok = reachy.setHeadRpyDeg( e * 4 * Math.sin(t * 3.1), e * 9 * Math.sin(t * 2.3) + 1.5 * Math.sin(t * 0.8), e * 7 * Math.sin(t * 1.7), ); } else { // Listening: slow, attentive sway. ok = reachy.setHeadRpyDeg( 1.5 * Math.sin(t * 0.7), 2.5 * Math.sin(t * 0.9), 4 * Math.sin(t * 0.45), ); } reachy.setAntennasDeg(this.ant, this.ant); } catch (err) { ok = false; if (now - this.lastLog > 3000) console.warn("[motion] send failed", err); } if (now - this.lastLog > 3000) { this.lastLog = now; console.log( `[motion] speaking=${this.speaking} level=${level.toFixed(2)} env=${e.toFixed(2)} ant=${this.ant.toFixed(1)} ok=${ok}`, ); } }; this.motionRaf = requestAnimationFrame(loop); } private stopMotion(): void { if (this.motionRaf !== null) { cancelAnimationFrame(this.motionRaf); this.motionRaf = null; } this.env = 0; this.ant = 0; try { const reachy = this.opts.getReachy(); reachy?.setHeadRpyDeg(0, 0, 0); reachy?.setAntennasDeg(0, 0); } catch { /* channel closed */ } } }