web/src/voice/controller.ts

import type { ReachyMiniInstance } from "@pollen-robotics/reachy-mini-sdk";
import { RealtimeSession } from "./realtime";
import { getRobotMicTrack, getRobotSpeakerTrack, primeRemoteTrack, resumeAudio, setPhoneOutput, ttsLevel } from "./audio";
import { countVideoReceivers } from "./camera";
import type { ToolContext } from "./tools";
import type { RobotGestures } from "../robot/gestures";
import type { Lang } from "../i18n";

export type VoiceStatus = "idle" | "connecting" | "listening" | "speaking" | "error";

/** Robot-only voice loop: Reachy's mic in, Reachy's speaker out, and smooth
 *  client-side head motion while it speaks or listens. There is deliberately
 *  no phone-mic/phone-speaker mode — the interaction is with the robot. */
export class VoiceController {
  private session: RealtimeSession | null = null;
  status: VoiceStatus = "idle";

  constructor(
    private opts: {
      getReachy: () => ReachyMiniInstance | null;
      gestures: RobotGestures | null;
      getApiKey: () => string | null;
      getLang: () => Lang;
      getName: () => string;
      getActivationMode: () => "always" | "name";
      getAllowInterrupt: () => boolean;
      toolCtx: ToolContext;
      onStatus: (s: VoiceStatus) => void;
      onError: (message: string) => void;
      onTranscript: (role: "user" | "assistant", text: string) => void;
    },
  ) {}

  get active(): boolean {
    return this.session !== null;
  }

  private setStatus(s: VoiceStatus): void {
    this.status = s;
    this.opts.onStatus(s);
  }

  async toggle(): Promise<void> {
    if (this.active) {
      this.stop();
    } else {
      await this.start();
    }
  }

  async start(): Promise<void> {
    const apiKey = this.opts.getApiKey();
    if (!apiKey) {
      this.opts.onError("Add your OpenAI key in Settings to talk to Reachy.");
      return;
    }
    const reachy = this.opts.getReachy();
    if (!reachy) {
      this.opts.onError("Connect to your Reachy first.");
      return;
    }
    this.setStatus("connecting");
    await resumeAudio();

    // Robot ears: the mic track the robot streams to us over WebRTC.
    const micTrack = getRobotMicTrack(reachy);
    if (!micTrack) {
      console.warn("[voice] no live robot audio receiver track found");
      this.opts.onError("Reachy's microphone isn't available — try leaving and reconnecting.");
      this.setStatus("idle");
      return;
    }
    // Chrome: remote tracks feed silence into WebAudio unless also consumed by
    // a media element. Prime it before the mic pump reads it.
    primeRemoteTrack(micTrack);

    // Robot voice: replace the SDK's silent placeholder on the WebRTC audio
    // sender with our TTS bus (SDK 1.8 contract — it never calls getUserMedia).
    if (!this.injectSpeakerTrack(reachy)) {
      this.opts.onError("Couldn't reach Reachy's speaker — try leaving and reconnecting.");
      this.setStatus("idle");
      return;
    }
    setPhoneOutput(false); // the robot is the only voice
    console.log(`[voice] video receivers: ${countVideoReceivers(reachy)}`);

    this.opts.gestures?.cancel();
    try {
      reachy.setMicMuted(false);
      reachy.sendRaw({ type: "set_wobbling", enabled: false }); // we own motion
    } catch {
      /* older daemon */
    }
    this.startMotion();

    this.session = new RealtimeSession({
      apiKey,
      lang: this.opts.getLang(),
      robotName: this.opts.getName(),
      activationMode: this.opts.getActivationMode(),
      allowInterrupt: this.opts.getAllowInterrupt(),
      micTrack,
      toolCtx: this.opts.toolCtx,
      callbacks: {
        onError: (m) => {
          this.opts.onError(m);
          this.setStatus("error");
        },
        onUserText: (t) => this.opts.onTranscript("user", t),
        onAssistantText: (t) => this.opts.onTranscript("assistant", t),
        onSpeakingChange: (speaking) => {
          this.speaking = speaking;
          this.setStatus(speaking ? "speaking" : "listening");
        },
        onClosed: () => {
          if (this.session) this.stop();
        },
      },
    });
    this.session.start();
    this.setStatus("listening");
  }

  stop(): void {
    this.session?.close();
    this.session = null;
    this.speaking = false;
    this.stopMotion();
    const reachy = this.opts.getReachy();
    try {
      this.restoreSpeakerTrack();
      reachy?.setMicMuted(true);
    } catch {
      /* channel closed */
    }
    this.setStatus("idle");
  }

  // ── Robot-speaker audio injection (replaceTrack on the WebRTC audio sender) ──
  private audioSender: RTCRtpSender | null = null;
  private placeholderTrack: MediaStreamTrack | null = null;

  /** Returns true if our TTS track was attached to the robot's audio sender. */
  private injectSpeakerTrack(reachy: ReachyMiniInstance | null): boolean {
    const pc = reachy?._pc ?? null;
    const ttsTrack = getRobotSpeakerTrack();
    if (!pc || !ttsTrack) return false;
    const sender = pc.getSenders().find((s) => s.track && s.track.kind === "audio") ?? null;
    if (!sender) return false;
    this.audioSender = sender;
    this.placeholderTrack = sender.track; // the SDK's silent placeholder
    void sender.replaceTrack(ttsTrack);
    return true;
  }

  private restoreSpeakerTrack(): void {
    if (this.audioSender) {
      void this.audioSender.replaceTrack(this.placeholderTrack);
      this.audioSender = null;
      this.placeholderTrack = null;
    }
  }

  // ── Client-side motion (~20 Hz, reachy-stories style) ──────────────────────
  // Speaking: head wobble scaled by a smoothed speech envelope (with a floor so
  // speech is always visibly animated) + antennas eased up slowly.
  // Listening: slow attentive sway so the robot feels alive.
  // Antennas are never oscillated at audio rate — that was the buzz.
  private speaking = false;
  private motionRaf: number | null = null;
  private env = 0;
  private ant = 0;
  private lastCmd = 0;
  private lastLog = 0;

  private startMotion(): void {
    if (this.motionRaf !== null) return;
    this.env = 0;
    this.ant = 0;
    const loop = (now: number) => {
      this.motionRaf = requestAnimationFrame(loop);
      if (now - this.lastCmd < 50) return; // ~20 Hz
      this.lastCmd = now;
      const reachy = this.opts.getReachy();
      if (!reachy) return;
      const t = now / 1000;
      const level = Math.min(ttsLevel(), 1);
      // Envelope: while speaking, never below 0.35 so motion is clearly visible
      // even in quiet passages; decays to 0 when not speaking.
      const target = this.speaking ? Math.max(level, 0.35) : 0;
      this.env += (target - this.env) * 0.15;
      const e = this.env;
      // Antennas ease toward a raised, slowly swaying pose while speaking.
      const antennaTarget = this.speaking ? 18 + 7 * Math.sin(t * 1.4) : 0;
      this.ant += (antennaTarget - this.ant) * 0.08;
      let ok = true;
      try {
        if (e > 0.03) {
          ok = reachy.setHeadRpyDeg(
            e * 4 * Math.sin(t * 3.1),
            e * 9 * Math.sin(t * 2.3) + 1.5 * Math.sin(t * 0.8),
            e * 7 * Math.sin(t * 1.7),
          );
        } else {
          // Listening: slow, attentive sway.
          ok = reachy.setHeadRpyDeg(
            1.5 * Math.sin(t * 0.7),
            2.5 * Math.sin(t * 0.9),
            4 * Math.sin(t * 0.45),
          );
        }
        reachy.setAntennasDeg(this.ant, this.ant);
      } catch (err) {
        ok = false;
        if (now - this.lastLog > 3000) console.warn("[motion] send failed", err);
      }
      if (now - this.lastLog > 3000) {
        this.lastLog = now;
        console.log(
          `[motion] speaking=${this.speaking} level=${level.toFixed(2)} env=${e.toFixed(2)} ant=${this.ant.toFixed(1)} ok=${ok}`,
        );
      }
    };
    this.motionRaf = requestAnimationFrame(loop);
  }

  private stopMotion(): void {
    if (this.motionRaf !== null) {
      cancelAnimationFrame(this.motionRaf);
      this.motionRaf = null;
    }
    this.env = 0;
    this.ant = 0;
    try {
      const reachy = this.opts.getReachy();
      reachy?.setHeadRpyDeg(0, 0, 0);
      reachy?.setAntennasDeg(0, 0);
    } catch {
      /* channel closed */
    }
  }
}