feat: circle states + head wobble agent driven by AI speech

- Add a `processing` state (user stopped, reply not ready yet) alongside
listening / user-speaking / ai-speaking, with distinct colors and
animations (dashed spinning ring, warm thinking throb).
- Make the user-speaking state react to the robot mic in real time via a
smoothed RMS monitor that writes `--audio-level` into CSS.
- Ship a parallel "head wobble" agent ported from the Python
`speech_tapper` + `head_wobbler` modules: hooks a Web Audio analyser
to the assistant MediaStreamTrack, runs hysteretic VAD + sinusoidal
sway at 20Hz, and drives robot.setHeadPose with a ~200ms latency so
the motion feels slaved to the voice.
- Reset the wobbler when the user barges in so the head settles during
listening, matching the reference conversation app.

Made-with: Cursor

src/head-wobbler.ts

@@ -0,0 +1,240 @@
+/**
+ * Head wobble generator - JS port of the Python
+ * `reachy_mini_conversation_app/audio/speech_tapper.py` + `head_wobbler.py`.
+ *
+ * Purpose: take the assistant's outgoing voice (the audio OpenAI plays back
+ * through the robot) and drive small, organic head sways + nods in sync with
+ * the speech, so Reachy "comes alive" while talking.
+ *
+ * The Python version sampled raw PCM base64 deltas over the OpenAI WebSocket
+ * API; we instead hook a Web Audio `AnalyserNode` to the remote
+ * `MediaStreamTrack` (which is how audio reaches us in WebRTC mode). Both
+ * yield the same loudness envelope we need to modulate the sine oscillators.
+ *
+ * The SDK only exposes rotational `setHeadPose(roll, pitch, yaw)` in degrees,
+ * so we drop the small (±mm-range) translations from the original Python
+ * pipeline - they're a cherry-on-top that isn't reachable through the public
+ * JS API anyway.
+ */
+
+// ─── Constants ported from speech_tapper.py ─────────────────────────────
+// Frame / hop timing. Hop drives how often we update offsets.
+const HOP_MS = 50; // produce one pose target every 50ms (20Hz)
+const FRAME_MS = 20; // RMS window in ms
+
+// Loudness mapping: RMS in dBFS is remapped to [0..1] with a slight gamma.
+const SWAY_DB_LOW = -46;
+const SWAY_DB_HIGH = -18;
+const SWAY_DB_GAMMA = 0.9;
+const SWAY_MASTER = 1.5;
+const SENS_DB_OFFSET = 4;
+
+// Hysteretic VAD thresholds on the sliding-frame dBFS envelope.
+const VAD_DB_ON = -35;
+const VAD_DB_OFF = -45;
+// Attack / release for the 0→1 envelope follower that multiplies the sway.
+const ENV_FOLLOW_GAIN = 0.65;
+
+// Sinusoid frequencies (Hz) + peak amplitudes (degrees).
+// All three axes use independent initial phases so the motion doesn't feel
+// mechanical when they align.
+const SWAY_F_PITCH = 2.2;
+const SWAY_A_PITCH_DEG = 4.5;
+
+const SWAY_F_YAW = 0.6;
+const SWAY_A_YAW_DEG = 7.5;
+
+const SWAY_F_ROLL = 1.3;
+const SWAY_A_ROLL_DEG = 2.25;
+
+// Deliberate delay between the audio we analyse and the matching head
+// motion. A touch of latency makes the motion feel slaved to the voice
+// (reactive, not predictive). Matches `MOVEMENT_LATENCY_S = 0.2` in Python.
+const MOVEMENT_LATENCY_S = 0.2;
+
+// ─── Types ──────────────────────────────────────────────────────────────
+
+export interface HeadOffsetsDeg {
+  roll: number;
+  pitch: number;
+  yaw: number;
+}
+
+export interface HeadWobblerOptions {
+  track: MediaStreamTrack;
+  onOffsets: (offsets: HeadOffsetsDeg) => void;
+  /**
+   * Master scalar applied on top of the per-axis amplitudes. Lets callers
+   * tone the whole effect down without recompiling constants. Defaults to 1.
+   */
+  gain?: number;
+}
+
+// ─── Implementation ─────────────────────────────────────────────────────
+
+/**
+ * Consumes a `MediaStreamTrack` (assistant voice) and emits head rotation
+ * offsets in degrees every hop (~50 ms), smoothly ramping in and out with
+ * the speech envelope.
+ *
+ * Call `start()` to begin, `stop()` to tear down, `reset()` to zero the
+ * internal sway state (useful when the user barges in on the assistant).
+ */
+export class HeadWobbler {
+  private readonly track: MediaStreamTrack;
+  private readonly onOffsets: (offsets: HeadOffsetsDeg) => void;
+  private readonly gain: number;
+
+  private audioCtx: AudioContext | null = null;
+  private analyser: AnalyserNode | null = null;
+  private sourceNode: MediaStreamAudioSourceNode | null = null;
+  private frameBuf: Float32Array<ArrayBuffer> | null = null;
+
+  private timer: number | null = null;
+
+  // Sway state
+  private t = 0; // seconds since start() for the sine oscillators
+  private swayEnv = 0; // 0..1 envelope follower on VAD-on/off
+  private vadOn = false;
+  private pendingOffsets: HeadOffsetsDeg[] = [];
+
+  // Per-axis phase offsets so the three sines don't line up on the downbeat.
+  private readonly phasePitch = 0.13 * Math.PI;
+  private readonly phaseYaw = 0.48 * Math.PI;
+  private readonly phaseRoll = 1.21 * Math.PI;
+
+  constructor(options: HeadWobblerOptions) {
+    this.track = options.track;
+    this.onOffsets = options.onOffsets;
+    this.gain = options.gain ?? 1.0;
+  }
+
+  start(): void {
+    if (this.audioCtx) return; // already running
+
+    // A dedicated AudioContext keeps the robot mic path untouched. We don't
+    // need the browser to actually play this back (the audio is also piped
+    // straight to the robot speaker via replaceTrack), we just want samples.
+    const ctx = new AudioContext();
+    const sourceStream = new MediaStream([this.track]);
+    const source = ctx.createMediaStreamSource(sourceStream);
+    const analyser = ctx.createAnalyser();
+    analyser.fftSize = 1024;
+    analyser.smoothingTimeConstant = 0;
+    source.connect(analyser);
+
+    const frameSamples = Math.max(
+      128,
+      Math.round((ctx.sampleRate * FRAME_MS) / 1000),
+    );
+    this.frameBuf = new Float32Array(new ArrayBuffer(frameSamples * 4));
+
+    this.audioCtx = ctx;
+    this.sourceNode = source;
+    this.analyser = analyser;
+
+    // Kick the tick loop on setInterval so the rate stays aligned with
+    // hop_ms regardless of requestAnimationFrame throttling when the tab
+    // is backgrounded. 20 Hz is low enough not to spam the robot.
+    this.timer = window.setInterval(() => this.tick(), HOP_MS);
+  }
+
+  stop(): void {
+    if (this.timer !== null) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    try {
+      this.sourceNode?.disconnect();
+      this.analyser?.disconnect();
+      this.audioCtx?.close();
+    } catch {
+      // ignored
+    }
+    this.sourceNode = null;
+    this.analyser = null;
+    this.audioCtx = null;
+    this.frameBuf = null;
+    this.pendingOffsets = [];
+
+    // Return to neutral head pose on teardown so the robot doesn't freeze
+    // mid-motion.
+    this.onOffsets({ roll: 0, pitch: 0, yaw: 0 });
+  }
+
+  /**
+   * Reset the sway envelope + latency queue. Call this when the user starts
+   * speaking while the assistant is still talking: we want motion to fade
+   * out smoothly instead of holding the last sine value.
+   */
+  reset(): void {
+    this.swayEnv = 0;
+    this.vadOn = false;
+    this.pendingOffsets = [];
+    this.onOffsets({ roll: 0, pitch: 0, yaw: 0 });
+  }
+
+  // ─── Internals ───────────────────────────────────────────────────────
+
+  private tick(): void {
+    if (!this.analyser || !this.frameBuf) return;
+
+    // 1. Measure current loudness in dBFS on the most recent ~20 ms.
+    this.analyser.getFloatTimeDomainData(this.frameBuf);
+    const db = rmsDbfs(this.frameBuf) + SENS_DB_OFFSET;
+
+    // 2. Hysteretic VAD on the dB envelope.
+    if (this.vadOn) {
+      if (db < VAD_DB_OFF) this.vadOn = false;
+    } else {
+      if (db > VAD_DB_ON) this.vadOn = true;
+    }
+
+    // 3. Smooth 0..1 envelope the sines are modulated by.
+    const target = this.vadOn ? 1 : 0;
+    this.swayEnv += (target - this.swayEnv) * ENV_FOLLOW_GAIN;
+
+    // 4. Loudness gain - how much of the max amplitude we reach on this hop.
+    const loud = loudnessGain(db);
+
+    // 5. Time-evolve the oscillators.
+    this.t += HOP_MS / 1000;
+    const twoPiT = 2 * Math.PI * this.t;
+    const mod = loud * this.swayEnv * this.gain * SWAY_MASTER;
+
+    const pitchDeg =
+      SWAY_A_PITCH_DEG * mod * Math.sin(twoPiT * SWAY_F_PITCH + this.phasePitch);
+    const yawDeg =
+      SWAY_A_YAW_DEG * mod * Math.sin(twoPiT * SWAY_F_YAW + this.phaseYaw);
+    const rollDeg =
+      SWAY_A_ROLL_DEG * mod * Math.sin(twoPiT * SWAY_F_ROLL + this.phaseRoll);
+
+    // 6. Enqueue with latency, then emit the oldest ready sample.
+    this.pendingOffsets.push({ roll: rollDeg, pitch: pitchDeg, yaw: yawDeg });
+    const maxQueue = Math.ceil((MOVEMENT_LATENCY_S * 1000) / HOP_MS);
+    while (this.pendingOffsets.length > maxQueue) {
+      const next = this.pendingOffsets.shift();
+      if (next) this.onOffsets(next);
+    }
+  }
+}
+
+// ─── Pure helpers ───────────────────────────────────────────────────────
+
+function rmsDbfs(samples: Float32Array<ArrayBuffer>): number {
+  let sum = 0;
+  for (let i = 0; i < samples.length; i++) {
+    const s = samples[i];
+    sum += s * s;
+  }
+  const rms = Math.sqrt(sum / Math.max(1, samples.length));
+  if (rms <= 1e-8) return -120;
+  return 20 * Math.log10(rms);
+}
+
+function loudnessGain(db: number): number {
+  // Linear remap [SWAY_DB_LOW..SWAY_DB_HIGH] -> [0..1] with a gamma.
+  const norm = (db - SWAY_DB_LOW) / (SWAY_DB_HIGH - SWAY_DB_LOW);
+  const clamped = Math.max(0, Math.min(1, norm));
+  return Math.pow(clamped, SWAY_DB_GAMMA);
+}

src/main.ts

@@ -21,6 +21,7 @@
 import "./style.css";
 
 import { OpenaiRealtimeClient } from "./openai-realtime.ts";
+import { HeadWobbler } from "./head-wobbler.ts";
 import type {
   ReachyMiniInstance,
   RobotInfo,
@@ -79,6 +80,7 @@ type AppState =
   | "starting"
   | "listening"
   | "user-speaking"
+  | "processing"
   | "ai-speaking"
   | "error";
 
@@ -129,6 +131,11 @@ const STATE_VIEWS: Record<AppState, StateView> = {
     sublabel: "Go on, I'm hearing you",
     disabled: false,
   },
+  processing: {
+    label: "Thinking…",
+    sublabel: "Reachy is coming up with a reply",
+    disabled: false,
+  },
   "ai-speaking": {
     label: "Speaking…",
     sublabel: "Reachy is answering",
@@ -151,6 +158,7 @@ const STATE_CLASS: Record<AppState, string> = {
   starting: "state-starting",
   listening: "state-listening",
   "user-speaking": "state-user-speaking",
+  processing: "state-processing",
   "ai-speaking": "state-ai-speaking",
   error: "state-error",
 };
@@ -194,6 +202,14 @@ let openai: OpenaiRealtimeClient | null = null;
 // keeps the MediaStream alive and actually decodes the incoming track).
 let openaiSink: HTMLAudioElement | null = null;
 
+// Head-motion agent: samples the assistant's voice and drives small head
+// sways via robot.setHeadPose. Lives for the duration of a session.
+let wobbler: HeadWobbler | null = null;
+
+// Mic level monitor: feeds a CSS custom property `--audio-level` in [0,1]
+// so the circle breathes/glows in reaction to the user's voice in real time.
+let micLevel: MicLevelMonitor | null = null;
+
 // ─── UI rendering ───────────────────────────────────────────────────────
 
 function setState(next: AppState): void {
@@ -385,6 +401,8 @@ async function doStart(): Promise<void> {
     return;
   }
 
+  startMicLevelMonitor(robotMicTrack);
+
   setStatus("Connecting to OpenAI Realtime…");
 
   openai = new OpenaiRealtimeClient({
@@ -395,7 +413,10 @@ async function doStart(): Promise<void> {
     inputTrack: robotMicTrack,
   });
 
-  openai.on("outputTrack", ({ track }) => routeOpenaiToRobot(track));
+  openai.on("outputTrack", ({ track }) => {
+    routeOpenaiToRobot(track);
+    startWobbler(track);
+  });
 
   openai.on("status", ({ status }) => {
     switch (status) {
@@ -405,6 +426,13 @@ async function doStart(): Promise<void> {
         break;
       case "user-speaking":
         setState("user-speaking");
+        // User barges in → stop the assistant's sway immediately so the head
+        // settles while we listen, mirroring the reference app's
+        // head_wobbler.reset() on speech_started.
+        wobbler?.reset();
+        break;
+      case "processing":
+        setState("processing");
         break;
       case "ai-speaking":
         setState("ai-speaking");
@@ -480,6 +508,115 @@ function routeOpenaiToRobot(track: MediaStreamTrack): void {
   openaiSink.srcObject = new MediaStream([track]);
 }
 
+// ─── Head motion agent ──────────────────────────────────────────────────
+
+/**
+ * Spawn the head-motion agent from the assistant audio track. Each new
+ * session replaces the previous instance; no-op if the wobbler is already
+ * wired to this exact track.
+ */
+function startWobbler(assistantTrack: MediaStreamTrack): void {
+  if (!robot) return;
+
+  wobbler?.stop();
+  wobbler = new HeadWobbler({
+    track: assistantTrack,
+    onOffsets: ({ roll, pitch, yaw }) => {
+      // The SDK's setHeadPose expects degrees. Our offsets are already in
+      // degrees; we push them as absolute target poses around the neutral
+      // head position (no base pose is preserved, which keeps the motion
+      // unambiguously around "looking forward").
+      robot?.setHeadPose(roll, pitch, yaw);
+    },
+  });
+  wobbler.start();
+}
+
+function stopWobbler(): void {
+  wobbler?.stop();
+  wobbler = null;
+  // Ensure the head returns to neutral when the session ends.
+  robot?.setHeadPose(0, 0, 0);
+}
+
+// ─── Mic-level monitor (circle audio-reactivity) ────────────────────────
+
+/**
+ * Sample the robot's microphone to a CSS custom property `--audio-level` so
+ * the circle can breathe/glow in sync with the user's voice. The value is a
+ * smoothed normalized RMS in [0, 1].
+ */
+class MicLevelMonitor {
+  private ctx: AudioContext | null = null;
+  private analyser: AnalyserNode | null = null;
+  private source: MediaStreamAudioSourceNode | null = null;
+  private raf = 0;
+  private buf: Float32Array<ArrayBuffer> | null = null;
+  private level = 0;
+
+  start(track: MediaStreamTrack): void {
+    this.stop();
+    const ctx = new AudioContext();
+    const src = ctx.createMediaStreamSource(new MediaStream([track]));
+    const analyser = ctx.createAnalyser();
+    analyser.fftSize = 1024;
+    analyser.smoothingTimeConstant = 0.7;
+    src.connect(analyser);
+
+    this.ctx = ctx;
+    this.source = src;
+    this.analyser = analyser;
+    this.buf = new Float32Array(new ArrayBuffer(analyser.fftSize * 4));
+
+    const tick = () => {
+      if (!this.analyser || !this.buf) return;
+      this.analyser.getFloatTimeDomainData(this.buf);
+      let sum = 0;
+      for (let i = 0; i < this.buf.length; i++) sum += this.buf[i] * this.buf[i];
+      const rms = Math.sqrt(sum / this.buf.length);
+
+      // Map raw RMS (~0..0.2 for typical speech) to 0..1 with mild boost.
+      const boosted = Math.min(1, Math.pow(rms * 6, 0.7));
+      // Attack/release smoothing so the CSS animation stays fluid.
+      const attack = boosted > this.level ? 0.55 : 0.12;
+      this.level += (boosted - this.level) * attack;
+
+      document.documentElement.style.setProperty(
+        "--audio-level",
+        this.level.toFixed(3),
+      );
+      this.raf = requestAnimationFrame(tick);
+    };
+    this.raf = requestAnimationFrame(tick);
+  }
+
+  stop(): void {
+    cancelAnimationFrame(this.raf);
+    this.raf = 0;
+    try {
+      this.source?.disconnect();
+      this.analyser?.disconnect();
+      this.ctx?.close();
+    } catch {
+      // ignored
+    }
+    this.ctx = null;
+    this.source = null;
+    this.analyser = null;
+    this.buf = null;
+    document.documentElement.style.setProperty("--audio-level", "0");
+  }
+}
+
+function startMicLevelMonitor(track: MediaStreamTrack): void {
+  micLevel ??= new MicLevelMonitor();
+  micLevel.start(track);
+}
+
+function stopMicLevelMonitor(): void {
+  micLevel?.stop();
+}
+
 async function teardown(): Promise<void> {
   try {
     await openai?.close();
@@ -488,6 +625,9 @@ async function teardown(): Promise<void> {
   }
   openai = null;
 
+  stopWobbler();
+  stopMicLevelMonitor();
+
   if (openaiSink) {
     openaiSink.srcObject = null;
     openaiSink.remove();

src/openai-realtime.ts

@@ -25,6 +25,7 @@ export type RealtimeStatus =
   | "connecting"
   | "connected"
   | "user-speaking"
+  | "processing"
   | "ai-speaking"
   | "closed"
   | "error";
@@ -187,19 +188,34 @@ export class OpenaiRealtimeClient {
         break;
 
       case "input_audio_buffer.speech_stopped":
+        // User stopped talking; the model is computing its reply and will
+        // start audio shortly. We surface this as `processing` so the UI
+        // can show a "thinking" visual until audio actually arrives.
         if (this.status === "user-speaking") {
-          this.setStatus("connected");
+          this.setStatus("processing");
+        }
+        break;
+
+      case "response.audio.delta":
+      case "response.output_audio.delta":
+        // First audio chunk from the assistant — switch out of processing.
+        if (this.status !== "ai-speaking") {
+          this.setStatus("ai-speaking");
         }
         break;
 
       case "response.created":
       case "response.output_item.added":
-        this.setStatus("ai-speaking");
+        // No audio yet, just metadata; keep us in `processing` until the
+        // first audio delta lands.
+        if (this.status === "connected" || this.status === "user-speaking") {
+          this.setStatus("processing");
+        }
         break;
 
       case "response.done":
       case "response.cancelled":
-        if (this.status === "ai-speaking") {
+        if (this.status === "ai-speaking" || this.status === "processing") {
           this.setStatus("connected");
         }
         break;

src/style.css

@@ -12,9 +12,14 @@
   --accent-2: #22d3ee;
   --listening: #22d3ee;
   --speaking: #8b7dff;
+  --processing: #f59e0b;
   --error: #ff6a75;
   --success: #34d399;
 
+  /* Smoothed mic RMS in [0..1], updated every frame from JS while a session
+   * is active. Used by audio-reactive circle states. */
+  --audio-level: 0;
+
   --radius-sm: 8px;
   --radius-md: 14px;
   --radius-lg: 22px;
@@ -217,7 +222,9 @@ a:hover {
   text-align: center;
 }
 
-/* Pulse animation: used in "streaming" states */
+/* ─── Circle animation keyframes ──────────────────────────────────────── */
+
+/* Generic medium pulse on the outer glow. */
 @keyframes pulse {
   0%, 100% {
     transform: scale(1);
@@ -229,6 +236,7 @@ a:hover {
   }
 }
 
+/* Faster / stronger pulse used while speaking. */
 @keyframes pulse-fast {
   0%, 100% {
     transform: scale(1);
@@ -240,6 +248,7 @@ a:hover {
   }
 }
 
+/* Slow, subtle breathing for "warm idle" states. */
 @keyframes breathe {
   0%, 100% {
     transform: scale(1);
@@ -249,6 +258,71 @@ a:hover {
   }
 }
 
+/* Outer ring expanding and fading - conveys "I am producing audio". */
+@keyframes ring-pulse-outer {
+  0% {
+    transform: scale(0.95);
+    opacity: 0.5;
+  }
+  100% {
+    transform: scale(1.25);
+    opacity: 0;
+  }
+}
+
+/* Soft inner scale for the core while talking. */
+@keyframes core-breathe {
+  0%, 100% {
+    transform: scale(1);
+  }
+  50% {
+    transform: scale(1.03);
+  }
+}
+
+/* Opacity throb used for the "thinking" state - dimmer and more pensive
+ * than the speaking pulse. */
+@keyframes thinking {
+  0%, 100% {
+    opacity: 0.45;
+    transform: scale(0.98);
+  }
+  50% {
+    opacity: 0.8;
+    transform: scale(1.02);
+  }
+}
+
+/* Slow spin for the processing ring accent. */
+@keyframes ring-spin {
+  from { transform: rotate(0deg); }
+  to { transform: rotate(360deg); }
+}
+
+/* ─── Ring decorations (sit absolutely inside .circle) ────────────────── */
+
+.circle::before,
+.circle::after {
+  content: "";
+  position: absolute;
+  inset: 0;
+  border-radius: 50%;
+  pointer-events: none;
+}
+
+/* Inner thin ring - shows up in speaking states, hidden by default. */
+.circle::before {
+  border: 1px solid var(--glow, var(--accent));
+  opacity: 0;
+  transition: opacity 0.25s;
+}
+
+/* Outer expanding ring - hidden by default, animated in speaking. */
+.circle::after {
+  border: 2px solid var(--glow, var(--accent));
+  opacity: 0;
+}
+
 /* ─── State-specific colors ──────────────────────────────────────────── */
 
 .circle.state-signed-out {
@@ -300,13 +374,40 @@ a:hover {
   opacity: 0.7;
 }
 
+/* User is speaking — scale + opacity driven in real time by the mic RMS
+ * via `--audio-level`. The CSS doesn't need a keyframe: JS smoothly
+ * interpolates the value, and `transition` handles the visual smoothing. */
 .circle.state-user-speaking {
   --glow: var(--listening);
   --core-from: #0e2a36;
   --core-to: #091a22;
 }
 .circle.state-user-speaking .circle-glow {
-  animation: pulse-fast 0.9s ease-in-out infinite;
+  animation: none;
+  opacity: calc(0.55 + 0.45 * var(--audio-level));
+  transform: scale(calc(1 + 0.18 * var(--audio-level)));
+  transition: transform 0.08s linear, opacity 0.08s linear;
+}
+.circle.state-user-speaking::before {
+  opacity: calc(0.25 + 0.5 * var(--audio-level));
+  transform: scale(calc(1.02 + 0.08 * var(--audio-level)));
+  transition: transform 0.08s linear, opacity 0.08s linear;
+}
+
+/* Assistant is computing its answer: warm dim pulse + slow spinning accent.
+ * No audio yet, so no reactive modulation. */
+.circle.state-processing {
+  --glow: var(--processing);
+  --core-from: #3a2f12;
+  --core-to: #1e1a0c;
+}
+.circle.state-processing .circle-glow {
+  animation: thinking 1.6s ease-in-out infinite;
+}
+.circle.state-processing::before {
+  opacity: 0.4;
+  border-style: dashed;
+  animation: ring-spin 6s linear infinite;
 }
 
 .circle.state-ai-speaking {
@@ -317,6 +418,15 @@ a:hover {
 .circle.state-ai-speaking .circle-glow {
   animation: pulse-fast 1.05s ease-in-out infinite;
 }
+.circle.state-ai-speaking .circle-core {
+  animation: core-breathe 1.6s ease-in-out infinite;
+}
+.circle.state-ai-speaking::before {
+  opacity: 0.5;
+}
+.circle.state-ai-speaking::after {
+  animation: ring-pulse-outer 1.6s ease-out infinite;
+}
 
 .circle.state-error {
   --glow: var(--error);