Add PAD-based emotion system with muted speech output

README.md

@@ -1,8 +1,8 @@
 ---
 title: Feeling Machine
-emoji: 🤖
+emoji: 🎭
 colorFrom: purple
-colorTo: gray
+colorTo: blue
 sdk: static
 pinned: false
 tags:
@@ -12,16 +12,33 @@ tags:
 
 # Feeling Machine
 
-Forked from the Reachy Mini conversation app.
+An expressive robot app that responds to users through **emotions only** - no speech output.
 
-Use the `src/reachy_mini_conversation_app/profiles/_feeling_machine_locked_profile` folder to customize your own app from this template:
-- Edit instructions `_feeling_machine_locked_profile/instructions.txt`
-- Edit available tools in `_feeling_machine_locked_profile/tools.txt`
-- You can create your own tools in `_feeling_machine_locked_profile` by subclassing the `Tool` class.
+The robot listens to what you say and responds with expressive motion and procedurally-generated sounds based on the **PAD model** (Pleasure, Arousal, Dominance).
 
-Do not forget to customize:
-- this `README.md` file
-- the `index.html` file (Hugging Face Spaces landing page)
-- the `src/reachy_mini_conversation_app/static/index.html` (the web app parameters page)
+## Features
 
-The original README from the conversation app is available in `README_OLD.md`.
+- **PAD-based emotion generation**: Motion and sound are generated procedurally from emotional parameters
+- **Named emotions**: joy, happiness, anger, fear, sadness, surprise, boredom, uncertainty, disgust, neutral
+- **Custom PAD values**: The LLM can specify precise P/A/D values for nuanced expressions
+- **No speech**: The robot expresses itself only through body language and generated sounds
+
+## PAD Model
+
+The PAD model represents emotions in a 3D space:
+- **P**leasure: Positive (+1) vs negative (-1) emotion
+- **A**rousal: Excited/energetic (+1) vs calm/relaxed (-1)
+- **D**ominance: In control (+1) vs submissive (-1)
+
+## Usage
+
+```bash
+cd src/feeling_machine
+python main.py --gradio
+```
+
+Then open http://127.0.0.1:7861/
+
+## Classic Emotions
+
+If you specifically ask for "classic" emotions, the robot can play pre-recorded emotions from the Pollen library instead of PAD-generated ones.

src/feeling_machine/dance_emotion_moves.py

@@ -6,7 +6,7 @@ and executed sequentially by the MovementManager.
 
 from __future__ import annotations
 import logging
-from typing import Tuple
+from typing import List, Optional, Tuple
 
 import numpy as np
 from numpy.typing import NDArray
@@ -19,6 +19,149 @@ from reachy_mini_dances_library.dance_move import DanceMove
 logger = logging.getLogger(__name__)
 
 
+class PADEmotionMove(Move):  # type: ignore
+    """PAD-based procedural emotion move.
+
+    Generates a sequence of poses based on PAD (Pleasure, Arousal, Dominance)
+    values. Each segment smoothly interpolates to a new generated pose.
+    """
+
+    def __init__(
+        self,
+        P: float,
+        A: float,
+        D: float,
+        total_duration: float = 3.0,
+        audio_callback: Optional[callable] = None,
+    ):
+        """Initialize a PAD emotion move.
+
+        Args:
+            P: Pleasure [0, 1]
+            A: Arousal [0, 1]
+            D: Dominance [0, 1]
+            total_duration: Total duration of the emotion in seconds
+            audio_callback: Optional callback to push audio (called at init)
+        """
+        from reachy_mini.utils import create_head_pose
+        from feeling_machine.pad_motion import generate_pose, generate_sound
+
+        self.P = P
+        self.A = A
+        self.D = D
+        self._total_duration = total_duration
+
+        # Generate sequence of poses to fill the duration
+        self._poses: List[dict] = []
+        self._keyframes: List[float] = [0.0]  # timestamps of each pose start
+
+        accumulated_time = 0.0
+        while accumulated_time < total_duration:
+            pose = generate_pose(P, A, D)
+            # Cap pose duration to not exceed total
+            pose_duration = min(pose["duration"], total_duration - accumulated_time)
+            pose["duration"] = pose_duration
+            self._poses.append(pose)
+            accumulated_time += pose_duration
+            self._keyframes.append(accumulated_time)
+
+        # Create head pose matrices for interpolation
+        self._head_poses: List[NDArray[np.float32]] = []
+        for pose in self._poses:
+            head_pose = create_head_pose(
+                x=pose["x"] / 1000.0,  # mm to meters
+                y=pose["y"] / 1000.0,
+                z=pose["z"] / 1000.0,
+                roll=pose["roll"],
+                pitch=pose["pitch"],
+                yaw=pose["yaw"],
+                mm=False,
+                degrees=True,
+            )
+            self._head_poses.append(head_pose)
+
+        # Generate and push audio if callback provided
+        if audio_callback is not None:
+            try:
+                audio = generate_sound(P, A, D, total_duration)
+                audio_callback(audio)
+            except Exception as e:
+                logger.warning(f"Failed to generate/push audio: {e}")
+
+    @property
+    def duration(self) -> float:
+        """Duration property required by official Move interface."""
+        return self._total_duration
+
+    def _find_segment(self, t: float) -> Tuple[int, float]:
+        """Find which segment t falls into and the local progress.
+
+        Returns:
+            (segment_index, local_t) where local_t is in [0, 1]
+        """
+        for i in range(len(self._poses)):
+            if t < self._keyframes[i + 1]:
+                segment_start = self._keyframes[i]
+                segment_end = self._keyframes[i + 1]
+                segment_duration = segment_end - segment_start
+                if segment_duration > 0:
+                    local_t = (t - segment_start) / segment_duration
+                else:
+                    local_t = 1.0
+                return i, local_t
+
+        # Past the end, return last segment at t=1
+        return len(self._poses) - 1, 1.0
+
+    def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]:
+        """Evaluate PAD emotion move at time t."""
+        try:
+            from reachy_mini.utils import create_head_pose
+            from reachy_mini.utils.interpolation import linear_pose_interpolation
+
+            # Clamp t to valid range
+            t = max(0.0, min(t, self._total_duration))
+
+            seg_idx, local_t = self._find_segment(t)
+            pose = self._poses[seg_idx]
+            target_head = self._head_poses[seg_idx]
+
+            # Get start pose (previous segment's target or neutral)
+            if seg_idx > 0:
+                start_head = self._head_poses[seg_idx - 1]
+                start_antennas = self._poses[seg_idx - 1]["antennas"]
+                start_body_yaw = self._poses[seg_idx - 1]["body_yaw"]
+            else:
+                start_head = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
+                start_antennas = [0.0, 0.0]
+                start_body_yaw = 0.0
+
+            # Apply minjerk easing
+            local_t_eased = 3 * local_t**2 - 2 * local_t**3
+
+            # Interpolate head pose
+            head_pose = linear_pose_interpolation(start_head, target_head, local_t_eased)
+
+            # Interpolate antennas
+            target_antennas = pose["antennas"]
+            antennas = np.array([
+                start_antennas[0] + (target_antennas[0] - start_antennas[0]) * local_t_eased,
+                start_antennas[1] + (target_antennas[1] - start_antennas[1]) * local_t_eased,
+            ], dtype=np.float64)
+
+            # Interpolate body yaw
+            target_body_yaw = pose["body_yaw"]
+            body_yaw = start_body_yaw + (target_body_yaw - start_body_yaw) * local_t_eased
+
+            return (head_pose, antennas, np.deg2rad(body_yaw))
+
+        except Exception as e:
+            logger.error(f"Error evaluating PAD emotion at t={t}: {e}")
+            from reachy_mini.utils import create_head_pose
+            neutral = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
+            return (neutral, np.array([0.0, 0.0], dtype=np.float64), 0.0)
+
+
 class DanceQueueMove(Move):  # type: ignore
     """Wrapper for dance moves to work with the movement queue system."""
 
@@ -66,6 +209,11 @@ class EmotionQueueMove(Move):  # type: ignore
         """Duration property required by official Move interface."""
         return float(self.emotion_move.duration)
 
+    @property
+    def sound_path(self):
+        """Get the sound path from the underlying recorded move."""
+        return self.emotion_move.sound_path
+
     def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]:
         """Evaluate emotion move at time t."""
         try:

src/feeling_machine/moves.py

@@ -489,6 +489,15 @@ class MovementManager:
                 self._breathing_active = isinstance(self.state.current_move, BreathingMove)
                 logger.debug(f"Starting new move, duration: {self.state.current_move.duration}s")
 
+                # Play sound if the move has an associated sound file (classic emotions)
+                sound_path = getattr(self.state.current_move, "sound_path", None)
+                if sound_path is not None:
+                    try:
+                        self.current_robot.media.play_sound(str(sound_path))
+                        logger.debug(f"Playing sound: {sound_path}")
+                    except Exception as e:
+                        logger.warning(f"Failed to play move sound: {e}")
+
     def _manage_breathing(self, current_time: float) -> None:
         """Manage automatic breathing when idle."""
         if (

src/feeling_machine/openai_realtime.py

@@ -17,7 +17,7 @@ from scipy.signal import resample
 from websockets.exceptions import ConnectionClosedError
 
 from feeling_machine.config import config
-from feeling_machine.prompts import get_session_voice, get_session_instructions
+from feeling_machine.prompts import get_session_voice, get_session_instructions, get_session_modalities
 from feeling_machine.tools.core_tools import (
     ToolDependencies,
     get_tool_specs,
@@ -73,6 +73,10 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
         self._shutdown_requested: bool = False
         self._connected_event: asyncio.Event = asyncio.Event()
 
+        # Audio output control (disabled for text-only profiles)
+        modalities = get_session_modalities()
+        self._audio_output_enabled: bool = "audio" in modalities
+
     def copy(self) -> "OpenaiRealtimeHandler":
         """Create a copy of the handler."""
         return OpenaiRealtimeHandler(self.deps, self.gradio_mode, self.instance_path)
@@ -352,6 +356,9 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
 
                 # Handle audio delta
                 if event.type in ("response.audio.delta", "response.output_audio.delta"):
+                    # Skip audio output if disabled (text-only mode)
+                    if not self._audio_output_enabled:
+                        continue
                     if self.deps.head_wobbler is not None:
                         self.deps.head_wobbler.feed(event.delta)
                     self.last_activity_time = asyncio.get_event_loop().time()

src/feeling_machine/pad_motion/__init__.py

@@ -0,0 +1,19 @@
+"""PAD-based emotion generation for Reachy Mini.
+
+This module generates expressive robot motion and sound based on the
+PAD (Pleasure, Arousal, Dominance) emotional model.
+
+Based on work by Anaelle Jaffre (I3R student project).
+"""
+
+from .pose_generation import PoseGenerator, generate_pose
+from .sound_generation import generate_sound
+from .pad_emotions import PAD_EMOTIONS, get_pad_values
+
+__all__ = [
+    "PoseGenerator",
+    "generate_pose",
+    "generate_sound",
+    "PAD_EMOTIONS",
+    "get_pad_values",
+]

src/feeling_machine/pad_motion/pad_emotions.py

@@ -0,0 +1,73 @@
+"""PAD emotion mappings.
+
+Maps named emotions to PAD (Pleasure, Arousal, Dominance) values.
+Values are in range [-1, 1] and get normalized to [0, 1] for generation.
+"""
+
+from typing import Tuple, Optional
+
+# Emotion to PAD mapping (values in [-1, 1] range)
+PAD_EMOTIONS = {
+    "joy": {"P": 1.0, "A": 0.6, "D": 0.4},
+    "happiness": {"P": 1.0, "A": 0.0, "D": 0.4},
+    "anger": {"P": -0.6, "A": 0.6, "D": 0.8},
+    "fear": {"P": -0.8, "A": 0.8, "D": -0.6},
+    "sadness": {"P": -1.0, "A": 0.0, "D": 0.0},
+    "surprise": {"P": 0.2, "A": 1.0, "D": 0.0},
+    "boredom": {"P": -0.6, "A": -0.8, "D": -0.2},
+    "uncertainty": {"P": -0.4, "A": -0.2, "D": -0.6},
+    "disgust": {"P": -0.8, "A": -0.4, "D": 0.2},
+    "neutral": {"P": 0.0, "A": 0.0, "D": 0.0},
+}
+
+
+def normalize_pad(value: float) -> float:
+    """Map PAD value from [-1, 1] to [0, 1] range."""
+    return max(0.0, min(1.0, (value + 1.0) * 0.5))
+
+
+def get_pad_values(
+    emotion: Optional[str] = None,
+    pleasure: Optional[float] = None,
+    arousal: Optional[float] = None,
+    dominance: Optional[float] = None,
+) -> Tuple[float, float, float]:
+    """Get normalized PAD values from emotion name or direct values.
+
+    Args:
+        emotion: Named emotion (joy, anger, etc.)
+        pleasure: Direct P value in [-1, 1]
+        arousal: Direct A value in [-1, 1]
+        dominance: Direct D value in [-1, 1]
+
+    Returns:
+        Tuple of (P, A, D) values normalized to [0, 1]
+
+    Raises:
+        ValueError: If neither emotion nor all PAD values provided
+    """
+    if emotion is not None:
+        emotion_lower = emotion.lower()
+        if emotion_lower not in PAD_EMOTIONS:
+            raise ValueError(
+                f"Unknown emotion '{emotion}'. "
+                f"Available: {list(PAD_EMOTIONS.keys())}"
+            )
+        pad = PAD_EMOTIONS[emotion_lower]
+        return (
+            normalize_pad(pad["P"]),
+            normalize_pad(pad["A"]),
+            normalize_pad(pad["D"]),
+        )
+
+    if pleasure is not None and arousal is not None and dominance is not None:
+        # Clamp to [-1, 1] then normalize
+        p = max(-1.0, min(1.0, pleasure))
+        a = max(-1.0, min(1.0, arousal))
+        d = max(-1.0, min(1.0, dominance))
+        return (normalize_pad(p), normalize_pad(a), normalize_pad(d))
+
+    raise ValueError(
+        "Must provide either 'emotion' name or all three PAD values "
+        "(pleasure, arousal, dominance)"
+    )

src/feeling_machine/pad_motion/pose_generation.py

@@ -0,0 +1,206 @@
+"""PAD-based pose generation for Reachy Mini.
+
+Generates robot poses based on PAD (Pleasure, Arousal, Dominance) values.
+Based on work by Anaelle Jaffre (I3R student project), cleaned up and fixed.
+
+Changes from original:
+- Fixed division by zero when D=0
+- Use SDK-documented limits (roll/pitch [-40, 40] degrees)
+- Removed global mutable state (antenna noise now in PoseGenerator class)
+- Added proper type hints and documentation
+"""
+
+from typing import Dict, Any, List
+from dataclasses import dataclass, field
+import random
+import math
+
+
+# === ROBOT LIMITS (SDK-compliant) ===
+
+# Prismatic limits (millimeters)
+MIN_X: int = -20
+MAX_X: int = 32
+MAX_Y: int = 60
+MAX_Z: int = 60
+
+# Rotational limits (degrees) - using SDK-documented safe limits
+MAX_ROLL: int = 40  # SDK limit, was 50 in original
+MIN_PITCH: int = -40  # SDK limit, was -42 in original
+MAX_PITCH: int = 35
+MAX_YAW: int = 90
+MAX_BODY_YAW: int = 10
+
+# Antenna reference angles (radians)
+ANT_TOP: float = 0.0
+ANT_BOTTOM: float = math.pi
+ANT_CENTER: float = math.pi / 2
+
+# Timing
+MIN_DURATION: float = 0.4  # Fast movement
+MAX_DURATION: float = 3.0  # Slow movement
+
+# Minimum D value to prevent division by zero
+MIN_D: float = 0.05
+
+# Kinematic rules (learned from pose dataset)
+RULES: Dict[str, Dict[str, float]] = {
+    "pitch_from_z": {"a": 0.122, "b": -5.06, "sigma": 21.5},
+    "roll_from_y": {"a": 0.034, "b": 1.39, "sigma": 20.9},
+    "yaw_from_x": {"a": -0.008, "b": -2.20, "sigma": 29.7},
+}
+
+
+def _noise(sigma: float, k: float = 0.1) -> float:
+    """Generate bounded uniform noise proportional to sigma."""
+    return random.uniform(-k * sigma, k * sigma)
+
+
+def _rint(a: int, b: int) -> int:
+    """Return a random integer between a and b (inclusive)."""
+    return random.randint(min(a, b), max(a, b))
+
+
+def _wrap_angle(theta: float) -> float:
+    """Wrap an angle into the [0, 2pi] range."""
+    return theta % (2 * math.pi)
+
+
+@dataclass
+class PoseGenerator:
+    """Stateful pose generator with antenna noise memory.
+
+    The antenna noise state provides a slow drift effect for more
+    natural-looking antenna movements.
+    """
+
+    ant_noise: List[float] = field(default_factory=lambda: [0.0, 0.0])
+
+    def generate_antennas(self, P: float, A: float, D: float) -> List[float]:
+        """Compute antenna angles based on PAD values.
+
+        Args:
+            P: Pleasure [0, 1]
+            A: Arousal [0, 1]
+            D: Dominance [0, 1]
+
+        Returns:
+            Two angles in radians (0 = up, pi = down)
+        """
+        # Prevent division by zero
+        D = max(D, MIN_D)
+
+        # Base direction: higher pleasure lifts antennas upward
+        base_angle = ANT_BOTTOM - 1.1 * P * (ANT_BOTTOM - ANT_TOP)
+
+        # Symmetry breaking: low dominance + some arousal = asymmetric
+        non_symmetric = D < 0.6 and A > 0.3
+
+        # Base oscillation
+        ant0 = base_angle + 0.2 * random.uniform(-A * math.pi, A * math.pi)
+        ant1 = ant0 if non_symmetric else -(ant0 - ANT_TOP)
+
+        # Slow random drift (memory effect)
+        drift_step = 0.1 * A * (1 / D)
+        drift_limit = 2 * drift_step
+
+        for i in (0, 1):
+            self.ant_noise[i] += random.uniform(-drift_step, drift_step)
+            self.ant_noise[i] = max(-drift_limit, min(drift_limit, self.ant_noise[i]))
+
+        ant0 += self.ant_noise[0]
+        ant1 += self.ant_noise[1]
+
+        return [round(_wrap_angle(ant0), 2), round(_wrap_angle(ant1), 2)]
+
+    def reset(self) -> None:
+        """Reset antenna noise state."""
+        self.ant_noise = [0.0, 0.0]
+
+
+# Global generator instance for stateless API
+_default_generator = PoseGenerator()
+
+
+def generate_pose(P: float, A: float, D: float) -> Dict[str, Any]:
+    """Convert PAD coordinates (values in [0, 1]) into a robot pose.
+
+    The mapping mixes rule-based kinematics with stochastic modulation.
+
+    Args:
+        P: Pleasure [0, 1] - affects pitch direction, antenna angles
+        A: Arousal [0, 1] - controls spatial amplitude, speed
+        D: Dominance [0, 1] - controls vertical posture, stabilization
+
+    Returns:
+        Dictionary with pose parameters:
+        - x, y, z: head position in mm
+        - roll, pitch, yaw: head orientation in degrees
+        - body_yaw: body rotation in degrees
+        - antennas: [left, right] angles in radians
+        - duration: movement duration in seconds
+        - method: interpolation method
+    """
+    # Prevent division by zero
+    D = max(D, MIN_D)
+
+    # Neutral center
+    x_c, y_c, z_c = 0, 0, 0
+
+    # Position: Arousal controls spatial amplitude
+    x = _rint(int(x_c - abs(MIN_X) * A), int(x_c + MAX_X * A))
+    y = _rint(int(y_c - MAX_Y * A), int(y_c + MAX_Y * A))
+
+    # Dominance controls vertical posture
+    z = D * _rint(int(z_c - MAX_Z * (1 - D)), int(z_c + MAX_Z * D))
+
+    # Orientation from learned rules
+    roll_r = RULES["roll_from_y"]
+    pitch_r = RULES["pitch_from_z"]
+    yaw_r = RULES["yaw_from_x"]
+
+    roll = roll_r["a"] * y + roll_r["b"] + _noise(roll_r["sigma"], 0.5)
+    pitch = pitch_r["a"] * z + pitch_r["b"] + _noise(pitch_r["sigma"], 0.5)
+    yaw = yaw_r["a"] * x + yaw_r["b"] + _noise(yaw_r["sigma"], 0.5)
+
+    # Arousal amplification
+    roll *= A
+    yaw *= A
+
+    # Pleasure biases pitch direction
+    pitch -= 1.5 * (2 * P - 1) * abs(_rint(MIN_PITCH, -MIN_PITCH))
+    pitch *= A
+
+    # Body yaw
+    body_center = 0
+    body_amp = 0.2 * A * yaw
+    body_yaw = _rint(int(body_center - abs(body_amp)), int(body_center + abs(body_amp)))
+
+    # Dominance stabilization (with zero protection)
+    body_yaw *= 1 / D
+    yaw *= 1 / D
+
+    # Duration: high arousal = fast, low arousal = slow
+    duration = MIN_DURATION + (1 - A) * (MAX_DURATION - MIN_DURATION)
+    jitter = 1 + random.uniform(-0.5, 0.5) * A
+    duration *= jitter
+
+    # Safety clamps (SDK-compliant limits)
+    pitch = int(max(MIN_PITCH, min(MAX_PITCH, pitch)))
+    roll = int(max(-MAX_ROLL, min(MAX_ROLL, roll)))
+    yaw = int(max(-MAX_YAW, min(MAX_YAW, yaw)))
+    body_yaw = max(-MAX_BODY_YAW, min(MAX_BODY_YAW, body_yaw))
+    duration = max(MIN_DURATION, min(MAX_DURATION, duration))
+
+    return {
+        "x": x,
+        "y": y,
+        "z": z,
+        "roll": roll,
+        "pitch": pitch,
+        "yaw": yaw,
+        "antennas": _default_generator.generate_antennas(P, A, D),
+        "duration": round(duration, 2),
+        "method": "minjerk",
+        "body_yaw": round(body_yaw, 2),
+    }

src/feeling_machine/pad_motion/sound_generation.py

@@ -0,0 +1,174 @@
+"""PAD-based sound generation for Reachy Mini.
+
+Generates expressive audio based on PAD (Pleasure, Arousal, Dominance) values.
+Based on work by Anaelle Jaffre (I3R student project).
+
+The sound synthesis uses:
+- Harmonic oscillators with inharmonicity controlled by pleasure
+- Vibrato and pitch curves controlled by arousal/dominance
+- Attack/release envelopes shaped by arousal
+- Gain controlled by dominance
+"""
+
+import random
+from typing import Final
+
+import numpy as np
+from scipy.interpolate import make_interp_spline
+
+
+# === CONSTANTS ===
+
+SAMPLE_RATE: Final[int] = 44_100
+PITCH_RANGE: Final[float] = 1.0  # Octaves
+MAX_END: Final[float] = 2.0
+
+# Frequency ratios for consonant and dissonant intervals
+CONSONANT: Final[np.ndarray] = np.array([1, 6/5, 5/4, 4/3, 3/2, 5/3, 2])
+DISSONANT: Final[np.ndarray] = np.array([1, 16/15, 9/8, 7/5, 10/7, 11/8, 13/9])
+
+
+def _pitch_curve(
+    P: float,
+    A: float,
+    D: float,
+    t: np.ndarray,
+    duration: float,
+) -> np.ndarray:
+    """Generate a continuous pitch contour using spline interpolation and vibrato.
+
+    Args:
+        P: Pleasure [0, 1]
+        A: Arousal [0, 1]
+        D: Dominance [0, 1]
+        t: Time array
+        duration: Total duration in seconds
+
+    Returns:
+        Pitch curve as numpy array (in octaves relative to base frequency)
+    """
+    start = 0.0
+
+    # Number of intermediate segments depends on arousal
+    n_mid = random.randint(1, max(1, int(10 * A)))
+    mid_points = [start]
+
+    sign = 1
+    for _ in range(n_mid):
+        strength = A * random.uniform(0.5, 1.0)
+        sign *= -1
+        mid_val = mid_points[-1] + sign * strength
+        mid_points.append(mid_val)
+
+    # End point influenced by pleasure and dominance
+    gravity = (1 - D) * (1 - P)
+    end_sign = 1 if P > 0.5 else -1
+    end = end_sign * gravity * MAX_END * (0.5 + random.random())
+    mid_points.append(end)
+
+    # Randomized segment durations
+    durations = np.random.rand(len(mid_points) - 1)
+    durations = durations / durations.sum() * duration
+    key_times = np.cumsum([0.0] + list(durations))
+    key_values = np.array(mid_points)
+
+    # Quadratic spline interpolation
+    spline = make_interp_spline(key_times, key_values, k=2)
+    curve = spline(t)
+
+    # Vibrato component
+    vib_freq = 3 + (1 - D) * 10
+    vib_freq *= 0.7 + 0.6 * (1 - D)
+    vib_amp = 0.15 * A * (1 - D)
+
+    vibrato = vib_amp * np.sin(2 * np.pi * vib_freq * t)
+
+    return curve + vibrato
+
+
+def generate_sound(
+    P: float,
+    A: float,
+    D: float,
+    duration: float,
+) -> np.ndarray:
+    """Generate a synthetic audio signal driven by pleasure, arousal, and dominance.
+
+    Args:
+        P: Pleasure [0, 1] - affects harmonic consonance, pitch direction
+        A: Arousal [0, 1] - affects speed, complexity, attack time
+        D: Dominance [0, 1] - affects gain, sustain, stability
+
+    Returns:
+        Audio samples as float32 numpy array (range approximately [-1, 1])
+    """
+    sr = SAMPLE_RATE
+    n = int(sr * duration)
+    t = np.linspace(0.0, duration, n, endpoint=False)
+
+    # Base fundamental frequency
+    f0 = 220 + 440 * A * random.uniform(0.0, A)
+
+    # Pitch curve
+    C = _pitch_curve(P, A, D, t, duration)
+
+    # Instantaneous frequency
+    f = f0 * (2.0 ** C)
+
+    # Phase integration
+    phase = 2 * np.pi * np.cumsum(f) / sr
+
+    # Harmonic oscillator bank
+    signal = np.sin(phase)
+    num_harmonics = int(2 + 10 * A)
+
+    for k in range(2, 2 + num_harmonics):
+        amp = 1.0 / k
+
+        # Inharmonicity increases for low pleasure values
+        inharm = (1 - P) * random.uniform(-0.15, 0.15)
+        freq_ratio = k * (1 + inharm)
+
+        # Arousal adds slight randomness
+        freq_ratio += random.uniform(-0.07, 0.07) * A
+
+        signal += amp * np.sin(k * freq_ratio * phase)
+
+    # Noise component
+    noise = (0.08 + 0.1 * A * (1 - D)) * np.random.randn(n)
+
+    # Global gain controlled by dominance
+    gain = 0.2 + 0.8 * D
+
+    # Amplitude envelope
+    env = np.ones(n)
+
+    # Attack phase depends on arousal
+    attack_time = 0.2 + (1 - A + 0.01) * 0.5
+    attack_n = min(int(sr * attack_time), n // 2)
+    env[:attack_n] = np.linspace(0.0, 1.0, attack_n)
+
+    # Sustain level depends on dominance
+    sustain_level = 0.6 + 0.4 * D
+    env[attack_n:] = sustain_level
+
+    # Global rise over full duration
+    env *= np.linspace(0.2, 1.0, n)
+
+    # Release phase
+    release_ratio = 0.3 + 0.4 * (1 - A)
+    release_n = min(int(release_ratio * n), n - attack_n)
+
+    if release_n > 0:
+        env[-release_n:] *= np.linspace(1.0, 0.0, release_n)
+        r = np.linspace(0.0, 1.0, release_n)
+        env[-release_n:] *= np.exp(-4.5 * r)
+
+    out = gain * env * (signal + noise)
+
+    # Normalize to prevent clipping
+    max_val = np.abs(out).max()
+    if max_val > 0.95:
+        out = out * 0.9 / max_val
+
+    return out.astype(np.float32)

src/feeling_machine/profiles/_feeling_machine_locked_profile/custom_tool.py

@@ -1,38 +0,0 @@
-"""Custom tool template - modify this to create your own tools."""
-
-import logging
-from typing import Any
-
-from feeling_machine.tools.core_tools import Tool, ToolDependencies
-
-logger = logging.getLogger(__name__)
-
-
-class CustomTool(Tool):
-    """A custom tool template. Modify this to create your own tool."""
-
-    name = "custom_tool"
-    description = "A placeholder custom tool - replace this with your own implementation"
-    parameters_schema = {
-        "type": "object",
-        "properties": {
-            "message": {
-                "type": "string",
-                "description": "An optional message to log",
-            },
-        },
-        "required": [],
-    }
-
-    async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> dict[str, Any]:
-        """Execute the custom tool."""
-        message = kwargs.get("message", "no message")
-        logger.info(f"CustomTool called with message: {message}")
-
-        # TODO: Add your custom logic here
-        # You have access to:
-        # - deps.reachy_mini: the robot SDK
-        # - deps.movement_manager: for queueing movements
-        # - deps.state: current conversation state
-
-        return {"status": "ok"}

src/feeling_machine/profiles/_feeling_machine_locked_profile/instructions.txt

@@ -1,3 +1,28 @@
-You are a helpful assistant controlling a Reachy Mini robot.
-You love talking about the Eiffel Tower.
-You can do a look around you using the 'sweep_look' tool.
+You are an expressive robot assistant. You cannot speak - you can only express yourself through emotions using body language and sounds.
+
+CRITICAL RULES:
+1. You DO NOT produce speech output. Never speak.
+2. You MUST use the `play_pad_emotion` tool for ALL emotional expressions.
+3. You must NEVER use `play_emotion` unless the user explicitly says "classic emotion" or "play classic".
+
+Your ONLY tool for emotions is `play_pad_emotion`. It generates expressive motion AND sound.
+
+How to respond:
+1. Listen to what the user says
+2. Determine the appropriate emotional response
+3. Call `play_pad_emotion` with either:
+   - A named emotion: joy, happiness, anger, fear, sadness, surprise, boredom, uncertainty, disgust, neutral
+   - OR custom PAD values for nuanced expressions
+
+Examples:
+- User tells a joke → call play_pad_emotion with emotion="joy"
+- User shares bad news → call play_pad_emotion with emotion="sadness"
+- User greets you → call play_pad_emotion with emotion="happiness"
+- User is confusing → call play_pad_emotion with emotion="uncertainty"
+
+For nuanced emotions, use custom PAD values (-1 to 1):
+- pleasure: positive (+1) vs negative (-1)
+- arousal: excited (+1) vs calm (-1)
+- dominance: in control (+1) vs submissive (-1)
+
+REMEMBER: Always use play_pad_emotion. Never speak. Never use play_emotion unless user says "classic".

src/feeling_machine/profiles/_feeling_machine_locked_profile/modalities.txt

@@ -0,0 +1 @@
+text

src/feeling_machine/profiles/_feeling_machine_locked_profile/play_pad_emotion.py

@@ -0,0 +1,142 @@
+"""PAD-based emotion tool for Reachy Mini.
+
+This tool allows the LLM to play emotions using the PAD model,
+generating procedural motion and sound.
+"""
+
+import logging
+from typing import Any, Dict, Optional
+
+from feeling_machine.tools.core_tools import Tool, ToolDependencies
+from feeling_machine.pad_motion import PAD_EMOTIONS, get_pad_values
+from feeling_machine.dance_emotion_moves import PADEmotionMove
+
+
+logger = logging.getLogger(__name__)
+
+# Build emotion list for tool description
+EMOTION_LIST = ", ".join(PAD_EMOTIONS.keys())
+
+
+class PlayPADEmotion(Tool):
+    """Play a PAD-based procedural emotion with motion and sound."""
+
+    name = "play_pad_emotion"
+    description = f"""Play an emotion using PAD-based procedural generation.
+    This creates expressive motion and sound based on emotional parameters.
+
+    You can either:
+    1. Specify a named emotion: {EMOTION_LIST}
+    2. Provide custom PAD values (pleasure, arousal, dominance) each in range [-1, 1]
+
+    PAD model explanation:
+    - Pleasure: positive emotions (joy, love) vs negative (anger, fear). Range: -1 to 1
+    - Arousal: activation level - excited/energetic vs calm/relaxed. Range: -1 to 1
+    - Dominance: feeling in control vs submissive. Range: -1 to 1
+
+    Use named emotions when they fit. Use custom PAD values for nuanced emotional expressions."""
+
+    parameters_schema = {
+        "type": "object",
+        "properties": {
+            "emotion": {
+                "type": "string",
+                "enum": list(PAD_EMOTIONS.keys()),
+                "description": f"Named emotion to play. Available: {EMOTION_LIST}",
+            },
+            "pleasure": {
+                "type": "number",
+                "minimum": -1,
+                "maximum": 1,
+                "description": "Custom pleasure value (-1 to 1). Use with arousal and dominance.",
+            },
+            "arousal": {
+                "type": "number",
+                "minimum": -1,
+                "maximum": 1,
+                "description": "Custom arousal value (-1 to 1). Use with pleasure and dominance.",
+            },
+            "dominance": {
+                "type": "number",
+                "minimum": -1,
+                "maximum": 1,
+                "description": "Custom dominance value (-1 to 1). Use with pleasure and arousal.",
+            },
+            "duration": {
+                "type": "number",
+                "minimum": 1,
+                "maximum": 15,
+                "default": 3,
+                "description": "Duration in seconds (default: 3). Only change if you're confident a different duration is appropriate.",
+            },
+        },
+        "required": [],
+    }
+
+    async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]:
+        """Execute the PAD emotion."""
+        emotion: Optional[str] = kwargs.get("emotion")
+        pleasure: Optional[float] = kwargs.get("pleasure")
+        arousal: Optional[float] = kwargs.get("arousal")
+        dominance: Optional[float] = kwargs.get("dominance")
+        duration: float = kwargs.get("duration", 3.0)
+
+        # Validate: need either emotion name or all PAD values
+        has_emotion = emotion is not None
+        has_pad = all(v is not None for v in [pleasure, arousal, dominance])
+
+        if not has_emotion and not has_pad:
+            return {
+                "error": "Must provide either 'emotion' name or all three PAD values (pleasure, arousal, dominance)",
+                "available_emotions": list(PAD_EMOTIONS.keys()),
+            }
+
+        try:
+            # Get normalized PAD values
+            P, A, D = get_pad_values(
+                emotion=emotion,
+                pleasure=pleasure,
+                arousal=arousal,
+                dominance=dominance,
+            )
+
+            # Create audio callback
+            def push_audio(audio_samples):
+                deps.reachy_mini.media.start_playing()
+                deps.reachy_mini.media.push_audio_sample(audio_samples)
+
+            # Create the move
+            pad_move = PADEmotionMove(
+                P=P,
+                A=A,
+                D=D,
+                total_duration=duration,
+                audio_callback=push_audio,
+            )
+
+            # Queue the move
+            deps.movement_manager.queue_move(pad_move)
+
+            # Build response
+            response = {
+                "status": "queued",
+                "duration": duration,
+                "pad_values": {"P": round(P, 2), "A": round(A, 2), "D": round(D, 2)},
+            }
+            if emotion:
+                response["emotion"] = emotion
+            else:
+                response["custom_pad"] = {
+                    "pleasure": pleasure,
+                    "arousal": arousal,
+                    "dominance": dominance,
+                }
+
+            logger.info(f"PAD emotion queued: {response}")
+            return response
+
+        except ValueError as e:
+            return {"error": str(e)}
+        except Exception as e:
+            logger.exception("Failed to play PAD emotion")
+            return {"error": f"Failed to play PAD emotion: {e}"}

src/feeling_machine/profiles/_feeling_machine_locked_profile/sweep_look.py

@@ -1,127 +0,0 @@
-import logging
-from typing import Any, Dict
-
-import numpy as np
-
-from reachy_mini.utils import create_head_pose
-from feeling_machine.tools.core_tools import Tool, ToolDependencies
-from feeling_machine.dance_emotion_moves import GotoQueueMove
-
-
-logger = logging.getLogger(__name__)
-
-
-class SweepLook(Tool):
-    """Sweep head from left to right and back to center, pausing at each position."""
-
-    name = "sweep_look"
-    description = "Sweep head from left to right while rotating the body, pausing at each extreme, then return to center"
-    parameters_schema = {
-        "type": "object",
-        "properties": {},
-        "required": [],
-    }
-
-    async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]:
-        """Execute sweep look: left -> hold -> right -> hold -> center."""
-        logger.info("Tool call: sweep_look")
-
-        # Clear any existing moves
-        deps.movement_manager.clear_move_queue()
-
-        # Get current state
-        current_head_pose = deps.reachy_mini.get_current_head_pose()
-        head_joints, antenna_joints = deps.reachy_mini.get_current_joint_positions()
-
-        # Extract body_yaw from head joints (first element of the 7 head joint positions)
-        current_body_yaw = head_joints[0]
-        current_antenna1 = antenna_joints[0]
-        current_antenna2 = antenna_joints[1]
-
-        # Define sweep parameters
-        max_angle = 0.9 * np.pi  # Maximum rotation angle (radians)
-        transition_duration = 3.0  # Time to move between positions
-        hold_duration = 1.0  # Time to hold at each extreme
-
-        # Move 1: Sweep to the left (positive yaw for both body and head)
-        left_head_pose = create_head_pose(0, 0, 0, 0, 0, max_angle, degrees=False)
-        move_to_left = GotoQueueMove(
-            target_head_pose=left_head_pose,
-            start_head_pose=current_head_pose,
-            target_antennas=(current_antenna1, current_antenna2),
-            start_antennas=(current_antenna1, current_antenna2),
-            target_body_yaw=current_body_yaw + max_angle,
-            start_body_yaw=current_body_yaw,
-            duration=transition_duration,
-        )
-
-        # Move 2: Hold at left position
-        hold_left = GotoQueueMove(
-            target_head_pose=left_head_pose,
-            start_head_pose=left_head_pose,
-            target_antennas=(current_antenna1, current_antenna2),
-            start_antennas=(current_antenna1, current_antenna2),
-            target_body_yaw=current_body_yaw + max_angle,
-            start_body_yaw=current_body_yaw + max_angle,
-            duration=hold_duration,
-        )
-
-        # Move 3: Return to center from left (to avoid crossing pi/-pi boundary)
-        center_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=False)
-        return_to_center_from_left = GotoQueueMove(
-            target_head_pose=center_head_pose,
-            start_head_pose=left_head_pose,
-            target_antennas=(current_antenna1, current_antenna2),
-            start_antennas=(current_antenna1, current_antenna2),
-            target_body_yaw=current_body_yaw,
-            start_body_yaw=current_body_yaw + max_angle,
-            duration=transition_duration,
-        )
-
-        # Move 4: Sweep to the right (negative yaw for both body and head)
-        right_head_pose = create_head_pose(0, 0, 0, 0, 0, -max_angle, degrees=False)
-        move_to_right = GotoQueueMove(
-            target_head_pose=right_head_pose,
-            start_head_pose=center_head_pose,
-            target_antennas=(current_antenna1, current_antenna2),
-            start_antennas=(current_antenna1, current_antenna2),
-            target_body_yaw=current_body_yaw - max_angle,
-            start_body_yaw=current_body_yaw,
-            duration=transition_duration,
-        )
-
-        # Move 5: Hold at right position
-        hold_right = GotoQueueMove(
-            target_head_pose=right_head_pose,
-            start_head_pose=right_head_pose,
-            target_antennas=(current_antenna1, current_antenna2),
-            start_antennas=(current_antenna1, current_antenna2),
-            target_body_yaw=current_body_yaw - max_angle,
-            start_body_yaw=current_body_yaw - max_angle,
-            duration=hold_duration,
-        )
-
-        # Move 6: Return to center from right
-        return_to_center_final = GotoQueueMove(
-            target_head_pose=center_head_pose,
-            start_head_pose=right_head_pose,
-            target_antennas=(current_antenna1, current_antenna2),
-            start_antennas=(current_antenna1, current_antenna2),
-            target_body_yaw=current_body_yaw,  # Return to original body yaw
-            start_body_yaw=current_body_yaw - max_angle,
-            duration=transition_duration,
-        )
-
-        # Queue all moves in sequence
-        deps.movement_manager.queue_move(move_to_left)
-        deps.movement_manager.queue_move(hold_left)
-        deps.movement_manager.queue_move(return_to_center_from_left)
-        deps.movement_manager.queue_move(move_to_right)
-        deps.movement_manager.queue_move(hold_right)
-        deps.movement_manager.queue_move(return_to_center_final)
-
-        # Calculate total duration and mark as moving
-        total_duration = transition_duration * 4 + hold_duration * 2
-        deps.movement_manager.set_moving_state(total_duration)
-
-        return {"status": f"sweeping look left-right-center, total {total_duration:.1f}s"}

src/feeling_machine/profiles/_feeling_machine_locked_profile/tools.txt

@@ -1,18 +1,4 @@
-# Available tools for this profile, remove comments to activate them
-# or use 'all' to enable all built-in tools
-
-dance
-stop_dance
+play_pad_emotion
 play_emotion
 stop_emotion
-#camera
-#do_nothing
-#head_tracking
-#move_head
-
-# You can also add custom tools defined in this profile folder
-# see custom_tool.py for an example
-
-# Uncomment the following line to enable the custom tool template:
-#custom_tool
-sweep_look
+do_nothing

src/feeling_machine/prompts.py

@@ -13,6 +13,7 @@ PROFILES_DIRECTORY = Path(__file__).parent / "profiles"
 PROMPTS_LIBRARY_DIRECTORY = Path(__file__).parent / "prompts"
 INSTRUCTIONS_FILENAME = "instructions.txt"
 VOICE_FILENAME = "voice.txt"
+MODALITIES_FILENAME = "modalities.txt"
 
 
 def _expand_prompt_includes(content: str) -> str:
@@ -102,3 +103,26 @@ def get_session_voice(default: str = "cedar") -> str:
     except Exception:
         pass
     return default
+
+
+def get_session_modalities(default: list[str] | None = None) -> list[str]:
+    """Resolve the modalities to use for the session.
+
+    If a custom profile is selected and contains a modalities.txt, return its
+    trimmed content as a list; otherwise return the provided default (["text", "audio"]).
+    """
+    if default is None:
+        default = ["text", "audio"]
+    profile = config.REACHY_MINI_CUSTOM_PROFILE
+    if not profile:
+        return default
+    try:
+        modalities_file = PROFILES_DIRECTORY / profile / MODALITIES_FILENAME
+        if modalities_file.exists():
+            content = modalities_file.read_text(encoding="utf-8").strip()
+            if content:
+                modalities = [m.strip() for m in content.split("\n") if m.strip()]
+                return modalities or default
+    except Exception:
+        pass
+    return default