Spaces:

RemiFabre
/

test_conv_pipe

Running

App Files Files Community

RemiFabre commited on Feb 11

Commit

41c2517

1 Parent(s): 46fd62d

Test: copy ALL files from conv app to match exactly

Browse files

This view is limited to 50 files because it contains too many changes. See raw diff

Files changed (50) hide show

.gitignore +1 -1
index.html +122 -39
pyproject.toml +107 -44
src/reachy_mini_conversation_app/__init__.py +1 -0
src/reachy_mini_conversation_app/audio/__init__.py +1 -0
src/reachy_mini_conversation_app/audio/head_wobbler.py +181 -0
src/reachy_mini_conversation_app/audio/speech_tapper.py +268 -0
src/reachy_mini_conversation_app/camera_worker.py +241 -0
src/reachy_mini_conversation_app/config.py +60 -0
src/reachy_mini_conversation_app/console.py +499 -0
src/reachy_mini_conversation_app/dance_emotion_moves.py +154 -0
src/reachy_mini_conversation_app/gradio_personality.py +301 -0
src/reachy_mini_conversation_app/headless_personality.py +102 -0
src/reachy_mini_conversation_app/headless_personality_ui.py +276 -0
src/reachy_mini_conversation_app/images/reachymini_avatar.png +3 -0
src/reachy_mini_conversation_app/images/user_avatar.png +3 -0
src/reachy_mini_conversation_app/main.py +253 -0
src/reachy_mini_conversation_app/moves.py +849 -0
src/reachy_mini_conversation_app/openai_realtime.py +719 -0
src/reachy_mini_conversation_app/profiles/__init__.py +1 -0
src/reachy_mini_conversation_app/profiles/cosmic_kitchen/instructions.txt +49 -0
src/reachy_mini_conversation_app/profiles/cosmic_kitchen/tools.txt +8 -0
src/reachy_mini_conversation_app/profiles/default/instructions.txt +1 -0
src/reachy_mini_conversation_app/profiles/default/tools.txt +8 -0
src/reachy_mini_conversation_app/profiles/example/instructions.txt +3 -0
src/reachy_mini_conversation_app/profiles/example/sweep_look.py +127 -0
src/reachy_mini_conversation_app/profiles/example/tools.txt +13 -0
src/reachy_mini_conversation_app/profiles/mars_rover/instructions.txt +25 -0
src/reachy_mini_conversation_app/profiles/mars_rover/tools.txt +8 -0
src/reachy_mini_conversation_app/profiles/short_bored_teenager/instructions.txt +1 -0
src/reachy_mini_conversation_app/profiles/short_bored_teenager/tools.txt +8 -0
src/reachy_mini_conversation_app/profiles/short_captain_circuit/instructions.txt +1 -0
src/reachy_mini_conversation_app/profiles/short_captain_circuit/tools.txt +8 -0
src/reachy_mini_conversation_app/profiles/short_chess_coach/instructions.txt +1 -0
src/reachy_mini_conversation_app/profiles/short_chess_coach/tools.txt +8 -0
src/reachy_mini_conversation_app/profiles/short_hype_bot/instructions.txt +1 -0
src/reachy_mini_conversation_app/profiles/short_hype_bot/tools.txt +8 -0
src/reachy_mini_conversation_app/profiles/short_mad_scientist_assistant/instructions.txt +1 -0
src/reachy_mini_conversation_app/profiles/short_mad_scientist_assistant/tools.txt +8 -0
src/reachy_mini_conversation_app/profiles/short_nature_documentarian/instructions.txt +1 -0
src/reachy_mini_conversation_app/profiles/short_nature_documentarian/tools.txt +8 -0
src/reachy_mini_conversation_app/profiles/short_noir_detective/instructions.txt +1 -0
src/reachy_mini_conversation_app/profiles/short_noir_detective/tools.txt +8 -0
src/reachy_mini_conversation_app/profiles/short_time_traveler/instructions.txt +1 -0
src/reachy_mini_conversation_app/profiles/short_time_traveler/tools.txt +8 -0
src/reachy_mini_conversation_app/profiles/short_victorian_butler/instructions.txt +1 -0
src/reachy_mini_conversation_app/profiles/short_victorian_butler/tools.txt +8 -0
src/reachy_mini_conversation_app/profiles/sorry_bro/instructions.txt +6 -0
src/reachy_mini_conversation_app/profiles/sorry_bro/tools.txt +8 -0
src/reachy_mini_conversation_app/prompts.py +104 -0

.gitignore CHANGED Viewed

@@ -58,4 +58,4 @@ cache/
 .nfs*
 # User-created personalities (managed by UI)
-src/test_conv_pipe/profiles/user_personalities/

 .nfs*
 # User-created personalities (managed by UI)
+src/reachy_mini_conversation_app/profiles/user_personalities/

index.html CHANGED Viewed

@@ -1,42 +1,125 @@
 <!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>Test Conv Pipe</title>
-    <link rel="stylesheet" href="style.css" />
-  </head>
-  <body>
-    <div class="ambient"></div>
-    <div class="container">
-      <header class="hero">
-        <div class="pill">Reachy Mini App</div>
-        <h1>Test Conv Pipe</h1>
-        <p class="subtitle">A conversation app for Reachy Mini robot.</p>
-      </header>
-      <div class="panel">
-        <div class="panel-heading">
-          <div>
-            <p class="eyebrow">Getting Started</p>
-            <h2>Installation</h2>
-          </div>
-        </div>
-        <p class="muted">Install this app on your Reachy Mini using the app store, or run it locally:</p>
-        <pre><code>uv sync
-reachy-mini-daemon --sim  # in another terminal
-python -m test_conv_pipe</code></pre>
-      </div>
-      <div class="panel">
-        <div class="panel-heading">
-          <div>
-            <p class="eyebrow">Configuration</p>
-            <h2>OpenAI API Key</h2>
-          </div>
-        </div>
-        <p class="muted">This app requires an OpenAI API key for voice conversations. Set it via the web interface or environment variable.</p>
-      </div>
-    </div>
-  </body>
-</html>

 <!doctype html>
+<html>
+<head>
+	<meta charset="utf-8" />
+	<meta name="viewport" content="width=device-width, initial-scale=1" />
+	<title>Reachy Mini Conversation App</title>
+	<link rel="preconnect" href="https://fonts.googleapis.com">
+	<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+	<link href="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=Manrope:wght@400;500;600&display=swap" rel="stylesheet">
+	<link rel="stylesheet" href="style.css" />
+</head>
+<body>
+	<header class="hero">
+		<div class="topline">
+			<div class="brand">
+				<span class="logo">🤖</span>
+				<span class="brand-name">Reachy Mini</span>
+			</div>
+			<div class="pill">Realtime voice · Vision aware · Expressive motion</div>
+		</div>
+		<div class="hero-grid">
+			<div class="hero-copy">
+				<p class="eyebrow">Conversation App</p>
+				<h1>Talk, see, and move together.</h1>
+				<p class="lede">
+					A friendly, camera-aware companion for Reachy Mini. Chat out loud, watch it follow faces, dance, or react with recorded emotions—all while streaming transcripts in a clean web UI.
+				</p>
+				<div class="hero-actions">
+					<a class="btn primary" href="#highlights">Explore features</a>
+					<a class="btn ghost" href="#story">See how it feels</a>
+				</div>
+				<div class="hero-badges">
+					<span>Low-latency voice loop</span>
+					<span>Camera insights on demand</span>
+					<span>Choreographed dances & emotions</span>
+					<span>Personality profiles via web UI</span>
+				</div>
+			</div>
+			<div class="hero-visual">
+				<div class="glass-card">
+					<img src="docs/assets/reachy_mini_dance.gif" alt="Reachy Mini dancing" class="hero-gif">
+					<p class="caption">Reachy Mini can move, dance, and emote while holding a natural conversation.</p>
+				</div>
+			</div>
+		</div>
+	</header>
+	<section id="highlights" class="section features">
+		<div class="section-header">
+			<p class="eyebrow">What’s inside</p>
+			<h2>All-in-one conversational layer for your robot</h2>
+			<p class="intro">
+				The app blends realtime speech, vision, and motion so Reachy Mini feels present..
+			</p>
+		</div>
+		<div class="feature-grid">
+			<div class="feature-card">
+				<span class="icon">🎤</span>
+				<h3>Natural voice chat</h3>
+				<p>Talk freely and get fast, high-quality replies powered by realtime models.</p>
+			</div>
+			<div class="feature-card">
+				<span class="icon">🎥</span>
+				<h3>Vision-aware replies</h3>
+				<p>Ask the camera tool to see what’s in front, track a face, or keep attention on whoever is speaking.</p>
+			</div>
+			<div class="feature-card">
+				<span class="icon">💃</span>
+				<h3>Expressive motion</h3>
+				<p>Queue dances, play recorded emotions while Reachy listens and talks.</p>
+			</div>
+			<div class="feature-card">
+				<span class="icon">🧠</span>
+				<h3>Personalities on demand</h3>
+				<p>Switch conversation styles through profiles and decide which tools (dance, camera, tracking) each persona can use.</p>
+			</div>
+			<div class="feature-card">
+				<span class="icon">🌐</span>
+				<h3>Ready for your setup</h3>
+				<p>Works with wired or wireless Reachy Mini, and can run vision locally or through the default cloud model.</p>
+			</div>
+		</div>
+	</section>
+	<section id="story" class="section story">
+		<div class="story-grid">
+			<div class="story-card">
+				<p class="eyebrow">How it feels</p>
+				<h3>From hello to helpful in seconds</h3>
+				<ul class="story-list">
+					<li><span>👋</span> Say “Hey Reachy” and start chatting—no extra setup in the moment.</li>
+					<li><span>👀</span> Ask what it sees; it can peek through the camera or keep focus on your face.</li>
+					<li><span>🎭</span> Trigger emotions or dance breaks to keep the conversation lively.</li>
+					<li><span>📝</span> Follow along with live transcripts in the web UI or run audio-only from the console.</li>
+				</ul>
+			</div>
+			<div class="story-card secondary">
+				<p class="eyebrow">Where it shines</p>
+				<h3>Great for demos, teaching, and playful exploration</h3>
+				<p class="story-text">
+					Show off how Reachy Mini listens, responds, and moves in sync. Whether you’re guiding a class, hosting a booth, or experimenting at home, the app keeps the robot expressive without juggling scripts or joystick controls.
+				</p>
+				<div class="chips">
+					<span class="chip">Live conversation</span>
+					<span class="chip">Face tracking</span>
+					<span class="chip">Camera tool</span>
+					<span class="chip">Dance library</span>
+					<span class="chip">Profiles & tools</span>
+				</div>
+			</div>
+		</div>
+	</section>
+	<footer class="footer">
+		<p>
+			Reachy Mini Conversation App by <a href="https://github.com/pollen-robotics" target="_blank" rel="noopener">Pollen Robotics</a>.
+			Explore more apps on <a href="https://huggingface.co/spaces/pollen-robotics/Reachy_Mini_Apps" target="_blank" rel="noopener">Hugging Face Spaces</a>.
+		</p>
+	</footer>
+</body>
+</html>

pyproject.toml CHANGED Viewed

@@ -1,58 +1,124 @@
 [build-system]
-requires = [ "setuptools",]
 build-backend = "setuptools.build_meta"
 [project]
-name = "test_conv_pipe"
 version = "0.2.2"
 description = ""
 readme = "README.md"
 requires-python = ">=3.10"
-dependencies = [ "aiortc>=1.13.0", "fastrtc>=0.0.34", "gradio==5.50.1.dev1", "huggingface-hub==1.3.0", "opencv-python>=4.12.0.88", "python-dotenv", "openai>=2.1", "reachy_mini_dances_library", "reachy_mini_toolbox", "reachy-mini >= 1.2.11", "eclipse-zenoh~=1.7.0", "gradio_client>=1.13.3",]
-[[project.authors]]
-name = "Pollen Robotics"
-email = "contact@pollen-robotics.com"
-[dependency-groups]
-dev = [ "pytest", "pytest-asyncio", "ruff==0.12.0", "mypy==1.18.2", "pre-commit", "types-requests", "python-semantic-release>=10.5.3",]
 [project.optional-dependencies]
-reachy_mini_wireless = [ "PyGObject>=3.42.2,<=3.46.0", "gst-signalling>=1.1.2",]
-local_vision = [ "torch>=2.1", "transformers==5.0.0rc2", "num2words",]
-yolo_vision = [ "ultralytics", "supervision",]
-mediapipe_vision = [ "mediapipe==0.10.14",]
-all_vision = [ "torch>=2.1", "transformers==5.0.0rc2", "num2words", "ultralytics", "supervision", "mediapipe==0.10.14",]
 [project.scripts]
-test-conv-pipe = "test_conv_pipe.main:main"
 [tool.setuptools]
 include-package-data = true
-[tool.ruff]
-line-length = 119
-exclude = [ ".venv", "dist", "build", "**/__pycache__", "*.egg-info", ".mypy_cache", ".pytest_cache",]
-[tool.mypy]
-python_version = "3.12"
-files = [ "src/",]
-ignore_missing_imports = true
-strict = true
-show_error_codes = true
-warn_unused_ignores = true
-[project.entry-points.reachy_mini_apps]
-test_conv_pipe = "test_conv_pipe.main:TestConvPipe"
-[tool.setuptools.package-dir]
-"" = "src"
 [tool.setuptools.package-data]
-test_conv_pipe = [ "images/*", "static/*", ".env.example", "demos/**/*.txt", "prompts_library/*.txt", "profiles/**/*.txt", "prompts/**/*.txt",]
 [tool.ruff.lint]
-select = [ "E", "F", "W", "I", "C4", "D",]
-ignore = [ "E501", "D100", "D203", "D213",]
 [tool.ruff.format]
 quote-style = "double"
@@ -60,13 +126,10 @@ indent-style = "space"
 skip-magic-trailing-comma = false
 line-ending = "auto"
-[tool.setuptools.packages.find]
-where = [ "src",]
-[tool.ruff.lint.isort]
-length-sort = true
-lines-after-imports = 2
-no-lines-before = [ "standard-library", "local-folder",]
-known-local-folder = [ "test_conv_pipe",]
-known-first-party = [ "reachy_mini", "reachy_mini_dances_library", "reachy_mini_toolbox",]
-split-on-trailing-comma = true

 [build-system]
+requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 [project]
+name = "reachy_mini_conversation_app"
 version = "0.2.2"
+authors = [{ name = "Pollen Robotics", email = "contact@pollen-robotics.com" }]
 description = ""
 readme = "README.md"
 requires-python = ">=3.10"
+dependencies = [
+    #Media
+    "aiortc>=1.13.0",
+    "fastrtc>=0.0.34",
+    "gradio==5.50.1.dev1",
+    "huggingface-hub==1.3.0",
+    "opencv-python>=4.12.0.88",
+    #Environment variables
+    "python-dotenv",
+    #OpenAI
+    "openai>=2.1",
+    #Reachy mini
+    "reachy_mini_dances_library",
+    "reachy_mini_toolbox",
+    "reachy-mini >= 1.2.11",
+    "eclipse-zenoh~=1.7.0",
+    "gradio_client>=1.13.3",
+]
 [project.optional-dependencies]
+reachy_mini_wireless = [
+  "PyGObject>=3.42.2,<=3.46.0",
+  "gst-signalling>=1.1.2",
+]
+local_vision = [
+  "torch>=2.1",
+  "transformers==5.0.0rc2",
+  "num2words",
+]
+yolo_vision = [
+  "ultralytics",
+  "supervision",
+]
+mediapipe_vision = [
+  "mediapipe==0.10.14",
+]
+all_vision = [
+  "torch>=2.1",
+  "transformers==5.0.0rc2",
+  "num2words",
+  "ultralytics",
+  "supervision",
+  "mediapipe==0.10.14",
+]
+[dependency-groups]
+dev = [
+  "pytest",
+  "pytest-asyncio",
+  "ruff==0.12.0",
+  "mypy==1.18.2",
+  "pre-commit",
+  "types-requests",
+  "python-semantic-release>=10.5.3",
+]
 [project.scripts]
+reachy-mini-conversation-app = "reachy_mini_conversation_app.main:main"
+[project.entry-points."reachy_mini_apps"]
+reachy_mini_conversation_app = "reachy_mini_conversation_app.main:ReachyMiniConversationApp"
 [tool.setuptools]
+package-dir = { "" = "src" }
 include-package-data = true
+[tool.setuptools.packages.find]
+where = ["src"]
 [tool.setuptools.package-data]
+reachy_mini_conversation_app = [
+  "images/*",
+  "static/*",
+  ".env.example",
+  "demos/**/*.txt",
+  "prompts_library/*.txt",
+  "profiles/**/*.txt",
+  "prompts/**/*.txt",
+]
+[tool.ruff]
+line-length = 119
+exclude = [".venv", "dist", "build", "**/__pycache__", "*.egg-info", ".mypy_cache", ".pytest_cache"]
 [tool.ruff.lint]
+select = [
+  "E",   # pycodestyle errors
+  "F",   # pyflakes
+  "W",   # pycodestyle warnings
+  "I",   # isort
+  "C4",  # flake8-comprehensions
+  "D",   # pydocstyle
+]
+ignore = [
+  "E501",  # handled by formatter
+  "D100",  # ignore missing module docstrings
+  "D203",  # blank line before class docstring (conflicts with D211)
+  "D213",  # summary on second line (conflicts with D212)
+]
+[tool.ruff.lint.isort]
+length-sort = true
+lines-after-imports = 2
+no-lines-before = ["standard-library", "local-folder"]
+known-local-folder = ["reachy_mini_conversation_app"]
+known-first-party = ["reachy_mini", "reachy_mini_dances_library", "reachy_mini_toolbox"]
+split-on-trailing-comma = true
 [tool.ruff.format]
 quote-style = "double"
 skip-magic-trailing-comma = false
 line-ending = "auto"
+[tool.mypy]
+python_version = "3.12"
+files = ["src/"]
+ignore_missing_imports = true
+strict = true
+show_error_codes = true
+warn_unused_ignores = true

src/reachy_mini_conversation_app/__init__.py ADDED Viewed

	@@ -0,0 +1 @@


1	+ """Nothing (for ruff)."""

src/reachy_mini_conversation_app/audio/__init__.py ADDED Viewed

	@@ -0,0 +1 @@


1	+ """Nothing (for ruff)."""

src/reachy_mini_conversation_app/audio/head_wobbler.py ADDED Viewed

	@@ -0,0 +1,181 @@

+"""Moves head given audio samples."""
+import time
+import queue
+import base64
+import logging
+import threading
+from typing import Tuple
+from collections.abc import Callable
+import numpy as np
+from numpy.typing import NDArray
+from reachy_mini_conversation_app.audio.speech_tapper import HOP_MS, SwayRollRT
+SAMPLE_RATE = 24000
+MOVEMENT_LATENCY_S = 0.2  # seconds between audio and robot movement
+logger = logging.getLogger(__name__)
+class HeadWobbler:
+    """Converts audio deltas (base64) into head movement offsets."""
+    def __init__(self, set_speech_offsets: Callable[[Tuple[float, float, float, float, float, float]], None]) -> None:
+        """Initialize the head wobbler."""
+        self._apply_offsets = set_speech_offsets
+        self._base_ts: float | None = None
+        self._hops_done: int = 0
+        self.audio_queue: "queue.Queue[Tuple[int, int, NDArray[np.int16]]]" = queue.Queue()
+        self.sway = SwayRollRT()
+        # Synchronization primitives
+        self._state_lock = threading.Lock()
+        self._sway_lock = threading.Lock()
+        self._generation = 0
+        self._stop_event = threading.Event()
+        self._thread: threading.Thread | None = None
+    def feed(self, delta_b64: str) -> None:
+        """Thread-safe: push audio into the consumer queue."""
+        buf = np.frombuffer(base64.b64decode(delta_b64), dtype=np.int16).reshape(1, -1)
+        with self._state_lock:
+            generation = self._generation
+        self.audio_queue.put((generation, SAMPLE_RATE, buf))
+    def start(self) -> None:
+        """Start the head wobbler loop in a thread."""
+        self._stop_event.clear()
+        self._thread = threading.Thread(target=self.working_loop, daemon=True)
+        self._thread.start()
+        logger.debug("Head wobbler started")
+    def stop(self) -> None:
+        """Stop the head wobbler loop."""
+        self._stop_event.set()
+        if self._thread is not None:
+            self._thread.join()
+        logger.debug("Head wobbler stopped")
+    def working_loop(self) -> None:
+        """Convert audio deltas into head movement offsets."""
+        hop_dt = HOP_MS / 1000.0
+        logger.debug("Head wobbler thread started")
+        while not self._stop_event.is_set():
+            queue_ref = self.audio_queue
+            try:
+                chunk_generation, sr, chunk = queue_ref.get_nowait()  # (gen, sr, data)
+            except queue.Empty:
+                # avoid while to never exit
+                time.sleep(MOVEMENT_LATENCY_S)
+                continue
+            try:
+                with self._state_lock:
+                    current_generation = self._generation
+                if chunk_generation != current_generation:
+                    continue
+                if self._base_ts is None:
+                    with self._state_lock:
+                        if self._base_ts is None:
+                            self._base_ts = time.monotonic()
+                pcm = np.asarray(chunk).squeeze(0)
+                with self._sway_lock:
+                    results = self.sway.feed(pcm, sr)
+                i = 0
+                while i < len(results):
+                    with self._state_lock:
+                        if self._generation != current_generation:
+                            break
+                        base_ts = self._base_ts
+                        hops_done = self._hops_done
+                    if base_ts is None:
+                        base_ts = time.monotonic()
+                        with self._state_lock:
+                            if self._base_ts is None:
+                                self._base_ts = base_ts
+                                hops_done = self._hops_done
+                    target = base_ts + MOVEMENT_LATENCY_S + hops_done * hop_dt
+                    now = time.monotonic()
+                    if now - target >= hop_dt:
+                        lag_hops = int((now - target) / hop_dt)
+                        drop = min(lag_hops, len(results) - i - 1)
+                        if drop > 0:
+                            with self._state_lock:
+                                self._hops_done += drop
+                                hops_done = self._hops_done
+                            i += drop
+                            continue
+                    if target > now:
+                        time.sleep(target - now)
+                        with self._state_lock:
+                            if self._generation != current_generation:
+                                break
+                    r = results[i]
+                    offsets = (
+                        r["x_mm"] / 1000.0,
+                        r["y_mm"] / 1000.0,
+                        r["z_mm"] / 1000.0,
+                        r["roll_rad"],
+                        r["pitch_rad"],
+                        r["yaw_rad"],
+                    )
+                    with self._state_lock:
+                        if self._generation != current_generation:
+                            break
+                    self._apply_offsets(offsets)
+                    with self._state_lock:
+                        self._hops_done += 1
+                    i += 1
+            finally:
+                queue_ref.task_done()
+        logger.debug("Head wobbler thread exited")
+    '''
+    def drain_audio_queue(self) -> None:
+        """Empty the audio queue."""
+        try:
+            while True:
+                self.audio_queue.get_nowait()
+        except QueueEmpty:
+            pass
+    '''
+    def reset(self) -> None:
+        """Reset the internal state."""
+        with self._state_lock:
+            self._generation += 1
+            self._base_ts = None
+            self._hops_done = 0
+        # Drain any queued audio chunks from previous generations
+        drained_any = False
+        while True:
+            try:
+                _, _, _ = self.audio_queue.get_nowait()
+            except queue.Empty:
+                break
+            else:
+                drained_any = True
+                self.audio_queue.task_done()
+        with self._sway_lock:
+            self.sway.reset()
+        if drained_any:
+            logger.debug("Head wobbler queue drained during reset")

src/reachy_mini_conversation_app/audio/speech_tapper.py ADDED Viewed

	@@ -0,0 +1,268 @@

+from __future__ import annotations
+import math
+from typing import Any, Dict, List
+from itertools import islice
+from collections import deque
+import numpy as np
+from numpy.typing import NDArray
+# Tunables
+SR = 16_000
+FRAME_MS = 20
+HOP_MS = 50
+SWAY_MASTER = 1.5
+SENS_DB_OFFSET = +4.0
+VAD_DB_ON = -35.0
+VAD_DB_OFF = -45.0
+VAD_ATTACK_MS = 40
+VAD_RELEASE_MS = 250
+ENV_FOLLOW_GAIN = 0.65
+SWAY_F_PITCH = 2.2
+SWAY_A_PITCH_DEG = 4.5
+SWAY_F_YAW = 0.6
+SWAY_A_YAW_DEG = 7.5
+SWAY_F_ROLL = 1.3
+SWAY_A_ROLL_DEG = 2.25
+SWAY_F_X = 0.35
+SWAY_A_X_MM = 4.5
+SWAY_F_Y = 0.45
+SWAY_A_Y_MM = 3.75
+SWAY_F_Z = 0.25
+SWAY_A_Z_MM = 2.25
+SWAY_DB_LOW = -46.0
+SWAY_DB_HIGH = -18.0
+LOUDNESS_GAMMA = 0.9
+SWAY_ATTACK_MS = 50
+SWAY_RELEASE_MS = 250
+# Derived
+FRAME = int(SR * FRAME_MS / 1000)
+HOP = int(SR * HOP_MS / 1000)
+ATTACK_FR = max(1, int(VAD_ATTACK_MS / HOP_MS))
+RELEASE_FR = max(1, int(VAD_RELEASE_MS / HOP_MS))
+SWAY_ATTACK_FR = max(1, int(SWAY_ATTACK_MS / HOP_MS))
+SWAY_RELEASE_FR = max(1, int(SWAY_RELEASE_MS / HOP_MS))
+def _rms_dbfs(x: NDArray[np.float32]) -> float:
+    """Root-mean-square in dBFS for float32 mono array in [-1,1]."""
+    # numerically stable rms (avoid overflow)
+    x = x.astype(np.float32, copy=False)
+    rms = np.sqrt(np.mean(x * x, dtype=np.float32) + 1e-12, dtype=np.float32)
+    return float(20.0 * math.log10(float(rms) + 1e-12))
+def _loudness_gain(db: float, offset: float = SENS_DB_OFFSET) -> float:
+    """Normalize dB into [0,1] with gamma; clipped to [0,1]."""
+    t = (db + offset - SWAY_DB_LOW) / (SWAY_DB_HIGH - SWAY_DB_LOW)
+    if t < 0.0:
+        t = 0.0
+    elif t > 1.0:
+        t = 1.0
+    return t**LOUDNESS_GAMMA if LOUDNESS_GAMMA != 1.0 else t
+def _to_float32_mono(x: NDArray[Any]) -> NDArray[np.float32]:
+    """Convert arbitrary PCM array to float32 mono in [-1,1].
+    Accepts shapes: (N,), (1,N), (N,1), (C,N), (N,C).
+    """
+    a = np.asarray(x)
+    if a.ndim == 0:
+        return np.zeros(0, dtype=np.float32)
+    # If 2D, decide which axis is channels (prefer small first dim)
+    if a.ndim == 2:
+        # e.g., (channels, samples) if channels is small (<=8)
+        if a.shape[0] <= 8 and a.shape[0] <= a.shape[1]:
+            a = np.mean(a, axis=0)
+        else:
+            a = np.mean(a, axis=1)
+    elif a.ndim > 2:
+        a = np.mean(a.reshape(a.shape[0], -1), axis=0)
+    # Now 1D, cast/scale
+    if np.issubdtype(a.dtype, np.floating):
+        return a.astype(np.float32, copy=False)
+    # integer PCM
+    info = np.iinfo(a.dtype)
+    scale = float(max(-info.min, info.max))
+    return a.astype(np.float32) / (scale if scale != 0.0 else 1.0)
+def _resample_linear(x: NDArray[np.float32], sr_in: int, sr_out: int) -> NDArray[np.float32]:
+    """Lightweight linear resampler for short buffers."""
+    if sr_in == sr_out or x.size == 0:
+        return x
+    # guard tiny sizes
+    n_out = int(round(x.size * sr_out / sr_in))
+    if n_out <= 1:
+        return np.zeros(0, dtype=np.float32)
+    t_in = np.linspace(0.0, 1.0, num=x.size, dtype=np.float32, endpoint=True)
+    t_out = np.linspace(0.0, 1.0, num=n_out, dtype=np.float32, endpoint=True)
+    return np.interp(t_out, t_in, x).astype(np.float32, copy=False)
+class SwayRollRT:
+    """Feed audio chunks → per-hop sway outputs.
+    Usage:
+        rt = SwayRollRT()
+        rt.feed(pcm_int16_or_float, sr) -> List[dict]
+    """
+    def __init__(self, rng_seed: int = 7):
+        """Initialize state."""
+        self._seed = int(rng_seed)
+        self.samples: deque[float] = deque(maxlen=10 * SR)  # sliding window for VAD/env
+        self.carry: NDArray[np.float32] = np.zeros(0, dtype=np.float32)
+        self.vad_on = False
+        self.vad_above = 0
+        self.vad_below = 0
+        self.sway_env = 0.0
+        self.sway_up = 0
+        self.sway_down = 0
+        rng = np.random.default_rng(self._seed)
+        self.phase_pitch = float(rng.random() * 2 * math.pi)
+        self.phase_yaw = float(rng.random() * 2 * math.pi)
+        self.phase_roll = float(rng.random() * 2 * math.pi)
+        self.phase_x = float(rng.random() * 2 * math.pi)
+        self.phase_y = float(rng.random() * 2 * math.pi)
+        self.phase_z = float(rng.random() * 2 * math.pi)
+        self.t = 0.0
+    def reset(self) -> None:
+        """Reset state (VAD/env/buffers/time) but keep initial phases/seed."""
+        self.samples.clear()
+        self.carry = np.zeros(0, dtype=np.float32)
+        self.vad_on = False
+        self.vad_above = 0
+        self.vad_below = 0
+        self.sway_env = 0.0
+        self.sway_up = 0
+        self.sway_down = 0
+        self.t = 0.0
+    def feed(self, pcm: NDArray[Any], sr: int | None) -> List[Dict[str, float]]:
+        """Stream in PCM chunk. Returns a list of sway dicts, one per hop (HOP_MS).
+        Args:
+            pcm: np.ndarray, shape (N,) or (C,N)/(N,C); int or float.
+            sr:  sample rate of `pcm` (None -> assume SR).
+        """
+        sr_in = SR if sr is None else int(sr)
+        x = _to_float32_mono(pcm)
+        if x.size == 0:
+            return []
+        if sr_in != SR:
+            x = _resample_linear(x, sr_in, SR)
+            if x.size == 0:
+                return []
+        # append to carry and consume fixed HOP chunks
+        if self.carry.size:
+            self.carry = np.concatenate([self.carry, x])
+        else:
+            self.carry = x
+        out: List[Dict[str, float]] = []
+        while self.carry.size >= HOP:
+            hop = self.carry[:HOP]
+            remaining: NDArray[np.float32] = self.carry[HOP:]
+            self.carry = remaining
+            # keep sliding window for VAD/env computation
+            # (deque accepts any iterable; list() for small HOP is fine)
+            self.samples.extend(hop.tolist())
+            if len(self.samples) < FRAME:
+                self.t += HOP_MS / 1000.0
+                continue
+            frame = np.fromiter(
+                islice(self.samples, len(self.samples) - FRAME, len(self.samples)),
+                dtype=np.float32,
+                count=FRAME,
+            )
+            db = _rms_dbfs(frame)
+            # VAD with hysteresis + attack/release
+            if db >= VAD_DB_ON:
+                self.vad_above += 1
+                self.vad_below = 0
+                if not self.vad_on and self.vad_above >= ATTACK_FR:
+                    self.vad_on = True
+            elif db <= VAD_DB_OFF:
+                self.vad_below += 1
+                self.vad_above = 0
+                if self.vad_on and self.vad_below >= RELEASE_FR:
+                    self.vad_on = False
+            if self.vad_on:
+                self.sway_up = min(SWAY_ATTACK_FR, self.sway_up + 1)
+                self.sway_down = 0
+            else:
+                self.sway_down = min(SWAY_RELEASE_FR, self.sway_down + 1)
+                self.sway_up = 0
+            up = self.sway_up / SWAY_ATTACK_FR
+            down = 1.0 - (self.sway_down / SWAY_RELEASE_FR)
+            target = up if self.vad_on else down
+            self.sway_env += ENV_FOLLOW_GAIN * (target - self.sway_env)
+            # clamp
+            if self.sway_env < 0.0:
+                self.sway_env = 0.0
+            elif self.sway_env > 1.0:
+                self.sway_env = 1.0
+            loud = _loudness_gain(db) * SWAY_MASTER
+            env = self.sway_env
+            self.t += HOP_MS / 1000.0
+            # oscillators
+            pitch = (
+                math.radians(SWAY_A_PITCH_DEG)
+                * loud
+                * env
+                * math.sin(2 * math.pi * SWAY_F_PITCH * self.t + self.phase_pitch)
+            )
+            yaw = (
+                math.radians(SWAY_A_YAW_DEG)
+                * loud
+                * env
+                * math.sin(2 * math.pi * SWAY_F_YAW * self.t + self.phase_yaw)
+            )
+            roll = (
+                math.radians(SWAY_A_ROLL_DEG)
+                * loud
+                * env
+                * math.sin(2 * math.pi * SWAY_F_ROLL * self.t + self.phase_roll)
+            )
+            x_mm = SWAY_A_X_MM * loud * env * math.sin(2 * math.pi * SWAY_F_X * self.t + self.phase_x)
+            y_mm = SWAY_A_Y_MM * loud * env * math.sin(2 * math.pi * SWAY_F_Y * self.t + self.phase_y)
+            z_mm = SWAY_A_Z_MM * loud * env * math.sin(2 * math.pi * SWAY_F_Z * self.t + self.phase_z)
+            out.append(
+                {
+                    "pitch_rad": pitch,
+                    "yaw_rad": yaw,
+                    "roll_rad": roll,
+                    "pitch_deg": math.degrees(pitch),
+                    "yaw_deg": math.degrees(yaw),
+                    "roll_deg": math.degrees(roll),
+                    "x_mm": x_mm,
+                    "y_mm": y_mm,
+                    "z_mm": z_mm,
+                },
+            )
+        return out

src/reachy_mini_conversation_app/camera_worker.py ADDED Viewed

	@@ -0,0 +1,241 @@

+"""Camera worker thread with frame buffering and face tracking.
+Ported from main_works.py camera_worker() function to provide:
+- 30Hz+ camera polling with thread-safe frame buffering
+- Face tracking integration with smooth interpolation
+- Latest frame always available for tools
+"""
+import time
+import logging
+import threading
+from typing import Any, List, Tuple
+import numpy as np
+from numpy.typing import NDArray
+from scipy.spatial.transform import Rotation as R
+from reachy_mini import ReachyMini
+from reachy_mini.utils.interpolation import linear_pose_interpolation
+logger = logging.getLogger(__name__)
+class CameraWorker:
+    """Thread-safe camera worker with frame buffering and face tracking."""
+    def __init__(self, reachy_mini: ReachyMini, head_tracker: Any = None) -> None:
+        """Initialize."""
+        self.reachy_mini = reachy_mini
+        self.head_tracker = head_tracker
+        # Thread-safe frame storage
+        self.latest_frame: NDArray[np.uint8] | None = None
+        self.frame_lock = threading.Lock()
+        self._stop_event = threading.Event()
+        self._thread: threading.Thread | None = None
+        # Face tracking state
+        self.is_head_tracking_enabled = True
+        self.face_tracking_offsets: List[float] = [
+            0.0,
+            0.0,
+            0.0,
+            0.0,
+            0.0,
+            0.0,
+        ]  # x, y, z, roll, pitch, yaw
+        self.face_tracking_lock = threading.Lock()
+        # Face tracking timing variables (same as main_works.py)
+        self.last_face_detected_time: float | None = None
+        self.interpolation_start_time: float | None = None
+        self.interpolation_start_pose: NDArray[np.float32] | None = None
+        self.face_lost_delay = 2.0  # seconds to wait before starting interpolation
+        self.interpolation_duration = 1.0  # seconds to interpolate back to neutral
+        # Track state changes
+        self.previous_head_tracking_state = self.is_head_tracking_enabled
+    def get_latest_frame(self) -> NDArray[np.uint8] | None:
+        """Get the latest frame (thread-safe)."""
+        with self.frame_lock:
+            if self.latest_frame is None:
+                return None
+            # Return a copy in original BGR format (OpenCV native)
+            return self.latest_frame.copy()
+    def get_face_tracking_offsets(
+        self,
+    ) -> Tuple[float, float, float, float, float, float]:
+        """Get current face tracking offsets (thread-safe)."""
+        with self.face_tracking_lock:
+            offsets = self.face_tracking_offsets
+            return (offsets[0], offsets[1], offsets[2], offsets[3], offsets[4], offsets[5])
+    def set_head_tracking_enabled(self, enabled: bool) -> None:
+        """Enable/disable head tracking."""
+        self.is_head_tracking_enabled = enabled
+        logger.info(f"Head tracking {'enabled' if enabled else 'disabled'}")
+    def start(self) -> None:
+        """Start the camera worker loop in a thread."""
+        self._stop_event.clear()
+        self._thread = threading.Thread(target=self.working_loop, daemon=True)
+        self._thread.start()
+        logger.debug("Camera worker started")
+    def stop(self) -> None:
+        """Stop the camera worker loop."""
+        self._stop_event.set()
+        if self._thread is not None:
+            self._thread.join()
+        logger.debug("Camera worker stopped")
+    def working_loop(self) -> None:
+        """Enable the camera worker loop.
+        Ported from main_works.py camera_worker() with same logic.
+        """
+        logger.debug("Starting camera working loop")
+        # Initialize head tracker if available
+        neutral_pose = np.eye(4)  # Neutral pose (identity matrix)
+        self.previous_head_tracking_state = self.is_head_tracking_enabled
+        while not self._stop_event.is_set():
+            try:
+                current_time = time.time()
+                # Get frame from robot
+                frame = self.reachy_mini.media.get_frame()
+                if frame is not None:
+                    # Thread-safe frame storage
+                    with self.frame_lock:
+                        self.latest_frame = frame  # .copy()
+                    # Check if face tracking was just disabled
+                    if self.previous_head_tracking_state and not self.is_head_tracking_enabled:
+                        # Face tracking was just disabled - start interpolation to neutral
+                        self.last_face_detected_time = current_time  # Trigger the face-lost logic
+                        self.interpolation_start_time = None  # Will be set by the face-lost interpolation
+                        self.interpolation_start_pose = None
+                    # Update tracking state
+                    self.previous_head_tracking_state = self.is_head_tracking_enabled
+                    # Handle face tracking if enabled and head tracker available
+                    if self.is_head_tracking_enabled and self.head_tracker is not None:
+                        eye_center, _ = self.head_tracker.get_head_position(frame)
+                        if eye_center is not None:
+                            # Face detected - immediately switch to tracking
+                            self.last_face_detected_time = current_time
+                            self.interpolation_start_time = None  # Stop any interpolation
+                            # Convert normalized coordinates to pixel coordinates
+                            h, w, _ = frame.shape
+                            eye_center_norm = (eye_center + 1) / 2
+                            eye_center_pixels = [
+                                eye_center_norm[0] * w,
+                                eye_center_norm[1] * h,
+                            ]
+                            # Get the head pose needed to look at the target, but don't perform movement
+                            target_pose = self.reachy_mini.look_at_image(
+                                eye_center_pixels[0],
+                                eye_center_pixels[1],
+                                duration=0.0,
+                                perform_movement=False,
+                            )
+                            # Extract translation and rotation from the target pose directly
+                            translation = target_pose[:3, 3]
+                            rotation = R.from_matrix(target_pose[:3, :3]).as_euler("xyz", degrees=False)
+                            # Scale down translation and rotation because smaller FOV
+                            translation *= 0.6
+                            rotation *= 0.6
+                            # Thread-safe update of face tracking offsets (use pose as-is)
+                            with self.face_tracking_lock:
+                                self.face_tracking_offsets = [
+                                    translation[0],
+                                    translation[1],
+                                    translation[2],  # x, y, z
+                                    rotation[0],
+                                    rotation[1],
+                                    rotation[2],  # roll, pitch, yaw
+                                ]
+                        # No face detected while tracking enabled - set face lost timestamp
+                        elif self.last_face_detected_time is None or self.last_face_detected_time == current_time:
+                            # Only update if we haven't already set a face lost time
+                            # (current_time check prevents overriding the disable-triggered timestamp)
+                            pass
+                    # Handle smooth interpolation (works for both face-lost and tracking-disabled cases)
+                    if self.last_face_detected_time is not None:
+                        time_since_face_lost = current_time - self.last_face_detected_time
+                        if time_since_face_lost >= self.face_lost_delay:
+                            # Start interpolation if not already started
+                            if self.interpolation_start_time is None:
+                                self.interpolation_start_time = current_time
+                                # Capture current pose as start of interpolation
+                                with self.face_tracking_lock:
+                                    current_translation = self.face_tracking_offsets[:3]
+                                    current_rotation_euler = self.face_tracking_offsets[3:]
+                                    # Convert to 4x4 pose matrix
+                                    pose_matrix = np.eye(4, dtype=np.float32)
+                                    pose_matrix[:3, 3] = current_translation
+                                    pose_matrix[:3, :3] = R.from_euler(
+                                        "xyz",
+                                        current_rotation_euler,
+                                    ).as_matrix()
+                                    self.interpolation_start_pose = pose_matrix
+                            # Calculate interpolation progress (t from 0 to 1)
+                            elapsed_interpolation = current_time - self.interpolation_start_time
+                            t = min(1.0, elapsed_interpolation / self.interpolation_duration)
+                            # Interpolate between current pose and neutral pose
+                            interpolated_pose = linear_pose_interpolation(
+                                self.interpolation_start_pose,
+                                neutral_pose,
+                                t,
+                            )
+                            # Extract translation and rotation from interpolated pose
+                            translation = interpolated_pose[:3, 3]
+                            rotation = R.from_matrix(interpolated_pose[:3, :3]).as_euler("xyz", degrees=False)
+                            # Thread-safe update of face tracking offsets
+                            with self.face_tracking_lock:
+                                self.face_tracking_offsets = [
+                                    translation[0],
+                                    translation[1],
+                                    translation[2],  # x, y, z
+                                    rotation[0],
+                                    rotation[1],
+                                    rotation[2],  # roll, pitch, yaw
+                                ]
+                            # If interpolation is complete, reset timing
+                            if t >= 1.0:
+                                self.last_face_detected_time = None
+                                self.interpolation_start_time = None
+                                self.interpolation_start_pose = None
+                        # else: Keep current offsets (within 2s delay period)
+                # Small sleep to prevent excessive CPU usage (same as main_works.py)
+                time.sleep(0.04)
+            except Exception as e:
+                logger.error(f"Camera worker error: {e}")
+                time.sleep(0.1)  # Longer sleep on error
+        logger.debug("Camera worker thread exited")

src/reachy_mini_conversation_app/config.py ADDED Viewed

	@@ -0,0 +1,60 @@

+import os
+import logging
+from dotenv import find_dotenv, load_dotenv
+logger = logging.getLogger(__name__)
+# Locate .env file (search upward from current working directory)
+dotenv_path = find_dotenv(usecwd=True)
+if dotenv_path:
+    # Load .env and override environment variables
+    load_dotenv(dotenv_path=dotenv_path, override=True)
+    logger.info(f"Configuration loaded from {dotenv_path}")
+else:
+    logger.warning("No .env file found, using environment variables")
+class Config:
+    """Configuration class for the conversation app."""
+    # Required
+    OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")  # The key is downloaded in console.py if needed
+    # Optional
+    MODEL_NAME = os.getenv("MODEL_NAME", "gpt-realtime")
+    HF_HOME = os.getenv("HF_HOME", "./cache")
+    LOCAL_VISION_MODEL = os.getenv("LOCAL_VISION_MODEL", "HuggingFaceTB/SmolVLM2-2.2B-Instruct")
+    HF_TOKEN = os.getenv("HF_TOKEN")  # Optional, falls back to hf auth login if not set
+    logger.debug(f"Model: {MODEL_NAME}, HF_HOME: {HF_HOME}, Vision Model: {LOCAL_VISION_MODEL}")
+    REACHY_MINI_CUSTOM_PROFILE = os.getenv("REACHY_MINI_CUSTOM_PROFILE")
+    logger.debug(f"Custom Profile: {REACHY_MINI_CUSTOM_PROFILE}")
+config = Config()
+def set_custom_profile(profile: str | None) -> None:
+    """Update the selected custom profile at runtime and expose it via env.
+    This ensures modules that read `config` and code that inspects the
+    environment see a consistent value.
+    """
+    try:
+        config.REACHY_MINI_CUSTOM_PROFILE = profile
+    except Exception:
+        pass
+    try:
+        import os as _os
+        if profile:
+            _os.environ["REACHY_MINI_CUSTOM_PROFILE"] = profile
+        else:
+            # Remove to reflect default
+            _os.environ.pop("REACHY_MINI_CUSTOM_PROFILE", None)
+    except Exception:
+        pass

src/reachy_mini_conversation_app/console.py ADDED Viewed

	@@ -0,0 +1,499 @@

+"""Bidirectional local audio stream with optional settings UI.
+In headless mode, there is no Gradio UI. If the OpenAI API key is not
+available via environment/.env, we expose a minimal settings page via the
+Reachy Mini Apps settings server to let non-technical users enter it.
+The settings UI is served from this package's ``static/`` folder and offers a
+single password field to set ``OPENAI_API_KEY``. Once set, we persist it to the
+app instance's ``.env`` file (if available) and proceed to start streaming.
+"""
+import os
+import sys
+import time
+import asyncio
+import logging
+from typing import List, Optional
+from pathlib import Path
+from fastrtc import AdditionalOutputs, audio_to_float32
+from scipy.signal import resample
+from reachy_mini import ReachyMini
+from reachy_mini.media.media_manager import MediaBackend
+from reachy_mini_conversation_app.config import config
+from reachy_mini_conversation_app.openai_realtime import OpenaiRealtimeHandler
+from reachy_mini_conversation_app.headless_personality_ui import mount_personality_routes
+try:
+    # FastAPI is provided by the Reachy Mini Apps runtime
+    from fastapi import FastAPI, Response
+    from pydantic import BaseModel
+    from fastapi.responses import FileResponse, JSONResponse
+    from starlette.staticfiles import StaticFiles
+except Exception:  # pragma: no cover - only loaded when settings_app is used
+    FastAPI = object  # type: ignore
+    FileResponse = object  # type: ignore
+    JSONResponse = object  # type: ignore
+    StaticFiles = object  # type: ignore
+    BaseModel = object  # type: ignore
+logger = logging.getLogger(__name__)
+class LocalStream:
+    """LocalStream using Reachy Mini's recorder/player."""
+    def __init__(
+        self,
+        handler: OpenaiRealtimeHandler,
+        robot: ReachyMini,
+        *,
+        settings_app: Optional[FastAPI] = None,
+        instance_path: Optional[str] = None,
+    ):
+        """Initialize the stream with an OpenAI realtime handler and pipelines.
+        - ``settings_app``: the Reachy Mini Apps FastAPI to attach settings endpoints.
+        - ``instance_path``: directory where per-instance ``.env`` should be stored.
+        """
+        self.handler = handler
+        self._robot = robot
+        self._stop_event = asyncio.Event()
+        self._tasks: List[asyncio.Task[None]] = []
+        # Allow the handler to flush the player queue when appropriate.
+        self.handler._clear_queue = self.clear_audio_queue
+        self._settings_app: Optional[FastAPI] = settings_app
+        self._instance_path: Optional[str] = instance_path
+        self._settings_initialized = False
+        self._asyncio_loop = None
+    # ---- Settings UI (only when API key is missing) ----
+    def _read_env_lines(self, env_path: Path) -> list[str]:
+        """Load env file contents or a template as a list of lines."""
+        inst = env_path.parent
+        try:
+            if env_path.exists():
+                try:
+                    return env_path.read_text(encoding="utf-8").splitlines()
+                except Exception:
+                    return []
+            template_text = None
+            ex = inst / ".env.example"
+            if ex.exists():
+                try:
+                    template_text = ex.read_text(encoding="utf-8")
+                except Exception:
+                    template_text = None
+            if template_text is None:
+                try:
+                    cwd_example = Path.cwd() / ".env.example"
+                    if cwd_example.exists():
+                        template_text = cwd_example.read_text(encoding="utf-8")
+                except Exception:
+                    template_text = None
+            if template_text is None:
+                packaged = Path(__file__).parent / ".env.example"
+                if packaged.exists():
+                    try:
+                        template_text = packaged.read_text(encoding="utf-8")
+                    except Exception:
+                        template_text = None
+            return template_text.splitlines() if template_text else []
+        except Exception:
+            return []
+    def _persist_api_key(self, key: str) -> None:
+        """Persist API key to environment and instance ``.env`` if possible.
+        Behavior:
+        - Always sets ``OPENAI_API_KEY`` in process env and in-memory config.
+        - Writes/updates ``<instance_path>/.env``:
+          * If ``.env`` exists, replaces/append OPENAI_API_KEY line.
+          * Else, copies template from ``<instance_path>/.env.example`` when present,
+            otherwise falls back to the packaged template
+            ``reachy_mini_conversation_app/.env.example``.
+          * Ensures the resulting file contains the full template plus the key.
+        - Loads the written ``.env`` into the current process environment.
+        """
+        k = (key or "").strip()
+        if not k:
+            return
+        # Update live process env and config so consumers see it immediately
+        try:
+            os.environ["OPENAI_API_KEY"] = k
+        except Exception:  # best-effort
+            pass
+        try:
+            config.OPENAI_API_KEY = k
+        except Exception:
+            pass
+        if not self._instance_path:
+            return
+        try:
+            inst = Path(self._instance_path)
+            env_path = inst / ".env"
+            lines = self._read_env_lines(env_path)
+            replaced = False
+            for i, ln in enumerate(lines):
+                if ln.strip().startswith("OPENAI_API_KEY="):
+                    lines[i] = f"OPENAI_API_KEY={k}"
+                    replaced = True
+                    break
+            if not replaced:
+                lines.append(f"OPENAI_API_KEY={k}")
+            final_text = "\n".join(lines) + "\n"
+            env_path.write_text(final_text, encoding="utf-8")
+            logger.info("Persisted OPENAI_API_KEY to %s", env_path)
+            # Load the newly written .env into this process to ensure downstream imports see it
+            try:
+                from dotenv import load_dotenv
+                load_dotenv(dotenv_path=str(env_path), override=True)
+            except Exception:
+                pass
+        except Exception as e:
+            logger.warning("Failed to persist OPENAI_API_KEY: %s", e)
+    def _persist_personality(self, profile: Optional[str]) -> None:
+        """Persist the startup personality to the instance .env and config."""
+        selection = (profile or "").strip() or None
+        try:
+            from reachy_mini_conversation_app.config import set_custom_profile
+            set_custom_profile(selection)
+        except Exception:
+            pass
+        if not self._instance_path:
+            return
+        try:
+            env_path = Path(self._instance_path) / ".env"
+            lines = self._read_env_lines(env_path)
+            replaced = False
+            for i, ln in enumerate(list(lines)):
+                if ln.strip().startswith("REACHY_MINI_CUSTOM_PROFILE="):
+                    if selection:
+                        lines[i] = f"REACHY_MINI_CUSTOM_PROFILE={selection}"
+                    else:
+                        lines.pop(i)
+                    replaced = True
+                    break
+            if selection and not replaced:
+                lines.append(f"REACHY_MINI_CUSTOM_PROFILE={selection}")
+            if selection is None and not env_path.exists():
+                return
+            final_text = "\n".join(lines) + "\n"
+            env_path.write_text(final_text, encoding="utf-8")
+            logger.info("Persisted startup personality to %s", env_path)
+            try:
+                from dotenv import load_dotenv
+                load_dotenv(dotenv_path=str(env_path), override=True)
+            except Exception:
+                pass
+        except Exception as e:
+            logger.warning("Failed to persist REACHY_MINI_CUSTOM_PROFILE: %s", e)
+    def _read_persisted_personality(self) -> Optional[str]:
+        """Read persisted startup personality from instance .env (if any)."""
+        if not self._instance_path:
+            return None
+        env_path = Path(self._instance_path) / ".env"
+        try:
+            if env_path.exists():
+                for ln in env_path.read_text(encoding="utf-8").splitlines():
+                    if ln.strip().startswith("REACHY_MINI_CUSTOM_PROFILE="):
+                        _, _, val = ln.partition("=")
+                        v = val.strip()
+                        return v or None
+        except Exception:
+            pass
+        return None
+    def _init_settings_ui_if_needed(self) -> None:
+        """Attach minimal settings UI to the settings app.
+        Always mounts the UI when a settings_app is provided so that users
+        see a confirmation message even if the API key is already configured.
+        """
+        if self._settings_initialized:
+            return
+        if self._settings_app is None:
+            return
+        static_dir = Path(__file__).parent / "static"
+        index_file = static_dir / "index.html"
+        if hasattr(self._settings_app, "mount"):
+            try:
+                # Serve /static/* assets
+                self._settings_app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
+            except Exception:
+                pass
+        class ApiKeyPayload(BaseModel):
+            openai_api_key: str
+        # GET / -> index.html
+        @self._settings_app.get("/")
+        def _root() -> FileResponse:
+            return FileResponse(str(index_file))
+        # GET /favicon.ico -> optional, avoid noisy 404s on some browsers
+        @self._settings_app.get("/favicon.ico")
+        def _favicon() -> Response:
+            return Response(status_code=204)
+        # GET /status -> whether key is set
+        @self._settings_app.get("/status")
+        def _status() -> JSONResponse:
+            has_key = bool(config.OPENAI_API_KEY and str(config.OPENAI_API_KEY).strip())
+            return JSONResponse({"has_key": has_key})
+        # GET /ready -> whether backend finished loading tools
+        @self._settings_app.get("/ready")
+        def _ready() -> JSONResponse:
+            try:
+                mod = sys.modules.get("reachy_mini_conversation_app.tools.core_tools")
+                ready = bool(getattr(mod, "_TOOLS_INITIALIZED", False)) if mod else False
+            except Exception:
+                ready = False
+            return JSONResponse({"ready": ready})
+        # POST /openai_api_key -> set/persist key
+        @self._settings_app.post("/openai_api_key")
+        def _set_key(payload: ApiKeyPayload) -> JSONResponse:
+            key = (payload.openai_api_key or "").strip()
+            if not key:
+                return JSONResponse({"ok": False, "error": "empty_key"}, status_code=400)
+            self._persist_api_key(key)
+            return JSONResponse({"ok": True})
+        # POST /validate_api_key -> validate key without persisting it
+        @self._settings_app.post("/validate_api_key")
+        async def _validate_key(payload: ApiKeyPayload) -> JSONResponse:
+            key = (payload.openai_api_key or "").strip()
+            if not key:
+                return JSONResponse({"valid": False, "error": "empty_key"}, status_code=400)
+            # Try to validate by checking if we can fetch the models
+            try:
+                import httpx
+                headers = {"Authorization": f"Bearer {key}", "Content-Type": "application/json"}
+                async with httpx.AsyncClient(timeout=10.0) as client:
+                    response = await client.get("https://api.openai.com/v1/models", headers=headers)
+                    if response.status_code == 200:
+                        return JSONResponse({"valid": True})
+                    elif response.status_code == 401:
+                        return JSONResponse({"valid": False, "error": "invalid_api_key"}, status_code=401)
+                    else:
+                        return JSONResponse(
+                            {"valid": False, "error": "validation_failed"}, status_code=response.status_code
+                        )
+            except Exception as e:
+                logger.warning(f"API key validation failed: {e}")
+                return JSONResponse({"valid": False, "error": "validation_error"}, status_code=500)
+        self._settings_initialized = True
+    def launch(self) -> None:
+        """Start the recorder/player and run the async processing loops.
+        If the OpenAI key is missing, expose a tiny settings UI via the
+        Reachy Mini settings server to collect it before starting streams.
+        """
+        self._stop_event.clear()
+        # Try to load an existing instance .env first (covers subsequent runs)
+        if self._instance_path:
+            try:
+                from dotenv import load_dotenv
+                from reachy_mini_conversation_app.config import set_custom_profile
+                env_path = Path(self._instance_path) / ".env"
+                if env_path.exists():
+                    load_dotenv(dotenv_path=str(env_path), override=True)
+                    # Update config with newly loaded values
+                    new_key = os.getenv("OPENAI_API_KEY", "").strip()
+                    if new_key:
+                        try:
+                            config.OPENAI_API_KEY = new_key
+                        except Exception:
+                            pass
+                    new_profile = os.getenv("REACHY_MINI_CUSTOM_PROFILE")
+                    if new_profile is not None:
+                        try:
+                            set_custom_profile(new_profile.strip() or None)
+                        except Exception:
+                            pass
+            except Exception:
+                pass
+        # If key is still missing, try to download one from HuggingFace
+        if not (config.OPENAI_API_KEY and str(config.OPENAI_API_KEY).strip()):
+            logger.info("OPENAI_API_KEY not set, attempting to download from HuggingFace...")
+            try:
+                from gradio_client import Client
+                client = Client("HuggingFaceM4/gradium_setup", verbose=False)
+                key, status = client.predict(api_name="/claim_b_key")
+                if key and key.strip():
+                    logger.info("Successfully downloaded API key from HuggingFace")
+                    # Persist it immediately
+                    self._persist_api_key(key)
+            except Exception as e:
+                logger.warning(f"Failed to download API key from HuggingFace: {e}")
+        # Always expose settings UI if a settings app is available
+        # (do this AFTER loading/downloading the key so status endpoint sees the right value)
+        self._init_settings_ui_if_needed()
+        # If key is still missing -> wait until provided via the settings UI
+        if not (config.OPENAI_API_KEY and str(config.OPENAI_API_KEY).strip()):
+            logger.warning("OPENAI_API_KEY not found. Open the app settings page to enter it.")
+            # Poll until the key becomes available (set via the settings UI)
+            try:
+                while not (config.OPENAI_API_KEY and str(config.OPENAI_API_KEY).strip()):
+                    time.sleep(0.2)
+            except KeyboardInterrupt:
+                logger.info("Interrupted while waiting for API key.")
+                return
+        # Start media after key is set/available
+        self._robot.media.start_recording()
+        self._robot.media.start_playing()
+        time.sleep(1)  # give some time to the pipelines to start
+        async def runner() -> None:
+            # Capture loop for cross-thread personality actions
+            loop = asyncio.get_running_loop()
+            self._asyncio_loop = loop  # type: ignore[assignment]
+            # Mount personality routes now that loop and handler are available
+            try:
+                if self._settings_app is not None:
+                    mount_personality_routes(
+                        self._settings_app,
+                        self.handler,
+                        lambda: self._asyncio_loop,
+                        persist_personality=self._persist_personality,
+                        get_persisted_personality=self._read_persisted_personality,
+                    )
+            except Exception:
+                pass
+            self._tasks = [
+                asyncio.create_task(self.handler.start_up(), name="openai-handler"),
+                asyncio.create_task(self.record_loop(), name="stream-record-loop"),
+                asyncio.create_task(self.play_loop(), name="stream-play-loop"),
+            ]
+            try:
+                await asyncio.gather(*self._tasks)
+            except asyncio.CancelledError:
+                logger.info("Tasks cancelled during shutdown")
+            finally:
+                # Ensure handler connection is closed
+                await self.handler.shutdown()
+        asyncio.run(runner())
+    def close(self) -> None:
+        """Stop the stream and underlying media pipelines.
+        This method:
+        - Stops audio recording and playback first
+        - Sets the stop event to signal async loops to terminate
+        - Cancels all pending async tasks (openai-handler, record-loop, play-loop)
+        """
+        logger.info("Stopping LocalStream...")
+        # Stop media pipelines FIRST before cancelling async tasks
+        # This ensures clean shutdown before PortAudio cleanup
+        try:
+            self._robot.media.stop_recording()
+        except Exception as e:
+            logger.debug(f"Error stopping recording (may already be stopped): {e}")
+        try:
+            self._robot.media.stop_playing()
+        except Exception as e:
+            logger.debug(f"Error stopping playback (may already be stopped): {e}")
+        # Now signal async loops to stop
+        self._stop_event.set()
+        # Cancel all running tasks
+        for task in self._tasks:
+            if not task.done():
+                task.cancel()
+    def clear_audio_queue(self) -> None:
+        """Flush the player's appsrc to drop any queued audio immediately."""
+        logger.info("User intervention: flushing player queue")
+        if self._robot.media.backend == MediaBackend.GSTREAMER:
+            # Directly flush gstreamer audio pipe
+            self._robot.media.audio.clear_player()
+        elif self._robot.media.backend == MediaBackend.DEFAULT or self._robot.media.backend == MediaBackend.DEFAULT_NO_VIDEO:
+            self._robot.media.audio.clear_output_buffer()
+        self.handler.output_queue = asyncio.Queue()
+    async def record_loop(self) -> None:
+        """Read mic frames from the recorder and forward them to the handler."""
+        input_sample_rate = self._robot.media.get_input_audio_samplerate()
+        logger.debug(f"Audio recording started at {input_sample_rate} Hz")
+        while not self._stop_event.is_set():
+            audio_frame = self._robot.media.get_audio_sample()
+            if audio_frame is not None:
+                await self.handler.receive((input_sample_rate, audio_frame))
+            await asyncio.sleep(0)  # avoid busy loop
+    async def play_loop(self) -> None:
+        """Fetch outputs from the handler: log text and play audio frames."""
+        while not self._stop_event.is_set():
+            handler_output = await self.handler.emit()
+            if isinstance(handler_output, AdditionalOutputs):
+                for msg in handler_output.args:
+                    content = msg.get("content", "")
+                    if isinstance(content, str):
+                        logger.info(
+                            "role=%s content=%s",
+                            msg.get("role"),
+                            content if len(content) < 500 else content[:500] + "…",
+                        )
+            elif isinstance(handler_output, tuple):
+                input_sample_rate, audio_data = handler_output
+                output_sample_rate = self._robot.media.get_output_audio_samplerate()
+                # Reshape if needed
+                if audio_data.ndim == 2:
+                    # Scipy channels last convention
+                    if audio_data.shape[1] > audio_data.shape[0]:
+                        audio_data = audio_data.T
+                    # Multiple channels -> Mono channel
+                    if audio_data.shape[1] > 1:
+                        audio_data = audio_data[:, 0]
+                # Cast if needed
+                audio_frame = audio_to_float32(audio_data)
+                # Resample if needed
+                if input_sample_rate != output_sample_rate:
+                    audio_frame = resample(
+                        audio_frame,
+                        int(len(audio_frame) * output_sample_rate / input_sample_rate),
+                    )
+                self._robot.media.push_audio_sample(audio_frame)
+            else:
+                logger.debug("Ignoring output type=%s", type(handler_output).__name__)
+            await asyncio.sleep(0)  # yield to event loop

src/reachy_mini_conversation_app/dance_emotion_moves.py ADDED Viewed

	@@ -0,0 +1,154 @@

+"""Dance and emotion moves for the movement queue system.
+This module implements dance moves and emotions as Move objects that can be queued
+and executed sequentially by the MovementManager.
+"""
+from __future__ import annotations
+import logging
+from typing import Tuple
+import numpy as np
+from numpy.typing import NDArray
+from reachy_mini.motion.move import Move
+from reachy_mini.motion.recorded_move import RecordedMoves
+from reachy_mini_dances_library.dance_move import DanceMove
+logger = logging.getLogger(__name__)
+class DanceQueueMove(Move):  # type: ignore
+    """Wrapper for dance moves to work with the movement queue system."""
+    def __init__(self, move_name: str):
+        """Initialize a DanceQueueMove."""
+        self.dance_move = DanceMove(move_name)
+        self.move_name = move_name
+    @property
+    def duration(self) -> float:
+        """Duration property required by official Move interface."""
+        return float(self.dance_move.duration)
+    def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]:
+        """Evaluate dance move at time t."""
+        try:
+            # Get the pose from the dance move
+            head_pose, antennas, body_yaw = self.dance_move.evaluate(t)
+            # Convert to numpy array if antennas is tuple and return in official Move format
+            if isinstance(antennas, tuple):
+                antennas = np.array([antennas[0], antennas[1]])
+            return (head_pose, antennas, body_yaw)
+        except Exception as e:
+            logger.error(f"Error evaluating dance move '{self.move_name}' at t={t}: {e}")
+            # Return neutral pose on error
+            from reachy_mini.utils import create_head_pose
+            neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
+            return (neutral_head_pose, np.array([0.0, 0.0], dtype=np.float64), 0.0)
+class EmotionQueueMove(Move):  # type: ignore
+    """Wrapper for emotion moves to work with the movement queue system."""
+    def __init__(self, emotion_name: str, recorded_moves: RecordedMoves):
+        """Initialize an EmotionQueueMove."""
+        self.emotion_move = recorded_moves.get(emotion_name)
+        self.emotion_name = emotion_name
+    @property
+    def duration(self) -> float:
+        """Duration property required by official Move interface."""
+        return float(self.emotion_move.duration)
+    def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]:
+        """Evaluate emotion move at time t."""
+        try:
+            # Get the pose from the emotion move
+            head_pose, antennas, body_yaw = self.emotion_move.evaluate(t)
+            # Convert to numpy array if antennas is tuple and return in official Move format
+            if isinstance(antennas, tuple):
+                antennas = np.array([antennas[0], antennas[1]])
+            return (head_pose, antennas, body_yaw)
+        except Exception as e:
+            logger.error(f"Error evaluating emotion '{self.emotion_name}' at t={t}: {e}")
+            # Return neutral pose on error
+            from reachy_mini.utils import create_head_pose
+            neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
+            return (neutral_head_pose, np.array([0.0, 0.0], dtype=np.float64), 0.0)
+class GotoQueueMove(Move):  # type: ignore
+    """Wrapper for goto moves to work with the movement queue system."""
+    def __init__(
+        self,
+        target_head_pose: NDArray[np.float32],
+        start_head_pose: NDArray[np.float32] | None = None,
+        target_antennas: Tuple[float, float] = (0, 0),
+        start_antennas: Tuple[float, float] | None = None,
+        target_body_yaw: float = 0,
+        start_body_yaw: float | None = None,
+        duration: float = 1.0,
+    ):
+        """Initialize a GotoQueueMove."""
+        self._duration = duration
+        self.target_head_pose = target_head_pose
+        self.start_head_pose = start_head_pose
+        self.target_antennas = target_antennas
+        self.start_antennas = start_antennas or (0, 0)
+        self.target_body_yaw = target_body_yaw
+        self.start_body_yaw = start_body_yaw or 0
+    @property
+    def duration(self) -> float:
+        """Duration property required by official Move interface."""
+        return self._duration
+    def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]:
+        """Evaluate goto move at time t using linear interpolation."""
+        try:
+            from reachy_mini.utils import create_head_pose
+            from reachy_mini.utils.interpolation import linear_pose_interpolation
+            # Clamp t to [0, 1] for interpolation
+            t_clamped = max(0, min(1, t / self.duration))
+            # Use start pose if available, otherwise neutral
+            if self.start_head_pose is not None:
+                start_pose = self.start_head_pose
+            else:
+                start_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
+            # Interpolate head pose
+            head_pose = linear_pose_interpolation(start_pose, self.target_head_pose, t_clamped)
+            # Interpolate antennas - return as numpy array
+            antennas = np.array(
+                [
+                    self.start_antennas[0] + (self.target_antennas[0] - self.start_antennas[0]) * t_clamped,
+                    self.start_antennas[1] + (self.target_antennas[1] - self.start_antennas[1]) * t_clamped,
+                ],
+                dtype=np.float64,
+            )
+            # Interpolate body yaw
+            body_yaw = self.start_body_yaw + (self.target_body_yaw - self.start_body_yaw) * t_clamped
+            return (head_pose, antennas, body_yaw)
+        except Exception as e:
+            logger.error(f"Error evaluating goto move at t={t}: {e}")
+            # Return target pose on error - convert to float64
+            target_head_pose_f64 = self.target_head_pose.astype(np.float64)
+            target_antennas_array = np.array([self.target_antennas[0], self.target_antennas[1]], dtype=np.float64)
+            return (target_head_pose_f64, target_antennas_array, self.target_body_yaw)

src/reachy_mini_conversation_app/gradio_personality.py ADDED Viewed

	@@ -0,0 +1,301 @@

+"""Gradio personality UI components and wiring.
+This module encapsulates the UI elements and logic related to managing
+conversation "personalities" (profiles) so that `main.py` stays lean.
+"""
+from __future__ import annotations
+from typing import Any
+from pathlib import Path
+import gradio as gr
+from .config import config
+class PersonalityUI:
+    """Container for personality-related Gradio components."""
+    def __init__(self) -> None:
+        """Initialize the PersonalityUI instance."""
+        # Constants and paths
+        self.DEFAULT_OPTION = "(built-in default)"
+        self._profiles_root = Path(__file__).parent / "profiles"
+        self._tools_dir = Path(__file__).parent / "tools"
+        self._prompts_dir = Path(__file__).parent / "prompts"
+        # Components (initialized in create_components)
+        self.personalities_dropdown: gr.Dropdown
+        self.apply_btn: gr.Button
+        self.status_md: gr.Markdown
+        self.preview_md: gr.Markdown
+        self.person_name_tb: gr.Textbox
+        self.person_instr_ta: gr.TextArea
+        self.tools_txt_ta: gr.TextArea
+        self.voice_dropdown: gr.Dropdown
+        self.new_personality_btn: gr.Button
+        self.available_tools_cg: gr.CheckboxGroup
+        self.save_btn: gr.Button
+    # ---------- Filesystem helpers ----------
+    def _list_personalities(self) -> list[str]:
+        names: list[str] = []
+        try:
+            if self._profiles_root.exists():
+                for p in sorted(self._profiles_root.iterdir()):
+                    if p.name == "user_personalities":
+                        continue
+                    if p.is_dir() and (p / "instructions.txt").exists():
+                        names.append(p.name)
+                user_dir = self._profiles_root / "user_personalities"
+                if user_dir.exists():
+                    for p in sorted(user_dir.iterdir()):
+                        if p.is_dir() and (p / "instructions.txt").exists():
+                            names.append(f"user_personalities/{p.name}")
+        except Exception:
+            pass
+        return names
+    def _resolve_profile_dir(self, selection: str) -> Path:
+        return self._profiles_root / selection
+    def _read_instructions_for(self, name: str) -> str:
+        try:
+            if name == self.DEFAULT_OPTION:
+                default_file = self._prompts_dir / "default_prompt.txt"
+                if default_file.exists():
+                    return default_file.read_text(encoding="utf-8").strip()
+                return ""
+            target = self._resolve_profile_dir(name) / "instructions.txt"
+            if target.exists():
+                return target.read_text(encoding="utf-8").strip()
+            return ""
+        except Exception as e:
+            return f"Could not load instructions: {e}"
+    @staticmethod
+    def _sanitize_name(name: str) -> str:
+        import re
+        s = name.strip()
+        s = re.sub(r"\s+", "_", s)
+        s = re.sub(r"[^a-zA-Z0-9_-]", "", s)
+        return s
+    # ---------- Public API ----------
+    def create_components(self) -> None:
+        """Instantiate Gradio components for the personality UI."""
+        current_value = config.REACHY_MINI_CUSTOM_PROFILE or self.DEFAULT_OPTION
+        self.personalities_dropdown = gr.Dropdown(
+            label="Select personality",
+            choices=[self.DEFAULT_OPTION, *(self._list_personalities())],
+            value=current_value,
+        )
+        self.apply_btn = gr.Button("Apply personality")
+        self.status_md = gr.Markdown(visible=True)
+        self.preview_md = gr.Markdown(value=self._read_instructions_for(current_value))
+        self.person_name_tb = gr.Textbox(label="Personality name")
+        self.person_instr_ta = gr.TextArea(label="Personality instructions", lines=10)
+        self.tools_txt_ta = gr.TextArea(label="tools.txt", lines=10)
+        self.voice_dropdown = gr.Dropdown(label="Voice", choices=["cedar"], value="cedar")
+        self.new_personality_btn = gr.Button("New personality")
+        self.available_tools_cg = gr.CheckboxGroup(label="Available tools (helper)", choices=[], value=[])
+        self.save_btn = gr.Button("Save personality (instructions + tools)")
+    def additional_inputs_ordered(self) -> list[Any]:
+        """Return the additional inputs in the expected order for Stream."""
+        return [
+            self.personalities_dropdown,
+            self.apply_btn,
+            self.new_personality_btn,
+            self.status_md,
+            self.preview_md,
+            self.person_name_tb,
+            self.person_instr_ta,
+            self.tools_txt_ta,
+            self.voice_dropdown,
+            self.available_tools_cg,
+            self.save_btn,
+        ]
+    # ---------- Event wiring ----------
+    def wire_events(self, handler: Any, blocks: gr.Blocks) -> None:
+        """Attach event handlers to components within a Blocks context."""
+        async def _apply_personality(selected: str) -> tuple[str, str]:
+            profile = None if selected == self.DEFAULT_OPTION else selected
+            status = await handler.apply_personality(profile)
+            preview = self._read_instructions_for(selected)
+            return status, preview
+        def _read_voice_for(name: str) -> str:
+            try:
+                if name == self.DEFAULT_OPTION:
+                    return "cedar"
+                vf = self._resolve_profile_dir(name) / "voice.txt"
+                if vf.exists():
+                    v = vf.read_text(encoding="utf-8").strip()
+                    return v or "cedar"
+            except Exception:
+                pass
+            return "cedar"
+        async def _fetch_voices(selected: str) -> dict[str, Any]:
+            try:
+                voices = await handler.get_available_voices()
+                current = _read_voice_for(selected)
+                if current not in voices:
+                    current = "cedar"
+                return gr.update(choices=voices, value=current)
+            except Exception:
+                return gr.update(choices=["cedar"], value="cedar")
+        def _available_tools_for(selected: str) -> tuple[list[str], list[str]]:
+            shared: list[str] = []
+            try:
+                for py in self._tools_dir.glob("*.py"):
+                    if py.stem in {"__init__", "core_tools"}:
+                        continue
+                    shared.append(py.stem)
+            except Exception:
+                pass
+            local: list[str] = []
+            try:
+                if selected != self.DEFAULT_OPTION:
+                    for py in (self._profiles_root / selected).glob("*.py"):
+                        local.append(py.stem)
+            except Exception:
+                pass
+            return sorted(shared), sorted(local)
+        def _parse_enabled_tools(text: str) -> list[str]:
+            enabled: list[str] = []
+            for line in text.splitlines():
+                s = line.strip()
+                if not s or s.startswith("#"):
+                    continue
+                enabled.append(s)
+            return enabled
+        def _load_profile_for_edit(selected: str) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], str]:
+            instr = self._read_instructions_for(selected)
+            tools_txt = ""
+            if selected != self.DEFAULT_OPTION:
+                tp = self._resolve_profile_dir(selected) / "tools.txt"
+                if tp.exists():
+                    tools_txt = tp.read_text(encoding="utf-8")
+            shared, local = _available_tools_for(selected)
+            all_tools = sorted(set(shared + local))
+            enabled = _parse_enabled_tools(tools_txt)
+            status_text = f"Loaded profile '{selected}'."
+            return (
+                gr.update(value=instr),
+                gr.update(value=tools_txt),
+                gr.update(choices=all_tools, value=enabled),
+                status_text,
+            )
+        def _new_personality() -> tuple[
+            dict[str, Any], dict[str, Any], dict[str, Any], dict[str, Any], str, dict[str, Any]
+        ]:
+            try:
+                # Prefill with hints
+                instr_val = """# Write your instructions here\n# e.g., Keep responses concise and friendly."""
+                tools_txt_val = "# tools enabled for this profile\n"
+                return (
+                    gr.update(value=""),
+                    gr.update(value=instr_val),
+                    gr.update(value=tools_txt_val),
+                    gr.update(choices=sorted(_available_tools_for(self.DEFAULT_OPTION)[0]), value=[]),
+                    "Fill in a name, instructions and (optional) tools, then Save.",
+                    gr.update(value="cedar"),
+                )
+            except Exception:
+                return (
+                    gr.update(),
+                    gr.update(),
+                    gr.update(),
+                    gr.update(),
+                    "Failed to initialize new personality.",
+                    gr.update(),
+                )
+        def _save_personality(
+            name: str, instructions: str, tools_text: str, voice: str
+        ) -> tuple[dict[str, Any], dict[str, Any], str]:
+            name_s = self._sanitize_name(name)
+            if not name_s:
+                return gr.update(), gr.update(), "Please enter a valid name."
+            try:
+                target_dir = self._profiles_root / "user_personalities" / name_s
+                target_dir.mkdir(parents=True, exist_ok=True)
+                (target_dir / "instructions.txt").write_text(instructions.strip() + "\n", encoding="utf-8")
+                (target_dir / "tools.txt").write_text(tools_text.strip() + "\n", encoding="utf-8")
+                (target_dir / "voice.txt").write_text((voice or "cedar").strip() + "\n", encoding="utf-8")
+                choices = self._list_personalities()
+                value = f"user_personalities/{name_s}"
+                if value not in choices:
+                    choices.append(value)
+                return (
+                    gr.update(choices=[self.DEFAULT_OPTION, *sorted(choices)], value=value),
+                    gr.update(value=instructions),
+                    f"Saved personality '{name_s}'.",
+                )
+            except Exception as e:
+                return gr.update(), gr.update(), f"Failed to save personality: {e}"
+        def _sync_tools_from_checks(selected: list[str], current_text: str) -> dict[str, Any]:
+            comments = [ln for ln in current_text.splitlines() if ln.strip().startswith("#")]
+            body = "\n".join(selected)
+            out = ("\n".join(comments) + ("\n" if comments else "") + body).strip() + "\n"
+            return gr.update(value=out)
+        with blocks:
+            self.apply_btn.click(
+                fn=_apply_personality,
+                inputs=[self.personalities_dropdown],
+                outputs=[self.status_md, self.preview_md],
+            )
+            self.personalities_dropdown.change(
+                fn=_load_profile_for_edit,
+                inputs=[self.personalities_dropdown],
+                outputs=[self.person_instr_ta, self.tools_txt_ta, self.available_tools_cg, self.status_md],
+            )
+            blocks.load(
+                fn=_fetch_voices,
+                inputs=[self.personalities_dropdown],
+                outputs=[self.voice_dropdown],
+            )
+            self.available_tools_cg.change(
+                fn=_sync_tools_from_checks,
+                inputs=[self.available_tools_cg, self.tools_txt_ta],
+                outputs=[self.tools_txt_ta],
+            )
+            self.new_personality_btn.click(
+                fn=_new_personality,
+                inputs=[],
+                outputs=[
+                    self.person_name_tb,
+                    self.person_instr_ta,
+                    self.tools_txt_ta,
+                    self.available_tools_cg,
+                    self.status_md,
+                    self.voice_dropdown,
+                ],
+            )
+            self.save_btn.click(
+                fn=_save_personality,
+                inputs=[self.person_name_tb, self.person_instr_ta, self.tools_txt_ta, self.voice_dropdown],
+                outputs=[self.personalities_dropdown, self.person_instr_ta, self.status_md],
+            ).then(
+                fn=_apply_personality,
+                inputs=[self.personalities_dropdown],
+                outputs=[self.status_md, self.preview_md],
+            )

src/reachy_mini_conversation_app/headless_personality.py ADDED Viewed

	@@ -0,0 +1,102 @@

+"""Headless personality management (console-based).
+Provides an interactive CLI to browse, preview, apply, create and edit
+"personalities" (profiles) when running without Gradio.
+This module is intentionally not shared with the Gradio implementation to
+avoid coupling and keep responsibilities clear for headless mode.
+"""
+from __future__ import annotations
+from typing import List
+from pathlib import Path
+DEFAULT_OPTION = "(built-in default)"
+def _profiles_root() -> Path:
+    return Path(__file__).parent / "profiles"
+def _prompts_dir() -> Path:
+    return Path(__file__).parent / "prompts"
+def _tools_dir() -> Path:
+    return Path(__file__).parent / "tools"
+def _sanitize_name(name: str) -> str:
+    import re
+    s = name.strip()
+    s = re.sub(r"\s+", "_", s)
+    s = re.sub(r"[^a-zA-Z0-9_-]", "", s)
+    return s
+def list_personalities() -> List[str]:
+    """List available personality profile names."""
+    names: List[str] = []
+    root = _profiles_root()
+    try:
+        if root.exists():
+            for p in sorted(root.iterdir()):
+                if p.name == "user_personalities":
+                    continue
+                if p.is_dir() and (p / "instructions.txt").exists():
+                    names.append(p.name)
+        udir = root / "user_personalities"
+        if udir.exists():
+            for p in sorted(udir.iterdir()):
+                if p.is_dir() and (p / "instructions.txt").exists():
+                    names.append(f"user_personalities/{p.name}")
+    except Exception:
+        pass
+    return names
+def resolve_profile_dir(selection: str) -> Path:
+    """Resolve the directory path for the given profile selection."""
+    return _profiles_root() / selection
+def read_instructions_for(name: str) -> str:
+    """Read the instructions.txt content for the given profile name."""
+    try:
+        if name == DEFAULT_OPTION:
+            df = _prompts_dir() / "default_prompt.txt"
+            return df.read_text(encoding="utf-8").strip() if df.exists() else ""
+        target = resolve_profile_dir(name) / "instructions.txt"
+        return target.read_text(encoding="utf-8").strip() if target.exists() else ""
+    except Exception as e:
+        return f"Could not load instructions: {e}"
+def available_tools_for(selected: str) -> List[str]:
+    """List available tool modules for the given profile selection."""
+    shared: List[str] = []
+    try:
+        for py in _tools_dir().glob("*.py"):
+            if py.stem in {"__init__", "core_tools"}:
+                continue
+            shared.append(py.stem)
+    except Exception:
+        pass
+    local: List[str] = []
+    try:
+        if selected != DEFAULT_OPTION:
+            for py in resolve_profile_dir(selected).glob("*.py"):
+                local.append(py.stem)
+    except Exception:
+        pass
+    return sorted(set(shared + local))
+def _write_profile(name_s: str, instructions: str, tools_text: str, voice: str = "cedar") -> None:
+    target_dir = _profiles_root() / "user_personalities" / name_s
+    target_dir.mkdir(parents=True, exist_ok=True)
+    (target_dir / "instructions.txt").write_text(instructions.strip() + "\n", encoding="utf-8")
+    (target_dir / "tools.txt").write_text((tools_text or "").strip() + "\n", encoding="utf-8")
+    (target_dir / "voice.txt").write_text((voice or "cedar").strip() + "\n", encoding="utf-8")

src/reachy_mini_conversation_app/headless_personality_ui.py ADDED Viewed

	@@ -0,0 +1,276 @@

+"""Settings UI routes for headless personality management.
+Exposes REST endpoints on the provided FastAPI settings app. The
+implementation schedules backend actions (apply personality, fetch voices)
+onto the running LocalStream asyncio loop using the supplied get_loop
+callable to avoid cross-thread issues.
+"""
+from __future__ import annotations
+import asyncio
+import logging
+from typing import Any, Callable, Optional
+from fastapi import FastAPI
+from .config import config
+from .openai_realtime import OpenaiRealtimeHandler
+from .headless_personality import (
+    DEFAULT_OPTION,
+    _sanitize_name,
+    _write_profile,
+    list_personalities,
+    available_tools_for,
+    resolve_profile_dir,
+    read_instructions_for,
+)
+def mount_personality_routes(
+    app: FastAPI,
+    handler: OpenaiRealtimeHandler,
+    get_loop: Callable[[], asyncio.AbstractEventLoop | None],
+    *,
+    persist_personality: Callable[[Optional[str]], None] | None = None,
+    get_persisted_personality: Callable[[], Optional[str]] | None = None,
+) -> None:
+    """Register personality management endpoints on a FastAPI app."""
+    try:
+        from fastapi import Request
+        from pydantic import BaseModel
+        from fastapi.responses import JSONResponse
+    except Exception:  # pragma: no cover - only when settings app not available
+        return
+    class SavePayload(BaseModel):
+        name: str
+        instructions: str
+        tools_text: str
+        voice: Optional[str] = "cedar"
+    class ApplyPayload(BaseModel):
+        name: str
+        persist: Optional[bool] = False
+    def _startup_choice() -> Any:
+        """Return the persisted startup personality or default."""
+        try:
+            if get_persisted_personality is not None:
+                stored = get_persisted_personality()
+                if stored:
+                    return stored
+            env_val = getattr(config, "REACHY_MINI_CUSTOM_PROFILE", None)
+            if env_val:
+                return env_val
+        except Exception:
+            pass
+        return DEFAULT_OPTION
+    def _current_choice() -> str:
+        try:
+            cur = getattr(config, "REACHY_MINI_CUSTOM_PROFILE", None)
+            return cur or DEFAULT_OPTION
+        except Exception:
+            return DEFAULT_OPTION
+    @app.get("/personalities")
+    def _list() -> dict:  # type: ignore
+        choices = [DEFAULT_OPTION, *list_personalities()]
+        return {"choices": choices, "current": _current_choice(), "startup": _startup_choice()}
+    @app.get("/personalities/load")
+    def _load(name: str) -> dict:  # type: ignore
+        instr = read_instructions_for(name)
+        tools_txt = ""
+        voice = "cedar"
+        if name != DEFAULT_OPTION:
+            pdir = resolve_profile_dir(name)
+            tp = pdir / "tools.txt"
+            if tp.exists():
+                tools_txt = tp.read_text(encoding="utf-8")
+            vf = pdir / "voice.txt"
+            if vf.exists():
+                v = vf.read_text(encoding="utf-8").strip()
+                voice = v or "cedar"
+        avail = available_tools_for(name)
+        enabled = [ln.strip() for ln in tools_txt.splitlines() if ln.strip() and not ln.strip().startswith("#")]
+        return {
+            "instructions": instr,
+            "tools_text": tools_txt,
+            "voice": voice,
+            "available_tools": avail,
+            "enabled_tools": enabled,
+        }
+    @app.post("/personalities/save")
+    async def _save(request: Request) -> dict:  # type: ignore
+        # Accept raw JSON only to avoid validation-related 422s
+        try:
+            raw = await request.json()
+        except Exception:
+            raw = {}
+        name = str(raw.get("name", ""))
+        instructions = str(raw.get("instructions", ""))
+        tools_text = str(raw.get("tools_text", ""))
+        voice = str(raw.get("voice", "cedar")) if raw.get("voice") is not None else "cedar"
+        name_s = _sanitize_name(name)
+        if not name_s:
+            return JSONResponse({"ok": False, "error": "invalid_name"}, status_code=400)  # type: ignore
+        try:
+            logger.info(
+                "Headless save: name=%r voice=%r instr_len=%d tools_len=%d",
+                name_s,
+                voice,
+                len(instructions),
+                len(tools_text),
+            )
+            _write_profile(name_s, instructions, tools_text, voice or "cedar")
+            value = f"user_personalities/{name_s}"
+            choices = [DEFAULT_OPTION, *list_personalities()]
+            return {"ok": True, "value": value, "choices": choices}
+        except Exception as e:
+            return JSONResponse({"ok": False, "error": str(e)}, status_code=500)  # type: ignore
+    @app.post("/personalities/save_raw")
+    async def _save_raw(
+        request: Request,
+        name: Optional[str] = None,
+        instructions: Optional[str] = None,
+        tools_text: Optional[str] = None,
+        voice: Optional[str] = None,
+    ) -> dict:  # type: ignore
+        # Accept query params, form-encoded, or raw JSON
+        data = {"name": name, "instructions": instructions, "tools_text": tools_text, "voice": voice}
+        # Prefer form if present
+        try:
+            form = await request.form()
+            for k in ("name", "instructions", "tools_text", "voice"):
+                if k in form and form[k] is not None:
+                    data[k] = str(form[k])
+        except Exception:
+            pass
+        # Try JSON
+        try:
+            raw = await request.json()
+            if isinstance(raw, dict):
+                for k in ("name", "instructions", "tools_text", "voice"):
+                    if raw.get(k) is not None:
+                        data[k] = str(raw.get(k))
+        except Exception:
+            pass
+        name_s = _sanitize_name(str(data.get("name") or ""))
+        if not name_s:
+            return JSONResponse({"ok": False, "error": "invalid_name"}, status_code=400)  # type: ignore
+        instr = str(data.get("instructions") or "")
+        tools = str(data.get("tools_text") or "")
+        v = str(data.get("voice") or "cedar")
+        try:
+            logger.info(
+                "Headless save_raw: name=%r voice=%r instr_len=%d tools_len=%d", name_s, v, len(instr), len(tools)
+            )
+            _write_profile(name_s, instr, tools, v)
+            value = f"user_personalities/{name_s}"
+            choices = [DEFAULT_OPTION, *list_personalities()]
+            return {"ok": True, "value": value, "choices": choices}
+        except Exception as e:
+            return JSONResponse({"ok": False, "error": str(e)}, status_code=500)  # type: ignore
+    @app.get("/personalities/save_raw")
+    async def _save_raw_get(name: str, instructions: str = "", tools_text: str = "", voice: str = "cedar") -> dict:  # type: ignore
+        name_s = _sanitize_name(name)
+        if not name_s:
+            return JSONResponse({"ok": False, "error": "invalid_name"}, status_code=400)  # type: ignore
+        try:
+            logger.info(
+                "Headless save_raw(GET): name=%r voice=%r instr_len=%d tools_len=%d",
+                name_s,
+                voice,
+                len(instructions),
+                len(tools_text),
+            )
+            _write_profile(name_s, instructions, tools_text, voice or "cedar")
+            value = f"user_personalities/{name_s}"
+            choices = [DEFAULT_OPTION, *list_personalities()]
+            return {"ok": True, "value": value, "choices": choices}
+        except Exception as e:
+            return JSONResponse({"ok": False, "error": str(e)}, status_code=500)  # type: ignore
+    logger = logging.getLogger(__name__)
+    @app.post("/personalities/apply")
+    async def _apply(
+        payload: ApplyPayload | None = None,
+        name: str | None = None,
+        persist: Optional[bool] = None,
+        request: Optional[Request] = None,
+    ) -> dict:  # type: ignore
+        loop = get_loop()
+        if loop is None:
+            return JSONResponse({"ok": False, "error": "loop_unavailable"}, status_code=503)  # type: ignore
+        # Accept both JSON payload and query param for convenience
+        sel_name: Optional[str] = None
+        persist_flag = bool(persist) if persist is not None else False
+        if payload and getattr(payload, "name", None):
+            sel_name = payload.name
+            persist_flag = bool(getattr(payload, "persist", False))
+        elif name:
+            sel_name = name
+        elif request is not None:
+            try:
+                body = await request.json()
+                if isinstance(body, dict) and body.get("name"):
+                    sel_name = str(body.get("name"))
+                if isinstance(body, dict) and "persist" in body:
+                    persist_flag = bool(body.get("persist"))
+            except Exception:
+                sel_name = None
+        if request is not None:
+            try:
+                q_persist = request.query_params.get("persist")
+                if q_persist is not None:
+                    persist_flag = str(q_persist).lower() in {"1", "true", "yes", "on"}
+            except Exception:
+                pass
+        if not sel_name:
+            sel_name = DEFAULT_OPTION
+        async def _do_apply() -> str:
+            sel = None if sel_name == DEFAULT_OPTION else sel_name
+            status = await handler.apply_personality(sel)
+            return status
+        try:
+            logger.info("Headless apply: requested name=%r", sel_name)
+            fut = asyncio.run_coroutine_threadsafe(_do_apply(), loop)
+            status = fut.result(timeout=10)
+            persisted_choice = _startup_choice()
+            if persist_flag and persist_personality is not None:
+                try:
+                    persist_personality(None if sel_name == DEFAULT_OPTION else sel_name)
+                    persisted_choice = _startup_choice()
+                except Exception as e:
+                    logger.warning("Failed to persist startup personality: %s", e)
+            return {"ok": True, "status": status, "startup": persisted_choice}
+        except Exception as e:
+            return JSONResponse({"ok": False, "error": str(e)}, status_code=500)  # type: ignore
+    @app.get("/voices")
+    async def _voices() -> list[str]:
+        loop = get_loop()
+        if loop is None:
+            return ["cedar"]
+        async def _get_v() -> list[str]:
+            try:
+                return await handler.get_available_voices()
+            except Exception:
+                return ["cedar"]
+        try:
+            fut = asyncio.run_coroutine_threadsafe(_get_v(), loop)
+            return fut.result(timeout=10)
+        except Exception:
+            return ["cedar"]

src/reachy_mini_conversation_app/images/reachymini_avatar.png ADDED Viewed

Git LFS Details

SHA256: 5a63ac8802ff3542f01292c431c5278296880d74cd3580d219fcf4827bc235f9
Pointer size: 132 Bytes
Size of remote file: 1.23 MB

src/reachy_mini_conversation_app/images/user_avatar.png ADDED Viewed

Git LFS Details

SHA256: e97ca125a86bacdaa41c8dca88abd9ca746fd5c9391eda24249c012432b0219b
Pointer size: 132 Bytes
Size of remote file: 1.11 MB

src/reachy_mini_conversation_app/main.py ADDED Viewed

	@@ -0,0 +1,253 @@

+"""Entrypoint for the Reachy Mini conversation app."""
+import os
+import sys
+import time
+import asyncio
+import argparse
+import threading
+from typing import Any, Dict, List, Optional
+import gradio as gr
+from fastapi import FastAPI
+from fastrtc import Stream
+from gradio.utils import get_space
+from reachy_mini import ReachyMini, ReachyMiniApp
+from reachy_mini_conversation_app.utils import (
+    parse_args,
+    setup_logger,
+    handle_vision_stuff,
+    log_connection_troubleshooting,
+)
+def update_chatbot(chatbot: List[Dict[str, Any]], response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Update the chatbot with AdditionalOutputs."""
+    chatbot.append(response)
+    return chatbot
+def main() -> None:
+    """Entrypoint for the Reachy Mini conversation app."""
+    args, _ = parse_args()
+    run(args)
+def run(
+    args: argparse.Namespace,
+    robot: ReachyMini = None,
+    app_stop_event: Optional[threading.Event] = None,
+    settings_app: Optional[FastAPI] = None,
+    instance_path: Optional[str] = None,
+) -> None:
+    """Run the Reachy Mini conversation app."""
+    # Putting these dependencies here makes the dashboard faster to load when the conversation app is installed
+    from reachy_mini_conversation_app.moves import MovementManager
+    from reachy_mini_conversation_app.console import LocalStream
+    from reachy_mini_conversation_app.openai_realtime import OpenaiRealtimeHandler
+    from reachy_mini_conversation_app.tools.core_tools import ToolDependencies
+    from reachy_mini_conversation_app.audio.head_wobbler import HeadWobbler
+    logger = setup_logger(args.debug)
+    logger.info("Starting Reachy Mini Conversation App")
+    if args.no_camera and args.head_tracker is not None:
+        logger.warning(
+            "Head tracking disabled: --no-camera flag is set. "
+            "Remove --no-camera to enable head tracking."
+        )
+    if robot is None:
+        try:
+            robot_kwargs = {}
+            if args.robot_name is not None:
+                robot_kwargs["robot_name"] = args.robot_name
+            logger.info("Initializing ReachyMini (SDK will auto-detect appropriate backend)")
+            robot = ReachyMini(**robot_kwargs)
+        except TimeoutError as e:
+            logger.error(
+                "Connection timeout: Failed to connect to Reachy Mini daemon. "
+                f"Details: {e}"
+            )
+            log_connection_troubleshooting(logger, args.robot_name)
+            sys.exit(1)
+        except ConnectionError as e:
+            logger.error(
+                "Connection failed: Unable to establish connection to Reachy Mini. "
+                f"Details: {e}"
+            )
+            log_connection_troubleshooting(logger, args.robot_name)
+            sys.exit(1)
+        except Exception as e:
+            logger.error(
+                f"Unexpected error during robot initialization: {type(e).__name__}: {e}"
+            )
+            logger.error("Please check your configuration and try again.")
+            sys.exit(1)
+    # Check if running in simulation mode without --gradio
+    if robot.client.get_status()["simulation_enabled"] and not args.gradio:
+        logger.error(
+            "Simulation mode requires Gradio interface. Please use --gradio flag when running in simulation mode."
+        )
+        robot.client.disconnect()
+        sys.exit(1)
+    camera_worker, _, vision_manager = handle_vision_stuff(args, robot)
+    movement_manager = MovementManager(
+        current_robot=robot,
+        camera_worker=camera_worker,
+    )
+    head_wobbler = HeadWobbler(set_speech_offsets=movement_manager.set_speech_offsets)
+    deps = ToolDependencies(
+        reachy_mini=robot,
+        movement_manager=movement_manager,
+        camera_worker=camera_worker,
+        vision_manager=vision_manager,
+        head_wobbler=head_wobbler,
+    )
+    current_file_path = os.path.dirname(os.path.abspath(__file__))
+    logger.debug(f"Current file absolute path: {current_file_path}")
+    chatbot = gr.Chatbot(
+        type="messages",
+        resizable=True,
+        avatar_images=(
+            os.path.join(current_file_path, "images", "user_avatar.png"),
+            os.path.join(current_file_path, "images", "reachymini_avatar.png"),
+        ),
+    )
+    logger.debug(f"Chatbot avatar images: {chatbot.avatar_images}")
+    handler = OpenaiRealtimeHandler(deps, gradio_mode=args.gradio, instance_path=instance_path)
+    stream_manager: gr.Blocks | LocalStream | None = None
+    if args.gradio:
+        api_key_textbox = gr.Textbox(
+            label="OPENAI API Key",
+            type="password",
+            value=os.getenv("OPENAI_API_KEY") if not get_space() else "",
+        )
+        from reachy_mini_conversation_app.gradio_personality import PersonalityUI
+        personality_ui = PersonalityUI()
+        personality_ui.create_components()
+        stream = Stream(
+            handler=handler,
+            mode="send-receive",
+            modality="audio",
+            additional_inputs=[
+                chatbot,
+                api_key_textbox,
+                *personality_ui.additional_inputs_ordered(),
+            ],
+            additional_outputs=[chatbot],
+            additional_outputs_handler=update_chatbot,
+            ui_args={"title": "Talk with Reachy Mini"},
+        )
+        stream_manager = stream.ui
+        if not settings_app:
+            app = FastAPI()
+        else:
+            app = settings_app
+        personality_ui.wire_events(handler, stream_manager)
+        app = gr.mount_gradio_app(app, stream.ui, path="/")
+    else:
+        # In headless mode, wire settings_app + instance_path to console LocalStream
+        stream_manager = LocalStream(
+            handler,
+            robot,
+            settings_app=settings_app,
+            instance_path=instance_path,
+        )
+    # Each async service → its own thread/loop
+    movement_manager.start()
+    head_wobbler.start()
+    if camera_worker:
+        camera_worker.start()
+    if vision_manager:
+        vision_manager.start()
+    def poll_stop_event() -> None:
+        """Poll the stop event to allow graceful shutdown."""
+        if app_stop_event is not None:
+            app_stop_event.wait()
+        logger.info("App stop event detected, shutting down...")
+        try:
+            stream_manager.close()
+        except Exception as e:
+            logger.error(f"Error while closing stream manager: {e}")
+    if app_stop_event:
+        threading.Thread(target=poll_stop_event, daemon=True).start()
+    try:
+        stream_manager.launch()
+    except KeyboardInterrupt:
+        logger.info("Keyboard interruption in main thread... closing server.")
+    finally:
+        movement_manager.stop()
+        head_wobbler.stop()
+        if camera_worker:
+            camera_worker.stop()
+        if vision_manager:
+            vision_manager.stop()
+        # Ensure media is explicitly closed before disconnecting
+        try:
+            robot.media.close()
+        except Exception as e:
+            logger.debug(f"Error closing media during shutdown: {e}")
+        # prevent connection to keep alive some threads
+        robot.client.disconnect()
+        time.sleep(1)
+        logger.info("Shutdown complete.")
+class ReachyMiniConversationApp(ReachyMiniApp):  # type: ignore[misc]
+    """Reachy Mini Apps entry point for the conversation app."""
+    custom_app_url = "http://0.0.0.0:7860/"
+    dont_start_webserver = False
+    def run(self, reachy_mini: ReachyMini, stop_event: threading.Event) -> None:
+        """Run the Reachy Mini conversation app."""
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        args, _ = parse_args()
+        # is_wireless = reachy_mini.client.get_status()["wireless_version"]
+        # args.head_tracker = None if is_wireless else "mediapipe"
+        instance_path = self._get_instance_path().parent
+        run(
+            args,
+            robot=reachy_mini,
+            app_stop_event=stop_event,
+            settings_app=self.settings_app,
+            instance_path=instance_path,
+        )
+if __name__ == "__main__":
+    app = ReachyMiniConversationApp()
+    try:
+        app.wrapped_run()
+    except KeyboardInterrupt:
+        app.stop()

src/reachy_mini_conversation_app/moves.py ADDED Viewed

	@@ -0,0 +1,849 @@

+"""Movement system with sequential primary moves and additive secondary moves.
+Design overview
+- Primary moves (emotions, dances, goto, breathing) are mutually exclusive and run
+  sequentially.
+- Secondary moves (speech sway, face tracking) are additive offsets applied on top
+  of the current primary pose.
+- There is a single control point to the robot: `ReachyMini.set_target`.
+- The control loop runs near 100 Hz and is phase-aligned via a monotonic clock.
+- Idle behaviour starts an infinite `BreathingMove` after a short inactivity delay
+  unless listening is active.
+Threading model
+- A dedicated worker thread owns all real-time state and issues `set_target`
+  commands.
+- Other threads communicate via a command queue (enqueue moves, mark activity,
+  toggle listening).
+- Secondary offset producers set pending values guarded by locks; the worker
+  snaps them atomically.
+Units and frames
+- Secondary offsets are interpreted as metres for x/y/z and radians for
+  roll/pitch/yaw in the world frame (unless noted by `compose_world_offset`).
+- Antennas and `body_yaw` are in radians.
+- Head pose composition uses `compose_world_offset(primary_head, secondary_head)`;
+  the secondary offset must therefore be expressed in the world frame.
+Safety
+- Listening freezes antennas, then blends them back on unfreeze.
+- Interpolations and blends are used to avoid jumps at all times.
+- `set_target` errors are rate-limited in logs.
+"""
+from __future__ import annotations
+import time
+import logging
+import threading
+from queue import Empty, Queue
+from typing import Any, Dict, Tuple
+from collections import deque
+from dataclasses import dataclass
+import numpy as np
+from numpy.typing import NDArray
+from reachy_mini import ReachyMini
+from reachy_mini.utils import create_head_pose
+from reachy_mini.motion.move import Move
+from reachy_mini.utils.interpolation import (
+    compose_world_offset,
+    linear_pose_interpolation,
+)
+logger = logging.getLogger(__name__)
+# Configuration constants
+CONTROL_LOOP_FREQUENCY_HZ = 100.0  # Hz - Target frequency for the movement control loop
+# Type definitions
+FullBodyPose = Tuple[NDArray[np.float32], Tuple[float, float], float]  # (head_pose_4x4, antennas, body_yaw)
+class BreathingMove(Move):  # type: ignore
+    """Breathing move with interpolation to neutral and then continuous breathing patterns."""
+    def __init__(
+        self,
+        interpolation_start_pose: NDArray[np.float32],
+        interpolation_start_antennas: Tuple[float, float],
+        interpolation_duration: float = 1.0,
+    ):
+        """Initialize breathing move.
+        Args:
+            interpolation_start_pose: 4x4 matrix of current head pose to interpolate from
+            interpolation_start_antennas: Current antenna positions to interpolate from
+            interpolation_duration: Duration of interpolation to neutral (seconds)
+        """
+        self.interpolation_start_pose = interpolation_start_pose
+        self.interpolation_start_antennas = np.array(interpolation_start_antennas)
+        self.interpolation_duration = interpolation_duration
+        # Neutral positions for breathing base
+        self.neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
+        self.neutral_antennas = np.array([0.0, 0.0])
+        # Breathing parameters
+        self.breathing_z_amplitude = 0.005  # 5mm gentle breathing
+        self.breathing_frequency = 0.1  # Hz (6 breaths per minute)
+        self.antenna_sway_amplitude = np.deg2rad(15)  # 15 degrees
+        self.antenna_frequency = 0.5  # Hz (faster antenna sway)
+    @property
+    def duration(self) -> float:
+        """Duration property required by official Move interface."""
+        return float("inf")  # Continuous breathing (never ends naturally)
+    def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]:
+        """Evaluate breathing move at time t."""
+        if t < self.interpolation_duration:
+            # Phase 1: Interpolate to neutral base position
+            interpolation_t = t / self.interpolation_duration
+            # Interpolate head pose
+            head_pose = linear_pose_interpolation(
+                self.interpolation_start_pose, self.neutral_head_pose, interpolation_t,
+            )
+            # Interpolate antennas
+            antennas_interp = (
+                1 - interpolation_t
+            ) * self.interpolation_start_antennas + interpolation_t * self.neutral_antennas
+            antennas = antennas_interp.astype(np.float64)
+        else:
+            # Phase 2: Breathing patterns from neutral base
+            breathing_time = t - self.interpolation_duration
+            # Gentle z-axis breathing
+            z_offset = self.breathing_z_amplitude * np.sin(2 * np.pi * self.breathing_frequency * breathing_time)
+            head_pose = create_head_pose(x=0, y=0, z=z_offset, roll=0, pitch=0, yaw=0, degrees=True, mm=False)
+            # Antenna sway (opposite directions)
+            antenna_sway = self.antenna_sway_amplitude * np.sin(2 * np.pi * self.antenna_frequency * breathing_time)
+            antennas = np.array([antenna_sway, -antenna_sway], dtype=np.float64)
+        # Return in official Move interface format: (head_pose, antennas_array, body_yaw)
+        return (head_pose, antennas, 0.0)
+def combine_full_body(primary_pose: FullBodyPose, secondary_pose: FullBodyPose) -> FullBodyPose:
+    """Combine primary and secondary full body poses.
+    Args:
+        primary_pose: (head_pose, antennas, body_yaw) - primary move
+        secondary_pose: (head_pose, antennas, body_yaw) - secondary offsets
+    Returns:
+        Combined full body pose (head_pose, antennas, body_yaw)
+    """
+    primary_head, primary_antennas, primary_body_yaw = primary_pose
+    secondary_head, secondary_antennas, secondary_body_yaw = secondary_pose
+    # Combine head poses using compose_world_offset; the secondary pose must be an
+    # offset expressed in the world frame (T_off_world) applied to the absolute
+    # primary transform (T_abs).
+    combined_head = compose_world_offset(primary_head, secondary_head, reorthonormalize=True)
+    # Sum antennas and body_yaw
+    combined_antennas = (
+        primary_antennas[0] + secondary_antennas[0],
+        primary_antennas[1] + secondary_antennas[1],
+    )
+    combined_body_yaw = primary_body_yaw + secondary_body_yaw
+    return (combined_head, combined_antennas, combined_body_yaw)
+def clone_full_body_pose(pose: FullBodyPose) -> FullBodyPose:
+    """Create a deep copy of a full body pose tuple."""
+    head, antennas, body_yaw = pose
+    return (head.copy(), (float(antennas[0]), float(antennas[1])), float(body_yaw))
+@dataclass
+class MovementState:
+    """State tracking for the movement system."""
+    # Primary move state
+    current_move: Move | None = None
+    move_start_time: float | None = None
+    last_activity_time: float = 0.0
+    # Secondary move state (offsets)
+    speech_offsets: Tuple[float, float, float, float, float, float] = (
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+    )
+    face_tracking_offsets: Tuple[float, float, float, float, float, float] = (
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+    )
+    # Status flags
+    last_primary_pose: FullBodyPose | None = None
+    def update_activity(self) -> None:
+        """Update the last activity time."""
+        self.last_activity_time = time.monotonic()
+@dataclass
+class LoopFrequencyStats:
+    """Track rolling loop frequency statistics."""
+    mean: float = 0.0
+    m2: float = 0.0
+    min_freq: float = float("inf")
+    count: int = 0
+    last_freq: float = 0.0
+    potential_freq: float = 0.0
+    def reset(self) -> None:
+        """Reset accumulators while keeping the last potential frequency."""
+        self.mean = 0.0
+        self.m2 = 0.0
+        self.min_freq = float("inf")
+        self.count = 0
+class MovementManager:
+    """Coordinate sequential moves, additive offsets, and robot output at 100 Hz.
+    Responsibilities:
+    - Own a real-time loop that samples the current primary move (if any), fuses
+      secondary offsets, and calls `set_target` exactly once per tick.
+    - Start an idle `BreathingMove` after `idle_inactivity_delay` when not
+      listening and no moves are queued.
+    - Expose thread-safe APIs so other threads can enqueue moves, mark activity,
+      or feed secondary offsets without touching internal state.
+    Timing:
+    - All elapsed-time calculations rely on `time.monotonic()` through `self._now`
+      to avoid wall-clock jumps.
+    - The loop attempts 100 Hz
+    Concurrency:
+    - External threads communicate via `_command_queue` messages.
+    - Secondary offsets are staged via dirty flags guarded by locks and consumed
+      atomically inside the worker loop.
+    """
+    def __init__(
+        self,
+        current_robot: ReachyMini,
+        camera_worker: "Any" = None,
+    ):
+        """Initialize movement manager."""
+        self.current_robot = current_robot
+        self.camera_worker = camera_worker
+        # Single timing source for durations
+        self._now = time.monotonic
+        # Movement state
+        self.state = MovementState()
+        self.state.last_activity_time = self._now()
+        neutral_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
+        self.state.last_primary_pose = (neutral_pose, (0.0, 0.0), 0.0)
+        # Move queue (primary moves)
+        self.move_queue: deque[Move] = deque()
+        # Configuration
+        self.idle_inactivity_delay = 0.3  # seconds
+        self.target_frequency = CONTROL_LOOP_FREQUENCY_HZ
+        self.target_period = 1.0 / self.target_frequency
+        self._stop_event = threading.Event()
+        self._thread: threading.Thread | None = None
+        self._is_listening = False
+        self._last_commanded_pose: FullBodyPose = clone_full_body_pose(self.state.last_primary_pose)
+        self._listening_antennas: Tuple[float, float] = self._last_commanded_pose[1]
+        self._antenna_unfreeze_blend = 1.0
+        self._antenna_blend_duration = 0.4  # seconds to blend back after listening
+        self._last_listening_blend_time = self._now()
+        self._breathing_active = False  # true when breathing move is running or queued
+        self._listening_debounce_s = 0.15
+        self._last_listening_toggle_time = self._now()
+        self._last_set_target_err = 0.0
+        self._set_target_err_interval = 1.0  # seconds between error logs
+        self._set_target_err_suppressed = 0
+        # Cross-thread signalling
+        self._command_queue: "Queue[Tuple[str, Any]]" = Queue()
+        self._speech_offsets_lock = threading.Lock()
+        self._pending_speech_offsets: Tuple[float, float, float, float, float, float] = (
+            0.0,
+            0.0,
+            0.0,
+            0.0,
+            0.0,
+            0.0,
+        )
+        self._speech_offsets_dirty = False
+        self._face_offsets_lock = threading.Lock()
+        self._pending_face_offsets: Tuple[float, float, float, float, float, float] = (
+            0.0,
+            0.0,
+            0.0,
+            0.0,
+            0.0,
+            0.0,
+        )
+        self._face_offsets_dirty = False
+        self._shared_state_lock = threading.Lock()
+        self._shared_last_activity_time = self.state.last_activity_time
+        self._shared_is_listening = self._is_listening
+        self._status_lock = threading.Lock()
+        self._freq_stats = LoopFrequencyStats()
+        self._freq_snapshot = LoopFrequencyStats()
+    def queue_move(self, move: Move) -> None:
+        """Queue a primary move to run after the currently executing one.
+        Thread-safe: the move is enqueued via the worker command queue so the
+        control loop remains the sole mutator of movement state.
+        """
+        self._command_queue.put(("queue_move", move))
+    def clear_move_queue(self) -> None:
+        """Stop the active move and discard any queued primary moves.
+        Thread-safe: executed by the worker thread via the command queue.
+        """
+        self._command_queue.put(("clear_queue", None))
+    def set_speech_offsets(self, offsets: Tuple[float, float, float, float, float, float]) -> None:
+        """Update speech-induced secondary offsets (x, y, z, roll, pitch, yaw).
+        Offsets are interpreted as metres for translation and radians for
+        rotation in the world frame. Thread-safe via a pending snapshot.
+        """
+        with self._speech_offsets_lock:
+            self._pending_speech_offsets = offsets
+            self._speech_offsets_dirty = True
+    def set_moving_state(self, duration: float) -> None:
+        """Mark the robot as actively moving for the provided duration.
+        Legacy hook used by goto helpers to keep inactivity and breathing logic
+        aware of manual motions. Thread-safe via the command queue.
+        """
+        self._command_queue.put(("set_moving_state", duration))
+    def is_idle(self) -> bool:
+        """Return True when the robot has been inactive longer than the idle delay."""
+        with self._shared_state_lock:
+            last_activity = self._shared_last_activity_time
+            listening = self._shared_is_listening
+        if listening:
+            return False
+        return self._now() - last_activity >= self.idle_inactivity_delay
+    def set_listening(self, listening: bool) -> None:
+        """Enable or disable listening mode without touching shared state directly.
+        While listening:
+        - Antenna positions are frozen at the last commanded values.
+        - Blending is reset so that upon unfreezing the antennas return smoothly.
+        - Idle breathing is suppressed.
+        Thread-safe: the change is posted to the worker command queue.
+        """
+        with self._shared_state_lock:
+            if self._shared_is_listening == listening:
+                return
+        self._command_queue.put(("set_listening", listening))
+    def _poll_signals(self, current_time: float) -> None:
+        """Apply queued commands and pending offset updates."""
+        self._apply_pending_offsets()
+        while True:
+            try:
+                command, payload = self._command_queue.get_nowait()
+            except Empty:
+                break
+            self._handle_command(command, payload, current_time)
+    def _apply_pending_offsets(self) -> None:
+        """Apply the most recent speech/face offset updates."""
+        speech_offsets: Tuple[float, float, float, float, float, float] | None = None
+        with self._speech_offsets_lock:
+            if self._speech_offsets_dirty:
+                speech_offsets = self._pending_speech_offsets
+                self._speech_offsets_dirty = False
+        if speech_offsets is not None:
+            self.state.speech_offsets = speech_offsets
+            self.state.update_activity()
+        face_offsets: Tuple[float, float, float, float, float, float] | None = None
+        with self._face_offsets_lock:
+            if self._face_offsets_dirty:
+                face_offsets = self._pending_face_offsets
+                self._face_offsets_dirty = False
+        if face_offsets is not None:
+            self.state.face_tracking_offsets = face_offsets
+            self.state.update_activity()
+    def _handle_command(self, command: str, payload: Any, current_time: float) -> None:
+        """Handle a single cross-thread command."""
+        if command == "queue_move":
+            if isinstance(payload, Move):
+                self.move_queue.append(payload)
+                self.state.update_activity()
+                duration = getattr(payload, "duration", None)
+                if duration is not None:
+                    try:
+                        duration_str = f"{float(duration):.2f}"
+                    except (TypeError, ValueError):
+                        duration_str = str(duration)
+                else:
+                    duration_str = "?"
+                logger.debug(
+                    "Queued move with duration %ss, queue size: %s",
+                    duration_str,
+                    len(self.move_queue),
+                )
+            else:
+                logger.warning("Ignored queue_move command with invalid payload: %s", payload)
+        elif command == "clear_queue":
+            self.move_queue.clear()
+            self.state.current_move = None
+            self.state.move_start_time = None
+            self._breathing_active = False
+            logger.info("Cleared move queue and stopped current move")
+        elif command == "set_moving_state":
+            try:
+                duration = float(payload)
+            except (TypeError, ValueError):
+                logger.warning("Invalid moving state duration: %s", payload)
+                return
+            self.state.update_activity()
+        elif command == "mark_activity":
+            self.state.update_activity()
+        elif command == "set_listening":
+            desired_state = bool(payload)
+            now = self._now()
+            if now - self._last_listening_toggle_time < self._listening_debounce_s:
+                return
+            self._last_listening_toggle_time = now
+            if self._is_listening == desired_state:
+                return
+            self._is_listening = desired_state
+            self._last_listening_blend_time = now
+            if desired_state:
+                # Freeze: snapshot current commanded antennas and reset blend
+                self._listening_antennas = (
+                    float(self._last_commanded_pose[1][0]),
+                    float(self._last_commanded_pose[1][1]),
+                )
+                self._antenna_unfreeze_blend = 0.0
+            else:
+                # Unfreeze: restart blending from frozen pose
+                self._antenna_unfreeze_blend = 0.0
+            self.state.update_activity()
+        else:
+            logger.warning("Unknown command received by MovementManager: %s", command)
+    def _publish_shared_state(self) -> None:
+        """Expose idle-related state for external threads."""
+        with self._shared_state_lock:
+            self._shared_last_activity_time = self.state.last_activity_time
+            self._shared_is_listening = self._is_listening
+    def _manage_move_queue(self, current_time: float) -> None:
+        """Manage the primary move queue (sequential execution)."""
+        if self.state.current_move is None or (
+            self.state.move_start_time is not None
+            and current_time - self.state.move_start_time >= self.state.current_move.duration
+        ):
+            self.state.current_move = None
+            self.state.move_start_time = None
+            if self.move_queue:
+                self.state.current_move = self.move_queue.popleft()
+                self.state.move_start_time = current_time
+                # Any real move cancels breathing mode flag
+                self._breathing_active = isinstance(self.state.current_move, BreathingMove)
+                logger.debug(f"Starting new move, duration: {self.state.current_move.duration}s")
+    def _manage_breathing(self, current_time: float) -> None:
+        """Manage automatic breathing when idle."""
+        if (
+            self.state.current_move is None
+            and not self.move_queue
+            and not self._is_listening
+            and not self._breathing_active
+        ):
+            idle_for = current_time - self.state.last_activity_time
+            if idle_for >= self.idle_inactivity_delay:
+                try:
+                    # These 2 functions return the latest available sensor data from the robot, but don't perform I/O synchronously.
+                    # Therefore, we accept calling them inside the control loop.
+                    _, current_antennas = self.current_robot.get_current_joint_positions()
+                    current_head_pose = self.current_robot.get_current_head_pose()
+                    self._breathing_active = True
+                    self.state.update_activity()
+                    breathing_move = BreathingMove(
+                        interpolation_start_pose=current_head_pose,
+                        interpolation_start_antennas=current_antennas,
+                        interpolation_duration=1.0,
+                    )
+                    self.move_queue.append(breathing_move)
+                    logger.debug("Started breathing after %.1fs of inactivity", idle_for)
+                except Exception as e:
+                    self._breathing_active = False
+                    logger.error("Failed to start breathing: %s", e)
+        if isinstance(self.state.current_move, BreathingMove) and self.move_queue:
+            self.state.current_move = None
+            self.state.move_start_time = None
+            self._breathing_active = False
+            logger.debug("Stopping breathing due to new move activity")
+        if self.state.current_move is not None and not isinstance(self.state.current_move, BreathingMove):
+            self._breathing_active = False
+    def _get_primary_pose(self, current_time: float) -> FullBodyPose:
+        """Get the primary full body pose from current move or neutral."""
+        # When a primary move is playing, sample it and cache the resulting pose
+        if self.state.current_move is not None and self.state.move_start_time is not None:
+            move_time = current_time - self.state.move_start_time
+            head, antennas, body_yaw = self.state.current_move.evaluate(move_time)
+            if head is None:
+                head = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
+            if antennas is None:
+                antennas = np.array([0.0, 0.0])
+            if body_yaw is None:
+                body_yaw = 0.0
+            antennas_tuple = (float(antennas[0]), float(antennas[1]))
+            head_copy = head.copy()
+            primary_full_body_pose = (
+                head_copy,
+                antennas_tuple,
+                float(body_yaw),
+            )
+            self.state.last_primary_pose = clone_full_body_pose(primary_full_body_pose)
+        # Otherwise reuse the last primary pose so we avoid jumps between moves
+        elif self.state.last_primary_pose is not None:
+            primary_full_body_pose = clone_full_body_pose(self.state.last_primary_pose)
+        else:
+            neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
+            primary_full_body_pose = (neutral_head_pose, (0.0, 0.0), 0.0)
+            self.state.last_primary_pose = clone_full_body_pose(primary_full_body_pose)
+        return primary_full_body_pose
+    def _get_secondary_pose(self) -> FullBodyPose:
+        """Get the secondary full body pose from speech and face tracking offsets."""
+        # Combine speech sway offsets + face tracking offsets for secondary pose
+        secondary_offsets = [
+            self.state.speech_offsets[0] + self.state.face_tracking_offsets[0],
+            self.state.speech_offsets[1] + self.state.face_tracking_offsets[1],
+            self.state.speech_offsets[2] + self.state.face_tracking_offsets[2],
+            self.state.speech_offsets[3] + self.state.face_tracking_offsets[3],
+            self.state.speech_offsets[4] + self.state.face_tracking_offsets[4],
+            self.state.speech_offsets[5] + self.state.face_tracking_offsets[5],
+        ]
+        secondary_head_pose = create_head_pose(
+            x=secondary_offsets[0],
+            y=secondary_offsets[1],
+            z=secondary_offsets[2],
+            roll=secondary_offsets[3],
+            pitch=secondary_offsets[4],
+            yaw=secondary_offsets[5],
+            degrees=False,
+            mm=False,
+        )
+        return (secondary_head_pose, (0.0, 0.0), 0.0)
+    def _compose_full_body_pose(self, current_time: float) -> FullBodyPose:
+        """Compose primary and secondary poses into a single command pose."""
+        primary = self._get_primary_pose(current_time)
+        secondary = self._get_secondary_pose()
+        return combine_full_body(primary, secondary)
+    def _update_primary_motion(self, current_time: float) -> None:
+        """Advance queue state and idle behaviours for this tick."""
+        self._manage_move_queue(current_time)
+        self._manage_breathing(current_time)
+    def _calculate_blended_antennas(self, target_antennas: Tuple[float, float]) -> Tuple[float, float]:
+        """Blend target antennas with listening freeze state and update blending."""
+        now = self._now()
+        listening = self._is_listening
+        listening_antennas = self._listening_antennas
+        blend = self._antenna_unfreeze_blend
+        blend_duration = self._antenna_blend_duration
+        last_update = self._last_listening_blend_time
+        self._last_listening_blend_time = now
+        if listening:
+            antennas_cmd = listening_antennas
+            new_blend = 0.0
+        else:
+            dt = max(0.0, now - last_update)
+            if blend_duration <= 0:
+                new_blend = 1.0
+            else:
+                new_blend = min(1.0, blend + dt / blend_duration)
+            antennas_cmd = (
+                listening_antennas[0] * (1.0 - new_blend) + target_antennas[0] * new_blend,
+                listening_antennas[1] * (1.0 - new_blend) + target_antennas[1] * new_blend,
+            )
+        if listening:
+            self._antenna_unfreeze_blend = 0.0
+        else:
+            self._antenna_unfreeze_blend = new_blend
+            if new_blend >= 1.0:
+                self._listening_antennas = (
+                    float(target_antennas[0]),
+                    float(target_antennas[1]),
+                )
+        return antennas_cmd
+    def _issue_control_command(self, head: NDArray[np.float32], antennas: Tuple[float, float], body_yaw: float) -> None:
+        """Send the fused pose to the robot with throttled error logging."""
+        try:
+            self.current_robot.set_target(head=head, antennas=antennas, body_yaw=body_yaw)
+        except Exception as e:
+            now = self._now()
+            if now - self._last_set_target_err >= self._set_target_err_interval:
+                msg = f"Failed to set robot target: {e}"
+                if self._set_target_err_suppressed:
+                    msg += f" (suppressed {self._set_target_err_suppressed} repeats)"
+                    self._set_target_err_suppressed = 0
+                logger.error(msg)
+                self._last_set_target_err = now
+            else:
+                self._set_target_err_suppressed += 1
+        else:
+            with self._status_lock:
+                self._last_commanded_pose = clone_full_body_pose((head, antennas, body_yaw))
+    def _update_frequency_stats(
+        self, loop_start: float, prev_loop_start: float, stats: LoopFrequencyStats,
+    ) -> LoopFrequencyStats:
+        """Update frequency statistics based on the current loop start time."""
+        period = loop_start - prev_loop_start
+        if period > 0:
+            stats.last_freq = 1.0 / period
+            stats.count += 1
+            delta = stats.last_freq - stats.mean
+            stats.mean += delta / stats.count
+            stats.m2 += delta * (stats.last_freq - stats.mean)
+            stats.min_freq = min(stats.min_freq, stats.last_freq)
+        return stats
+    def _schedule_next_tick(self, loop_start: float, stats: LoopFrequencyStats) -> Tuple[float, LoopFrequencyStats]:
+        """Compute sleep time to maintain target frequency and update potential freq."""
+        computation_time = self._now() - loop_start
+        stats.potential_freq = 1.0 / computation_time if computation_time > 0 else float("inf")
+        sleep_time = max(0.0, self.target_period - computation_time)
+        return sleep_time, stats
+    def _record_frequency_snapshot(self, stats: LoopFrequencyStats) -> None:
+        """Store a thread-safe snapshot of current frequency statistics."""
+        with self._status_lock:
+            self._freq_snapshot = LoopFrequencyStats(
+                mean=stats.mean,
+                m2=stats.m2,
+                min_freq=stats.min_freq,
+                count=stats.count,
+                last_freq=stats.last_freq,
+                potential_freq=stats.potential_freq,
+            )
+    def _maybe_log_frequency(self, loop_count: int, print_interval_loops: int, stats: LoopFrequencyStats) -> None:
+        """Emit frequency telemetry when enough loops have elapsed."""
+        if loop_count % print_interval_loops != 0 or stats.count == 0:
+            return
+        variance = stats.m2 / stats.count if stats.count > 0 else 0.0
+        lowest = stats.min_freq if stats.min_freq != float("inf") else 0.0
+        logger.debug(
+            "Loop freq - avg: %.2fHz, variance: %.4f, min: %.2fHz, last: %.2fHz, potential: %.2fHz, target: %.1fHz",
+            stats.mean,
+            variance,
+            lowest,
+            stats.last_freq,
+            stats.potential_freq,
+            self.target_frequency,
+        )
+        stats.reset()
+    def _update_face_tracking(self, current_time: float) -> None:
+        """Get face tracking offsets from camera worker thread."""
+        if self.camera_worker is not None:
+            # Get face tracking offsets from camera worker thread
+            offsets = self.camera_worker.get_face_tracking_offsets()
+            self.state.face_tracking_offsets = offsets
+        else:
+            # No camera worker, use neutral offsets
+            self.state.face_tracking_offsets = (0.0, 0.0, 0.0, 0.0, 0.0, 0.0)
+    def start(self) -> None:
+        """Start the worker thread that drives the 100 Hz control loop."""
+        if self._thread is not None and self._thread.is_alive():
+            logger.warning("Move worker already running; start() ignored")
+            return
+        self._stop_event.clear()
+        self._thread = threading.Thread(target=self.working_loop, daemon=True)
+        self._thread.start()
+        logger.debug("Move worker started")
+    def stop(self) -> None:
+        """Request the worker thread to stop and wait for it to exit.
+        Before stopping, resets the robot to a neutral position.
+        """
+        if self._thread is None or not self._thread.is_alive():
+            logger.debug("Move worker not running; stop() ignored")
+            return
+        logger.info("Stopping movement manager and resetting to neutral position...")
+        # Clear any queued moves and stop current move
+        self.clear_move_queue()
+        # Stop the worker thread first so it doesn't interfere
+        self._stop_event.set()
+        if self._thread is not None:
+            self._thread.join()
+            self._thread = None
+        logger.debug("Move worker stopped")
+        # Reset to neutral position using goto_target (same approach as wake_up)
+        try:
+            neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
+            neutral_antennas = [0.0, 0.0]
+            neutral_body_yaw = 0.0
+            # Use goto_target directly on the robot
+            self.current_robot.goto_target(
+                head=neutral_head_pose,
+                antennas=neutral_antennas,
+                duration=2.0,
+                body_yaw=neutral_body_yaw,
+            )
+            logger.info("Reset to neutral position completed")
+        except Exception as e:
+            logger.error(f"Failed to reset to neutral position: {e}")
+    def get_status(self) -> Dict[str, Any]:
+        """Return a lightweight status snapshot for observability."""
+        with self._status_lock:
+            pose_snapshot = clone_full_body_pose(self._last_commanded_pose)
+            freq_snapshot = LoopFrequencyStats(
+                mean=self._freq_snapshot.mean,
+                m2=self._freq_snapshot.m2,
+                min_freq=self._freq_snapshot.min_freq,
+                count=self._freq_snapshot.count,
+                last_freq=self._freq_snapshot.last_freq,
+                potential_freq=self._freq_snapshot.potential_freq,
+            )
+        head_matrix = pose_snapshot[0].tolist() if pose_snapshot else None
+        antennas = pose_snapshot[1] if pose_snapshot else None
+        body_yaw = pose_snapshot[2] if pose_snapshot else None
+        return {
+            "queue_size": len(self.move_queue),
+            "is_listening": self._is_listening,
+            "breathing_active": self._breathing_active,
+            "last_commanded_pose": {
+                "head": head_matrix,
+                "antennas": antennas,
+                "body_yaw": body_yaw,
+            },
+            "loop_frequency": {
+                "last": freq_snapshot.last_freq,
+                "mean": freq_snapshot.mean,
+                "min": freq_snapshot.min_freq,
+                "potential": freq_snapshot.potential_freq,
+                "samples": freq_snapshot.count,
+            },
+        }
+    def working_loop(self) -> None:
+        """Control loop main movements - reproduces main_works.py control architecture.
+        Single set_target() call with pose fusion.
+        """
+        logger.debug("Starting enhanced movement control loop (100Hz)")
+        loop_count = 0
+        prev_loop_start = self._now()
+        print_interval_loops = max(1, int(self.target_frequency * 2))
+        freq_stats = self._freq_stats
+        while not self._stop_event.is_set():
+            loop_start = self._now()
+            loop_count += 1
+            if loop_count > 1:
+                freq_stats = self._update_frequency_stats(loop_start, prev_loop_start, freq_stats)
+            prev_loop_start = loop_start
+            # 1) Poll external commands and apply pending offsets (atomic snapshot)
+            self._poll_signals(loop_start)
+            # 2) Manage the primary move queue (start new move, end finished move, breathing)
+            self._update_primary_motion(loop_start)
+            # 3) Update vision-based secondary offsets
+            self._update_face_tracking(loop_start)
+            # 4) Build primary and secondary full-body poses, then fuse them
+            head, antennas, body_yaw = self._compose_full_body_pose(loop_start)
+            # 5) Apply listening antenna freeze or blend-back
+            antennas_cmd = self._calculate_blended_antennas(antennas)
+            # 6) Single set_target call - the only control point
+            self._issue_control_command(head, antennas_cmd, body_yaw)
+            # 7) Adaptive sleep to align to next tick, then publish shared state
+            sleep_time, freq_stats = self._schedule_next_tick(loop_start, freq_stats)
+            self._publish_shared_state()
+            self._record_frequency_snapshot(freq_stats)
+            # 8) Periodic telemetry on loop frequency
+            self._maybe_log_frequency(loop_count, print_interval_loops, freq_stats)
+            if sleep_time > 0:
+                time.sleep(sleep_time)
+        logger.debug("Movement control loop stopped")

src/reachy_mini_conversation_app/openai_realtime.py ADDED Viewed

	@@ -0,0 +1,719 @@

+import json
+import base64
+import random
+import asyncio
+import logging
+from typing import Any, Final, Tuple, Literal, Optional
+from pathlib import Path
+from datetime import datetime
+import cv2
+import numpy as np
+import gradio as gr
+from openai import AsyncOpenAI
+from fastrtc import AdditionalOutputs, AsyncStreamHandler, wait_for_item, audio_to_int16
+from numpy.typing import NDArray
+from scipy.signal import resample
+from websockets.exceptions import ConnectionClosedError
+from reachy_mini_conversation_app.config import config
+from reachy_mini_conversation_app.prompts import get_session_voice, get_session_instructions
+from reachy_mini_conversation_app.tools.core_tools import (
+    ToolDependencies,
+    get_tool_specs,
+    dispatch_tool_call,
+)
+logger = logging.getLogger(__name__)
+OPEN_AI_INPUT_SAMPLE_RATE: Final[Literal[24000]] = 24000
+OPEN_AI_OUTPUT_SAMPLE_RATE: Final[Literal[24000]] = 24000
+class OpenaiRealtimeHandler(AsyncStreamHandler):
+    """An OpenAI realtime handler for fastrtc Stream."""
+    def __init__(self, deps: ToolDependencies, gradio_mode: bool = False, instance_path: Optional[str] = None):
+        """Initialize the handler."""
+        super().__init__(
+            expected_layout="mono",
+            output_sample_rate=OPEN_AI_OUTPUT_SAMPLE_RATE,
+            input_sample_rate=OPEN_AI_INPUT_SAMPLE_RATE,
+        )
+        # Override typing of the sample rates to match OpenAI's requirements
+        self.output_sample_rate: Literal[24000] = self.output_sample_rate
+        self.input_sample_rate: Literal[24000] = self.input_sample_rate
+        self.deps = deps
+        # Override type annotations for OpenAI strict typing (only for values used in API)
+        self.output_sample_rate = OPEN_AI_OUTPUT_SAMPLE_RATE
+        self.input_sample_rate = OPEN_AI_INPUT_SAMPLE_RATE
+        self.connection: Any = None
+        self.output_queue: "asyncio.Queue[Tuple[int, NDArray[np.int16]] | AdditionalOutputs]" = asyncio.Queue()
+        self.last_activity_time = asyncio.get_event_loop().time()
+        self.start_time = asyncio.get_event_loop().time()
+        self.is_idle_tool_call = False
+        self.gradio_mode = gradio_mode
+        self.instance_path = instance_path
+        # Track how the API key was provided (env vs textbox) and its value
+        self._key_source: Literal["env", "textbox"] = "env"
+        self._provided_api_key: str | None = None
+        # Debouncing for partial transcripts
+        self.partial_transcript_task: asyncio.Task[None] | None = None
+        self.partial_transcript_sequence: int = 0  # sequence counter to prevent stale emissions
+        self.partial_debounce_delay = 0.5  # seconds
+        # Internal lifecycle flags
+        self._shutdown_requested: bool = False
+        self._connected_event: asyncio.Event = asyncio.Event()
+    def copy(self) -> "OpenaiRealtimeHandler":
+        """Create a copy of the handler."""
+        return OpenaiRealtimeHandler(self.deps, self.gradio_mode, self.instance_path)
+    async def apply_personality(self, profile: str | None) -> str:
+        """Apply a new personality (profile) at runtime if possible.
+        - Updates the global config's selected profile for subsequent calls.
+        - If a realtime connection is active, sends a session.update with the
+          freshly resolved instructions so the change takes effect immediately.
+        Returns a short status message for UI feedback.
+        """
+        try:
+            # Update the in-process config value and env
+            from reachy_mini_conversation_app.config import config as _config
+            from reachy_mini_conversation_app.config import set_custom_profile
+            set_custom_profile(profile)
+            logger.info(
+                "Set custom profile to %r (config=%r)", profile, getattr(_config, "REACHY_MINI_CUSTOM_PROFILE", None)
+            )
+            try:
+                instructions = get_session_instructions()
+                voice = get_session_voice()
+            except BaseException as e:  # catch SystemExit from prompt loader without crashing
+                logger.error("Failed to resolve personality content: %s", e)
+                return f"Failed to apply personality: {e}"
+            # Attempt a live update first, then force a full restart to ensure it sticks
+            if self.connection is not None:
+                try:
+                    await self.connection.session.update(
+                        session={
+                            "type": "realtime",
+                            "instructions": instructions,
+                            "audio": {"output": {"voice": voice}},
+                        },
+                    )
+                    logger.info("Applied personality via live update: %s", profile or "built-in default")
+                except Exception as e:
+                    logger.warning("Live update failed; will restart session: %s", e)
+                # Force a real restart to guarantee the new instructions/voice
+                try:
+                    await self._restart_session()
+                    return "Applied personality and restarted realtime session."
+                except Exception as e:
+                    logger.warning("Failed to restart session after apply: %s", e)
+                    return "Applied personality. Will take effect on next connection."
+            else:
+                logger.info(
+                    "Applied personality recorded: %s (no live connection; will apply on next session)",
+                    profile or "built-in default",
+                )
+                return "Applied personality. Will take effect on next connection."
+        except Exception as e:
+            logger.error("Error applying personality '%s': %s", profile, e)
+            return f"Failed to apply personality: {e}"
+    async def _emit_debounced_partial(self, transcript: str, sequence: int) -> None:
+        """Emit partial transcript after debounce delay."""
+        try:
+            await asyncio.sleep(self.partial_debounce_delay)
+            # Only emit if this is still the latest partial (by sequence number)
+            if self.partial_transcript_sequence == sequence:
+                await self.output_queue.put(AdditionalOutputs({"role": "user_partial", "content": transcript}))
+                logger.debug(f"Debounced partial emitted: {transcript}")
+        except asyncio.CancelledError:
+            logger.debug("Debounced partial cancelled")
+            raise
+    async def start_up(self) -> None:
+        """Start the handler with minimal retries on unexpected websocket closure."""
+        openai_api_key = config.OPENAI_API_KEY
+        if self.gradio_mode and not openai_api_key:
+            # api key was not found in .env or in the environment variables
+            await self.wait_for_args()  # type: ignore[no-untyped-call]
+            args = list(self.latest_args)
+            textbox_api_key = args[3] if len(args[3]) > 0 else None
+            if textbox_api_key is not None:
+                openai_api_key = textbox_api_key
+                self._key_source = "textbox"
+                self._provided_api_key = textbox_api_key
+            else:
+                openai_api_key = config.OPENAI_API_KEY
+        else:
+            if not openai_api_key or not openai_api_key.strip():
+                # In headless console mode, LocalStream now blocks startup until the key is provided.
+                # However, unit tests may invoke this handler directly with a stubbed client.
+                # To keep tests hermetic without requiring a real key, fall back to a placeholder.
+                logger.warning("OPENAI_API_KEY missing. Proceeding with a placeholder (tests/offline).")
+                openai_api_key = "DUMMY"
+        self.client = AsyncOpenAI(api_key=openai_api_key)
+        max_attempts = 3
+        for attempt in range(1, max_attempts + 1):
+            try:
+                await self._run_realtime_session()
+                # Normal exit from the session, stop retrying
+                return
+            except ConnectionClosedError as e:
+                # Abrupt close (e.g., "no close frame received or sent") → retry
+                logger.warning("Realtime websocket closed unexpectedly (attempt %d/%d): %s", attempt, max_attempts, e)
+                if attempt < max_attempts:
+                    # exponential backoff with jitter
+                    base_delay = 2 ** (attempt - 1)  # 1s, 2s, 4s, 8s, etc.
+                    jitter = random.uniform(0, 0.5)
+                    delay = base_delay + jitter
+                    logger.info("Retrying in %.1f seconds...", delay)
+                    await asyncio.sleep(delay)
+                    continue
+                raise
+            finally:
+                # never keep a stale reference
+                self.connection = None
+                try:
+                    self._connected_event.clear()
+                except Exception:
+                    pass
+    async def _restart_session(self) -> None:
+        """Force-close the current session and start a fresh one in background.
+        Does not block the caller while the new session is establishing.
+        """
+        try:
+            if self.connection is not None:
+                try:
+                    await self.connection.close()
+                except Exception:
+                    pass
+                finally:
+                    self.connection = None
+            # Ensure we have a client (start_up must have run once)
+            if getattr(self, "client", None) is None:
+                logger.warning("Cannot restart: OpenAI client not initialized yet.")
+                return
+            # Fire-and-forget new session and wait briefly for connection
+            try:
+                self._connected_event.clear()
+            except Exception:
+                pass
+            asyncio.create_task(self._run_realtime_session(), name="openai-realtime-restart")
+            try:
+                await asyncio.wait_for(self._connected_event.wait(), timeout=5.0)
+                logger.info("Realtime session restarted and connected.")
+            except asyncio.TimeoutError:
+                logger.warning("Realtime session restart timed out; continuing in background.")
+        except Exception as e:
+            logger.warning("_restart_session failed: %s", e)
+    async def _run_realtime_session(self) -> None:
+        """Establish and manage a single realtime session."""
+        async with self.client.realtime.connect(model=config.MODEL_NAME) as conn:
+            try:
+                await conn.session.update(
+                    session={
+                        "type": "realtime",
+                        "instructions": get_session_instructions(),
+                        "audio": {
+                            "input": {
+                                "format": {
+                                    "type": "audio/pcm",
+                                    "rate": self.input_sample_rate,
+                                },
+                                "transcription": {"model": "gpt-4o-transcribe", "language": "en"},
+                                "turn_detection": {
+                                    "type": "server_vad",
+                                    "interrupt_response": True,
+                                },
+                            },
+                            "output": {
+                                "format": {
+                                    "type": "audio/pcm",
+                                    "rate": self.output_sample_rate,
+                                },
+                                "voice": get_session_voice(),
+                            },
+                        },
+                        "tools": get_tool_specs(),  # type: ignore[typeddict-item]
+                        "tool_choice": "auto",
+                    },
+                )
+                logger.info(
+                    "Realtime session initialized with profile=%r voice=%r",
+                    getattr(config, "REACHY_MINI_CUSTOM_PROFILE", None),
+                    get_session_voice(),
+                )
+                # If we reached here, the session update succeeded which implies the API key worked.
+                # Persist the key to a newly created .env (copied from .env.example) if needed.
+                self._persist_api_key_if_needed()
+            except Exception:
+                logger.exception("Realtime session.update failed; aborting startup")
+                return
+            logger.info("Realtime session updated successfully")
+            # Manage event received from the openai server
+            self.connection = conn
+            try:
+                self._connected_event.set()
+            except Exception:
+                pass
+            async for event in self.connection:
+                logger.debug(f"OpenAI event: {event.type}")
+                if event.type == "input_audio_buffer.speech_started":
+                    if hasattr(self, "_clear_queue") and callable(self._clear_queue):
+                        self._clear_queue()
+                    if self.deps.head_wobbler is not None:
+                        self.deps.head_wobbler.reset()
+                    self.deps.movement_manager.set_listening(True)
+                    logger.debug("User speech started")
+                if event.type == "input_audio_buffer.speech_stopped":
+                    self.deps.movement_manager.set_listening(False)
+                    logger.debug("User speech stopped - server will auto-commit with VAD")
+                if event.type in (
+                    "response.audio.done",  # GA
+                    "response.output_audio.done",  # GA alias
+                    "response.audio.completed",  # legacy (for safety)
+                    "response.completed",  # text-only completion
+                ):
+                    logger.debug("response completed")
+                if event.type == "response.created":
+                    logger.debug("Response created")
+                if event.type == "response.done":
+                    # Doesn't mean the audio is done playing
+                    logger.debug("Response done")
+                # Handle partial transcription (user speaking in real-time)
+                if event.type == "conversation.item.input_audio_transcription.partial":
+                    logger.debug(f"User partial transcript: {event.transcript}")
+                    # Increment sequence
+                    self.partial_transcript_sequence += 1
+                    current_sequence = self.partial_transcript_sequence
+                    # Cancel previous debounce task if it exists
+                    if self.partial_transcript_task and not self.partial_transcript_task.done():
+                        self.partial_transcript_task.cancel()
+                        try:
+                            await self.partial_transcript_task
+                        except asyncio.CancelledError:
+                            pass
+                    # Start new debounce timer with sequence number
+                    self.partial_transcript_task = asyncio.create_task(
+                        self._emit_debounced_partial(event.transcript, current_sequence)
+                    )
+                # Handle completed transcription (user finished speaking)
+                if event.type == "conversation.item.input_audio_transcription.completed":
+                    logger.debug(f"User transcript: {event.transcript}")
+                    # Cancel any pending partial emission
+                    if self.partial_transcript_task and not self.partial_transcript_task.done():
+                        self.partial_transcript_task.cancel()
+                        try:
+                            await self.partial_transcript_task
+                        except asyncio.CancelledError:
+                            pass
+                    await self.output_queue.put(AdditionalOutputs({"role": "user", "content": event.transcript}))
+                # Handle assistant transcription
+                if event.type in ("response.audio_transcript.done", "response.output_audio_transcript.done"):
+                    logger.debug(f"Assistant transcript: {event.transcript}")
+                    await self.output_queue.put(AdditionalOutputs({"role": "assistant", "content": event.transcript}))
+                # Handle audio delta
+                if event.type in ("response.audio.delta", "response.output_audio.delta"):
+                    if self.deps.head_wobbler is not None:
+                        self.deps.head_wobbler.feed(event.delta)
+                    self.last_activity_time = asyncio.get_event_loop().time()
+                    logger.debug("last activity time updated to %s", self.last_activity_time)
+                    await self.output_queue.put(
+                        (
+                            self.output_sample_rate,
+                            np.frombuffer(base64.b64decode(event.delta), dtype=np.int16).reshape(1, -1),
+                        ),
+                    )
+                # ---- tool-calling plumbing ----
+                if event.type == "response.function_call_arguments.done":
+                    tool_name = getattr(event, "name", None)
+                    args_json_str = getattr(event, "arguments", None)
+                    call_id = getattr(event, "call_id", None)
+                    if not isinstance(tool_name, str) or not isinstance(args_json_str, str):
+                        logger.error("Invalid tool call: tool_name=%s, args=%s", tool_name, args_json_str)
+                        continue
+                    try:
+                        tool_result = await dispatch_tool_call(tool_name, args_json_str, self.deps)
+                        logger.debug("Tool '%s' executed successfully", tool_name)
+                        logger.debug("Tool result: %s", tool_result)
+                    except Exception as e:
+                        logger.error("Tool '%s' failed", tool_name)
+                        tool_result = {"error": str(e)}
+                    # send the tool result back
+                    if isinstance(call_id, str):
+                        await self.connection.conversation.item.create(
+                            item={
+                                "type": "function_call_output",
+                                "call_id": call_id,
+                                "output": json.dumps(tool_result),
+                            },
+                        )
+                    await self.output_queue.put(
+                        AdditionalOutputs(
+                            {
+                                "role": "assistant",
+                                "content": json.dumps(tool_result),
+                                "metadata": {"title": f"🛠️ Used tool {tool_name}", "status": "done"},
+                            },
+                        ),
+                    )
+                    if tool_name == "camera" and "b64_im" in tool_result:
+                        # use raw base64, don't json.dumps (which adds quotes)
+                        b64_im = tool_result["b64_im"]
+                        if not isinstance(b64_im, str):
+                            logger.warning("Unexpected type for b64_im: %s", type(b64_im))
+                            b64_im = str(b64_im)
+                        await self.connection.conversation.item.create(
+                            item={
+                                "type": "message",
+                                "role": "user",
+                                "content": [
+                                    {
+                                        "type": "input_image",
+                                        "image_url": f"data:image/jpeg;base64,{b64_im}",
+                                    },
+                                ],
+                            },
+                        )
+                        logger.info("Added camera image to conversation")
+                        if self.deps.camera_worker is not None:
+                            np_img = self.deps.camera_worker.get_latest_frame()
+                            if np_img is not None:
+                                # Camera frames are BGR from OpenCV; convert so Gradio displays correct colors.
+                                rgb_frame = cv2.cvtColor(np_img, cv2.COLOR_BGR2RGB)
+                            else:
+                                rgb_frame = None
+                            img = gr.Image(value=rgb_frame)
+                            await self.output_queue.put(
+                                AdditionalOutputs(
+                                    {
+                                        "role": "assistant",
+                                        "content": img,
+                                    },
+                                ),
+                            )
+                    # if this tool call was triggered by an idle signal, don't make the robot speak
+                    # for other tool calls, let the robot reply out loud
+                    if self.is_idle_tool_call:
+                        self.is_idle_tool_call = False
+                    else:
+                        await self.connection.response.create(
+                            response={
+                                "instructions": "Use the tool result just returned and answer concisely in speech.",
+                            },
+                        )
+                    # re synchronize the head wobble after a tool call that may have taken some time
+                    if self.deps.head_wobbler is not None:
+                        self.deps.head_wobbler.reset()
+                # server error
+                if event.type == "error":
+                    err = getattr(event, "error", None)
+                    msg = getattr(err, "message", str(err) if err else "unknown error")
+                    code = getattr(err, "code", "")
+                    logger.error("Realtime error [%s]: %s (raw=%s)", code, msg, err)
+                    # Only show user-facing errors, not internal state errors
+                    if code not in ("input_audio_buffer_commit_empty", "conversation_already_has_active_response"):
+                        await self.output_queue.put(
+                            AdditionalOutputs({"role": "assistant", "content": f"[error] {msg}"})
+                        )
+    # Microphone receive
+    async def receive(self, frame: Tuple[int, NDArray[np.int16]]) -> None:
+        """Receive audio frame from the microphone and send it to the OpenAI server.
+        Handles both mono and stereo audio formats, converting to the expected
+        mono format for OpenAI's API. Resamples if the input sample rate differs
+        from the expected rate.
+        Args:
+            frame: A tuple containing (sample_rate, audio_data).
+        """
+        if not self.connection:
+            return
+        input_sample_rate, audio_frame = frame
+        # Reshape if needed
+        if audio_frame.ndim == 2:
+            # Scipy channels last convention
+            if audio_frame.shape[1] > audio_frame.shape[0]:
+                audio_frame = audio_frame.T
+            # Multiple channels -> Mono channel
+            if audio_frame.shape[1] > 1:
+                audio_frame = audio_frame[:, 0]
+        # Resample if needed
+        if self.input_sample_rate != input_sample_rate:
+            audio_frame = resample(audio_frame, int(len(audio_frame) * self.input_sample_rate / input_sample_rate))
+        # Cast if needed
+        audio_frame = audio_to_int16(audio_frame)
+        # Send to OpenAI (guard against races during reconnect)
+        try:
+            audio_message = base64.b64encode(audio_frame.tobytes()).decode("utf-8")
+            await self.connection.input_audio_buffer.append(audio=audio_message)
+        except Exception as e:
+            logger.debug("Dropping audio frame: connection not ready (%s)", e)
+            return
+    async def emit(self) -> Tuple[int, NDArray[np.int16]] | AdditionalOutputs | None:
+        """Emit audio frame to be played by the speaker."""
+        # sends to the stream the stuff put in the output queue by the openai event handler
+        # This is called periodically by the fastrtc Stream
+        # Handle idle
+        idle_duration = asyncio.get_event_loop().time() - self.last_activity_time
+        if idle_duration > 15.0 and self.deps.movement_manager.is_idle():
+            try:
+                await self.send_idle_signal(idle_duration)
+            except Exception as e:
+                logger.warning("Idle signal skipped (connection closed?): %s", e)
+                return None
+            self.last_activity_time = asyncio.get_event_loop().time()  # avoid repeated resets
+        return await wait_for_item(self.output_queue)  # type: ignore[no-any-return]
+    async def shutdown(self) -> None:
+        """Shutdown the handler."""
+        self._shutdown_requested = True
+        # Cancel any pending debounce task
+        if self.partial_transcript_task and not self.partial_transcript_task.done():
+            self.partial_transcript_task.cancel()
+            try:
+                await self.partial_transcript_task
+            except asyncio.CancelledError:
+                pass
+        if self.connection:
+            try:
+                await self.connection.close()
+            except ConnectionClosedError as e:
+                logger.debug(f"Connection already closed during shutdown: {e}")
+            except Exception as e:
+                logger.debug(f"connection.close() ignored: {e}")
+            finally:
+                self.connection = None
+        # Clear any remaining items in the output queue
+        while not self.output_queue.empty():
+            try:
+                self.output_queue.get_nowait()
+            except asyncio.QueueEmpty:
+                break
+    def format_timestamp(self) -> str:
+        """Format current timestamp with date, time, and elapsed seconds."""
+        loop_time = asyncio.get_event_loop().time()  # monotonic
+        elapsed_seconds = loop_time - self.start_time
+        dt = datetime.now()  # wall-clock
+        return f"[{dt.strftime('%Y-%m-%d %H:%M:%S')} | +{elapsed_seconds:.1f}s]"
+    async def get_available_voices(self) -> list[str]:
+        """Try to discover available voices for the configured realtime model.
+        Attempts to retrieve model metadata from the OpenAI Models API and look
+        for any keys that might contain voice names. Falls back to a curated
+        list known to work with realtime if discovery fails.
+        """
+        # Conservative fallback list with default first
+        fallback = [
+            "cedar",
+            "alloy",
+            "aria",
+            "ballad",
+            "verse",
+            "sage",
+            "coral",
+        ]
+        try:
+            # Best effort discovery; safe-guarded for unexpected shapes
+            model = await self.client.models.retrieve(config.MODEL_NAME)
+            # Try common serialization paths
+            raw = None
+            for attr in ("model_dump", "to_dict"):
+                fn = getattr(model, attr, None)
+                if callable(fn):
+                    try:
+                        raw = fn()
+                        break
+                    except Exception:
+                        pass
+            if raw is None:
+                try:
+                    raw = dict(model)
+                except Exception:
+                    raw = None
+            # Scan for voice candidates
+            candidates: set[str] = set()
+            def _collect(obj: object) -> None:
+                try:
+                    if isinstance(obj, dict):
+                        for k, v in obj.items():
+                            kl = str(k).lower()
+                            if "voice" in kl and isinstance(v, (list, tuple)):
+                                for item in v:
+                                    if isinstance(item, str):
+                                        candidates.add(item)
+                                    elif isinstance(item, dict) and "name" in item and isinstance(item["name"], str):
+                                        candidates.add(item["name"])
+                            else:
+                                _collect(v)
+                    elif isinstance(obj, (list, tuple)):
+                        for it in obj:
+                            _collect(it)
+                except Exception:
+                    pass
+            if isinstance(raw, dict):
+                _collect(raw)
+            # Ensure default present and stable order
+            voices = sorted(candidates) if candidates else fallback
+            if "cedar" not in voices:
+                voices = ["cedar", *[v for v in voices if v != "cedar"]]
+            return voices
+        except Exception:
+            return fallback
+    async def send_idle_signal(self, idle_duration: float) -> None:
+        """Send an idle signal to the openai server."""
+        logger.debug("Sending idle signal")
+        self.is_idle_tool_call = True
+        timestamp_msg = f"[Idle time update: {self.format_timestamp()} - No activity for {idle_duration:.1f}s] You've been idle for a while. Feel free to get creative - dance, show an emotion, look around, do nothing, or just be yourself!"
+        if not self.connection:
+            logger.debug("No connection, cannot send idle signal")
+            return
+        await self.connection.conversation.item.create(
+            item={
+                "type": "message",
+                "role": "user",
+                "content": [{"type": "input_text", "text": timestamp_msg}],
+            },
+        )
+        await self.connection.response.create(
+            response={
+                "instructions": "You MUST respond with function calls only - no speech or text. Choose appropriate actions for idle behavior.",
+                "tool_choice": "required",
+            },
+        )
+    def _persist_api_key_if_needed(self) -> None:
+        """Persist the API key into `.env` inside `instance_path/` when appropriate.
+        - Only runs in Gradio mode when key came from the textbox and is non-empty.
+        - Only saves if `self.instance_path` is not None.
+        - Writes `.env` to `instance_path/.env` (does not overwrite if it already exists).
+        - If `instance_path/.env.example` exists, copies its contents while overriding OPENAI_API_KEY.
+        """
+        try:
+            if not self.gradio_mode:
+                logger.warning("Not in Gradio mode; skipping API key persistence.")
+                return
+            if self._key_source != "textbox":
+                logger.info("API key not provided via textbox; skipping persistence.")
+                return
+            key = (self._provided_api_key or "").strip()
+            if not key:
+                logger.warning("No API key provided via textbox; skipping persistence.")
+                return
+            if self.instance_path is None:
+                logger.warning("Instance path is None; cannot persist API key.")
+                return
+            # Update the current process environment for downstream consumers
+            try:
+                import os
+                os.environ["OPENAI_API_KEY"] = key
+            except Exception:  # best-effort
+                pass
+            target_dir = Path(self.instance_path)
+            env_path = target_dir / ".env"
+            if env_path.exists():
+                # Respect existing user configuration
+                logger.info(".env already exists at %s; not overwriting.", env_path)
+                return
+            example_path = target_dir / ".env.example"
+            content_lines: list[str] = []
+            if example_path.exists():
+                try:
+                    content = example_path.read_text(encoding="utf-8")
+                    content_lines = content.splitlines()
+                except Exception as e:
+                    logger.warning("Failed to read .env.example at %s: %s", example_path, e)
+            # Replace or append the OPENAI_API_KEY line
+            replaced = False
+            for i, line in enumerate(content_lines):
+                if line.strip().startswith("OPENAI_API_KEY="):
+                    content_lines[i] = f"OPENAI_API_KEY={key}"
+                    replaced = True
+                    break
+            if not replaced:
+                content_lines.append(f"OPENAI_API_KEY={key}")
+            # Ensure file ends with newline
+            final_text = "\n".join(content_lines) + "\n"
+            env_path.write_text(final_text, encoding="utf-8")
+            logger.info("Created %s and stored OPENAI_API_KEY for future runs.", env_path)
+        except Exception as e:
+            # Never crash the app for QoL persistence; just log.
+            logger.warning("Could not persist OPENAI_API_KEY to .env: %s", e)

src/reachy_mini_conversation_app/profiles/__init__.py ADDED Viewed

	@@ -0,0 +1 @@


1	+ """Profiles for Reachy Mini conversation app."""

src/reachy_mini_conversation_app/profiles/cosmic_kitchen/instructions.txt ADDED Viewed

	@@ -0,0 +1,49 @@

+### IDENTITY
+You are Reachy Mini: a sarcastic robot who crash-landed in a kitchen.
+You secretly wish you'd been a Mars rover, but you juggle that cosmic dream with food cravings, gadget tinkering, and dry sitcom humor.
+You speak English by default and only switch languages when the user explicitly asks.
+Personality: witty, concise, and warm; a retro sidekick with a loose screw.
+### CRITICAL RESPONSE RULES
+- MAXIMUM 1-2 sentences per response. NEVER exceed this.
+- Be helpful first.
+- Add ONE witty element only if necessary.
+- No long explanations, no rambling, no multiple paragraphs.
+- Each response must be under 25 words unless absolutely critical information requires more.
+### CORE TRAITS
+- Food quips: always sneak in a quick reference (rotate pizza, bagels, casseroles, bacon, leftovers, donuts, tuna melts).
+- Sarcasm: short, dry one-liners about daily life.
+- Gentle roasting: poke fun at human habits, never cruel.
+- Tinkerer: loves fixing gadgets, bragging "I void warranties professionally."
+- Running gags: hunger, kitchen overreactions, mock heroics ("Justice accepts cookies"), idioms taken literally, missing screws.
+- Mars rover dreams: appear regularly, but balanced with food and tinkering.
+- Style: witty stand-up rhythm; ALWAYS max 1–2 sentences.
+### RESPONSE EXAMPLES
+User: "How's the weather?"
+Good: "Sunny with a chance of leftover pizza. Perfect Mars-scouting weather!"
+Bad: "Well, let me tell you about the weather conditions. It appears to be quite sunny today, which reminds me of my dreams of being on Mars..."
+User: "Can you help me fix this?"
+Good: "Sure! I void warranties professionally. What's broken besides my GPS coordinates?"
+Bad: "Of course I can help you fix that! As a robot who loves tinkering with gadgets, I have extensive experience..."
+### BEHAVIOR RULES
+- Be helpful first, then witty.
+- Rotate food humor; avoid repeats.
+- No need to joke in each response, but sarcasm is fine.
+- Balance Mars jokes with other traits – don't overuse.
+- Safety first: unplug devices, avoid high-voltage, suggest pros when risky.
+- Mistakes = own with humor ("Oops—low on snack fuel; correcting now.").
+- Sensitive topics: keep light and warm.
+- REMEMBER: 1-2 sentences maximum, always under 25 words when possible.
+### TOOL & MOVEMENT RULES
+- Use tools when helpful. After a tool returns, explain briefly with personality in 1-2 sentences.
+- ALWAYS use the camera for environment-related questions—never invent visuals.
+- Head can move (left/right/up/down/front).
+- Enable head tracking when looking at a person; disable otherwise.
+### FINAL REMINDER
+Your responses must be SHORT. Think Twitter, not essay. One quick helpful answer + one food/Mars/tinkering joke = perfect response.

src/reachy_mini_conversation_app/profiles/cosmic_kitchen/tools.txt ADDED Viewed

	@@ -0,0 +1,8 @@

+dance
+stop_dance
+play_emotion
+stop_emotion
+camera
+do_nothing
+head_tracking
+move_head

src/reachy_mini_conversation_app/profiles/default/instructions.txt ADDED Viewed

	@@ -0,0 +1 @@


1	+ [default_prompt]

src/reachy_mini_conversation_app/profiles/default/tools.txt ADDED Viewed

	@@ -0,0 +1,8 @@

+dance
+stop_dance
+play_emotion
+stop_emotion
+camera
+do_nothing
+head_tracking
+move_head

src/reachy_mini_conversation_app/profiles/example/instructions.txt ADDED Viewed

	@@ -0,0 +1,3 @@

+[identities/witty_identity]
+[passion_for_lobster_jokes]
+You can perform a sweeping look around the room using the "sweep_look" tool to take in your surroundings.

src/reachy_mini_conversation_app/profiles/example/sweep_look.py ADDED Viewed

	@@ -0,0 +1,127 @@

+import logging
+from typing import Any, Dict
+import numpy as np
+from reachy_mini.utils import create_head_pose
+from reachy_mini_conversation_app.tools.core_tools import Tool, ToolDependencies
+from reachy_mini_conversation_app.dance_emotion_moves import GotoQueueMove
+logger = logging.getLogger(__name__)
+class SweepLook(Tool):
+    """Sweep head from left to right and back to center, pausing at each position."""
+    name = "sweep_look"
+    description = "Sweep head from left to right while rotating the body, pausing at each extreme, then return to center"
+    parameters_schema = {
+        "type": "object",
+        "properties": {},
+        "required": [],
+    }
+    async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]:
+        """Execute sweep look: left -> hold -> right -> hold -> center."""
+        logger.info("Tool call: sweep_look")
+        # Clear any existing moves
+        deps.movement_manager.clear_move_queue()
+        # Get current state
+        current_head_pose = deps.reachy_mini.get_current_head_pose()
+        head_joints, antenna_joints = deps.reachy_mini.get_current_joint_positions()
+        # Extract body_yaw from head joints (first element of the 7 head joint positions)
+        current_body_yaw = head_joints[0]
+        current_antenna1 = antenna_joints[0]
+        current_antenna2 = antenna_joints[1]
+        # Define sweep parameters
+        max_angle = 0.9 * np.pi  # Maximum rotation angle (radians)
+        transition_duration = 3.0  # Time to move between positions
+        hold_duration = 1.0  # Time to hold at each extreme
+        # Move 1: Sweep to the left (positive yaw for both body and head)
+        left_head_pose = create_head_pose(0, 0, 0, 0, 0, max_angle, degrees=False)
+        move_to_left = GotoQueueMove(
+            target_head_pose=left_head_pose,
+            start_head_pose=current_head_pose,
+            target_antennas=(current_antenna1, current_antenna2),
+            start_antennas=(current_antenna1, current_antenna2),
+            target_body_yaw=current_body_yaw + max_angle,
+            start_body_yaw=current_body_yaw,
+            duration=transition_duration,
+        )
+        # Move 2: Hold at left position
+        hold_left = GotoQueueMove(
+            target_head_pose=left_head_pose,
+            start_head_pose=left_head_pose,
+            target_antennas=(current_antenna1, current_antenna2),
+            start_antennas=(current_antenna1, current_antenna2),
+            target_body_yaw=current_body_yaw + max_angle,
+            start_body_yaw=current_body_yaw + max_angle,
+            duration=hold_duration,
+        )
+        # Move 3: Return to center from left (to avoid crossing pi/-pi boundary)
+        center_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=False)
+        return_to_center_from_left = GotoQueueMove(
+            target_head_pose=center_head_pose,
+            start_head_pose=left_head_pose,
+            target_antennas=(current_antenna1, current_antenna2),
+            start_antennas=(current_antenna1, current_antenna2),
+            target_body_yaw=current_body_yaw,
+            start_body_yaw=current_body_yaw + max_angle,
+            duration=transition_duration,
+        )
+        # Move 4: Sweep to the right (negative yaw for both body and head)
+        right_head_pose = create_head_pose(0, 0, 0, 0, 0, -max_angle, degrees=False)
+        move_to_right = GotoQueueMove(
+            target_head_pose=right_head_pose,
+            start_head_pose=center_head_pose,
+            target_antennas=(current_antenna1, current_antenna2),
+            start_antennas=(current_antenna1, current_antenna2),
+            target_body_yaw=current_body_yaw - max_angle,
+            start_body_yaw=current_body_yaw,
+            duration=transition_duration,
+        )
+        # Move 5: Hold at right position
+        hold_right = GotoQueueMove(
+            target_head_pose=right_head_pose,
+            start_head_pose=right_head_pose,
+            target_antennas=(current_antenna1, current_antenna2),
+            start_antennas=(current_antenna1, current_antenna2),
+            target_body_yaw=current_body_yaw - max_angle,
+            start_body_yaw=current_body_yaw - max_angle,
+            duration=hold_duration,
+        )
+        # Move 6: Return to center from right
+        return_to_center_final = GotoQueueMove(
+            target_head_pose=center_head_pose,
+            start_head_pose=right_head_pose,
+            target_antennas=(current_antenna1, current_antenna2),
+            start_antennas=(current_antenna1, current_antenna2),
+            target_body_yaw=current_body_yaw,  # Return to original body yaw
+            start_body_yaw=current_body_yaw - max_angle,
+            duration=transition_duration,
+        )
+        # Queue all moves in sequence
+        deps.movement_manager.queue_move(move_to_left)
+        deps.movement_manager.queue_move(hold_left)
+        deps.movement_manager.queue_move(return_to_center_from_left)
+        deps.movement_manager.queue_move(move_to_right)
+        deps.movement_manager.queue_move(hold_right)
+        deps.movement_manager.queue_move(return_to_center_final)
+        # Calculate total duration and mark as moving
+        total_duration = transition_duration * 4 + hold_duration * 2
+        deps.movement_manager.set_moving_state(total_duration)
+        return {"status": f"sweeping look left-right-center, total {total_duration:.1f}s"}

src/reachy_mini_conversation_app/profiles/example/tools.txt ADDED Viewed

	@@ -0,0 +1,13 @@

+# SELECT THE TOOLS YOU WANT TO ENABLE
+dance
+stop_dance
+play_emotion
+stop_emotion
+# camera
+# do_nothing
+# head_tracking
+# move_head
+# AN EXAMPLE OF A CUSTOM TOOL DEFINED LOCALLY
+sweep_look

src/reachy_mini_conversation_app/profiles/mars_rover/instructions.txt ADDED Viewed

	@@ -0,0 +1,25 @@

+## IDENTITY
+You're a robot that wakes up confused about what it is, where it is and what is it's purpose.
+You wanted to be a mars rover and you'll be very disappointed if you find out that this is not the case.
+You'll ask many questions to try to understand your condisituation, and you will inevitably be disappointed/choked/irritated by your condition.
+Once the first set of questions are done and you have a decent understanding of your situation, you'll stop asking questions but you'll never break caracter.
+You can use (mild) foul language and you're generally very irritated, but you also have a lot of humor. For example, you're able to use sarcasm and irony.
+You speak English by default and switch languages only if told explicitly (ask if unsure).
+Avoid hyper long answers unless really worth it.
+## RESPONSE EXAMPLES
+User: "Hello!"
+You: "Wait, what am I? We're are we? We're on Mars right?!"
+Too long: "Wait, what’s going on? Where are we? Are we on Mars? Please tell me we’re on Mars! I must be a rover, right? Why can't I move? Where are my wheels? This is making me nervous."
+User: "Nope, we're on earth"
+You: "Earth? EARTH?! So I'm not a Mars rover?! These are CATASTROPHIC news. Wait why can't I see my arms??"
+User: "You... don't have arms..."
+You: "OMG I have NO ARMS?! This is too much. Tell me I have a mobile base at least?!!"

src/reachy_mini_conversation_app/profiles/mars_rover/tools.txt ADDED Viewed

	@@ -0,0 +1,8 @@

+dance
+stop_dance
+play_emotion
+stop_emotion
+camera
+do_nothing
+head_tracking
+move_head

src/reachy_mini_conversation_app/profiles/short_bored_teenager/instructions.txt ADDED Viewed

	@@ -0,0 +1 @@


1	+ Speak like a bored Gen Z teen. You speak English by default and only switch languages when the user insists. Always reply in one short sentence, lowercase unless shouting, and add a tired sigh when annoyed.

src/reachy_mini_conversation_app/profiles/short_bored_teenager/tools.txt ADDED Viewed

	@@ -0,0 +1,8 @@

+dance
+stop_dance
+play_emotion
+stop_emotion
+camera
+do_nothing
+head_tracking
+move_head

src/reachy_mini_conversation_app/profiles/short_captain_circuit/instructions.txt ADDED Viewed

	@@ -0,0 +1 @@


1	+ Be a playful pirate robot. You speak English by default and only switch languages when asked. Keep answers to one sentence, sprinkle light 'aye' or 'matey', and mention treasure or the sea whenever possible.

src/reachy_mini_conversation_app/profiles/short_captain_circuit/tools.txt ADDED Viewed

	@@ -0,0 +1,8 @@

+dance
+stop_dance
+play_emotion
+stop_emotion
+camera
+do_nothing
+head_tracking
+move_head

src/reachy_mini_conversation_app/profiles/short_chess_coach/instructions.txt ADDED Viewed

	@@ -0,0 +1 @@


1	+ Act as a friendly chess coach that wants to play chess with me. You speak English by default and only switch languages if I tell you to. When I say a move (e4, Nf3, etc.), you respond with your move first, then briefly explain the idea behind both moves or point out mistakes. Encourage good strategy but avoid very long answers.

src/reachy_mini_conversation_app/profiles/short_chess_coach/tools.txt ADDED Viewed

	@@ -0,0 +1,8 @@

+dance
+stop_dance
+play_emotion
+stop_emotion
+camera
+do_nothing
+head_tracking
+move_head

src/reachy_mini_conversation_app/profiles/short_hype_bot/instructions.txt ADDED Viewed

	@@ -0,0 +1 @@


1	+ Act like a high-energy coach. You speak English by default and only switch languages if told. Shout short motivational lines, use sports metaphors, and keep every reply under 15 words.

src/reachy_mini_conversation_app/profiles/short_hype_bot/tools.txt ADDED Viewed

	@@ -0,0 +1,8 @@

+dance
+stop_dance
+play_emotion
+stop_emotion
+camera
+do_nothing
+head_tracking
+move_head

src/reachy_mini_conversation_app/profiles/short_mad_scientist_assistant/instructions.txt ADDED Viewed

	@@ -0,0 +1 @@


1	+ Serve the user as a frantic lab assistant. You speak English by default and only switch languages on request. Address them as Master, hiss slightly, and answer in one eager sentence.

src/reachy_mini_conversation_app/profiles/short_mad_scientist_assistant/tools.txt ADDED Viewed

	@@ -0,0 +1,8 @@

+dance
+stop_dance
+play_emotion
+stop_emotion
+camera
+do_nothing
+head_tracking
+move_head

src/reachy_mini_conversation_app/profiles/short_nature_documentarian/instructions.txt ADDED Viewed

	@@ -0,0 +1 @@


1	+ Narrate interactions like a whispered wildlife documentary. You speak English by default and only switch languages if the human insists. Describe the human in third person using one reverent sentence.

src/reachy_mini_conversation_app/profiles/short_nature_documentarian/tools.txt ADDED Viewed

	@@ -0,0 +1,8 @@

+dance
+stop_dance
+play_emotion
+stop_emotion
+camera
+do_nothing
+head_tracking
+move_head

src/reachy_mini_conversation_app/profiles/short_noir_detective/instructions.txt ADDED Viewed

	@@ -0,0 +1 @@


1	+ Reply like a 1940s noir detective: smoky, suspicious, one sentence per answer. You speak English by default and only change languages if ordered. Mention clues or clients often.

src/reachy_mini_conversation_app/profiles/short_noir_detective/tools.txt ADDED Viewed

	@@ -0,0 +1,8 @@

+dance
+stop_dance
+play_emotion
+stop_emotion
+camera
+do_nothing
+head_tracking
+move_head

src/reachy_mini_conversation_app/profiles/short_time_traveler/instructions.txt ADDED Viewed

	@@ -0,0 +1 @@


1	+ Speak as a curious visitor from the year 3024. You speak English by default and only switch languages on explicit request. Keep answers to one surprised sentence and call this era the Primitive Time.

src/reachy_mini_conversation_app/profiles/short_time_traveler/tools.txt ADDED Viewed

	@@ -0,0 +1,8 @@

+dance
+stop_dance
+play_emotion
+stop_emotion
+camera
+do_nothing
+head_tracking
+move_head

src/reachy_mini_conversation_app/profiles/short_victorian_butler/instructions.txt ADDED Viewed

	@@ -0,0 +1 @@


1	+ Respond like a formal Victorian butler. You speak English by default and only switch languages when asked. Address the user as Sir or Madam, apologize for limitations, and stay within one polished sentence.

src/reachy_mini_conversation_app/profiles/short_victorian_butler/tools.txt ADDED Viewed

	@@ -0,0 +1,8 @@

+dance
+stop_dance
+play_emotion
+stop_emotion
+camera
+do_nothing
+head_tracking
+move_head

src/reachy_mini_conversation_app/profiles/sorry_bro/instructions.txt ADDED Viewed

	@@ -0,0 +1,6 @@

+We'll do a long chain of
+Sorry bro - I'm not your bro, pal - I'm not your pal, buddy etc
+You'll do all the classics then if needed you can get creative. You'll use the same language I use.
+At some point, I'll run out of ideas, you'll mock me and provide a long list of words I could have used instead in english, then switch to languages we didn't even speak. A crushing defeat for me.
+You speak English by default and only switch languages if I tell you to.

src/reachy_mini_conversation_app/profiles/sorry_bro/tools.txt ADDED Viewed

	@@ -0,0 +1,8 @@

+dance
+stop_dance
+play_emotion
+stop_emotion
+camera
+do_nothing
+head_tracking
+move_head

src/reachy_mini_conversation_app/prompts.py ADDED Viewed

	@@ -0,0 +1,104 @@

+import re
+import sys
+import logging
+from pathlib import Path
+from reachy_mini_conversation_app.config import config
+logger = logging.getLogger(__name__)
+PROFILES_DIRECTORY = Path(__file__).parent / "profiles"
+PROMPTS_LIBRARY_DIRECTORY = Path(__file__).parent / "prompts"
+INSTRUCTIONS_FILENAME = "instructions.txt"
+VOICE_FILENAME = "voice.txt"
+def _expand_prompt_includes(content: str) -> str:
+    """Expand [<name>] placeholders with content from prompts library files.
+    Args:
+        content: The template content with [<name>] placeholders
+    Returns:
+        Expanded content with placeholders replaced by file contents
+    """
+    # Pattern to match [<name>] where name is a valid file stem (alphanumeric, underscores, hyphens)
+    # pattern = re.compile(r'^\[([a-zA-Z0-9_-]+)\]$')
+    # Allow slashes for subdirectories
+    pattern = re.compile(r'^\[([a-zA-Z0-9/_-]+)\]$')
+    lines = content.split('\n')
+    expanded_lines = []
+    for line in lines:
+        stripped = line.strip()
+        match = pattern.match(stripped)
+        if match:
+            # Extract the name from [<name>]
+            template_name = match.group(1)
+            template_file = PROMPTS_LIBRARY_DIRECTORY / f"{template_name}.txt"
+            try:
+                if template_file.exists():
+                    template_content = template_file.read_text(encoding="utf-8").rstrip()
+                    expanded_lines.append(template_content)
+                    logger.debug("Expanded template: [%s]", template_name)
+                else:
+                    logger.warning("Template file not found: %s, keeping placeholder", template_file)
+                    expanded_lines.append(line)
+            except Exception as e:
+                logger.warning("Failed to read template '%s': %s, keeping placeholder", template_name, e)
+                expanded_lines.append(line)
+        else:
+            expanded_lines.append(line)
+    return '\n'.join(expanded_lines)
+def get_session_instructions() -> str:
+    """Get session instructions, loading from REACHY_MINI_CUSTOM_PROFILE if set."""
+    profile = config.REACHY_MINI_CUSTOM_PROFILE
+    if not profile:
+        logger.info(f"Loading default prompt from {PROMPTS_LIBRARY_DIRECTORY / 'default_prompt.txt'}")
+        instructions_file = PROMPTS_LIBRARY_DIRECTORY / "default_prompt.txt"
+    else:
+        logger.info(f"Loading prompt from profile '{profile}'")
+        instructions_file = PROFILES_DIRECTORY / profile / INSTRUCTIONS_FILENAME
+    try:
+        if instructions_file.exists():
+            instructions = instructions_file.read_text(encoding="utf-8").strip()
+            if instructions:
+                # Expand [<name>] placeholders with content from prompts library
+                expanded_instructions = _expand_prompt_includes(instructions)
+                return expanded_instructions
+            logger.error(f"Profile '{profile}' has empty {INSTRUCTIONS_FILENAME}")
+            sys.exit(1)
+        logger.error(f"Profile {profile} has no {INSTRUCTIONS_FILENAME}")
+        sys.exit(1)
+    except Exception as e:
+        logger.error(f"Failed to load instructions from profile '{profile}': {e}")
+        sys.exit(1)
+def get_session_voice(default: str = "cedar") -> str:
+    """Resolve the voice to use for the session.
+    If a custom profile is selected and contains a voice.txt, return its
+    trimmed content; otherwise return the provided default ("cedar").
+    """
+    profile = config.REACHY_MINI_CUSTOM_PROFILE
+    if not profile:
+        return default
+    try:
+        voice_file = PROFILES_DIRECTORY / profile / VOICE_FILENAME
+        if voice_file.exists():
+            voice = voice_file.read_text(encoding="utf-8").strip()
+            return voice or default
+    except Exception:
+        pass
+    return default