Spaces:
Running
Running
Update app docs
Browse files- docs/assets/conversation_app_arch.svg +0 -0
- docs/assets/reachy_mini_dance.gif +3 -0
- docs/scheme.mmd +58 -0
- index.html +9 -3
docs/assets/conversation_app_arch.svg
ADDED
|
|
docs/assets/reachy_mini_dance.gif
ADDED
|
Git LFS Details
|
docs/scheme.mmd
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
config:
|
| 3 |
+
layout: dagre
|
| 4 |
+
flowchart:
|
| 5 |
+
htmlLabels: true
|
| 6 |
+
---
|
| 7 |
+
flowchart TB
|
| 8 |
+
User(["<span style='font-size:16px;font-weight:bold;'>User</span><br><span style='font-size:13px;color:#01579b;'>Person interacting with system</span>"])
|
| 9 |
+
-- audio stream -->
|
| 10 |
+
UI@{ label: "<span style='font-size:16px;font-weight:bold;'>UI Layer</span><br><span style='font-size:13px;color:#0277bd;'>Gradio/Console</span>" }
|
| 11 |
+
|
| 12 |
+
UI -- audio stream -->
|
| 13 |
+
OpenAI@{ label: "<span style='font-size:17px;font-weight:bold;'>gpt-realtime API</span><br><span style='font-size:13px; color:#7b1fa2;'>Audio+Tool Calls+Vision</span>" }
|
| 14 |
+
|
| 15 |
+
OpenAI -- audio stream -->
|
| 16 |
+
Motion@{ label: "<span style='font-size:16px;font-weight:bold;'>Motion Control</span><br><span style='font-size:13px;color:#f57f17;'>Audio Sync + Tracking</span>" }
|
| 17 |
+
|
| 18 |
+
OpenAI -- tool calls -->
|
| 19 |
+
Handlers@{ label: "<span style='font-size:16px;font-weight:bold;'>Tool Handlers</span><br><span style='font-size:12px;color:#f9a825;'>move_head, camera, head_tracking,<br/>dance, play_emotion, do_nothing</span>" }
|
| 20 |
+
|
| 21 |
+
Handlers -- movement
|
| 22 |
+
requests --> Motion
|
| 23 |
+
|
| 24 |
+
Handlers -- camera frames, face tracking -->
|
| 25 |
+
Camera@{ label: "<span style='font-size:16px;font-weight:bold;'>Camera Worker</span><br><span style='font-size:13px;color:#f57f17;'>Frame Buffer + Face Tracking</span>" }
|
| 26 |
+
|
| 27 |
+
Handlers -. image for
|
| 28 |
+
analysis .-> OpenAI
|
| 29 |
+
|
| 30 |
+
Camera -- face tracking --> Motion
|
| 31 |
+
|
| 32 |
+
Camera -. frames .->
|
| 33 |
+
Vision@{ label: "<span style='font-size:16px;font-weight:bold;'>Vision Processor</span><br><span style='font-size:13px;color:#7b1fa2;'>Local VLM (optional)</span>" }
|
| 34 |
+
|
| 35 |
+
Vision -. description .-> Handlers
|
| 36 |
+
|
| 37 |
+
Robot@{ label: "<span style='font-size:16px;font-weight:bold;'>reachy_mini</span><br><span style='font-size:13px;color:#c62828;'>Robot Control Library</span>" }
|
| 38 |
+
-- camera
|
| 39 |
+
frames --> Camera
|
| 40 |
+
|
| 41 |
+
Motion -- commands --> Robot
|
| 42 |
+
|
| 43 |
+
Handlers -- results --> OpenAI
|
| 44 |
+
|
| 45 |
+
User:::userStyle
|
| 46 |
+
UI:::uiStyle
|
| 47 |
+
OpenAI:::aiStyle
|
| 48 |
+
Motion:::coreStyle
|
| 49 |
+
Handlers:::toolStyle
|
| 50 |
+
Camera:::coreStyle
|
| 51 |
+
Vision:::aiStyle
|
| 52 |
+
Robot:::hardwareStyle
|
| 53 |
+
classDef userStyle fill:#e1f5fe,stroke:#01579b,stroke-width:3px
|
| 54 |
+
classDef uiStyle fill:#b3e5fc,stroke:#0277bd,stroke-width:2px
|
| 55 |
+
classDef aiStyle fill:#e1bee7,stroke:#7b1fa2,stroke-width:3px
|
| 56 |
+
classDef coreStyle fill:#fff9c4,stroke:#f57f17,stroke-width:2px
|
| 57 |
+
classDef hardwareStyle fill:#ef9a9a,stroke:#c62828,stroke-width:3px
|
| 58 |
+
classDef toolStyle fill:#fffde7,stroke:#f9a825,stroke-width:1px
|
index.html
CHANGED
|
@@ -35,7 +35,7 @@
|
|
| 35 |
<span>Low-latency voice loop</span>
|
| 36 |
<span>Camera insights on demand</span>
|
| 37 |
<span>Choreographed dances & emotions</span>
|
| 38 |
-
<span>
|
| 39 |
</div>
|
| 40 |
</div>
|
| 41 |
<div class="hero-visual">
|
|
@@ -76,6 +76,11 @@
|
|
| 76 |
<h3>Personalities on demand</h3>
|
| 77 |
<p>Switch conversation styles through profiles and decide which tools (dance, camera, tracking) each persona can use.</p>
|
| 78 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
<div class="feature-card">
|
| 80 |
<span class="icon">🌐</span>
|
| 81 |
<h3>Ready for your setup</h3>
|
|
@@ -90,9 +95,10 @@
|
|
| 90 |
<p class="eyebrow">How it feels</p>
|
| 91 |
<h3>From hello to helpful in seconds</h3>
|
| 92 |
<ul class="story-list">
|
| 93 |
-
<li><span>👋</span> Say
|
| 94 |
<li><span>👀</span> Ask what it sees; it can peek through the camera or keep focus on your face.</li>
|
| 95 |
<li><span>🎭</span> Trigger emotions or dance breaks to keep the conversation lively.</li>
|
|
|
|
| 96 |
<li><span>📝</span> Follow along with live transcripts in the web UI or run audio-only from the console.</li>
|
| 97 |
</ul>
|
| 98 |
</div>
|
|
@@ -107,7 +113,7 @@
|
|
| 107 |
<span class="chip">Face tracking</span>
|
| 108 |
<span class="chip">Camera tool</span>
|
| 109 |
<span class="chip">Dance library</span>
|
| 110 |
-
<span class="chip">
|
| 111 |
</div>
|
| 112 |
</div>
|
| 113 |
</div>
|
|
|
|
| 35 |
<span>Low-latency voice loop</span>
|
| 36 |
<span>Camera insights on demand</span>
|
| 37 |
<span>Choreographed dances & emotions</span>
|
| 38 |
+
<span>MCP server integration</span>
|
| 39 |
</div>
|
| 40 |
</div>
|
| 41 |
<div class="hero-visual">
|
|
|
|
| 76 |
<h3>Personalities on demand</h3>
|
| 77 |
<p>Switch conversation styles through profiles and decide which tools (dance, camera, tracking) each persona can use.</p>
|
| 78 |
</div>
|
| 79 |
+
<div class="feature-card">
|
| 80 |
+
<span class="icon">🔌</span>
|
| 81 |
+
<h3>MCP server support</h3>
|
| 82 |
+
<p>Connect to external data via MCP—web search, weather, stock prices, sports scores, news headlines, and more.</p>
|
| 83 |
+
</div>
|
| 84 |
<div class="feature-card">
|
| 85 |
<span class="icon">🌐</span>
|
| 86 |
<h3>Ready for your setup</h3>
|
|
|
|
| 95 |
<p class="eyebrow">How it feels</p>
|
| 96 |
<h3>From hello to helpful in seconds</h3>
|
| 97 |
<ul class="story-list">
|
| 98 |
+
<li><span>👋</span> Say "Hey Reachy" and start chatting—no extra setup in the moment.</li>
|
| 99 |
<li><span>👀</span> Ask what it sees; it can peek through the camera or keep focus on your face.</li>
|
| 100 |
<li><span>🎭</span> Trigger emotions or dance breaks to keep the conversation lively.</li>
|
| 101 |
+
<li><span>🔌</span> Query live data—weather, news, stocks—through MCP server connections.</li>
|
| 102 |
<li><span>📝</span> Follow along with live transcripts in the web UI or run audio-only from the console.</li>
|
| 103 |
</ul>
|
| 104 |
</div>
|
|
|
|
| 113 |
<span class="chip">Face tracking</span>
|
| 114 |
<span class="chip">Camera tool</span>
|
| 115 |
<span class="chip">Dance library</span>
|
| 116 |
+
<span class="chip">MCP tools</span>
|
| 117 |
</div>
|
| 118 |
</div>
|
| 119 |
</div>
|