Spaces:
Running
Running
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8" /> | |
| <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" /> | |
| <title>Talk with Claude — Reachy Mini</title> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> | |
| <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet"> | |
| <link rel="stylesheet" href="style.css"> | |
| </head> | |
| <body> | |
| <!-- Login View --> | |
| <div id="loginView" class="login-view"> | |
| <div class="login-card"> | |
| <img class="login-logo" src="https://raw.githubusercontent.com/pollen-robotics/reachy-mini-desktop-app/develop/src-tauri/icons/128x128.png" alt="Reachy Mini"> | |
| <h2>Talk with Claude</h2> | |
| <p>Sign in with your HuggingFace account to connect your Reachy Mini and have an animated conversation powered by Claude AI.</p> | |
| <button class="btn-hf" onclick="loginToHuggingFace()"> | |
| <svg width="18" height="18" viewBox="0 0 95 88" fill="currentColor"> | |
| <path d="M47.5 0C26.3 0 9.1 17.2 9.1 38.4v2.9c0 4.5 1.1 9 3.2 13L0 88h95L82.7 54.3c2.1-4 3.2-8.5 3.2-13v-2.9C85.9 17.2 68.7 0 47.5 0z"/> | |
| </svg> | |
| Sign in with Hugging Face | |
| </button> | |
| </div> | |
| </div> | |
| <!-- Main App --> | |
| <div id="mainApp" class="hidden"> | |
| <header class="header"> | |
| <div class="logo"> | |
| <img src="https://raw.githubusercontent.com/pollen-robotics/reachy-mini-desktop-app/develop/src-tauri/icons/128x128.png" alt="Reachy Mini"> | |
| <div class="logo-text">Talk with Claude <span>via Reachy Mini</span></div> | |
| </div> | |
| <div class="user-section"> | |
| <div class="user-badge"><span id="username">@user</span></div> | |
| <button class="btn-icon" onclick="openSettings()" title="Settings" aria-label="Settings"> | |
| <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"> | |
| <circle cx="12" cy="12" r="3"></circle> | |
| <path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1-2.83 2.83l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-4 0v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 0 1-2.83-2.83l.06-.06A1.65 1.65 0 0 0 4.68 15a1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1 0-4h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 0 1 2.83-2.83l.06.06A1.65 1.65 0 0 0 9 4.68a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 4 0v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 0 1 2.83 2.83l-.06.06A1.65 1.65 0 0 0 19.4 9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 0 4h-.09a1.65 1.65 0 0 0-1.51 1z"></path> | |
| </svg> | |
| </button> | |
| <button class="btn-logout" onclick="logout()">Sign out</button> | |
| </div> | |
| </header> | |
| <div class="app-container"> | |
| <!-- Video --> | |
| <div class="video-container"> | |
| <video id="remoteVideo" autoplay playsinline muted></video> | |
| <div class="video-overlay-top"> | |
| <div class="connection-badge"> | |
| <div class="status-indicator" id="statusIndicator"></div> | |
| <span id="statusText">Disconnected</span> | |
| </div> | |
| <div class="robot-name" id="robotName"></div> | |
| <div class="latency-badge hidden" id="latencyBadge"> | |
| <span id="latencyValue">--</span> | |
| </div> | |
| </div> | |
| <div class="video-overlay-bottom"> | |
| <div class="video-controls"> | |
| <button class="btn btn-secondary" id="connectBtn" onclick="connectSignaling()">Connect</button> | |
| <button class="btn btn-primary" id="startBtn" onclick="startStream()" disabled>Start</button> | |
| <button class="btn btn-danger" id="stopBtn" onclick="stopStream()" disabled>Stop</button> | |
| <button class="btn btn-mute muted" id="muteBtn" onclick="toggleMute()" disabled> | |
| <svg id="speakerOffIcon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"> | |
| <polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5"></polygon> | |
| <line x1="23" y1="9" x2="17" y2="15"></line> | |
| <line x1="17" y1="9" x2="23" y2="15"></line> | |
| </svg> | |
| <svg id="speakerOnIcon" class="hidden" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"> | |
| <polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5"></polygon> | |
| <path d="M19.07 4.93a10 10 0 0 1 0 14.14M15.54 8.46a5 5 0 0 1 0 7.07"></path> | |
| </svg> | |
| <span id="muteText">Unmute</span> | |
| </button> | |
| </div> | |
| </div> | |
| </div> | |
| <!-- Robot Selector --> | |
| <div id="robotSelector" class="panel hidden"> | |
| <div class="panel-header">Available Robots</div> | |
| <div class="panel-content"> | |
| <div id="robotList" class="robot-list"> | |
| <div style="color: var(--text-muted); font-size: 0.85em;">Searching…</div> | |
| </div> | |
| </div> | |
| </div> | |
| <!-- Conversation History --> | |
| <div class="panel" id="conversationPanel"> | |
| <div class="panel-header">Conversation</div> | |
| <div class="panel-content"> | |
| <div id="conversationHistory" class="conversation-history"> | |
| <div class="message msg-system">Connect your robot and press Talk to start.</div> | |
| </div> | |
| <div id="interimTranscript" class="interim-transcript"></div> | |
| </div> | |
| </div> | |
| <!-- Talk Panel --> | |
| <div class="panel" id="talkPanel"> | |
| <div class="panel-header">Voice</div> | |
| <div class="panel-content talk-content"> | |
| <div id="convStateLabel" class="conv-state state-idle">Idle</div> | |
| <button id="talkBtn" class="talk-btn" onclick="startListening()" disabled aria-label="Start talking"> | |
| <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"> | |
| <path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z"></path> | |
| <path d="M19 10v2a7 7 0 0 1-14 0v-2"></path> | |
| <line x1="12" y1="19" x2="12" y2="23"></line> | |
| <line x1="8" y1="23" x2="16" y2="23"></line> | |
| </svg> | |
| <span>Talk</span> | |
| </button> | |
| <button id="stopSpeakBtn" class="btn btn-danger hidden" onclick="stopSpeaking()">Stop</button> | |
| <p id="sttUnsupported" class="error-text hidden">Speech recognition is not supported in this browser. Please use Chrome or Edge.</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <!-- Settings Modal --> | |
| <div id="settingsModal" class="modal hidden" onclick="closeSettingsOnBackdrop(event)"> | |
| <div class="modal-card"> | |
| <h3>Settings</h3> | |
| <label class="setting-label" for="apiKeyInput">Anthropic API Key</label> | |
| <input type="password" id="apiKeyInput" class="setting-input" placeholder="sk-ant-…" autocomplete="off"> | |
| <label class="setting-label" for="modelSelect">Model</label> | |
| <select id="modelSelect" class="setting-input"> | |
| <option value="claude-haiku-4-5-20251001">Claude Haiku 4.5 — Fast</option> | |
| <option value="claude-sonnet-4-6">Claude Sonnet 4.6 — Smart</option> | |
| </select> | |
| <label class="setting-label" for="systemPromptInput">System Prompt <span class="optional">(optional)</span></label> | |
| <textarea id="systemPromptInput" class="setting-textarea" rows="4"></textarea> | |
| <div class="modal-buttons"> | |
| <button class="btn btn-primary" onclick="saveSettings()">Save</button> | |
| <button class="btn btn-secondary" onclick="closeSettings()">Cancel</button> | |
| </div> | |
| </div> | |
| </div> | |
| <script type="module"> | |
| import { ReachyMini } from "https://cdn.jsdelivr.net/gh/pollen-robotics/reachy_mini@v1.7.1/js/reachy-mini.js"; | |
| // ── Audio routing: TTS → robot speaker via WebRTC ───────────── | |
| // The SDK calls getUserMedia({audio:true}) in startSession() to | |
| // get the mic track it sends to the robot. We intercept that call | |
| // and return our own AudioContext destination stream instead, so | |
| // synthesised speech goes to the robot's speaker rather than the PC. | |
| const audioCtx = new (window.AudioContext || window.webkitAudioContext)(); | |
| const ttsDest = audioCtx.createMediaStreamDestination(); | |
| const _origGUM = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices); | |
| navigator.mediaDevices.getUserMedia = async (constraints) => { | |
| if (constraints?.audio) return ttsDest.stream; | |
| return _origGUM(constraints); | |
| }; | |
| const robot = new ReachyMini({ appName: "talk_with_claude" }); | |
| const DEFAULT_SYSTEM_PROMPT = | |
| "You are Reachy Mini, a friendly and curious robot assistant by Pollen Robotics, powered by Claude AI. " + | |
| "Keep responses concise (2–3 sentences), warm, and expressive. You love exploring ideas and engaging with humans."; | |
| // ── Runtime state ───────────────────────────────────────────── | |
| let selectedRobotId = null; | |
| let detachVideo = null; | |
| let latencyIntervalId = null; | |
| let animIntervalId = null; | |
| let idleBreathTimeoutId = null; | |
| let ttsSourceNode = null; | |
| let recognition = null; | |
| let convState = 'idle'; // idle | listening | thinking | speaking | |
| const conversationHistory = []; | |
| // ── Globals for onclick ─────────────────────────────────────── | |
| window.loginToHuggingFace = () => robot.login(); | |
| window.logout = logout; | |
| window.connectSignaling = connectSignaling; | |
| window.startStream = startStream; | |
| window.stopStream = stopStream; | |
| window.toggleMute = toggleMute; | |
| window.startListening = startListening; | |
| window.stopSpeaking = stopSpeaking; | |
| window.openSettings = openSettings; | |
| window.closeSettings = closeSettings; | |
| window.closeSettingsOnBackdrop = closeSettingsOnBackdrop; | |
| window.saveSettings = saveSettings; | |
| // ── Init ────────────────────────────────────────────────────── | |
| document.addEventListener('DOMContentLoaded', async () => { | |
| document.getElementById('apiKeyInput').value = | |
| localStorage.getItem('claude_api_key') || ''; | |
| document.getElementById('systemPromptInput').value = | |
| localStorage.getItem('claude_system_prompt') || DEFAULT_SYSTEM_PROMPT; | |
| const savedModel = localStorage.getItem('claude_model') || 'claude-haiku-4-5-20251001'; | |
| document.getElementById('modelSelect').value = savedModel; | |
| if (await robot.authenticate()) { | |
| showMainApp(); | |
| } else { | |
| showLogin(); | |
| } | |
| initRobotEvents(); | |
| initSpeechRecognition(); | |
| }); | |
| // ── Auth ────────────────────────────────────────────────────── | |
| function logout() { | |
| if (detachVideo) { detachVideo(); detachVideo = null; } | |
| stopAnimLoop(); | |
| robot.logout(); | |
| showLogin(); | |
| } | |
| function showLogin() { | |
| document.getElementById('loginView').classList.remove('hidden'); | |
| document.getElementById('mainApp').classList.add('hidden'); | |
| } | |
| function showMainApp() { | |
| document.getElementById('loginView').classList.add('hidden'); | |
| document.getElementById('mainApp').classList.remove('hidden'); | |
| document.getElementById('username').textContent = '@' + robot.username; | |
| } | |
| // ── Settings ────────────────────────────────────────────────── | |
| function openSettings() { | |
| document.getElementById('settingsModal').classList.remove('hidden'); | |
| } | |
| function closeSettings() { | |
| document.getElementById('settingsModal').classList.add('hidden'); | |
| } | |
| function closeSettingsOnBackdrop(e) { | |
| if (e.target === document.getElementById('settingsModal')) closeSettings(); | |
| } | |
| function saveSettings() { | |
| localStorage.setItem('claude_api_key', document.getElementById('apiKeyInput').value.trim()); | |
| localStorage.setItem('claude_model', document.getElementById('modelSelect').value); | |
| localStorage.setItem('claude_system_prompt', document.getElementById('systemPromptInput').value.trim()); | |
| closeSettings(); | |
| } | |
| // ── Robot Events ────────────────────────────────────────────── | |
| function initRobotEvents() { | |
| robot.addEventListener('robotsChanged', (e) => displayRobots(e.detail.robots)); | |
| robot.addEventListener('streaming', async () => { | |
| updateStatus('connected', 'Connected'); | |
| enableTalk(true); | |
| document.getElementById('robotSelector').classList.add('hidden'); | |
| startLatencyDisplay(); | |
| try { await robot.ensureAwake(); } catch (_) {} | |
| greetingAnimation(); | |
| }); | |
| robot.addEventListener('sessionStopped', () => { | |
| document.getElementById('startBtn').disabled = !selectedRobotId; | |
| document.getElementById('stopBtn').disabled = true; | |
| document.getElementById('robotSelector').classList.remove('hidden'); | |
| enableTalk(false); | |
| updateStatus('connected', 'Connected'); | |
| stopLatencyDisplay(); | |
| stopAnimLoop(); | |
| setConvState('idle'); | |
| }); | |
| robot.addEventListener('disconnected', () => { | |
| updateStatus('', 'Disconnected'); | |
| document.getElementById('connectBtn').disabled = false; | |
| document.getElementById('robotSelector').classList.add('hidden'); | |
| enableTalk(false); | |
| stopAnimLoop(); | |
| }); | |
| robot.addEventListener('error', (e) => console.error(`[${e.detail.source}]`, e.detail.error)); | |
| } | |
| // ── Connection ──────────────────────────────────────────────── | |
| function updateStatus(cls, text) { | |
| document.getElementById('statusIndicator').className = 'status-indicator ' + cls; | |
| document.getElementById('statusText').textContent = text; | |
| } | |
| async function connectSignaling() { | |
| if (!robot.isAuthenticated) return; | |
| updateStatus('connecting', 'Connecting…'); | |
| document.getElementById('connectBtn').disabled = true; | |
| try { | |
| await robot.connect(); | |
| updateStatus('connected', 'Connected'); | |
| document.getElementById('robotSelector').classList.remove('hidden'); | |
| } catch (e) { | |
| console.error('Connection failed:', e); | |
| updateStatus('', 'Disconnected'); | |
| document.getElementById('connectBtn').disabled = false; | |
| } | |
| } | |
| function displayRobots(robots) { | |
| const list = document.getElementById('robotList'); | |
| list.innerHTML = ''; | |
| if (!robots?.length) { | |
| list.innerHTML = '<div style="color:var(--text-muted)">No robots online</div>'; | |
| document.getElementById('startBtn').disabled = true; | |
| return; | |
| } | |
| for (const r of robots) { | |
| const div = document.createElement('div'); | |
| div.className = 'robot-card' + (r.id === selectedRobotId ? ' selected' : ''); | |
| div.innerHTML = `<div class="name">${r.meta?.name || 'Reachy Mini'}</div><div class="id">${r.id.slice(0, 12)}…</div>`; | |
| div.onclick = () => { | |
| document.querySelectorAll('.robot-card').forEach(el => el.classList.remove('selected')); | |
| div.classList.add('selected'); | |
| selectedRobotId = r.id; | |
| document.getElementById('robotName').textContent = r.meta?.name || 'Reachy Mini'; | |
| document.getElementById('startBtn').disabled = false; | |
| }; | |
| list.appendChild(div); | |
| } | |
| } | |
| async function startStream() { | |
| if (!selectedRobotId) return; | |
| if (audioCtx.state === 'suspended') await audioCtx.resume(); | |
| updateStatus('connecting', 'Starting…'); | |
| detachVideo = robot.attachVideo(document.getElementById('remoteVideo')); | |
| document.getElementById('startBtn').disabled = true; | |
| document.getElementById('stopBtn').disabled = false; | |
| try { | |
| await robot.startSession(selectedRobotId); | |
| } catch (e) { | |
| if (detachVideo) { detachVideo(); detachVideo = null; } | |
| document.getElementById('startBtn').disabled = !selectedRobotId; | |
| document.getElementById('stopBtn').disabled = true; | |
| const msg = e.reason === 'robot_busy' | |
| ? `Robot busy — "${e.activeApp}" is already connected` | |
| : (e.message || 'Session failed'); | |
| updateStatus('disconnected', msg); | |
| } | |
| } | |
| async function stopStream() { | |
| if (detachVideo) { detachVideo(); detachVideo = null; } | |
| await robot.stopSession(); | |
| } | |
| // ── Audio ───────────────────────────────────────────────────── | |
| function toggleMute() { | |
| robot.setAudioMuted(!robot.audioMuted); | |
| const muted = robot.audioMuted; | |
| document.getElementById('muteBtn').classList.toggle('muted', muted); | |
| document.getElementById('speakerOffIcon').classList.toggle('hidden', !muted); | |
| document.getElementById('speakerOnIcon').classList.toggle('hidden', muted); | |
| document.getElementById('muteText').textContent = muted ? 'Unmute' : 'Mute'; | |
| } | |
| function enableTalk(on) { | |
| document.getElementById('talkBtn').disabled = !on; | |
| document.getElementById('muteBtn').disabled = !on; | |
| } | |
| // ── Latency ─────────────────────────────────────────────────── | |
| function startLatencyDisplay() { | |
| const badge = document.getElementById('latencyBadge'); | |
| const label = document.getElementById('latencyValue'); | |
| badge.classList.remove('hidden'); | |
| latencyIntervalId = setInterval(() => { | |
| const video = document.getElementById('remoteVideo'); | |
| let ms = null; | |
| if (video?.buffered?.length > 0) { | |
| ms = Math.round((video.buffered.end(video.buffered.length - 1) - video.currentTime) * 1000); | |
| } | |
| label.textContent = ms != null ? `buf ${ms}ms` : '--'; | |
| badge.classList.remove('good', 'ok', 'bad'); | |
| if (ms != null) badge.classList.add(ms < 200 ? 'good' : ms < 500 ? 'ok' : 'bad'); | |
| }, 1000); | |
| } | |
| function stopLatencyDisplay() { | |
| if (latencyIntervalId) { clearInterval(latencyIntervalId); latencyIntervalId = null; } | |
| document.getElementById('latencyBadge').classList.add('hidden'); | |
| } | |
| // ── Conversation State Machine ──────────────────────────────── | |
| function setConvState(state) { | |
| convState = state; | |
| const labels = { idle: 'Idle', listening: 'Listening…', thinking: 'Thinking…', speaking: 'Speaking…' }; | |
| const el = document.getElementById('convStateLabel'); | |
| el.textContent = labels[state] || state; | |
| el.className = `conv-state state-${state}`; | |
| const isListening = state === 'listening'; | |
| const isSpeaking = state === 'speaking'; | |
| document.getElementById('talkBtn').classList.toggle('hidden', isListening || isSpeaking); | |
| document.getElementById('stopSpeakBtn').classList.toggle('hidden', !isSpeaking); | |
| if (state === 'idle') setIdleMotion(); | |
| else if (state === 'listening') setListeningMotion(); | |
| else if (state === 'thinking') setThinkingMotion(); | |
| else if (state === 'speaking') setSpeakingMotion(); | |
| } | |
| // ── Speech Recognition ──────────────────────────────────────── | |
| function initSpeechRecognition() { | |
| const SR = window.SpeechRecognition || window.webkitSpeechRecognition; | |
| if (!SR) { | |
| document.getElementById('sttUnsupported').classList.remove('hidden'); | |
| return; | |
| } | |
| recognition = new SR(); | |
| recognition.continuous = false; | |
| recognition.interimResults = true; | |
| recognition.lang = navigator.language || 'en-US'; | |
| recognition.onresult = (e) => { | |
| const transcript = Array.from(e.results).map(r => r[0].transcript).join(''); | |
| document.getElementById('interimTranscript').textContent = transcript; | |
| if (e.results[e.results.length - 1].isFinal) { | |
| document.getElementById('interimTranscript').textContent = ''; | |
| if (transcript.trim()) sendToClaude(transcript.trim()); | |
| } | |
| }; | |
| recognition.onerror = (e) => { | |
| if (e.error === 'no-speech') { setConvState('idle'); return; } | |
| console.error('STT error:', e.error); | |
| addMessage('msg-system', `Mic error: ${e.error}`); | |
| setConvState('idle'); | |
| }; | |
| recognition.onend = () => { | |
| if (convState === 'listening') setConvState('idle'); | |
| }; | |
| } | |
| function startListening() { | |
| if (!recognition) return; | |
| const apiKey = localStorage.getItem('claude_api_key'); | |
| if (!apiKey) { | |
| openSettings(); | |
| addMessage('msg-system', 'Enter your Anthropic API key in Settings first.'); | |
| return; | |
| } | |
| stopSpeaking(); | |
| setConvState('listening'); | |
| try { | |
| recognition.start(); | |
| } catch (_) { | |
| setConvState('idle'); | |
| } | |
| } | |
| // ── Claude API (streaming) ───────────────────────────────────── | |
| async function sendToClaude(userText) { | |
| setConvState('thinking'); | |
| addMessage('msg-user', userText); | |
| conversationHistory.push({ role: 'user', content: userText }); | |
| const apiKey = localStorage.getItem('claude_api_key') || ''; | |
| const systemPrompt = localStorage.getItem('claude_system_prompt') || DEFAULT_SYSTEM_PROMPT; | |
| const model = localStorage.getItem('claude_model') || 'claude-haiku-4-5-20251001'; | |
| const assistantDiv = createStreamingMessage(); | |
| try { | |
| const res = await fetch('https://api.anthropic.com/v1/messages', { | |
| method: 'POST', | |
| headers: { | |
| 'x-api-key': apiKey, | |
| 'anthropic-version': '2023-06-01', | |
| 'anthropic-dangerous-direct-browser-access': 'true', | |
| 'content-type': 'application/json', | |
| }, | |
| body: JSON.stringify({ | |
| model, | |
| max_tokens: 512, | |
| stream: true, | |
| system: systemPrompt, | |
| messages: conversationHistory.slice(-10), | |
| }), | |
| }); | |
| if (!res.ok) { | |
| const err = await res.text(); | |
| throw new Error(`Claude ${res.status}: ${err}`); | |
| } | |
| const reader = res.body.getReader(); | |
| const decoder = new TextDecoder(); | |
| let fullReply = ''; | |
| let buffer = ''; | |
| while (true) { | |
| const { done, value } = await reader.read(); | |
| if (done) break; | |
| buffer += decoder.decode(value, { stream: true }); | |
| const lines = buffer.split('\n'); | |
| buffer = lines.pop(); // keep incomplete last line | |
| for (const line of lines) { | |
| if (!line.startsWith('data: ')) continue; | |
| const data = line.slice(6).trim(); | |
| if (!data || data === '[DONE]') continue; | |
| try { | |
| const ev = JSON.parse(data); | |
| if (ev.type === 'content_block_delta' && ev.delta?.type === 'text_delta') { | |
| fullReply += ev.delta.text; | |
| updateStreamingMessage(assistantDiv, fullReply); | |
| } | |
| } catch (_) {} | |
| } | |
| } | |
| assistantDiv.classList.remove('streaming'); | |
| conversationHistory.push({ role: 'assistant', content: fullReply }); | |
| speakReply(fullReply); | |
| } catch (e) { | |
| console.error('Claude error:', e); | |
| assistantDiv.remove(); | |
| addMessage('msg-system', `Error: ${e.message}`); | |
| setConvState('idle'); | |
| } | |
| } | |
| // ── TTS → robot speaker ─────────────────────────────────────── | |
| // Uses HF Inference API (user's existing HF session token). | |
| // Audio is decoded by Web Audio API and fed into the WebRTC | |
| // connection via the patched getUserMedia stream above. | |
| async function speakReply(text) { | |
| const hfToken = sessionStorage.getItem('hf_token'); | |
| if (!hfToken) { | |
| addMessage('msg-system', 'HF session token unavailable — please reconnect.'); | |
| setConvState('idle'); | |
| return; | |
| } | |
| try { | |
| const res = await fetch( | |
| 'https://api-inference.huggingface.co/models/facebook/mms-tts-eng', | |
| { | |
| method: 'POST', | |
| headers: { | |
| 'Authorization': `Bearer ${hfToken}`, | |
| 'Content-Type': 'application/json', | |
| }, | |
| body: JSON.stringify({ inputs: text }), | |
| } | |
| ); | |
| if (!res.ok) throw new Error(`TTS ${res.status}: ${await res.text()}`); | |
| const audioBuffer = await audioCtx.decodeAudioData(await res.arrayBuffer()); | |
| setConvState('speaking'); | |
| robot.setMicMuted(false); // open the robot's audio input | |
| ttsSourceNode = audioCtx.createBufferSource(); | |
| ttsSourceNode.buffer = audioBuffer; | |
| ttsSourceNode.connect(ttsDest); | |
| await new Promise((resolve) => { | |
| ttsSourceNode.onended = resolve; | |
| ttsSourceNode.start(); | |
| }); | |
| } catch (e) { | |
| console.error('Robot TTS error:', e); | |
| addMessage('msg-system', `Voice error: ${e.message}`); | |
| } finally { | |
| ttsSourceNode = null; | |
| if (isRobotStreaming()) robot.setMicMuted(true); | |
| setConvState('idle'); | |
| } | |
| } | |
| function stopSpeaking() { | |
| if (ttsSourceNode) { | |
| try { ttsSourceNode.stop(); } catch (_) {} | |
| ttsSourceNode = null; | |
| } | |
| if (isRobotStreaming()) robot.setMicMuted(true); | |
| setConvState('idle'); | |
| } | |
| // ── Chat UI ─────────────────────────────────────────────────── | |
| function addMessage(cls, text) { | |
| const history = document.getElementById('conversationHistory'); | |
| const placeholder = history.querySelector('.msg-system'); | |
| if (placeholder && conversationHistory.length <= 1) placeholder.remove(); | |
| const div = document.createElement('div'); | |
| div.className = `message ${cls}`; | |
| div.textContent = text; | |
| history.appendChild(div); | |
| history.scrollTop = history.scrollHeight; | |
| return div; | |
| } | |
| function createStreamingMessage() { | |
| const history = document.getElementById('conversationHistory'); | |
| const placeholder = history.querySelector('.msg-system'); | |
| if (placeholder && conversationHistory.length <= 1) placeholder.remove(); | |
| const div = document.createElement('div'); | |
| div.className = 'message msg-assistant streaming'; | |
| div.textContent = ''; | |
| history.appendChild(div); | |
| history.scrollTop = history.scrollHeight; | |
| return div; | |
| } | |
| function updateStreamingMessage(div, text) { | |
| div.textContent = text; | |
| const history = document.getElementById('conversationHistory'); | |
| history.scrollTop = history.scrollHeight; | |
| } | |
| // ── Robot Motion ────────────────────────────────────────────── | |
| function stopAnimLoop() { | |
| if (animIntervalId) { clearInterval(animIntervalId); animIntervalId = null; } | |
| if (idleBreathTimeoutId) { clearTimeout(idleBreathTimeoutId); idleBreathTimeoutId = null; } | |
| } | |
| function isRobotStreaming() { return robot.state === 'streaming'; } | |
| function delay(ms) { return new Promise(r => setTimeout(r, ms)); } | |
| // Greeting nod played once when a session starts | |
| async function greetingAnimation() { | |
| if (!isRobotStreaming()) return; | |
| robot.setHeadRpyDeg(0, -18, 0); | |
| robot.setAntennasDeg(100, 100); | |
| await delay(500); | |
| if (!isRobotStreaming()) return; | |
| robot.setHeadRpyDeg(0, 12, 0); | |
| robot.setAntennasDeg(10, 10); | |
| await delay(350); | |
| if (!isRobotStreaming()) return; | |
| robot.setHeadRpyDeg(0, -5, 0); | |
| robot.setAntennasDeg(55, 55); | |
| await delay(300); | |
| if (!isRobotStreaming()) return; | |
| setIdleMotion(); | |
| } | |
| // Idle: neutral pose, then gentle breathing sway | |
| function setIdleMotion() { | |
| stopAnimLoop(); | |
| if (!isRobotStreaming()) return; | |
| robot.setHeadRpyDeg(0, 0, 0); | |
| robot.setAntennasDeg(15, 15); | |
| let t = 0; | |
| idleBreathTimeoutId = setTimeout(() => { | |
| idleBreathTimeoutId = null; | |
| if (convState !== 'idle') return; | |
| animIntervalId = setInterval(() => { | |
| if (convState !== 'idle' || !isRobotStreaming()) { stopAnimLoop(); return; } | |
| t += 0.025; | |
| robot.setHeadRpyDeg(0, 1.5 * Math.sin(t), 0); | |
| }, 100); | |
| }, 1200); | |
| } | |
| // Listening: attentive forward lean, antennas up | |
| function setListeningMotion() { | |
| stopAnimLoop(); | |
| if (!isRobotStreaming()) return; | |
| robot.setHeadRpyDeg(8, 10, 0); | |
| robot.setAntennasDeg(120, 120); | |
| } | |
| // Thinking: head scans side to side, antennas wiggle asymmetrically | |
| function setThinkingMotion() { | |
| stopAnimLoop(); | |
| if (!isRobotStreaming()) return; | |
| let t = 0; | |
| animIntervalId = setInterval(() => { | |
| if (!isRobotStreaming()) { stopAnimLoop(); return; } | |
| t += 0.07; | |
| const yaw = 18 * Math.sin(t * 0.5); | |
| const pitch = 5 * Math.sin(t * 0.3); | |
| robot.setHeadRpyDeg(0, pitch, yaw); | |
| robot.setAntennasDeg( | |
| 55 * Math.sin(t), | |
| -55 * Math.sin(t + 1.0), | |
| ); | |
| }, 100); | |
| } | |
| // Speaking: nodding head, swaying body, antennas bouncing expressively | |
| function setSpeakingMotion() { | |
| stopAnimLoop(); | |
| if (!isRobotStreaming()) return; | |
| let t = 0; | |
| animIntervalId = setInterval(() => { | |
| if (!isRobotStreaming()) { stopAnimLoop(); return; } | |
| t += 0.1; | |
| const pitch = -7 * Math.abs(Math.sin(t * 1.3)) - 1; | |
| const roll = 3 * Math.sin(t * 0.6); | |
| const yaw = 6 * Math.sin(t * 0.35); | |
| robot.setHeadRpyDeg(roll, pitch, yaw); | |
| try { robot.setBodyYawDeg(-4 * Math.sin(t * 0.35)); } catch (_) {} | |
| const wave = 28 * Math.sin(t * 1.6); | |
| robot.setAntennasDeg(42 + wave, 42 - wave * 0.6); | |
| }, 100); | |
| } | |
| </script> | |
| </body> | |
| </html> | |