// Nova web client: passcode -> room ticket -> LiveKit room with the Pi's agent. // Audio: WebRTC via livekit-client. Controls: JSON on data topic "nova.control", // the same nova.* messages the in-car HMI speaks on /v1/realtime. (() => { "use strict"; const LK = window.LivekitClient; const TOPIC = "nova.control"; const AGENT_JOIN_TIMEOUT_MS = 20000; const $ = (id) => document.getElementById(id); const app = $("app"); let room = null; let agent = null; // Two sources of state: the Pi's wake gate (idle until "Hey Nova") and // LiveKit's agent state (listening / thinking / speaking). let gate = "idle"; let lkState = "listening"; let llmMode = null; let joinTimer = null; let analyser = null; const chime = new Audio("/static/wake.mp3"); // ---- UI helpers --------------------------------------------------------- const LABELS = { listening: "Listening", thinking: "Thinking", speaking: "Speaking", error: "Error" }; function setState(s) { app.dataset.state = s; $("phase").textContent = LABELS[s] || (s === "connecting" ? "Connecting" : "Idle"); } function say(el, text, bad = false) { el.textContent = text; el.classList.toggle("bad", bad); } function addLine(who, text) { const li = document.createElement("li"); li.className = who; li.textContent = text; $("transcript").append(li); li.scrollIntoView({ block: "end", behavior: "smooth" }); } function showLlm(snap) { llmMode = snap.mode; const b = $("llm"); b.hidden = false; b.dataset.mode = snap.mode; b.textContent = snap.mode === "local" ? "Local Hailo" : `Cloud · ${snap.llm_backend || "llm"}`; b.disabled = false; } // ---- control channel ---------------------------------------------------- function send(msg) { if (!room) return; const data = new TextEncoder().encode(JSON.stringify(msg)); room.localParticipant.publishData(data, { reliable: true, topic: TOPIC }).catch(() => {}); } function onControl(msg) { switch (msg.type) { case "nova.settings": showLlm(msg); if (msg.wake_state) { gate = msg.wake_state; render(); } break; case "nova.state": // idle -> listening is a wake: "Hey Nova", a tap, or (no wake word // configured) the session itself becoming ready. Chime on that edge. if (msg.state === "listening" && gate !== "listening") chime.play().catch(() => {}); gate = msg.state; render(); break; case "nova.llm_status": if (msg.status === "loading") { $("llm").disabled = true; $("llm").textContent = "Switching…"; } if (msg.status === "error") { say($("live-msg"), `Model switch failed: ${msg.error}`, true); send({ type: "nova.settings.get" }); } break; case "nova.turn_metrics": if (msg.ttfa_ms != null) $("metrics").textContent = `first audio ${Math.round(msg.ttfa_ms)} ms`; break; case "error": if (msg.error?.type === "session_limit_reached") { hangUp("Nova is in use right now (someone is talking to it in the car). Try again shortly.", true); } else { say($("live-msg"), msg.error?.message || "Error", true); } break; } } // ---- orb: agent audio level --------------------------------------------- function watchLevel(track) { try { const ctx = new (window.AudioContext || window.webkitAudioContext)(); const src = ctx.createMediaStreamSource(new MediaStream([track.mediaStreamTrack])); analyser = ctx.createAnalyser(); analyser.fftSize = 256; src.connect(analyser); } catch { analyser = null; } } function drawOrb() { const c = $("orb"), g = c.getContext("2d"), w = c.width, h = c.height; const accent = getComputedStyle(app).getPropertyValue("--accent").trim() || "#2a3242"; let level = 0; if (analyser) { const buf = new Uint8Array(analyser.frequencyBinCount); analyser.getByteFrequencyData(buf); level = buf.reduce((a, b) => a + b, 0) / (buf.length * 255); } const t = performance.now() / 1000; const r = w * 0.26 * (1 + 0.08 * Math.sin(t * 1.6) + 0.9 * level); g.clearRect(0, 0, w, h); const grad = g.createRadialGradient(w / 2, h / 2, r * 0.1, w / 2, h / 2, r * 1.5); grad.addColorStop(0, accent); grad.addColorStop(0.55, accent + "55"); grad.addColorStop(1, "transparent"); g.fillStyle = grad; g.beginPath(); g.arc(w / 2, h / 2, r * 1.5, 0, Math.PI * 2); g.fill(); if (room) requestAnimationFrame(drawOrb); } // ---- room lifecycle ----------------------------------------------------- function isAgent(p) { return p && (p.kind === LK.ParticipantKind?.AGENT || p.identity?.startsWith("agent-")); } function agentJoined(p) { agent = p; clearTimeout(joinTimer); say($("live-msg"), ""); send({ type: "nova.settings.get" }); agentState(p.attributes?.["lk.agent.state"]); } function agentState(s) { if (!s) return; lkState = s; render(); } function render() { if (!agent || lkState === "initializing") return setState("connecting"); const busy = lkState === "thinking" || lkState === "speaking"; // A closed gate shows Idle unless Nova is still talking (e.g. research). setState(gate === "idle" && !busy ? "idle" : lkState); say($("live-msg"), gate === "idle" && !busy ? "Say “Hey Nova” — or tap the orb." : ""); } async function connect(passcode) { say($("gate-msg"), "Connecting…"); $("connect").disabled = true; let ticket; try { const r = await fetch("/api/session", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ passcode }), }); const body = await r.json().catch(() => ({})); if (r.status === 401) throw new Error("Wrong passcode."); if (r.status === 429) throw new Error("Too many attempts — wait a few minutes."); if (!r.ok) throw new Error(body.error === "not_configured" ? "This page is not configured yet." : `Server error (${r.status}).`); ticket = body; } catch (e) { say($("gate-msg"), e.message, true); $("connect").disabled = false; return; } room = new LK.Room({ adaptiveStream: true, dynacast: true }); const E = LK.RoomEvent; room.on(E.TrackSubscribed, (track, _pub, p) => { if (track.kind !== "audio") return; const el = track.attach(); el.hidden = true; document.body.append(el); if (isAgent(p)) watchLevel(track); }); room.on(E.ParticipantConnected, (p) => { if (isAgent(p)) agentJoined(p); }); room.on(E.ParticipantDisconnected, (p) => { if (p === agent) hangUp("Nova left the room.", true); }); room.on(E.ParticipantAttributesChanged, (changed, p) => { if (isAgent(p) && "lk.agent.state" in changed) agentState(changed["lk.agent.state"]); }); room.on(E.DataReceived, (payload, _p, _kind, topic) => { if (topic !== TOPIC) return; try { onControl(JSON.parse(new TextDecoder().decode(payload))); } catch { /* ignore */ } }); room.on(E.Disconnected, () => { if (room) hangUp("Disconnected.", false); }); room.registerTextStreamHandler("lk.transcription", async (reader, info) => { const text = (await reader.readAll()).trim(); if (!text) return; const final = reader.info?.attributes?.["lk.transcription_final"]; const mine = info.identity === room?.localParticipant.identity; if (mine && final === "false") return; // show the user's settled words only addLine(mine ? "you" : "nova", text); }); try { await room.connect(ticket.url, ticket.token); await room.localParticipant.setMicrophoneEnabled(true); } catch (e) { const why = /permission|notallowed/i.test(String(e)) ? "Microphone permission was denied." : "Could not reach the voice server."; room = null; say($("gate-msg"), why, true); $("connect").disabled = false; return; } $("gate").hidden = true; $("live").hidden = false; setState("connecting"); say($("live-msg"), "Waiting for Nova to join from the Pi…"); requestAnimationFrame(drawOrb); const already = [...room.remoteParticipants.values()].find(isAgent); if (already) agentJoined(already); else joinTimer = setTimeout(() => { if (!agent) say($("live-msg"), "Nova hasn't joined — is the Pi on and online?", true); }, AGENT_JOIN_TIMEOUT_MS); } function hangUp(message = "", bad = false) { clearTimeout(joinTimer); const r = room; room = null; agent = null; gate = "idle"; lkState = "listening"; analyser = null; if (r) r.disconnect(); document.querySelectorAll("body > audio").forEach((a) => a.remove()); $("transcript").replaceChildren(); $("metrics").textContent = ""; $("llm").hidden = true; $("live").hidden = true; $("gate").hidden = false; $("connect").disabled = false; $("mute").setAttribute("aria-pressed", "false"); $("mute").textContent = "Mute"; setState("idle"); say($("gate-msg"), message, bad); } // ---- wiring ------------------------------------------------------------- $("login").addEventListener("submit", (e) => { e.preventDefault(); chime.load(); // a user gesture: lets the chime play later on iOS/Safari connect($("passcode").value); }); $("hangup").addEventListener("click", () => hangUp()); // Tap to talk: the same wake event as "Hey Nova", like the HMI's PTT. $("orb").addEventListener("click", () => { if (agent && gate === "idle") send({ type: "nova.wake" }); }); $("mute").addEventListener("click", async () => { if (!room) return; const muted = $("mute").getAttribute("aria-pressed") === "true"; await room.localParticipant.setMicrophoneEnabled(muted); $("mute").setAttribute("aria-pressed", String(!muted)); $("mute").textContent = muted ? "Mute" : "Unmute"; }); $("llm").addEventListener("click", () => { const next = llmMode === "local" ? "openrouter" : "local"; send({ type: "nova.settings.set", settings: { mode: next } }); }); window.addEventListener("beforeunload", () => room?.disconnect()); })();