// @ts-check /** * Minimal voice conversation app, talking to a Hugging Face speech-to-speech * backend over **WebSocket** or **WebRTC**. * * Click the orb -> we ask for the mic, connect (WS dial, or SDP handshake via * the /api/calls proxy), push session.update + mic audio, play back the TTS * audio. The orb visually reflects the live state (idle, connecting, * listening, user-speaking, processing, ai-speaking). * * The two client classes expose the same events and methods, so everything * below except the constructor pick is transport-agnostic. WebRTC is offered * only in env-pinned direct mode (the /api/calls proxy forwards exclusively * to SPEECH_TO_SPEECH_URL); LB mode and user-typed URLs stay on WebSocket. * * @typedef {"idle" | "connecting" | "queued" | "your-turn" | "listening" | "user-speaking" | "processing" | "ai-speaking" | "error"} AppState * @typedef {S2sWsRealtimeClient | S2sRtcRealtimeClient} RealtimeClient */ import { S2sWsRealtimeClient } from "./ws/s2s-ws-client.js"; import { S2sRtcRealtimeClient } from "./rtc/s2s-rtc-client.js"; import { $, truncateError, DEBUG } from "./ui/dom.js"; import { ChatView } from "./ui/chat.js"; import { Account } from "./ui/account.js"; const DEFAULT_VOICE = "Aiden"; const DEFAULT_INSTRUCTIONS = "You are a friendly voice assistant. " + "Keep replies short, warm, and spoken. Avoid long monologues."; const STARTUP_GREETING_PROMPT = "Start the conversation now with a brief, spontaneous greeting in character. " + "Keep it to one sentence, invite the user in naturally, and vary the wording each time."; // Appended to the user's instructions whenever at least one tool is enabled. // Stops the model from announcing capabilities ("Yes, I can search") and then // idling for the next turn — it should act immediately in the same response. const TOOL_USE_HINT = " When the user's request calls for one of your tools, do not describe your " + "capabilities or say you can do it and wait for another turn. Instead, say " + 'a brief acknowledgement like "Let me search for that..." and call the tool ' + "right away in the same response."; const STORAGE_KEYS = { // Direct s2s server URL, used only when the deploy has no LOAD_BALANCER_URL // (in LB mode the browser never learns the LB address — it POSTs /api/session). directUrl: "s2s.ws.directUrl", voice: "s2s.ws.voice", instructions: "s2s.ws.instructions", tools: "s2s.ws.tools", searchKey: "s2s.ws.searchKey", noiseGate: "s2s.ws.noiseGate", // "ws" | "webrtc". Not under the historical "s2s.ws." prefix — it selects // between the transports rather than configuring the WS one. transport: "s2s.transport", audioInputId: "s2s.audio.inputId", audioOutputId: "s2s.audio.outputId", }; // ── Noise gate ────────────────────────────────────────────────────────────── // The Settings cursor sets the gate's open threshold in dBFS. Its leftmost // position is an OFF detent (gate disabled, pure passthrough); the rest of the // travel is the active threshold. The cursor shares the meter's dB axis, so the // handle sits on the level bar — raise it until room noise stops lighting it up. // The slider range IS the shared axis: the live meter fill and the threshold // thumb both map across [GATE_OFF_DB, GATE_MAX_DB], so the thumb sits exactly // where the gate cuts on the same scale as the level bar. const GATE_OFF_DB = -66; // slider minimum = off / bottom of the meter axis const GATE_MAX_DB = -3; // slider maximum = most aggressive / top of the meter axis const GATE_DEFAULT_DB = -50; // first-run default: a gentle gate, enabled /** @param {number} thresholdDb @returns {import("./ws/s2s-ws-client.js").NoiseGate} */ function gateParams(thresholdDb) { return { enabled: thresholdDb > GATE_OFF_DB, thresholdDb }; } // ── Tools ───────────────────────────────────────────────────────────────── // Function tools we declare to the backend. The model decides when to call // one; the executor below runs it and returns the result (see runTool). /** @type {Record} */ const TOOL_DEFS = { web_search: { type: "function", name: "web_search", description: "Search the web for current or factual information you don't already know " + "(news, prices, facts, documentation). Returns the top results with titles, " + "snippets and URLs.", parameters: { type: "object", properties: { query: { type: "string", description: "The search query." } }, required: ["query"], }, }, camera_snapshot: { type: "function", name: "camera_snapshot", description: "Capture the current frame from the user's webcam so you can see what they " + "are showing you. Use it whenever the user refers to something visual or " + "asks you to look.", parameters: { type: "object", properties: {}, required: [] }, }, }; /** Longest edge of the snapshot sent to the VLM, in px (keeps payload sane). */ const SNAPSHOT_MAX_EDGE = 768; const SNAPSHOT_QUALITY = 0.7; // Over WebRTC the snapshot travels inside ONE data-channel message, and SCTP // messages above the negotiated max (64 KiB on the aiortc side) fail to send. // Budget for the data-URL portion, leaving headroom for the JSON envelope. const SNAPSHOT_DC_BUDGET_CHARS = 60_000; // Re-encode ladder walked until the frame fits the budget: quality first // (cheap wins), then resolution. The last rung is ~15–25 KB for any content, // so a frame that "fits" is deterministic, not content-dependent luck. const SNAPSHOT_LADDER = /** @type {[number, number][]} */ ([ [SNAPSHOT_MAX_EDGE, SNAPSHOT_QUALITY], [SNAPSHOT_MAX_EDGE, 0.5], [640, 0.45], [512, 0.4], [448, 0.35], [384, 0.3], ]); function loadSettings() { return { directUrl: localStorage.getItem(STORAGE_KEYS.directUrl) || "", voice: localStorage.getItem(STORAGE_KEYS.voice) || DEFAULT_VOICE, instructions: localStorage.getItem(STORAGE_KEYS.instructions) || DEFAULT_INSTRUCTIONS, noiseGate: loadGateThreshold(), // Default WebSocket: the proven path stays the first-run experience. transport: localStorage.getItem(STORAGE_KEYS.transport) === "webrtc" ? "webrtc" : "ws", audioInputId: localStorage.getItem(STORAGE_KEYS.audioInputId) || "", audioOutputId: localStorage.getItem(STORAGE_KEYS.audioOutputId) || "", }; } /** Stored gate threshold (dBFS), clamped to the slider range. Defaults to a * gentle enabled gate (GATE_DEFAULT_DB) when the user hasn't set one yet. */ function loadGateThreshold() { const stored = localStorage.getItem(STORAGE_KEYS.noiseGate); // getItem returns null when unset, and Number(null) === 0 (finite!), so guard // the missing/empty case explicitly before coercing — otherwise the default // never fires and 0 clamps to the slider max. if (stored === null || stored === "") return GATE_DEFAULT_DB; const raw = Number(stored); if (!Number.isFinite(raw)) return GATE_DEFAULT_DB; return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(raw))); } /** @param {ReturnType} s */ function saveSettings(s) { localStorage.setItem(STORAGE_KEYS.directUrl, s.directUrl); localStorage.setItem(STORAGE_KEYS.voice, s.voice); localStorage.setItem(STORAGE_KEYS.instructions, s.instructions); localStorage.setItem(STORAGE_KEYS.noiseGate, String(s.noiseGate)); localStorage.setItem(STORAGE_KEYS.transport, s.transport); localStorage.setItem(STORAGE_KEYS.audioInputId, s.audioInputId || ""); localStorage.setItem(STORAGE_KEYS.audioOutputId, s.audioOutputId || ""); } /** @returns {{ web_search: boolean, camera_snapshot: boolean }} */ function loadTools() { try { const raw = JSON.parse(localStorage.getItem(STORAGE_KEYS.tools) || "{}"); // Both tools default ON (web search still only activates when a key exists). // We never call getUserMedia on page load — the camera only actually starts // on a user gesture (conversation start), so a default-on flag doesn't // silently resume the webcam; an explicit saved `false` is respected. return { web_search: raw.web_search ?? true, camera_snapshot: raw.camera_snapshot ?? true, }; } catch { return { web_search: true, camera_snapshot: true }; } } function saveTools() { localStorage.setItem(STORAGE_KEYS.tools, JSON.stringify(toolsEnabled)); } /** @type {Record} */ const STATE_VIEWS = { idle: { caption: "Tap to start", disabled: false }, connecting: { caption: "Connecting", disabled: true }, queued: { caption: "Finding you a spot…", disabled: true }, "your-turn": { caption: "You're up! 🎉", disabled: true }, listening: { caption: "", disabled: false }, "user-speaking": { caption: "", disabled: false }, processing: { caption: "", disabled: false }, "ai-speaking": { caption: "", disabled: false }, error: { caption: "Tap to retry", disabled: false }, }; /** @type {Record} */ const STATE_CLASS = { idle: "state-idle", connecting: "state-connecting", queued: "state-queued", "your-turn": "state-your-turn", listening: "state-listening", "user-speaking": "state-user-speaking", processing: "state-processing", "ai-speaking": "state-ai-speaking", error: "state-error", }; /** @type {ReadonlySet} */ const LIVE_STATES = new Set(["listening", "user-speaking", "processing", "ai-speaking"]); /** @type {HTMLButtonElement} */ const circleBtn = $("#main-circle"); /** @type {HTMLParagraphElement} */ const circleCaption = $("#circle-caption"); /** @type {HTMLParagraphElement} */ const circleSubcaption = $("#circle-subcaption"); /** @type {HTMLElement} */ const orbWrap = $(".orb-wrap"); /** @type {HTMLButtonElement} */ const micBtn = $("#mic-btn"); /** @type {HTMLButtonElement} */ const stopBtn = $("#stop-btn"); /** @type {HTMLElement} */ const queueActions = $("#queue-actions"); /** @type {HTMLButtonElement} */ const joinQueueBtn = $("#join-queue-btn"); /** @type {HTMLButtonElement} */ const leaveQueueBtn = $("#leave-queue-btn"); /** @type {HTMLButtonElement} */ const settingsBtn = $("#settings-btn"); /** @type {HTMLDialogElement} */ const settingsModal = $("#settings-modal"); /** @type {HTMLButtonElement} */ const aboutBtn = $("#about-btn"); /** @type {HTMLDialogElement} */ const aboutModal = $("#about-modal"); /** @type {HTMLButtonElement} */ const aboutClose = $("#about-close"); /** @type {HTMLButtonElement} */ const toolsBtn = $("#tools-btn"); /** @type {HTMLDialogElement} */ const toolsModal = $("#tools-modal"); /** @type {HTMLButtonElement} */ const toolsClose = $("#tools-close"); /** @type {HTMLInputElement} */ const toolWebSwitch = $("#tool-web"); /** @type {HTMLInputElement} */ const toolCamSwitch = $("#tool-cam"); /** @type {HTMLElement} */ const toolWebRow = $("#tool-web-row"); /** @type {HTMLElement} */ const toolWebHint = $("#tool-web-hint"); /** @type {HTMLElement} */ const toolCamHint = $("#tool-cam-hint"); /** @type {HTMLInputElement} */ const searchKeyInput = $("#search-key"); /** @type {HTMLElement} */ const camPip = $("#cam-pip"); /** @type {HTMLVideoElement} */ const camVideo = $("#cam-video"); /** @type {HTMLInputElement} */ const inputLbUrl = $("#lb-url"); /** @type {HTMLElement} */ const connField = $("#conn-field"); /** @type {HTMLElement} */ const connHint = $("#conn-hint"); /** @type {HTMLElement} */ const transportField = $("#transport-field"); /** @type {HTMLSelectElement} */ const inputTransport = $("#transport"); /** @type {HTMLElement} */ const transportHint = $("#transport-hint"); /** @type {HTMLElement} */ const gateField = $("#gate-field"); /** @type {HTMLSelectElement} */ const inputVoice = $("#voice"); /** @type {HTMLSelectElement} */ const inputAudioInput = $("#audio-input"); /** @type {HTMLSelectElement} */ const inputAudioOutput = $("#audio-output"); /** @type {HTMLElement} */ const audioOutputHint = $("#audio-output-hint"); /** @type {HTMLTextAreaElement} */ const inputInstructions = $("#instructions"); /** @type {HTMLInputElement} */ const inputNoiseGate = $("#noise-gate"); /** @type {HTMLElement} */ const gateValue = $("#gate-value"); /** @type {HTMLElement} */ const gateMeterFill = $("#gate-meter-fill"); /** @type {HTMLElement} */ const micGate = $("#mic-gate"); const mgaArc = /** @type {SVGSVGElement} */ (document.querySelector("#mic-gate-arc")); const mgaTrack = /** @type {SVGPathElement} */ (document.querySelector("#mga-track")); const mgaFill = /** @type {SVGPathElement} */ (document.querySelector("#mga-fill")); const mgaHit = /** @type {SVGPathElement} */ (document.querySelector("#mga-hit")); const mgaHandle = /** @type {SVGCircleElement} */ (document.querySelector("#mga-handle")); /** @type {HTMLButtonElement} */ const restartBtn = $("#restart-conversation"); /** @type {HTMLElement} */ const restartHint = $("#restart-hint"); const settingsForm = /** @type {HTMLFormElement} */ (settingsModal.querySelector("form")); /** @type {AppState} */ let currentState = "idle"; let settings = loadSettings(); // ── Connection target ──────────────────────────────────────────────────────── // Three modes, decided by the deploy via /api/config: // • SPEECH_TO_SPEECH_URL set -> direct mode pinned by the deploy: the browser // connects straight to that URL, shown read-only in Settings. Overrides the // load balancer entirely. // • LOAD_BALANCER_URL set -> original flow: POST the same-origin /api/session // proxy (the server forwards to the LB; the LB address is never sent here). // • neither (allowDirect) -> the user sets a speech-to-speech server URL and // the browser connects to it directly (no load balancer, no /session). let lbMode = false; // Fail open: direct entry is allowed unless /api/config reports an LB URL. This // way a missing/unreachable config (e.g. static hosting) leaves the field // usable rather than locked. let allowDirect = true; // Deploy-pinned s2s URL (SPEECH_TO_SPEECH_URL). Non-empty -> locked direct // mode: the field displays it read-only and the saved user URL is untouched. let pinnedUrl = ""; // Whether the deploy offers the WebRTC transport (/api/config `rtc`; true // exactly when the URL is env-pinned, since /api/calls only forwards there). let rtcAvailable = false; /** @type {RTCIceServer[]} STUN/TURN servers for the browser peer connection * (deploy-provided via RTC_ICE_SERVERS; empty -> host candidates only). */ let iceServers = []; // Transport of the LIVE (or starting) conversation — as opposed to // `settings.transport`, which is what the NEXT one will use. Drives the // camera-snapshot size budget while a call is running. /** @type {"ws" | "webrtc"} */ let activeTransport = "ws"; // ── Tool state ────────────────────────────────────────────────────────────── let toolsEnabled = loadTools(); // Whether the server holds a Serper key (learned from /api/config on load). let serverSearchKey = false; // A user-supplied key (fallback when the deploy has none). localStorage only. let userSearchKey = localStorage.getItem(STORAGE_KEYS.searchKey) || ""; /** @type {MediaStream | null} */ let cameraStream = null; /** Search is usable if the server has a key or the user supplied one. */ function searchAvailable() { return serverSearchKey || !!userSearchKey; } /** Tool definitions for the currently-enabled (and usable) tools. */ function activeToolDefs() { const defs = []; if (toolsEnabled.web_search && searchAvailable()) defs.push(TOOL_DEFS.web_search); if (toolsEnabled.camera_snapshot) defs.push(TOOL_DEFS.camera_snapshot); return defs; } /** Instructions plus the hidden tool-use hint when any tool is active. */ function effectiveInstructions() { const base = settings.instructions; return activeToolDefs().length ? base + TOOL_USE_HINT : base; } /** Push the active tool set to a live session so toggles apply mid-call. */ function pushToolsToSession() { if (!client || !LIVE_STATES.has(currentState)) return; client.setTools(activeToolDefs()); // The hidden tool-use hint depends on whether any tool is active, so refresh // instructions alongside the tool set. client.updateSession({ instructions: effectiveInstructions() }); } // ── Chat view ─────────────────────────────────────────────────────────────── // Owns the history panel, the ephemeral bubbles, and all transcript/tool // streaming state. The client's events are forwarded to its on* methods. let userAudioReplaying = false; const chat = new ChatView({ onUserAudioPlaybackChange(playing) { userAudioReplaying = playing; syncMicMuteState(); }, }); // ── Account / limiter ───────────────────────────────────────────────────── // Login chip + daily-limit modal (inert unless the deploy is in LB mode). The // server meters conversation time; the client just heartbeats a live session // and tears down when the server reports the budget is spent. const account = new Account(); let limiterOn = false; let loginRequired = false; let heartbeatTimer = 0; let trackedSessionId = ""; let trackedTier = ""; // The waiting-queue ticket id while we're in line (else ""). Used to leave the // queue on teardown / tab-close so we don't hold a phantom place. let queuedTicketId = ""; /** @type {RealtimeClient | null} */ let client = null; /** @type {MediaStream | null} */ let micStream = null; let micMuted = false; /** Apply both the user's mute choice and the temporary replay guard. */ function syncMicMuteState() { const muted = micMuted || userAudioReplaying; for (const track of micStream?.getAudioTracks() ?? []) { track.enabled = !muted; } client?.setMuted(muted); } /** @param {AppState} next */ function setState(next) { currentState = next; const view = STATE_VIEWS[next]; circleBtn.disabled = view.disabled; circleBtn.className = `circle ${STATE_CLASS[next]}`; if (next !== "error") setCaption(view.caption); const live = LIVE_STATES.has(next); orbWrap.classList.toggle("live", live); micBtn.setAttribute("aria-hidden", live ? "false" : "true"); stopBtn.setAttribute("aria-hidden", live ? "false" : "true"); micBtn.tabIndex = live ? 0 : -1; stopBtn.tabIndex = live ? 0 : -1; // Queue affordances: "Leave queue" whenever we're in line; "Join now" only once // it's our turn (a slot is held for us). Both live under #queue-actions. const yourTurn = next === "your-turn"; const inLine = next === "queued" || yourTurn; queueActions.hidden = !inLine; joinQueueBtn.hidden = !yourTurn; joinQueueBtn.tabIndex = yourTurn ? 0 : -1; leaveQueueBtn.hidden = !inLine; leaveQueueBtn.tabIndex = inLine ? 0 : -1; if (!yourTurn) stopJoinCountdown(); // Warm reassurance under the terse position, only while waiting in line. if (next === "queued") { circleSubcaption.textContent = "Sorry, we overhugged! 🤗 Every slot is busy, so we saved you a spot. Hang tight, you're moving up."; circleSubcaption.hidden = false; } else { circleSubcaption.hidden = true; } updateRestartAvailability(); } function updateRestartAvailability() { // Restart works from any settled state — it tears down a live call (if any) // and reconnects with the current settings. Only block while mid-connect or // while waiting in the queue (restarting from there would just re-queue). restartBtn.disabled = currentState === "connecting" || currentState === "queued" || currentState === "your-turn"; restartHint.hidden = false; restartHint.textContent = LIVE_STATES.has(currentState) ? "Reconnects now with the settings above." : "Starts a conversation with the settings above."; } /** * @param {string} text * @param {"" | "error" | "muted"} [kind] */ function setCaption(text, kind = "") { const trimmed = text.trim(); circleCaption.textContent = trimmed; circleCaption.className = `circle-caption${kind ? ` ${kind}` : ""}${trimmed ? "" : " empty"}`; } function openSettings() { syncConnectionUi(); inputVoice.value = settings.voice; inputInstructions.value = settings.instructions; syncGateUi(); updateRestartAvailability(); void refreshAudioDeviceLists(); settingsModal.showModal(); } /** dB position (clamped to the slider axis) as a 0..1 fraction of the track. * @param {number} db */ function dbToFraction(db) { const clamped = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, db)); return (clamped - GATE_OFF_DB) / (GATE_MAX_DB - GATE_OFF_DB); } /** @param {number} f @returns {number} dB at a 0..1 position on the gate axis. */ function fractionToDb(f) { const clamped = Math.min(1, Math.max(0, f)); return Math.round(GATE_OFF_DB + clamped * (GATE_MAX_DB - GATE_OFF_DB)); } // ── Radial gate arc (around the mic button, live during a call) ───────────── // A 270° arc with the gap facing the orb (right). Fraction 0 (=Off) sits at the // bottom-ish start; 1 (=max) at the top-ish end. The level fill and the // threshold handle ride this same axis, mirroring the Settings widget. const ARC_R = 40; // A ~200° arc centred on the left (180°) so the wide gap faces the orb (right). const ARC_SPAN_DEG = 200; const ARC_START_DEG = 180 - ARC_SPAN_DEG / 2; // lower-left start; Off end /** Point at fraction f (0..1) and radius r, in the 0..100 viewBox. * @param {number} f @param {number} [r] */ function arcPoint(f, r = ARC_R) { const deg = ARC_START_DEG + f * ARC_SPAN_DEG; const rad = (deg * Math.PI) / 180; return { x: 50 + r * Math.cos(rad), y: 50 + r * Math.sin(rad) }; } /** SVG path `d` for the full 0..1 arc (clockwise). */ function fullArcD() { const a = arcPoint(0); const b = arcPoint(1); const largeArc = ARC_SPAN_DEG > 180 ? 1 : 0; return `M ${a.x} ${a.y} A ${ARC_R} ${ARC_R} 0 ${largeArc} 1 ${b.x} ${b.y}`; } /** One-time geometry: track, fill (dash-revealed) and the transparent hit band. */ function initGateArc() { const d = fullArcD(); mgaTrack.setAttribute("d", d); mgaFill.setAttribute("d", d); mgaHit.setAttribute("d", d); // pathLength 100 lets us reveal the fill by fraction via dashoffset. mgaFill.setAttribute("pathLength", "100"); mgaFill.style.strokeDasharray = "100 100"; mgaFill.style.strokeDashoffset = "100"; // empty until levels arrive renderGateHandle(); } /** Place the threshold bead on the arc at the stored threshold; flag off state. */ function renderGateHandle() { const off = settings.noiseGate <= GATE_OFF_DB; const p = arcPoint(dbToFraction(settings.noiseGate)); mgaHandle.setAttribute("cx", String(p.x)); mgaHandle.setAttribute("cy", String(p.y)); micGate.classList.toggle("gate-off", off); } /** Paint a 0..1 live level onto the arc fill (and the Settings meter if open). * Brightens the tick when the level crosses the threshold — i.e. the gate is * actually open — but only when gating is enabled. * @param {number} rms */ function paintInputLevel(rms) { const db = rms > 0 ? 20 * Math.log10(rms) : GATE_OFF_DB; const f = dbToFraction(db); mgaFill.style.strokeDashoffset = String(100 * (1 - f)); if (settingsModal.open) gateMeterFill.style.width = `${f * 100}%`; const enabled = settings.noiseGate > GATE_OFF_DB; micGate.classList.toggle("gate-open", enabled && f >= dbToFraction(settings.noiseGate)); } /** The single place that commits a new gate threshold: updates both controls, * persists, and applies live to the running session. * @param {number} db */ function setGateThreshold(db) { settings.noiseGate = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(db))); const off = settings.noiseGate <= GATE_OFF_DB; inputNoiseGate.value = String(settings.noiseGate); gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`; renderGateHandle(); localStorage.setItem(STORAGE_KEYS.noiseGate, String(settings.noiseGate)); if (client && LIVE_STATES.has(currentState)) { client.setNoiseGate(gateParams(settings.noiseGate)); } } /** Reflect the stored gate threshold into the slider, label and arc handle. */ function syncGateUi() { inputNoiseGate.value = String(settings.noiseGate); const off = settings.noiseGate <= GATE_OFF_DB; gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`; renderGateHandle(); } // Drag along the arc band to set the threshold (a tap on the glyph still mutes). let gateDragging = false; /** @param {PointerEvent} e */ function gatePointerToDb(e) { const rect = mgaArc.getBoundingClientRect(); const cx = rect.left + rect.width / 2; const cy = rect.top + rect.height / 2; let deg = (Math.atan2(e.clientY - cy, e.clientX - cx) * 180) / Math.PI; if (deg < 0) deg += 360; // Map the on-arc angle to a fraction; angles in the right-side gap fall // outside [0,1] and fractionToDb clamps them to the nearest end (just-below // start -> Off, just-past end -> max). const f = (deg - ARC_START_DEG) / ARC_SPAN_DEG; return fractionToDb(f); } mgaHit.addEventListener("pointerdown", (e) => { gateDragging = true; mgaHit.setPointerCapture(e.pointerId); setGateThreshold(gatePointerToDb(e)); }); mgaHit.addEventListener("pointermove", (e) => { if (gateDragging) setGateThreshold(gatePointerToDb(e)); }); const endGateDrag = (/** @type {PointerEvent} */ e) => { if (!gateDragging) return; gateDragging = false; try { mgaHit.releasePointerCapture(e.pointerId); } catch {} }; mgaHit.addEventListener("pointerup", endGateDrag); mgaHit.addEventListener("pointercancel", endGateDrag); settingsBtn.addEventListener("click", openSettings); // About panel: native , Esc closes for free; also close on the X and // on a click in the backdrop (a click whose target is the dialog itself). aboutBtn.addEventListener("click", () => aboutModal.showModal()); // Mobile twin of the (i), living in the right-hand control cluster. $("#about-btn-m").addEventListener("click", () => aboutModal.showModal()); aboutClose.addEventListener("click", () => aboutModal.close()); aboutModal.addEventListener("click", (e) => { if (e.target === aboutModal) aboutModal.close(); }); // ── Tools panel ─────────────────────────────────────────────────────────── /** Reflect the current tool state into the panel controls. */ function syncToolsUi() { const avail = searchAvailable(); toolWebSwitch.checked = toolsEnabled.web_search && avail; toolWebSwitch.disabled = !avail; toolWebRow.classList.toggle("disabled", !avail); toolCamSwitch.checked = toolsEnabled.camera_snapshot; if (serverSearchKey) { // Key lives server-side: show it as configured, never expose it. searchKeyInput.value = ""; searchKeyInput.placeholder = "•••••••• · provided by the server"; searchKeyInput.disabled = true; toolWebHint.textContent = "Ready. The search key is held server-side and never sent to your browser."; } else { searchKeyInput.disabled = false; searchKeyInput.value = userSearchKey; searchKeyInput.placeholder = "Paste a Serper key to enable web search"; toolWebHint.textContent = userSearchKey ? "Using your key — stored in this browser only." : "No server key configured. Add your own Serper key to enable web search."; } } toolsBtn.addEventListener("click", () => { syncToolsUi(); toolsModal.showModal(); }); toolsClose.addEventListener("click", () => toolsModal.close()); toolsModal.addEventListener("click", (e) => { if (e.target === toolsModal) toolsModal.close(); }); toolWebSwitch.addEventListener("change", () => { if (toolWebSwitch.checked && !searchAvailable()) { toolWebSwitch.checked = false; // guard: can't enable without a key return; } toolsEnabled.web_search = toolWebSwitch.checked; saveTools(); pushToolsToSession(); }); toolCamSwitch.addEventListener("change", async () => { if (toolCamSwitch.checked) { try { // Flipping the switch always re-requests the camera, so a permission that // was only dismissed earlier is asked again here. await enableCamera(); } catch (err) { toolCamSwitch.checked = false; const denied = err instanceof Error && (err.name === "NotAllowedError" || err.name === "SecurityError"); toolCamHint.textContent = denied ? "Camera blocked. Allow it from the camera icon in your browser's address bar — it switches on automatically." : `Camera unavailable${err instanceof Error ? `: ${err.message}` : ""}`; return; } toolsEnabled.camera_snapshot = true; toolCamHint.textContent = "Camera on. The assistant can take a snapshot when it needs to see."; } else { disableCamera(); toolsEnabled.camera_snapshot = false; toolCamHint.textContent = "Let the assistant see through your webcam."; } saveTools(); pushToolsToSession(); }); searchKeyInput.addEventListener("input", () => { if (serverSearchKey) return; userSearchKey = searchKeyInput.value.trim(); if (userSearchKey) localStorage.setItem(STORAGE_KEYS.searchKey, userSearchKey); else localStorage.removeItem(STORAGE_KEYS.searchKey); const avail = searchAvailable(); toolWebSwitch.disabled = !avail; toolWebRow.classList.toggle("disabled", !avail); // Losing the key disables a previously-enabled tool. if (!avail && toolsEnabled.web_search) { toolsEnabled.web_search = false; toolWebSwitch.checked = false; saveTools(); pushToolsToSession(); } toolWebHint.textContent = userSearchKey ? "Using your key — stored in this browser only." : "No server key configured. Add your own Serper key to enable web search."; }); // ── Camera ────────────────────────────────────────────────────────────────── async function enableCamera() { if (cameraStream) return; cameraStream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: "user" }, audio: false, }); camVideo.srcObject = cameraStream; try { await camVideo.play(); } catch { /* autoplay quirks; muted video is fine */ } camPip.classList.add("visible"); camPip.setAttribute("aria-hidden", "false"); // Lets the footer reflow to the bottom-right (and hide on mobile) while the // webcam preview occupies the bottom of the stage. document.body.classList.add("cam-on"); } function disableCamera() { if (cameraStream) { for (const t of cameraStream.getTracks()) t.stop(); cameraStream = null; } camVideo.srcObject = null; camPip.classList.remove("visible"); camPip.setAttribute("aria-hidden", "true"); document.body.classList.remove("cam-on"); } /** Auto-start the webcam on arrival (the camera tool is on by default). If the * user declines the permission, switch the tool off and reflect it in the UI * rather than nagging. */ async function autoStartCamera() { if (!toolsEnabled.camera_snapshot || cameraStream) return; try { await enableCamera(); } catch (err) { console.warn("[main] camera auto-start declined/failed:", err); toolsEnabled.camera_snapshot = false; saveTools(); syncToolsUi(); } } /** Track the browser's camera permission so a later re-grant (e.g. the user * unblocks it from the address bar after a denial) turns the camera back on * without another toggle, and a revoke turns it off. Best-effort: the * Permissions API doesn't support "camera" everywhere (e.g. Safari). */ async function watchCameraPermission() { try { const status = await navigator.permissions?.query?.({ name: /** @type {any} */ ("camera") }); if (!status) return; status.addEventListener("change", () => { if (status.state === "granted") { if (!toolsEnabled.camera_snapshot) { toolsEnabled.camera_snapshot = true; saveTools(); } void autoStartCamera(); syncToolsUi(); } else if (status.state === "denied") { disableCamera(); if (toolsEnabled.camera_snapshot) { toolsEnabled.camera_snapshot = false; saveTools(); } syncToolsUi(); } }); } catch { // Permissions API unavailable for "camera" — the toggle still re-asks. } } /** * Grab the current webcam frame as a downscaled JPEG data URL. The preview is * mirrored in CSS for a natural self-view, but we draw the raw (un-mirrored) * video here so the model sees the scene in its true orientation. * * Over WebRTC the frame must fit one data-channel message, so it's re-encoded * down the SNAPSHOT_LADDER until it's under SNAPSHOT_DC_BUDGET_CHARS; over * WebSocket the first (full-quality) rung is used as before. * @returns {string | null} */ function captureSnapshot() { if (!cameraStream || !camVideo.videoWidth) return null; const vw = camVideo.videoWidth; const vh = camVideo.videoHeight; /** @param {number} maxEdge @param {number} quality @returns {string | null} */ const encode = (maxEdge, quality) => { const scale = Math.min(1, maxEdge / Math.max(vw, vh)); const w = Math.max(1, Math.round(vw * scale)); const h = Math.max(1, Math.round(vh * scale)); const canvas = document.createElement("canvas"); canvas.width = w; canvas.height = h; const ctx = canvas.getContext("2d"); if (!ctx) return null; ctx.drawImage(camVideo, 0, 0, w, h); return canvas.toDataURL("image/jpeg", quality); }; if (activeTransport !== "webrtc") { return encode(SNAPSHOT_MAX_EDGE, SNAPSHOT_QUALITY); } let dataUrl = null; for (const [edge, quality] of SNAPSHOT_LADDER) { dataUrl = encode(edge, quality); if (!dataUrl) return null; if (dataUrl.length <= SNAPSHOT_DC_BUDGET_CHARS) return dataUrl; } // Even the last rung overflowed (shouldn't happen in practice) — send it // anyway; the client logs the failed send rather than killing the session. console.warn(`[tool] snapshot exceeds the data-channel budget after the full ladder (${dataUrl?.length} chars)`); return dataUrl; } /** Brief shutter flash on the preview so the user sees a snapshot was taken. */ function flashPreview() { camPip.classList.remove("flash"); void camPip.offsetWidth; // reflow so the animation restarts camPip.classList.add("flash"); } // ── Tool executor ───────────────────────────────────────────────────────── // Runs the function the model called, returns the result, and asks for a // response so the model speaks it. Errors come back as the tool output too, so // the model can recover gracefully instead of the turn stalling. /** * Run the function the model called, return its result to the backend, and ask * for a follow-up response. We also hand the result back to the caller so it * can be shown in the conversation once the tool has actually run. * @param {string} name @param {string} argsJson @param {string} callId * @returns {Promise<{ output: string, image?: string }>} */ async function runTool(name, argsJson, callId) { if (!client) return { output: "" }; let args = /** @type {Record} */ ({}); try { args = JSON.parse(argsJson || "{}"); } catch { /* keep {} */ } if (DEBUG) console.debug(`[tool] run name=${name} callId=${JSON.stringify(callId)} args=${argsJson}`); if (!callId) console.warn("[tool] empty call_id — the backend didn't tag the call, can't return a function_call_output"); /** @type {{ output: string, image?: string }} */ let result = { output: "" }; try { if (name === "web_search") { const query = typeof args.query === "string" ? args.query : ""; result.output = await execWebSearch(query); // Return the result and let the bare response.create (below) trigger the // spoken answer. client.sendToolOutput(callId, result.output); } else if (name === "camera_snapshot") { const dataUrl = captureSnapshot(); if (dataUrl) { if (DEBUG) console.debug(`[tool] camera_snapshot captured frame (${dataUrl.length} chars), sending image + output`); result = { output: "Snapshot captured from the webcam and attached as an image.", image: dataUrl }; // Return the tool output; the frame itself rides along with the // response.create below (sent right before it), so the model sees the // snapshot in the very response it's about to speak. client.sendToolOutput(callId, result.output); flashPreview(); } else { console.warn("[tool] camera_snapshot: no frame — camera off or not ready"); result.output = "The camera is not available right now."; client.sendToolOutput(callId, result.output); } } else { result.output = `Unknown tool: ${name}`; client.sendToolOutput(callId, result.output); } } catch (err) { const msg = err instanceof Error ? err.message : String(err); result.output = `Tool failed: ${msg}`; client.sendToolOutput(callId, result.output); } if (DEBUG) console.debug(`[tool] requesting model response after ${name}`); // Camera: the captured frame rides with the response.create (sent just before // it) so it's in context for the reply. Other tools: a bare create. client.requestResponse(result.image ? { image: result.image } : undefined); return result; } /** @param {string} query @returns {Promise} */ async function execWebSearch(query) { if (!query) return "No query provided."; /** @type {Record} */ const body = { query }; // Only send a user key when there's no server key (server prefers its own). if (!serverSearchKey && userSearchKey) body.key = userSearchKey; const res = await fetch("api/search", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), }); if (!res.ok) { let detail = String(res.status); try { const j = await res.json(); if (j.detail) detail = j.detail; } catch {} throw new Error(`search error (${detail})`); } const json = await res.json(); // Date-stamp the header so the model treats these as fresh realtime facts // rather than its (older) training knowledge. const today = new Date().toISOString().slice(0, 10); /** @type {string[]} */ const lines = [`Google search result from ${today}:`]; if (json.answer) lines.push(`Answer: ${json.answer}`); for (const r of json.results || []) { lines.push(`- ${r.title}: ${r.snippet} (${r.url})`); } return lines.length > 1 ? lines.join("\n") : `${lines[0]}\nNo results found.`; } /** Learn server config (search key + connection target), then refresh the UI. */ async function fetchConfig() { try { const res = await fetch("api/config"); if (res.ok) { const json = await res.json(); serverSearchKey = !!json.search; lbMode = !!json.lb; // Lock to LB mode only when the deploy reports a load balancer. allowDirect = json.allowDirect ?? !lbMode; // Deploy-pinned direct URL (overrides the LB server-side already). pinnedUrl = (json.s2sUrl || "").trim(); // WebRTC transport: offered only when the deploy pins the URL (the // /api/calls proxy refuses to forward anywhere else). rtcAvailable = !!json.rtc; iceServers = Array.isArray(json.iceServers) ? json.iceServers : []; loginRequired = !!json.requireLogin; // The conversation-time limiter rides on the LB being present. limiterOn = lbMode; } // Non-OK response: leave the fail-open default (allowDirect = true). } catch { // Config endpoint unreachable (e.g. static hosting): keep direct entry. } if (DEBUG) console.debug(`[ui] config: allowDirect=${allowDirect} lbMode=${lbMode}`); // Login chip + remaining-budget (no-op / hidden when the limiter is off). await account.refresh(); if (currentState === "idle" && loginRequired && !account.loggedIn) { setCaption("Sign in to start"); } syncToolsUi(); syncConnectionUi(); } /** * Resolve where to connect, per the deploy's mode: * • LB mode -> `{ sessionUrl }`, the client POSTs the same-origin /api/session * proxy and the server forwards to the LB (its address stays server-side). * • direct -> `{ directUrl }`, connect straight to the s2s WebSocket. * Throws a user-facing error if direct mode is on but no URL was entered. * @returns {{ sessionUrl: string } | { directUrl: string }} */ function connectionTarget() { if (!allowDirect) { return { sessionUrl: "api/session" }; } const directUrl = buildDirectWsUrl(pinnedUrl || settings.directUrl); if (!directUrl) { throw new Error("Enter a speech-to-speech server URL in Settings."); } return { directUrl }; } /** * Normalise a user-typed server address into a realtime WebSocket URL. * Accepts bare hosts (`localhost:8080`), http(s) URLs, or ws(s) URLs, and adds * the `/v1/realtime` path when none is given. A full connect URL (with path * and/or query) is preserved as-is. * @param {string} raw @returns {string} */ function buildDirectWsUrl(raw) { let s = (raw || "").trim(); if (!s) return ""; if (!/^wss?:\/\//i.test(s)) { if (/^https?:\/\//i.test(s)) { s = s.replace(/^http/i, "ws"); // http→ws, https→wss } else { const isLocal = /^(localhost|127\.0\.0\.1|\[::1\])(:|\/|$)/i.test(s); s = (isLocal ? "ws://" : "wss://") + s; } } try { const u = new URL(s); if (u.pathname === "" || u.pathname === "/") u.pathname = "/v1/realtime"; return u.toString(); } catch { return s; } } /** Create + resume an AudioContext synchronously (must run inside the user * gesture so iOS lets it start). Returns null if construction fails. */ function createResumedAudioContext() { try { const Ctx = window.AudioContext || /** @type {any} */ (window).webkitAudioContext; const ctx = new Ctx({ latencyHint: "interactive" }); if (ctx.state === "suspended") void ctx.resume().catch(() => {}); return /** @type {AudioContext} */ (ctx); } catch (err) { console.warn("[main] AudioContext init failed:", err); return null; } } /** Read the editable settings out of the form. The URL field is only honoured * in free direct mode — in LB mode it's hidden, and when the deploy pins a * URL it's read-only, so the user's saved URL survives either way. The * transport select is only honoured while selectable, so a saved "webrtc" * survives a visit to a deploy that can't offer it. */ function readSettingsFromForm() { return { directUrl: allowDirect && !pinnedUrl ? inputLbUrl.value.trim() : settings.directUrl, voice: inputVoice.value || DEFAULT_VOICE, instructions: inputInstructions.value.trim() || DEFAULT_INSTRUCTIONS, noiseGate: readGateThreshold(), transport: /** @type {"ws" | "webrtc"} */ ( transportSelectable() ? (inputTransport.value === "webrtc" ? "webrtc" : "ws") : settings.transport ), audioInputId: inputAudioInput.value || "", audioOutputId: inputAudioOutput.value || "", }; } /** Gate threshold (dBFS) currently shown on the slider, clamped to range. */ function readGateThreshold() { const v = Math.round(Number(inputNoiseGate.value)); if (!Number.isFinite(v)) return GATE_OFF_DB; return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, v)); } /** Whether the transport picker is live: env-pinned direct mode only. The * /api/calls proxy forwards exclusively to SPEECH_TO_SPEECH_URL, so without * the pin there is nowhere safe to send a WebRTC offer. */ function transportSelectable() { return allowDirect && !!pinnedUrl && rtcAvailable; } /** The transport the next conversation will actually use. */ function effectiveTransport() { return transportSelectable() && settings.transport === "webrtc" ? "webrtc" : "ws"; } /** Reflect transport availability + selection into Settings, and hide the * noise gate when WebRTC is picked (the gate lives in the WS capture * worklet; the WebRTC mic path sends the raw track). */ function syncTransportUi() { // Hidden in LB mode (nothing to choose); visible-but-locked in un-pinned // direct mode so the option is discoverable along with what unlocks it. transportField.hidden = !allowDirect; const selectable = transportSelectable(); inputTransport.disabled = !selectable; inputTransport.value = selectable && settings.transport === "webrtc" ? "webrtc" : "ws"; transportHint.textContent = selectable ? "How audio travels to the server. Applies on the next conversation." : "WebRTC needs a server URL pinned by the deployment (SPEECH_TO_SPEECH_URL)."; gateField.hidden = effectiveTransport() === "webrtc"; } /** Adapt the connection field to the mode learned from /api/config. */ function syncConnectionUi() { syncTransportUi(); if (pinnedUrl) { // Deploy-pinned URL: show it, but locked — the deployment owns it. connField.hidden = false; inputLbUrl.value = pinnedUrl; inputLbUrl.readOnly = true; connHint.classList.remove("error"); connHint.textContent = "Speech-to-speech server URL pinned by this deployment."; } else if (allowDirect) { // Direct mode: the user sets their own s2s server URL. connField.hidden = false; inputLbUrl.value = settings.directUrl; inputLbUrl.readOnly = false; inputLbUrl.placeholder = "http://localhost:port"; connHint.classList.remove("error"); connHint.textContent = "URL of your speech-to-speech server, e.g. http://localhost:8080 (the app adds /v1/realtime)."; } else { // LB mode: the load balancer URL is deployment-owned — hide it entirely so // its address is never exposed in Settings. connField.hidden = true; } } /** True when the user must supply a server URL before connecting (direct mode * with nothing set). */ function missingServerUrl() { return allowDirect && !pinnedUrl && !buildDirectWsUrl(settings.directUrl); } /** Open Settings and point the user at the empty server-URL field. */ function promptServerUrl() { if (settingsModal.open) syncConnectionUi(); else openSettings(); connHint.textContent = "Set the speech-to-speech server URL to start."; connHint.classList.add("error"); inputLbUrl.focus(); } settingsForm.addEventListener("submit", (event) => { const submitter = /** @type {HTMLButtonElement | null} */ ((/** @type {SubmitEvent} */ (event)).submitter); if (submitter?.value !== "save") return; settings = readSettingsFromForm(); saveSettings(settings); // Voice + instructions can apply to a live session without reconnecting; a // changed connection URL only takes effect on the next restart. Speaker // output can switch live when the browser supports AudioContext.setSinkId; // mic device changes need a Restart (new getUserMedia stream). if (client && LIVE_STATES.has(currentState)) { client.updateSession({ voice: settings.voice, instructions: effectiveInstructions() }); if (typeof client.setAudioOutputDevice === "function") { void client.setAudioOutputDevice(settings.audioOutputId); } } }); // The noise gate applies live (worklet param), so tune it without a restart: // update the label/marker, persist, and push straight to the running client. inputNoiseGate.addEventListener("input", () => { setGateThreshold(readGateThreshold()); }); // Transport persists on change (like the gate) and takes effect on the next // conversation; the gate field previews its WS-only availability right away. inputTransport.addEventListener("change", () => { if (!transportSelectable()) return; settings.transport = inputTransport.value === "webrtc" ? "webrtc" : "ws"; localStorage.setItem(STORAGE_KEYS.transport, settings.transport); syncTransportUi(); }); restartBtn.addEventListener("click", async () => { if (currentState === "connecting") return; // a connect is already underway if (loginRequired && !account.loggedIn) { account.showLoginRequired(); return; } settings = readSettingsFromForm(); saveSettings(settings); if (missingServerUrl()) { promptServerUrl(); return; } // keep settings open settingsModal.close(); // Grab the AudioContext NOW, inside the click gesture — teardown() awaits, and // creating it afterwards would fall outside the gesture (silent on iOS). const audioContext = createResumedAudioContext(); try { if (client) await teardown(); await doStart(audioContext); } catch (err) { await handleStartError(err); } }); circleBtn.addEventListener("click", async () => { try { if (currentState === "idle" || currentState === "error") { if (loginRequired && !account.loggedIn) { account.showLoginRequired(); return; } if (missingServerUrl()) { promptServerUrl(); return; } await doStart(); } } catch (err) { await handleStartError(err); } }); /** A failed start is either the daily limit (show the modal, return to idle) or * a real fault (surface it). doStart already closed any orphan AudioContext. * @param {any} err */ async function handleStartError(err) { if (err && err.code === "login-required") { await teardown(); account.showLoginRequired(err.loginUrl); return; } if (err && err.code === "limit") { await teardown(); account.showLimit(err.tier); return; } // The user left the queue (close() aborted the wait): teardown already reset // the UI to idle, so there's nothing to report. if (err && err.code === "aborted") return; // The whole waiting line is full: a warm, reassuring modal rather than an error. if (err && err.code === "queue-full") { await teardown(); account.showBusy(); return; } // Our place lapsed (ticket reaped, or the join window ran out). Recoverable, not // a fault: land on the retry state with a kind, plain-language reason. if (err && (err.code === "queue-expired" || err.code === "join-expired")) { await teardown(); setState("error"); setCaption( err.code === "join-expired" ? "Your spot expired. Tap to rejoin." : "That took a while. Tap to rejoin.", "error", ); return; } onFatalError(err); } micBtn.addEventListener("click", () => { if (!micStream || !client) return; micMuted = !micMuted; syncMicMuteState(); micBtn.classList.toggle("muted", micMuted); micBtn.setAttribute("aria-label", micMuted ? "Unmute" : "Mute"); micBtn.title = micMuted ? "Unmute" : "Mute"; }); stopBtn.addEventListener("click", async () => { await teardown(); }); // "Leave queue": tear down the pending connect (aborts the poll wait) and drop // our place in line. Same teardown path as stopping a live call. leaveQueueBtn.addEventListener("click", async () => { await teardown(); }); // "Join now": accept the held slot. The click is a user gesture, so the client // re-resumes the AudioContext here (iOS) before dialing. joinQueueBtn.addEventListener("click", () => { stopJoinCountdown(); if (client) client.join(); }); const MIC_CONSTRAINTS_BASE = { echoCancellation: true, noiseSuppression: true, autoGainControl: true, }; /** @returns {MediaStreamConstraints} */ function micConstraints() { /** @type {MediaTrackConstraints} */ const audio = { ...MIC_CONSTRAINTS_BASE }; if (settings.audioInputId) { // ideal (not exact): if the saved device was unplugged, fall back quietly. audio.deviceId = { ideal: settings.audioInputId }; } return { audio }; } /** True when Web Audio can route playback to a chosen output device. */ function supportsAudioOutputSelection() { const Ctx = window.AudioContext || /** @type {any} */ (window).webkitAudioContext; return typeof Ctx?.prototype?.setSinkId === "function"; } /** * Rebuild the mic/speaker