| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| import { S2sWsRealtimeClient } from "./ws/s2s-ws-client.js"; |
| import { $, truncateError, DEBUG } from "./ui/dom.js"; |
| import { ChatView } from "./ui/chat.js"; |
| import { Account } from "./ui/account.js"; |
|
|
| const DEFAULT_VOICE = "Aiden"; |
| const DEFAULT_INSTRUCTIONS = |
| "You are a friendly voice assistant. " + |
| "Keep replies short, warm, and spoken. Avoid long monologues."; |
|
|
| |
| |
| |
| const TOOL_USE_HINT = |
| " When the user's request calls for one of your tools, do not describe your " + |
| "capabilities or say you can do it and wait for another turn. Instead, say " + |
| 'a brief acknowledgement like "Let me search for that..." and call the tool ' + |
| "right away in the same response."; |
|
|
| const STORAGE_KEYS = { |
| |
| |
| directUrl: "s2s.ws.directUrl", |
| voice: "s2s.ws.voice", |
| instructions: "s2s.ws.instructions", |
| tools: "s2s.ws.tools", |
| searchKey: "s2s.ws.searchKey", |
| noiseGate: "s2s.ws.noiseGate", |
| }; |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| const GATE_OFF_DB = -66; |
| const GATE_MAX_DB = -3; |
| const GATE_DEFAULT_DB = -50; |
|
|
| |
| function gateParams(thresholdDb) { |
| return { enabled: thresholdDb > GATE_OFF_DB, thresholdDb }; |
| } |
|
|
| |
| |
| |
| |
| const TOOL_DEFS = { |
| web_search: { |
| type: "function", |
| name: "web_search", |
| description: |
| "Search the web for current or factual information you don't already know " + |
| "(news, prices, facts, documentation). Returns the top results with titles, " + |
| "snippets and URLs.", |
| parameters: { |
| type: "object", |
| properties: { query: { type: "string", description: "The search query." } }, |
| required: ["query"], |
| }, |
| }, |
| camera_snapshot: { |
| type: "function", |
| name: "camera_snapshot", |
| description: |
| "Capture the current frame from the user's webcam so you can see what they " + |
| "are showing you. Use it whenever the user refers to something visual or " + |
| "asks you to look.", |
| parameters: { type: "object", properties: {}, required: [] }, |
| }, |
| }; |
|
|
| |
| const SNAPSHOT_MAX_EDGE = 768; |
| const SNAPSHOT_QUALITY = 0.7; |
|
|
| function loadSettings() { |
| return { |
| directUrl: localStorage.getItem(STORAGE_KEYS.directUrl) || "", |
| voice: localStorage.getItem(STORAGE_KEYS.voice) || DEFAULT_VOICE, |
| instructions: localStorage.getItem(STORAGE_KEYS.instructions) || DEFAULT_INSTRUCTIONS, |
| noiseGate: loadGateThreshold(), |
| }; |
| } |
|
|
| |
| |
| function loadGateThreshold() { |
| const stored = localStorage.getItem(STORAGE_KEYS.noiseGate); |
| |
| |
| |
| if (stored === null || stored === "") return GATE_DEFAULT_DB; |
| const raw = Number(stored); |
| if (!Number.isFinite(raw)) return GATE_DEFAULT_DB; |
| return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(raw))); |
| } |
|
|
| |
| function saveSettings(s) { |
| localStorage.setItem(STORAGE_KEYS.directUrl, s.directUrl); |
| localStorage.setItem(STORAGE_KEYS.voice, s.voice); |
| localStorage.setItem(STORAGE_KEYS.instructions, s.instructions); |
| localStorage.setItem(STORAGE_KEYS.noiseGate, String(s.noiseGate)); |
| } |
|
|
| |
| function loadTools() { |
| try { |
| const raw = JSON.parse(localStorage.getItem(STORAGE_KEYS.tools) || "{}"); |
| |
| |
| |
| |
| return { |
| web_search: raw.web_search ?? true, |
| camera_snapshot: raw.camera_snapshot ?? true, |
| }; |
| } catch { |
| return { web_search: true, camera_snapshot: true }; |
| } |
| } |
|
|
| function saveTools() { |
| localStorage.setItem(STORAGE_KEYS.tools, JSON.stringify(toolsEnabled)); |
| } |
|
|
| |
| const STATE_VIEWS = { |
| idle: { caption: "Tap to start", disabled: false }, |
| connecting: { caption: "Connecting", disabled: true }, |
| queued: { caption: "Finding you a spot…", disabled: true }, |
| "your-turn": { caption: "You're up! 🎉", disabled: true }, |
| listening: { caption: "", disabled: false }, |
| "user-speaking": { caption: "", disabled: false }, |
| processing: { caption: "", disabled: false }, |
| "ai-speaking": { caption: "", disabled: false }, |
| error: { caption: "Tap to retry", disabled: false }, |
| }; |
|
|
| |
| const STATE_CLASS = { |
| idle: "state-idle", |
| connecting: "state-connecting", |
| queued: "state-queued", |
| "your-turn": "state-your-turn", |
| listening: "state-listening", |
| "user-speaking": "state-user-speaking", |
| processing: "state-processing", |
| "ai-speaking": "state-ai-speaking", |
| error: "state-error", |
| }; |
|
|
| |
| const LIVE_STATES = new Set(["listening", "user-speaking", "processing", "ai-speaking"]); |
|
|
| |
| const circleBtn = $("#main-circle"); |
| |
| const circleCaption = $("#circle-caption"); |
| |
| const circleSubcaption = $("#circle-subcaption"); |
| |
| const orbWrap = $(".orb-wrap"); |
| |
| const micBtn = $("#mic-btn"); |
| |
| const stopBtn = $("#stop-btn"); |
| |
| const queueActions = $("#queue-actions"); |
| |
| const joinQueueBtn = $("#join-queue-btn"); |
| |
| const leaveQueueBtn = $("#leave-queue-btn"); |
|
|
| |
| const settingsBtn = $("#settings-btn"); |
| |
| const settingsModal = $("#settings-modal"); |
|
|
| |
| const aboutBtn = $("#about-btn"); |
| |
| const aboutModal = $("#about-modal"); |
| |
| const aboutClose = $("#about-close"); |
|
|
| |
| const toolsBtn = $("#tools-btn"); |
| |
| const toolsModal = $("#tools-modal"); |
| |
| const toolsClose = $("#tools-close"); |
| |
| const toolWebSwitch = $("#tool-web"); |
| |
| const toolCamSwitch = $("#tool-cam"); |
| |
| const toolWebRow = $("#tool-web-row"); |
| |
| const toolWebHint = $("#tool-web-hint"); |
| |
| const toolCamHint = $("#tool-cam-hint"); |
| |
| const searchKeyInput = $("#search-key"); |
| |
| const camPip = $("#cam-pip"); |
| |
| const camVideo = $("#cam-video"); |
|
|
| |
| const inputLbUrl = $("#lb-url"); |
| |
| const connField = $("#conn-field"); |
| |
| const connHint = $("#conn-hint"); |
| |
| const inputVoice = $("#voice"); |
| |
| const inputInstructions = $("#instructions"); |
| |
| const inputNoiseGate = $("#noise-gate"); |
| |
| const gateValue = $("#gate-value"); |
| |
| const gateMeterFill = $("#gate-meter-fill"); |
| |
| const micGate = $("#mic-gate"); |
| const mgaArc = (document.querySelector("#mic-gate-arc")); |
| const mgaTrack = (document.querySelector("#mga-track")); |
| const mgaFill = (document.querySelector("#mga-fill")); |
| const mgaHit = (document.querySelector("#mga-hit")); |
| const mgaHandle = (document.querySelector("#mga-handle")); |
| |
| const restartBtn = $("#restart-conversation"); |
| |
| const restartHint = $("#restart-hint"); |
| const settingsForm = (settingsModal.querySelector("form")); |
|
|
| |
| let currentState = "idle"; |
| let settings = loadSettings(); |
|
|
| |
| |
| |
| |
| |
| |
| let lbMode = false; |
| |
| |
| |
| let allowDirect = true; |
|
|
| |
| let toolsEnabled = loadTools(); |
| |
| let serverSearchKey = false; |
| |
| let userSearchKey = localStorage.getItem(STORAGE_KEYS.searchKey) || ""; |
| |
| let cameraStream = null; |
|
|
| |
| function searchAvailable() { |
| return serverSearchKey || !!userSearchKey; |
| } |
|
|
| |
| function activeToolDefs() { |
| const defs = []; |
| if (toolsEnabled.web_search && searchAvailable()) defs.push(TOOL_DEFS.web_search); |
| if (toolsEnabled.camera_snapshot) defs.push(TOOL_DEFS.camera_snapshot); |
| return defs; |
| } |
|
|
| |
| function effectiveInstructions() { |
| const base = settings.instructions; |
| return activeToolDefs().length ? base + TOOL_USE_HINT : base; |
| } |
|
|
| |
| function pushToolsToSession() { |
| if (!client || !LIVE_STATES.has(currentState)) return; |
| client.setTools(activeToolDefs()); |
| |
| |
| client.updateSession({ instructions: effectiveInstructions() }); |
| } |
|
|
| |
| |
| |
| const chat = new ChatView(); |
|
|
| |
| |
| |
| |
| const account = new Account(); |
| let limiterOn = false; |
| let heartbeatTimer = 0; |
| let trackedSessionId = ""; |
| let trackedTier = ""; |
| |
| |
| let queuedTicketId = ""; |
|
|
| |
| let client = null; |
| |
| let micStream = null; |
| let micMuted = false; |
|
|
| |
| function setState(next) { |
| currentState = next; |
| const view = STATE_VIEWS[next]; |
| circleBtn.disabled = view.disabled; |
| circleBtn.className = `circle ${STATE_CLASS[next]}`; |
| if (next !== "error") setCaption(view.caption); |
|
|
| const live = LIVE_STATES.has(next); |
| orbWrap.classList.toggle("live", live); |
| micBtn.setAttribute("aria-hidden", live ? "false" : "true"); |
| stopBtn.setAttribute("aria-hidden", live ? "false" : "true"); |
| micBtn.tabIndex = live ? 0 : -1; |
| stopBtn.tabIndex = live ? 0 : -1; |
|
|
| |
| |
| const yourTurn = next === "your-turn"; |
| const inLine = next === "queued" || yourTurn; |
| queueActions.hidden = !inLine; |
| joinQueueBtn.hidden = !yourTurn; |
| joinQueueBtn.tabIndex = yourTurn ? 0 : -1; |
| leaveQueueBtn.hidden = !inLine; |
| leaveQueueBtn.tabIndex = inLine ? 0 : -1; |
| if (!yourTurn) stopJoinCountdown(); |
|
|
| |
| if (next === "queued") { |
| circleSubcaption.textContent = |
| "Sorry, we overhugged! 🤗 Every slot is busy, so we saved you a spot. Hang tight, you're moving up."; |
| circleSubcaption.hidden = false; |
| } else { |
| circleSubcaption.hidden = true; |
| } |
|
|
| updateRestartAvailability(); |
| } |
|
|
| function updateRestartAvailability() { |
| |
| |
| |
| restartBtn.disabled = |
| currentState === "connecting" || currentState === "queued" || currentState === "your-turn"; |
| restartHint.hidden = false; |
| restartHint.textContent = LIVE_STATES.has(currentState) |
| ? "Reconnects now with the settings above." |
| : "Starts a conversation with the settings above."; |
| } |
|
|
| |
| |
| |
| |
| function setCaption(text, kind = "") { |
| const trimmed = text.trim(); |
| circleCaption.textContent = trimmed; |
| circleCaption.className = `circle-caption${kind ? ` ${kind}` : ""}${trimmed ? "" : " empty"}`; |
| } |
|
|
| function openSettings() { |
| syncConnectionUi(); |
| inputVoice.value = settings.voice; |
| inputInstructions.value = settings.instructions; |
| syncGateUi(); |
| updateRestartAvailability(); |
| settingsModal.showModal(); |
| } |
|
|
| |
| |
| function dbToFraction(db) { |
| const clamped = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, db)); |
| return (clamped - GATE_OFF_DB) / (GATE_MAX_DB - GATE_OFF_DB); |
| } |
|
|
| |
| function fractionToDb(f) { |
| const clamped = Math.min(1, Math.max(0, f)); |
| return Math.round(GATE_OFF_DB + clamped * (GATE_MAX_DB - GATE_OFF_DB)); |
| } |
|
|
| |
| |
| |
| |
| const ARC_R = 40; |
| |
| const ARC_SPAN_DEG = 200; |
| const ARC_START_DEG = 180 - ARC_SPAN_DEG / 2; |
|
|
| |
| |
| function arcPoint(f, r = ARC_R) { |
| const deg = ARC_START_DEG + f * ARC_SPAN_DEG; |
| const rad = (deg * Math.PI) / 180; |
| return { x: 50 + r * Math.cos(rad), y: 50 + r * Math.sin(rad) }; |
| } |
|
|
| |
| function fullArcD() { |
| const a = arcPoint(0); |
| const b = arcPoint(1); |
| const largeArc = ARC_SPAN_DEG > 180 ? 1 : 0; |
| return `M ${a.x} ${a.y} A ${ARC_R} ${ARC_R} 0 ${largeArc} 1 ${b.x} ${b.y}`; |
| } |
|
|
| |
| function initGateArc() { |
| const d = fullArcD(); |
| mgaTrack.setAttribute("d", d); |
| mgaFill.setAttribute("d", d); |
| mgaHit.setAttribute("d", d); |
| |
| mgaFill.setAttribute("pathLength", "100"); |
| mgaFill.style.strokeDasharray = "100 100"; |
| mgaFill.style.strokeDashoffset = "100"; |
| renderGateHandle(); |
| } |
|
|
| |
| function renderGateHandle() { |
| const off = settings.noiseGate <= GATE_OFF_DB; |
| const p = arcPoint(dbToFraction(settings.noiseGate)); |
| mgaHandle.setAttribute("cx", String(p.x)); |
| mgaHandle.setAttribute("cy", String(p.y)); |
| micGate.classList.toggle("gate-off", off); |
| } |
|
|
| |
| |
| |
| |
| function paintInputLevel(rms) { |
| const db = rms > 0 ? 20 * Math.log10(rms) : GATE_OFF_DB; |
| const f = dbToFraction(db); |
| mgaFill.style.strokeDashoffset = String(100 * (1 - f)); |
| if (settingsModal.open) gateMeterFill.style.width = `${f * 100}%`; |
| const enabled = settings.noiseGate > GATE_OFF_DB; |
| micGate.classList.toggle("gate-open", enabled && f >= dbToFraction(settings.noiseGate)); |
| } |
|
|
| |
| |
| |
| function setGateThreshold(db) { |
| settings.noiseGate = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(db))); |
| const off = settings.noiseGate <= GATE_OFF_DB; |
| inputNoiseGate.value = String(settings.noiseGate); |
| gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`; |
| renderGateHandle(); |
| localStorage.setItem(STORAGE_KEYS.noiseGate, String(settings.noiseGate)); |
| if (client && LIVE_STATES.has(currentState)) { |
| client.setNoiseGate(gateParams(settings.noiseGate)); |
| } |
| } |
|
|
| |
| function syncGateUi() { |
| inputNoiseGate.value = String(settings.noiseGate); |
| const off = settings.noiseGate <= GATE_OFF_DB; |
| gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`; |
| renderGateHandle(); |
| } |
|
|
| |
| let gateDragging = false; |
| |
| function gatePointerToDb(e) { |
| const rect = mgaArc.getBoundingClientRect(); |
| const cx = rect.left + rect.width / 2; |
| const cy = rect.top + rect.height / 2; |
| let deg = (Math.atan2(e.clientY - cy, e.clientX - cx) * 180) / Math.PI; |
| if (deg < 0) deg += 360; |
| |
| |
| |
| const f = (deg - ARC_START_DEG) / ARC_SPAN_DEG; |
| return fractionToDb(f); |
| } |
| mgaHit.addEventListener("pointerdown", (e) => { |
| gateDragging = true; |
| mgaHit.setPointerCapture(e.pointerId); |
| setGateThreshold(gatePointerToDb(e)); |
| }); |
| mgaHit.addEventListener("pointermove", (e) => { |
| if (gateDragging) setGateThreshold(gatePointerToDb(e)); |
| }); |
| const endGateDrag = ( e) => { |
| if (!gateDragging) return; |
| gateDragging = false; |
| try { mgaHit.releasePointerCapture(e.pointerId); } catch {} |
| }; |
| mgaHit.addEventListener("pointerup", endGateDrag); |
| mgaHit.addEventListener("pointercancel", endGateDrag); |
|
|
| settingsBtn.addEventListener("click", openSettings); |
|
|
| |
| |
| aboutBtn.addEventListener("click", () => aboutModal.showModal()); |
| |
| $("#about-btn-m").addEventListener("click", () => aboutModal.showModal()); |
| aboutClose.addEventListener("click", () => aboutModal.close()); |
| aboutModal.addEventListener("click", (e) => { |
| if (e.target === aboutModal) aboutModal.close(); |
| }); |
|
|
| |
|
|
| |
| function syncToolsUi() { |
| const avail = searchAvailable(); |
| toolWebSwitch.checked = toolsEnabled.web_search && avail; |
| toolWebSwitch.disabled = !avail; |
| toolWebRow.classList.toggle("disabled", !avail); |
| toolCamSwitch.checked = toolsEnabled.camera_snapshot; |
|
|
| if (serverSearchKey) { |
| |
| searchKeyInput.value = ""; |
| searchKeyInput.placeholder = "•••••••• · provided by the server"; |
| searchKeyInput.disabled = true; |
| toolWebHint.textContent = "Ready. The search key is held server-side and never sent to your browser."; |
| } else { |
| searchKeyInput.disabled = false; |
| searchKeyInput.value = userSearchKey; |
| searchKeyInput.placeholder = "Paste a Serper key to enable web search"; |
| toolWebHint.textContent = userSearchKey |
| ? "Using your key — stored in this browser only." |
| : "No server key configured. Add your own Serper key to enable web search."; |
| } |
| } |
|
|
| toolsBtn.addEventListener("click", () => { syncToolsUi(); toolsModal.showModal(); }); |
| toolsClose.addEventListener("click", () => toolsModal.close()); |
| toolsModal.addEventListener("click", (e) => { |
| if (e.target === toolsModal) toolsModal.close(); |
| }); |
|
|
| toolWebSwitch.addEventListener("change", () => { |
| if (toolWebSwitch.checked && !searchAvailable()) { |
| toolWebSwitch.checked = false; |
| return; |
| } |
| toolsEnabled.web_search = toolWebSwitch.checked; |
| saveTools(); |
| pushToolsToSession(); |
| }); |
|
|
| toolCamSwitch.addEventListener("change", async () => { |
| if (toolCamSwitch.checked) { |
| try { |
| |
| |
| await enableCamera(); |
| } catch (err) { |
| toolCamSwitch.checked = false; |
| const denied = err instanceof Error && (err.name === "NotAllowedError" || err.name === "SecurityError"); |
| toolCamHint.textContent = denied |
| ? "Camera blocked. Allow it from the camera icon in your browser's address bar — it switches on automatically." |
| : `Camera unavailable${err instanceof Error ? `: ${err.message}` : ""}`; |
| return; |
| } |
| toolsEnabled.camera_snapshot = true; |
| toolCamHint.textContent = "Camera on. The assistant can take a snapshot when it needs to see."; |
| } else { |
| disableCamera(); |
| toolsEnabled.camera_snapshot = false; |
| toolCamHint.textContent = "Let the assistant see through your webcam."; |
| } |
| saveTools(); |
| pushToolsToSession(); |
| }); |
|
|
| searchKeyInput.addEventListener("input", () => { |
| if (serverSearchKey) return; |
| userSearchKey = searchKeyInput.value.trim(); |
| if (userSearchKey) localStorage.setItem(STORAGE_KEYS.searchKey, userSearchKey); |
| else localStorage.removeItem(STORAGE_KEYS.searchKey); |
|
|
| const avail = searchAvailable(); |
| toolWebSwitch.disabled = !avail; |
| toolWebRow.classList.toggle("disabled", !avail); |
| |
| if (!avail && toolsEnabled.web_search) { |
| toolsEnabled.web_search = false; |
| toolWebSwitch.checked = false; |
| saveTools(); |
| pushToolsToSession(); |
| } |
| toolWebHint.textContent = userSearchKey |
| ? "Using your key — stored in this browser only." |
| : "No server key configured. Add your own Serper key to enable web search."; |
| }); |
|
|
| |
|
|
| async function enableCamera() { |
| if (cameraStream) return; |
| cameraStream = await navigator.mediaDevices.getUserMedia({ |
| video: { facingMode: "user" }, |
| audio: false, |
| }); |
| camVideo.srcObject = cameraStream; |
| try { await camVideo.play(); } catch { } |
| camPip.classList.add("visible"); |
| camPip.setAttribute("aria-hidden", "false"); |
| |
| |
| document.body.classList.add("cam-on"); |
| } |
|
|
| function disableCamera() { |
| if (cameraStream) { |
| for (const t of cameraStream.getTracks()) t.stop(); |
| cameraStream = null; |
| } |
| camVideo.srcObject = null; |
| camPip.classList.remove("visible"); |
| camPip.setAttribute("aria-hidden", "true"); |
| document.body.classList.remove("cam-on"); |
| } |
|
|
| |
| |
| |
| async function autoStartCamera() { |
| if (!toolsEnabled.camera_snapshot || cameraStream) return; |
| try { |
| await enableCamera(); |
| } catch (err) { |
| console.warn("[main] camera auto-start declined/failed:", err); |
| toolsEnabled.camera_snapshot = false; |
| saveTools(); |
| syncToolsUi(); |
| } |
| } |
|
|
| |
| |
| |
| |
| async function watchCameraPermission() { |
| try { |
| const status = await navigator.permissions?.query?.({ name: ("camera") }); |
| if (!status) return; |
| status.addEventListener("change", () => { |
| if (status.state === "granted") { |
| if (!toolsEnabled.camera_snapshot) { toolsEnabled.camera_snapshot = true; saveTools(); } |
| void autoStartCamera(); |
| syncToolsUi(); |
| } else if (status.state === "denied") { |
| disableCamera(); |
| if (toolsEnabled.camera_snapshot) { toolsEnabled.camera_snapshot = false; saveTools(); } |
| syncToolsUi(); |
| } |
| }); |
| } catch { |
| |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| |
| function captureSnapshot() { |
| if (!cameraStream || !camVideo.videoWidth) return null; |
| const vw = camVideo.videoWidth; |
| const vh = camVideo.videoHeight; |
| const scale = Math.min(1, SNAPSHOT_MAX_EDGE / Math.max(vw, vh)); |
| const w = Math.max(1, Math.round(vw * scale)); |
| const h = Math.max(1, Math.round(vh * scale)); |
| const canvas = document.createElement("canvas"); |
| canvas.width = w; |
| canvas.height = h; |
| const ctx = canvas.getContext("2d"); |
| if (!ctx) return null; |
| ctx.drawImage(camVideo, 0, 0, w, h); |
| return canvas.toDataURL("image/jpeg", SNAPSHOT_QUALITY); |
| } |
|
|
| |
| function flashPreview() { |
| camPip.classList.remove("flash"); |
| void camPip.offsetWidth; |
| camPip.classList.add("flash"); |
| } |
|
|
| |
| |
| |
| |
|
|
| |
| |
| |
| |
| |
| |
| |
| async function runTool(name, argsJson, callId) { |
| if (!client) return { output: "" }; |
| let args = ({}); |
| try { args = JSON.parse(argsJson || "{}"); } catch { } |
|
|
| if (DEBUG) console.debug(`[tool] run name=${name} callId=${JSON.stringify(callId)} args=${argsJson}`); |
| if (!callId) console.warn("[tool] empty call_id — the backend didn't tag the call, can't return a function_call_output"); |
|
|
| |
| let result = { output: "" }; |
| try { |
| if (name === "web_search") { |
| const query = typeof args.query === "string" ? args.query : ""; |
| result.output = await execWebSearch(query); |
| |
| |
| client.sendToolOutput(callId, result.output); |
| } else if (name === "camera_snapshot") { |
| const dataUrl = captureSnapshot(); |
| if (dataUrl) { |
| if (DEBUG) console.debug(`[tool] camera_snapshot captured frame (${dataUrl.length} chars), sending image + output`); |
| result = { output: "Snapshot captured from the webcam and attached as an image.", image: dataUrl }; |
| |
| |
| |
| client.sendToolOutput(callId, result.output); |
| flashPreview(); |
| } else { |
| console.warn("[tool] camera_snapshot: no frame — camera off or not ready"); |
| result.output = "The camera is not available right now."; |
| client.sendToolOutput(callId, result.output); |
| } |
| } else { |
| result.output = `Unknown tool: ${name}`; |
| client.sendToolOutput(callId, result.output); |
| } |
| } catch (err) { |
| const msg = err instanceof Error ? err.message : String(err); |
| result.output = `Tool failed: ${msg}`; |
| client.sendToolOutput(callId, result.output); |
| } |
| if (DEBUG) console.debug(`[tool] requesting model response after ${name}`); |
| |
| |
| client.requestResponse(result.image ? { image: result.image } : undefined); |
| return result; |
| } |
|
|
| |
| async function execWebSearch(query) { |
| if (!query) return "No query provided."; |
| |
| const body = { query }; |
| |
| if (!serverSearchKey && userSearchKey) body.key = userSearchKey; |
|
|
| const res = await fetch("api/search", { |
| method: "POST", |
| headers: { "Content-Type": "application/json" }, |
| body: JSON.stringify(body), |
| }); |
| if (!res.ok) { |
| let detail = String(res.status); |
| try { const j = await res.json(); if (j.detail) detail = j.detail; } catch {} |
| throw new Error(`search error (${detail})`); |
| } |
| const json = await res.json(); |
| |
| |
| const today = new Date().toISOString().slice(0, 10); |
| |
| const lines = [`Google search result from ${today}:`]; |
| if (json.answer) lines.push(`Answer: ${json.answer}`); |
| for (const r of json.results || []) { |
| lines.push(`- ${r.title}: ${r.snippet} (${r.url})`); |
| } |
| return lines.length > 1 ? lines.join("\n") : `${lines[0]}\nNo results found.`; |
| } |
|
|
| |
| async function fetchConfig() { |
| try { |
| const res = await fetch("api/config"); |
| if (res.ok) { |
| const json = await res.json(); |
| serverSearchKey = !!json.search; |
| lbMode = !!json.lb; |
| |
| allowDirect = json.allowDirect ?? !lbMode; |
| |
| limiterOn = lbMode; |
| } |
| |
| } catch { |
| |
| } |
| if (DEBUG) console.debug(`[ui] config: allowDirect=${allowDirect} lbMode=${lbMode}`); |
| |
| void account.refresh(); |
| syncToolsUi(); |
| syncConnectionUi(); |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| function connectionTarget() { |
| if (!allowDirect) { |
| return { sessionUrl: "api/session" }; |
| } |
| const directUrl = buildDirectWsUrl(settings.directUrl); |
| if (!directUrl) { |
| throw new Error("Enter a speech-to-speech server URL in Settings."); |
| } |
| return { directUrl }; |
| } |
|
|
| |
| |
| |
| |
| |
| |
| |
| function buildDirectWsUrl(raw) { |
| let s = (raw || "").trim(); |
| if (!s) return ""; |
| if (!/^wss?:\/\//i.test(s)) { |
| if (/^https?:\/\//i.test(s)) { |
| s = s.replace(/^http/i, "ws"); |
| } else { |
| const isLocal = /^(localhost|127\.0\.0\.1|\[::1\])(:|\/|$)/i.test(s); |
| s = (isLocal ? "ws://" : "wss://") + s; |
| } |
| } |
| try { |
| const u = new URL(s); |
| if (u.pathname === "" || u.pathname === "/") u.pathname = "/v1/realtime"; |
| return u.toString(); |
| } catch { |
| return s; |
| } |
| } |
|
|
| |
| |
| function createResumedAudioContext() { |
| try { |
| const Ctx = window.AudioContext || (window).webkitAudioContext; |
| const ctx = new Ctx({ latencyHint: "interactive" }); |
| if (ctx.state === "suspended") void ctx.resume().catch(() => {}); |
| return (ctx); |
| } catch (err) { |
| console.warn("[main] AudioContext init failed:", err); |
| return null; |
| } |
| } |
|
|
| |
| |
| function readSettingsFromForm() { |
| return { |
| directUrl: allowDirect ? inputLbUrl.value.trim() : settings.directUrl, |
| voice: inputVoice.value || DEFAULT_VOICE, |
| instructions: inputInstructions.value.trim() || DEFAULT_INSTRUCTIONS, |
| noiseGate: readGateThreshold(), |
| }; |
| } |
|
|
| |
| function readGateThreshold() { |
| const v = Math.round(Number(inputNoiseGate.value)); |
| if (!Number.isFinite(v)) return GATE_OFF_DB; |
| return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, v)); |
| } |
|
|
| |
| function syncConnectionUi() { |
| if (allowDirect) { |
| |
| connField.hidden = false; |
| inputLbUrl.value = settings.directUrl; |
| inputLbUrl.placeholder = "http://localhost:port"; |
| connHint.classList.remove("error"); |
| connHint.textContent = |
| "URL of your speech-to-speech server, e.g. http://localhost:8080 (the app adds /v1/realtime)."; |
| } else { |
| |
| |
| connField.hidden = true; |
| } |
| } |
|
|
| |
| |
| function missingServerUrl() { |
| return allowDirect && !buildDirectWsUrl(settings.directUrl); |
| } |
|
|
| |
| function promptServerUrl() { |
| if (settingsModal.open) syncConnectionUi(); |
| else openSettings(); |
| connHint.textContent = "Set the speech-to-speech server URL to start."; |
| connHint.classList.add("error"); |
| inputLbUrl.focus(); |
| } |
|
|
| settingsForm.addEventListener("submit", (event) => { |
| const submitter = (( (event)).submitter); |
| if (submitter?.value !== "save") return; |
|
|
| settings = readSettingsFromForm(); |
| saveSettings(settings); |
|
|
| |
| |
| if (client && LIVE_STATES.has(currentState)) { |
| client.updateSession({ voice: settings.voice, instructions: effectiveInstructions() }); |
| } |
| }); |
|
|
| |
| |
| inputNoiseGate.addEventListener("input", () => { |
| setGateThreshold(readGateThreshold()); |
| }); |
|
|
| restartBtn.addEventListener("click", async () => { |
| if (currentState === "connecting") return; |
| settings = readSettingsFromForm(); |
| saveSettings(settings); |
| if (missingServerUrl()) { promptServerUrl(); return; } |
| settingsModal.close(); |
| |
| |
| const audioContext = createResumedAudioContext(); |
| try { |
| if (client) await teardown(); |
| await doStart(audioContext); |
| } catch (err) { |
| await handleStartError(err); |
| } |
| }); |
|
|
| circleBtn.addEventListener("click", async () => { |
| try { |
| if (currentState === "idle" || currentState === "error") { |
| if (missingServerUrl()) { promptServerUrl(); return; } |
| await doStart(); |
| } |
| } catch (err) { |
| await handleStartError(err); |
| } |
| }); |
|
|
| |
| |
| |
| async function handleStartError(err) { |
| if (err && err.code === "limit") { |
| await teardown(); |
| account.showLimit(err.tier); |
| return; |
| } |
| |
| |
| if (err && err.code === "aborted") return; |
| |
| if (err && err.code === "queue-full") { |
| await teardown(); |
| account.showBusy(); |
| return; |
| } |
| |
| |
| if (err && (err.code === "queue-expired" || err.code === "join-expired")) { |
| await teardown(); |
| setState("error"); |
| setCaption( |
| err.code === "join-expired" |
| ? "Your spot expired. Tap to rejoin." |
| : "That took a while. Tap to rejoin.", |
| "error", |
| ); |
| return; |
| } |
| onFatalError(err); |
| } |
|
|
| micBtn.addEventListener("click", () => { |
| if (!micStream || !client) return; |
| micMuted = !micMuted; |
| for (const track of micStream.getAudioTracks()) { |
| track.enabled = !micMuted; |
| } |
| client.setMuted(micMuted); |
| micBtn.classList.toggle("muted", micMuted); |
| micBtn.setAttribute("aria-label", micMuted ? "Unmute" : "Mute"); |
| micBtn.title = micMuted ? "Unmute" : "Mute"; |
| }); |
|
|
| stopBtn.addEventListener("click", async () => { |
| await teardown(); |
| }); |
|
|
| |
| |
| leaveQueueBtn.addEventListener("click", async () => { |
| await teardown(); |
| }); |
|
|
| |
| |
| joinQueueBtn.addEventListener("click", () => { |
| stopJoinCountdown(); |
| if (client) client.join(); |
| }); |
|
|
| const MIC_CONSTRAINTS = { |
| audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, |
| }; |
|
|
| |
| |
| |
| async function primeMicPermission() { |
| try { |
| const s = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS); |
| for (const track of s.getTracks()) track.stop(); |
| } catch (err) { |
| throw new Error( |
| `Microphone access denied${err instanceof Error ? `: ${err.message}` : ""}`, |
| ); |
| } |
| } |
|
|
| |
| |
| async function acquireMicStream() { |
| micStream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS); |
| return micStream; |
| } |
|
|
| |
| function onQueuePosition(position) { |
| const n = Number(position) || 0; |
| setCaption(n > 0 ? `You're #${n} in line` : "Finding you a spot…", "muted"); |
| } |
|
|
| |
| |
| |
| let joinCountdownTimer = 0; |
|
|
| |
| function startJoinCountdown(sec) { |
| stopJoinCountdown(); |
| let left = Math.max(0, Math.floor(sec)); |
| const paint = () => { |
| joinQueueBtn.textContent = left > 0 ? `Join now (${left}s)` : "Join now"; |
| }; |
| paint(); |
| joinCountdownTimer = window.setInterval(() => { |
| left -= 1; |
| if (left <= 0) { |
| stopJoinCountdown(); |
| joinQueueBtn.textContent = "Join now"; |
| return; |
| } |
| paint(); |
| }, 1000); |
| } |
|
|
| function stopJoinCountdown() { |
| if (joinCountdownTimer) { |
| clearInterval(joinCountdownTimer); |
| joinCountdownTimer = 0; |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| |
| async function doStart(audioContext = null) { |
| |
| |
| const target = connectionTarget(); |
|
|
| chat.clear(); |
| chat.reset(); |
| setState("connecting"); |
| setCaption("Asking for mic…", "muted"); |
|
|
| |
| |
| |
| |
| if (!audioContext) audioContext = createResumedAudioContext(); |
|
|
| |
| |
| |
| |
| try { |
| await primeMicPermission(); |
| } catch (err) { |
| if (audioContext) void audioContext.close().catch(() => {}); |
| throw err; |
| } |
|
|
| |
| |
|
|
| const c = new S2sWsRealtimeClient({ |
| ...target, |
| voice: settings.voice, |
| instructions: effectiveInstructions(), |
| acquireMic: acquireMicStream, |
| tools: activeToolDefs(), |
| noiseGate: gateParams(settings.noiseGate), |
| ...(audioContext ? { audioContext } : {}), |
| }); |
| client = c; |
|
|
| c.addEventListener("queue", (e) => { |
| const { position, queueId } = (e).detail; |
| if (queueId) queuedTicketId = queueId; |
| onQueuePosition(position); |
| }); |
|
|
| c.addEventListener("ready-to-join", (e) => { |
| const { info, expiresSec } = (e).detail; |
| |
| |
| |
| queuedTicketId = ""; |
| if (info?.sessionId) { |
| trackedSessionId = info.sessionId; |
| trackedTier = info.tier || "anon"; |
| } |
| startJoinCountdown(expiresSec); |
| }); |
|
|
| c.addEventListener("status", (e) => { |
| const detail = (e).detail; |
| onClientStatus(detail.status); |
| }); |
| c.addEventListener("transcript", (e) => { |
| const d = (e).detail; |
| chat.onTranscript(d); |
| }); |
|
|
| c.addEventListener("response-finished", (e) => { |
| const detail = (e).detail; |
| chat.onResponseFinished(detail); |
| }); |
|
|
| c.addEventListener("toolcall", (e) => { |
| const { name, arguments: args, callId } = (e).detail; |
| chat.onToolCall(name); |
| |
| |
| void runTool(name, args, callId).then(({ output, image }) => { |
| chat.onToolResult(name, args, output, image); |
| }); |
| }); |
| c.addEventListener("error", (e) => { |
| const detail = (e).detail; |
| onFatalError(detail.error); |
| }); |
| c.addEventListener("server-error", (e) => { |
| |
| |
| const detail = (e).detail; |
| const msg = detail.error instanceof Error ? detail.error.message : String(detail.error); |
| console.warn("[main] server error (non-fatal):", msg); |
| }); |
| c.addEventListener("session", (e) => { |
| const info = (e).detail.info; |
| console.log("[ws] session created:", info.sessionId); |
| |
| |
| queuedTicketId = ""; |
| |
| |
| if (info.limited && info.sessionId) { |
| trackedSessionId = info.sessionId; |
| trackedTier = info.tier || "anon"; |
| startHeartbeat(info.heartbeatSec || 5); |
| } |
| }); |
| c.addEventListener("input-level", (e) => { |
| const { rms } = (e).detail; |
| paintInputLevel(rms); |
| }); |
|
|
| try { |
| await c.connect(); |
| } catch (err) { |
| |
| |
| |
| if (audioContext) void audioContext.close().catch(() => {}); |
| throw err; |
| } |
| } |
|
|
| |
|
|
| |
| |
| |
| function startHeartbeat(sec) { |
| stopHeartbeat(); |
| heartbeatTimer = window.setInterval(async () => { |
| if (!trackedSessionId) return; |
| try { |
| const res = await fetch("api/session/heartbeat", { |
| method: "POST", |
| headers: { "Content-Type": "application/json" }, |
| body: JSON.stringify({ sessionId: trackedSessionId }), |
| keepalive: true, |
| }); |
| const json = await res.json().catch(() => ({})); |
| if (json.expired) await onLimitReached(); |
| } catch (err) { |
| |
| if (DEBUG) console.debug("[ui] heartbeat failed:", err); |
| } |
| }, Math.max(1, sec) * 1000); |
| } |
|
|
| function stopHeartbeat() { |
| if (heartbeatTimer) { |
| clearInterval(heartbeatTimer); |
| heartbeatTimer = 0; |
| } |
| } |
|
|
| |
| async function onLimitReached() { |
| const tier = trackedTier; |
| stopHeartbeat(); |
| await teardown(); |
| account.showLimit(tier); |
| } |
|
|
| |
| |
| function endTrackedSession() { |
| if (!trackedSessionId) return; |
| const body = JSON.stringify({ sessionId: trackedSessionId }); |
| try { |
| const blob = new Blob([body], { type: "application/json" }); |
| if (!navigator.sendBeacon("api/session/end", blob)) { |
| void fetch("api/session/end", { |
| method: "POST", headers: { "Content-Type": "application/json" }, body, keepalive: true, |
| }).catch(() => {}); |
| } |
| } catch { |
| |
| } |
| trackedSessionId = ""; |
| trackedTier = ""; |
| } |
|
|
| |
| |
| function endQueueTicket() { |
| if (!queuedTicketId) return; |
| const body = JSON.stringify({ queueId: queuedTicketId }); |
| try { |
| const blob = new Blob([body], { type: "application/json" }); |
| if (!navigator.sendBeacon("api/queue/end", blob)) { |
| void fetch("api/queue/end", { |
| method: "POST", headers: { "Content-Type": "application/json" }, body, keepalive: true, |
| }).catch(() => {}); |
| } |
| } catch { |
| |
| } |
| queuedTicketId = ""; |
| } |
|
|
| |
| function onClientStatus(status) { |
| switch (status) { |
| case "creating-session": |
| case "connecting": |
| setState("connecting"); |
| break; |
| case "queued": |
| setState("queued"); |
| break; |
| case "your-turn": |
| setState("your-turn"); |
| break; |
| case "connected": |
| setState("listening"); |
| break; |
| case "user-speaking": |
| setState("user-speaking"); |
| break; |
| case "processing": |
| setState("processing"); |
| break; |
| case "ai-speaking": |
| setState("ai-speaking"); |
| break; |
| case "closed": |
| |
| break; |
| case "error": |
| setState("error"); |
| break; |
| } |
| } |
|
|
| async function teardown() { |
| stopHeartbeat(); |
| stopJoinCountdown(); |
| endTrackedSession(); |
| endQueueTicket(); |
| chat.reset({ dismiss: true }); |
| if (client) { |
| try { |
| await client.close(); |
| } catch (err) { |
| console.warn("[main] error closing client:", err); |
| } |
| client = null; |
| } |
| if (micStream) { |
| for (const track of micStream.getTracks()) track.stop(); |
| micStream = null; |
| } |
| |
| |
| micMuted = false; |
| micBtn.classList.remove("muted"); |
| setState("idle"); |
| |
| if (limiterOn) void account.refresh(); |
| } |
|
|
| |
| function onFatalError(err) { |
| console.error("[main] fatal:", err); |
| setState("error"); |
| const message = err instanceof Error ? err.message : String(err); |
| setCaption(truncateError(message), "error"); |
| void teardown().catch(() => { |
| setState("error"); |
| setCaption(truncateError(message), "error"); |
| }); |
| } |
|
|
| setState("idle"); |
| chat.renderEmptyState(); |
| initGateArc(); |
| void fetchConfig(); |
| |
| |
| void autoStartCamera(); |
| void watchCameraPermission(); |
|
|
| |
| window.addEventListener("pagehide", () => { endTrackedSession(); endQueueTicket(); }); |
|
|
| requestAnimationFrame(() => { |
| document.body.classList.remove("booting"); |
| }); |
|
|