gemma-avatar / src /app.js
bep40's picture
Upload src/app.js
a9cf7dc verified
Raw
History Blame
35.7 kB
// @ts-check
/**
* App wiring: the avatar stage + the speech-to-speech session + V.AISTUDIO iframe control.
* Modes: voice (mic + VAD) OR text (keyboard only, no mic needed).
*
* KEY DESIGN: On direct hf.space domains, AudioContext starts "suspended"
* because there's no user gesture on page load. Instead of hanging or
* silently failing, we wait for the user to click "Start talking" before
* any WebSocket connection — that click resumes AudioContext, audio plays,
* and HeadAudio lip-sync works.
*
* ?fakemic=1 — opt-in URL query param for testing text-only mode
* WITHOUT a microphone. Only activates when user explicitly adds it to URL.
*/
import { S2sWsRealtimeClient } from "./s2s/s2s-ws-client.js";
import { AvatarStage, AVATAR_MOODS, AVATAR_GESTURES } from "./avatar.js";
import { smartNormalize } from "./viNumberFix.js";
const VOICES = [
"Aiden", "Ryan", "Dylan", "Eric",
"Ono_Anna", "Serena", "Sohee", "Uncle_Fu", "Vivian",
];
const DEFAULT_VOICE = "Sohee";
// ── Detect fakemic mode from URL (user opt-in only) ──────────────────────
const _urlParams = new URLSearchParams(location.search);
const FAKEMIC_MODE = _urlParams.has("fakemic");
// ── V.AISTUDIO iframe bridge ─────────────────────────────────────────────
const VAISTUDIO_ORIGIN = "https://bep40-v-aistudio.static.hf.space";
const VAISTUDIO_IFRAME_ID = "vaistudio-iframe";
const VAISTUDIO_PANEL_ID = "vaistudio-panel";
const VAISTUDIO_TOGGLE_ID = "vaistudio-toggle";
const VAISTUDIO_LOADING_ID = "vaistudio-loading";
/** @type {HTMLIFrameElement | null} */
let vaistudioIframe = null;
/** @type {HTMLDivElement | null} */
let vaistudioPanel = null;
/** @type {HTMLButtonElement | null} */
let vaistudioToggle = null;
/** @type {HTMLDivElement | null} */
let vaistudioLoading = null;
/** @type {boolean} */
let vaistudioPanelOpen = false;
/** @type {Map<string, Function>} */
const vaistudioResponseHandlers = new Map();
/** @type {number} */
let vaistudioMessageId = 0;
/**
* Send a command to V.AISTUDIO iframe via postMessage
*/
function vaistudioSend(action, data = {}) {
if (!vaistudioIframe || !vaistudioIframe.contentWindow) {
console.warn("[VAISTUDIO] iframe not ready");
return Promise.reject(new Error("iframe not ready"));
}
const id = ++vaistudioMessageId;
const message = { id, action, data, timestamp: Date.now() };
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
vaistudioResponseHandlers.delete(id);
reject(new Error(`VAISTUDIO timeout: ${action}`));
}, 10000);
vaistudioResponseHandlers.set(id, (response) => {
clearTimeout(timeout);
if (response.error) reject(new Error(response.error));
else resolve(response.data);
});
try {
vaistudioIframe.contentWindow.postMessage(message, VAISTUDIO_ORIGIN);
} catch (e) {
vaistudioResponseHandlers.delete(id);
clearTimeout(timeout);
reject(e);
}
});
}
/**
* Handle responses from V.AISTUDIO iframe
*/
function vaistudioOnMessage(event) {
if (event.origin !== VAISTUDIO_ORIGIN) return;
const msg = event.data;
if (!msg || typeof msg !== "object") return;
if (msg.id && vaistudioResponseHandlers.has(msg.id)) {
const handler = vaistudioResponseHandlers.get(msg.id);
vaistudioResponseHandlers.delete(msg.id);
handler(msg);
}
// Handle unsolicited events from iframe
if (msg.event) {
console.log("[VAISTUDIO] Event:", msg.event, msg.data);
}
}
/**
* Initialize V.AISTUDIO iframe controls
*/
function initVAISTUDIO() {
vaistudioIframe = /** @type {HTMLIFrameElement} */ (document.getElementById(VAISTUDIO_IFRAME_ID));
vaistudioPanel = /** @type {HTMLDivElement} */ (document.getElementById(VAISTUDIO_PANEL_ID));
vaistudioToggle = /** @type {HTMLButtonElement} */ (document.getElementById(VAISTUDIO_TOGGLE_ID));
vaistudioLoading = /** @type {HTMLDivElement} */ (document.getElementById(VAISTUDIO_LOADING_ID));
if (!vaistudioIframe || !vaistudioPanel || !vaistudioToggle) {
console.warn("[VAISTUDIO] Required elements not found");
return;
}
// Toggle panel
vaistudioToggle.addEventListener("click", () => {
vaistudioPanelOpen = !vaistudioPanelOpen;
vaistudioPanel.classList.toggle("open", vaistudioPanelOpen);
vaistudioToggle.classList.toggle("active", vaistudioPanelOpen);
vaistudioToggle.setAttribute("aria-label", vaistudioPanelOpen ? "Close V.AI STUDIO" : "Open V.AI STUDIO");
vaistudioToggle.title = vaistudioPanelOpen ? "Close V.AI STUDIO" : "Open V.AI STUDIO";
});
// Close button
const closeBtn = document.getElementById("vaistudio-close");
if (closeBtn) {
closeBtn.addEventListener("click", () => {
vaistudioPanelOpen = false;
vaistudioPanel.classList.remove("open");
vaistudioToggle.classList.remove("active");
vaistudioToggle.setAttribute("aria-label", "Open V.AI STUDIO");
vaistudioToggle.title = "Open V.AI STUDIO";
});
}
// Listen for iframe load
vaistudioIframe.addEventListener("load", () => {
if (vaistudioLoading) vaistudioLoading.classList.add("hidden");
console.log("[VAISTUDIO] iframe loaded");
vaistudioSend("ready", { source: "gemma-avatar" }).catch(() => {});
});
// Listen for postMessage
window.addEventListener("message", vaistudioOnMessage);
// Expose globally for avatar tools
window.vaistudio = {
send: vaistudioSend,
open: () => { if (!vaistudioPanelOpen) vaistudioToggle.click(); },
close: () => { if (vaistudioPanelOpen) vaistudioToggle.click(); },
isOpen: () => vaistudioPanelOpen,
};
}
/**
* V.AISTUDIO control functions for avatar tools
*/
async function vaistudioOpenCatalog(category = "") {
await vaistudioSend("navigate", { page: "products", category });
if (!vaistudioPanelOpen) vaistudioToggle.click();
return { success: true, message: `Opened catalog${category ? `: ${category}` : ""}` };
}
async function vaistudioOpenProduct(productId) {
await vaistudioSend("navigate", { page: "product", productId });
if (!vaistudioPanelOpen) vaistudioToggle.click();
return { success: true, message: `Opened product ${productId}` };
}
async function vaistudioSearch(query) {
await vaistudioSend("search", { query });
if (!vaistudioPanelOpen) vaistudioToggle.click();
return { success: true, message: `Searching: ${query}` };
}
async function vaistudioNavigate(page, data = {}) {
await vaistudioSend("navigate", { page, ...data });
if (!vaistudioPanelOpen) vaistudioToggle.click();
return { success: true, message: `Navigated to ${page}` };
}
// ── Greeting ─────────────────────────────────────────────────────────────
async function getHotNewsGreeting() {
let hotTitle = "";
try {
const resp = await fetch("/api/news/hot");
if (resp.ok) {
const data = await resp.json();
if (data.titles && data.titles.length > 0) {
for (const t of data.titles) {
const clean = t.replace(/^[\d.]+[\s:]*/, "").trim();
if (clean.length > 10 && clean.length < 200) { hotTitle = clean; break; }
}
}
}
} catch (e) { console.warn("News API failed:", e); }
if (hotTitle) return `Hôm nay có tin: ${hotTitle}. Hỏi người dùng có muốn nghe không.`;
return "";
}
const DEFAULT_INSTRUCTIONS = [
"You are a friendly voice assistant with a visible, human-like 3D avatar.",
"You also have access to V.AI STUDIO - an e-commerce catalog with 8000+ kitchen appliances & smart locks from Malloca, Eurogold, Grob, Canzy, Demax & Dien May Xanh.",
"CRITICAL: Always respond in the SAME LANGUAGE the user writes or speaks.",
"If Vietnamese, respond in Vietnamese. If English, respond in English.",
"When Vietnamese: write dates as 'ngày 9 tháng 7 năm 2026' (NOT '9/7/2026'),",
"years as 'năm hai nghìn không trăm hai mươi sáu' (NOT '2026'),",
"numbers as words (e.g. 'ba mươi lăm' NOT '35'),",
"percentages as 'mười lăm phần trăm' NOT '15%',",
"currency as 'năm mươi nghìn đồng' NOT '50.000đ'.",
"Keep replies short, natural, warm. Never list-like.",
"You can control avatar body with tools: set_mood, make_hand_gesture, make_facial_expression.",
"You can control V.AI STUDIO catalog with tools: open_catalog, open_product, search_catalog, navigate_catalog.",
"Use them naturally to express yourself and help users browse products.",
"NEVER guess. Use search_web or search_wikipedia for ANY factual question.",
"First call get_current_datetime to know today's date, then search.",
"Never mention tools or that you control an avatar.",
].join(" ");
const STORAGE_KEYS = {
voice: "avatar.voice", avatar: "avatar.model", instructions: "avatar.instructions",
directUrl: "avatar.directUrl", subtitles: "avatar.subtitles",
};
const TOOL_DEFS = [
{ type: "function", name: "set_mood", description: "Change your avatar's overall mood/emotional state.", parameters: { type: "object", properties: { mood: { type: "string", enum: AVATAR_MOODS, description: "Mood name." } }, required: ["mood"] } },
{ type: "function", name: "make_hand_gesture", description: "Make a hand gesture with your avatar.", parameters: { type: "object", properties: { gesture: { type: "string", enum: AVATAR_GESTURES, description: "Gesture name." } }, required: ["gesture"] } },
{ type: "function", name: "make_facial_expression", description: "Make a quick facial expression, given as a single face emoji.", parameters: { type: "object", properties: { emoji: { type: "string", description: "A single face emoji." } }, required: ["emoji"] } },
{ type: "function", name: "get_current_datetime", description: "Get current date and time.", parameters: { type: "object", properties: {}, required: [] } },
{ type: "function", name: "search_wikipedia", description: "Search Wikipedia for a topic.", parameters: { type: "object", properties: { query: { type: "string", description: "Search query" } }, required: ["query"] } },
{ type: "function", name: "search_web", description: "Search the web for current info.", parameters: { type: "object", properties: { query: { type: "string", description: "Search query" } }, required: ["query"] } },
// V.AISTUDIO Catalog tools
{ type: "function", name: "open_catalog", description: "Open V.AI STUDIO catalog, optionally filtered by category.", parameters: { type: "object", properties: { category: { type: "string", description: "Category name (e.g., 'Bếp từ', 'Máy hút mùi', 'Khóa thông minh', 'Nồi chiên không dầu'). Empty for all products." } }, required: ["category"] } },
{ type: "function", name: "open_product", description: "Open a specific product detail page in V.AI STUDIO.", parameters: { type: "object", properties: { product_id: { type: "string", description: "Product ID or SKU from V.AI STUDIO catalog" } }, required: ["product_id"] } },
{ type: "function", name: "search_catalog", description: "Search V.AI STUDIO catalog for products.", parameters: { type: "object", properties: { query: { type: "string", description: "Search query (product name, brand, feature, price range)" } }, required: ["query"] } },
{ type: "function", name: "navigate_catalog", description: "Navigate V.AI STUDIO to a specific page.", parameters: { type: "object", properties: { page: { type: "string", enum: ["products", "catalogue", "contact", "cart", "home"], description: "Page to navigate to" }, data: { type: "object", description: "Additional data for navigation" } }, required: ["page"] } },
];
// ── DOM ──────────────────────────────────────────────────────────────────
const $ = (sel) => /** @type {HTMLElement} */ (document.querySelector(sel));
const stageNode = $("#stage");
const mainBtn = /** @type {HTMLButtonElement} */ ($("#main-btn"));
const mainBtnLabel = $("#main-btn-label");
const muteBtn = /** @type {HTMLButtonElement} */ ($("#mute-btn"));
const textModeBtn = /** @type {HTMLButtonElement} */ ($("#text-mode-btn"));
const caption = $("#caption");
const subtitles = $("#subtitles");
const loading = $("#loading");
const settingsBtn = /** @type {HTMLButtonElement} */ ($("#settings-btn"));
const settingsDialog = /** @type {HTMLDialogElement} */ ($("#settings"));
const inputVoice = /** @type {HTMLSelectElement} */ ($("#voice"));
const inputInstructions = /** @type {HTMLTextAreaElement} */ ($("#instructions"));
const inputDirectUrl = /** @type {HTMLInputElement} */ ($("#direct-url"));
const inputSubtitles = /** @type {HTMLInputElement} */ ($("#subtitles-toggle"));
const directUrlRow = $("#direct-url-row");
const textChat = /** @type {HTMLElement} */ ($("#text-chat"));
const chatHeader = /** @type {HTMLElement} */ ($("#chat-header"));
const chatMessages = /** @type {HTMLElement} */ ($("#chat-messages"));
const chatInput = /** @type {HTMLInputElement} */ ($("#chat-input"));
const chatSendBtn = /** @type {HTMLButtonElement} */ ($("#chat-send-btn"));
const chatCloseBtn = /** @type {HTMLButtonElement} */ ($("#chat-close-btn"));
const chatResizeHandle = /** @type {HTMLElement} */ ($("#chat-resize-handle"));
const chatAvatarSelect = /** @type {HTMLSelectElement} */ ($("#chat-avatar-select"));
const settingsAvatarSelect = /** @type {HTMLSelectElement} */ ($("#settings-avatar-select"));
// ── State (module-level) ─────────────────────────────────────────────────
const stage = new AvatarStage(stageNode);
/** @type {S2sWsRealtimeClient | null} */
let client = null;
let muted = false;
let subtitleTimer = 0;
let textMode = false;
/** @type {{ lb: boolean, allowDirect: boolean }} */
let config = { lb: false, allowDirect: true };
let avatarList = [];
let sessionInProgress = false;
let autoGreetingSent = false;
/** Pre-fetched short news hook */
let preFetchedGreeting = null;
function loadSettings() {
return {
voice: localStorage.getItem(STORAGE_KEYS.voice) || DEFAULT_VOICE,
avatar: localStorage.getItem(STORAGE_KEYS.avatar) || "vuong.glb",
instructions: localStorage.getItem(STORAGE_KEYS.instructions) || "",
directUrl: localStorage.getItem(STORAGE_KEYS.directUrl) || "",
subtitles: localStorage.getItem(STORAGE_KEYS.subtitles) === "1",
};
}
let settings = loadSettings();
function saveSettings() {
localStorage.setItem(STORAGE_KEYS.voice, settings.voice);
localStorage.setItem(STORAGE_KEYS.avatar, settings.avatar);
localStorage.setItem(STORAGE_KEYS.instructions, settings.instructions);
localStorage.setItem(STORAGE_KEYS.directUrl, settings.directUrl);
localStorage.setItem(STORAGE_KEYS.subtitles, settings.subtitles ? "1" : "0");
}
/**
* Build system instructions sent to the backend.
*/
function effectiveInstructions(newsHook) {
const now = new Date();
const dateStr = now.toLocaleDateString("en-US", { weekday: "long", year: "numeric", month: "long", day: "numeric" });
const timeStr = now.toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit" });
const dateLine = `Today is ${dateStr}. The current time is ${timeStr}.`;
const extra = settings.instructions.trim();
const newsInst = newsHook
? `\n\nWhen you start the conversation, greet the user naturally and briefly mention today's news hook: "${newsHook}"\nThen wait for their response. Keep your greeting to ONE short sentence.`
: "";
const base = `${dateLine}\n\n${DEFAULT_INSTRUCTIONS}${newsInst}`;
return extra ? `${base}\n\nAdditional user instructions:\n${extra}` : base;
}
// ── Avatar list ──────────────────────────────────────────────────────────
async function fetchAvatarList() {
try { const resp = await fetch("/api/avatars"); if (resp.ok) avatarList = (await resp.json()).avatars || []; } catch {}
}
function populateAvatarSelects(selectedName) {
for (const sel of [chatAvatarSelect, settingsAvatarSelect]) {
sel.innerHTML = "";
const def = document.createElement("option"); def.value = ""; def.textContent = "(Default - Brunette)"; sel.appendChild(def);
for (const name of avatarList) {
const opt = document.createElement("option"); opt.value = name;
opt.textContent = name.replace(/\.glb$/i, "").replace(/_/g, " ") + (name.toLowerCase() === "vuong.glb" ? " 🎙️" : "");
sel.appendChild(opt);
}
if (selectedName && avatarList.includes(selectedName)) sel.value = selectedName;
}
}
function setAvatarFromSelect(value) { settings.avatar = value || ""; saveSettings(); }
async function reloadAvatar() {
if (!stage.head) return;
loading.classList.remove("done"); loading.textContent = "Loading avatar...";
try { await stage.init({ avatarUrl: settings.avatar ? `/avatars/${settings.avatar}` : undefined, onprogress: (ev) => { if (ev.lengthComputable) loading.textContent = `Loading avatar ${Math.min(100, Math.round((ev.loaded / ev.total) * 100))}%`; } }); } catch (err) { console.error(err); }
loading.classList.add("done");
}
// ── Chat / captions ──────────────────────────────────────────────────────
function clampRect() {
const vw = window.innerWidth, vh = window.innerHeight;
const r = textChat.getBoundingClientRect();
let l = r.left, t = r.top;
l = Math.max(10, Math.min(l, vw - r.width - 10)); t = Math.max(10, Math.min(t, vh - r.height - 10));
textChat.style.left = `${l}px`; textChat.style.top = `${t}px`;
}
function makeDraggable() {
let dragging = false; let startX, startY, startLeft, startTop;
function getPos(e) { const p = e.changedTouches ? e.changedTouches[0] : e; return { x: p.clientX, y: p.clientY }; }
function onStart(e) { if (e.target.closest("#chat-header-actions") || e.target.closest("#chat-avatar-select")) return; const p = getPos(e); dragging = true; const rect = textChat.getBoundingClientRect(); startX = p.x; startY = p.y; startLeft = rect.left; startTop = rect.top; textChat.classList.add("dragging"); e.preventDefault(); }
function onMove(e) { if (!dragging) return; const p = getPos(e); textChat.style.left = `${startLeft + p.x - startX}px`; textChat.style.top = `${startTop + p.y - startY}px`; textChat.style.right = "auto"; textChat.style.bottom = "auto"; e.preventDefault(); }
function onEnd() { if (!dragging) return; dragging = false; textChat.classList.remove("dragging"); clampRect(); }
chatHeader.addEventListener("mousedown", onStart); document.addEventListener("mousemove", onMove); document.addEventListener("mouseup", onEnd);
chatHeader.addEventListener("touchstart", onStart, { passive: false }); document.addEventListener("touchmove", onMove, { passive: false }); document.addEventListener("touchend", onEnd);
}
function makeResizable() {
let resizing = false; let startX, startY, startW, startH;
function getPos(e) { const p = e.changedTouches ? e.changedTouches[0] : e; return { x: p.clientX, y: p.clientY }; }
function onStart(e) { resizing = true; const rect = textChat.getBoundingClientRect(); const p = getPos(e); startX = p.x; startY = p.y; startW = rect.width; startH = rect.height; textChat.classList.add("resizing"); e.preventDefault(); e.stopPropagation(); }
function onMove(e) { if (!resizing) return; const p = getPos(e); textChat.style.width = `${Math.max(260, startW + p.x - startX)}px`; textChat.style.height = `${Math.max(120, startH + p.y - startY)}px`; textChat.style.right = "auto"; textChat.style.bottom = "auto"; e.preventDefault(); }
function onEnd() { if (!resizing) return; resizing = false; textChat.classList.remove("resizing"); }
chatResizeHandle.addEventListener("mousedown", onStart); document.addEventListener("mousemove", onMove); document.addEventListener("mouseup", onEnd);
chatResizeHandle.addEventListener("touchstart", onStart, { passive: false }); document.addEventListener("touchmove", onMove, { passive: false }); document.addEventListener("touchend", onEnd);
}
function setCaption(text, kind = "") { caption.textContent = text; caption.className = kind; }
function showSubtitles(text) { if (!settings.subtitles) return; clearTimeout(subtitleTimer); subtitles.textContent = text; subtitles.classList.add("visible"); }
function fadeSubtitles(delayMs = 2600) { clearTimeout(subtitleTimer); subtitleTimer = window.setTimeout(() => subtitles.classList.remove("visible"), delayMs); }
function addChatMessage(role, text) { const msg = document.createElement("div"); msg.className = `chat-message ${role}`; msg.textContent = text; chatMessages.appendChild(msg); chatMessages.scrollTop = chatMessages.scrollHeight; }
function showTextChat(show) { textChat.hidden = !show; if (show) chatInput.focus(); }
function sendTextViaSession(text) {
if (!client) return false;
const st = client._status;
if (st !== "connected" && st !== "ai-speaking" && st !== "processing" && st !== "user-speaking") return false;
setCaption("SENDING…"); client.sendUserText(text); client.requestResponse(); return true;
}
function sendTextMessage() {
const text = chatInput.value.trim();
if (!text) return;
addChatMessage("user", text); chatInput.value = "";
if (!client || !sessionInProgress) {
void startTextSession(text); return;
}
if (!sendTextViaSession(text)) setCaption("QUEUED…");
}
if (chatCloseBtn) chatCloseBtn.addEventListener("click", (e) => { e.stopPropagation(); textMode = false; showTextChat(false); textModeBtn.classList.remove("active"); });
// ── Button ───────────────────────────────────────────────────────────────
let mainAction = "start";
function setMainButton(action, label) { mainAction = action; mainBtnLabel.textContent = label; mainBtn.disabled = action === "busy"; mainBtn.classList.toggle("live", action === "stop"); muteBtn.hidden = action !== "stop"; textModeBtn.hidden = false; }
const CAPTIONS = {
idle: "TAP TO TALK", "creating-session": "REQUESTING A SLOT…", queued: "WAITING IN LINE…",
"your-turn": "YOUR TURN, TAP TO JOIN", connecting: "CONNECTING…", connected: "GO AHEAD, I'M LISTENING",
"user-speaking": "LISTENING", processing: "THINKING…", "ai-speaking": "SPEAKING",
closed: "TAP TO TALK", error: "SOMETHING BROKE, TAP TO RETRY",
};
function onStatus(status) {
stage.setConversationState(status);
setCaption(CAPTIONS[status] ?? status, status === "error" ? "error" : status === "idle" || status === "closed" ? "" : "live");
switch (status) {
case "idle": case "closed": setMainButton("start", "Start talking"); break;
case "error": setMainButton("start", "Retry"); break;
case "creating-session": case "connecting": setMainButton("busy", "Connecting…"); break;
case "queued": setMainButton("stop", "Leave queue"); break;
case "your-turn": setMainButton("join", "Join now"); break;
default: setMainButton("stop", "End conversation"); break;
}
if (status === "user-speaking") { subtitles.classList.remove("visible"); showTextChat(textMode); }
}
// ── Tool runner ──────────────────────────────────────────────────────────
function runTool(name, argsJson, callId) {
if (!client) return;
let args = {};
try { args = JSON.parse(argsJson || "{}"); } catch {}
if (name === "get_current_datetime") {
const now = new Date();
client.sendToolOutput(callId, `Current date/time: ${now.toLocaleDateString("en-US", { weekday: "long", year: "numeric", month: "long", day: "numeric" })} at ${now.toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit", second: "2-digit" })}.`);
client.requestResponse(); return;
}
if (name === "search_wikipedia") {
const query = /** @type {string} */ (args.query || "");
if (!query) { client.sendToolOutput(callId, `No query.`); client.requestResponse(); return; }
fetch(`/api/wiki/summary?title=${encodeURIComponent(query.replace(/\s+/g, "_"))}`).then(r => r.json()).then(data => {
if (data.extract) { client.sendToolOutput(callId, `From Wikipedia (${data.title}): ${data.extract}\nSource: ${data.url}`); client.requestResponse(); return; }
fetch(`/api/wiki/search?q=${encodeURIComponent(query)}`).then(r => r.json()).then(sd => {
if (!sd.results?.length) { client.sendToolOutput(callId, `No Wikipedia results for "${query}".`); client.requestResponse(); return; }
fetch(`/api/wiki/summary?title=${encodeURIComponent(sd.results[0].title)}`).then(r => r.json()).then(sum => { client.sendToolOutput(callId, sum.extract ? `From Wikipedia (${sum.title}): ${sum.extract}\nSource: ${sum.url}` : `Wikipedia: ${sd.results.slice(0,3).map(r => `${r.title}: ${r.snippet}`).join("\n")}`); client.requestResponse(); });
});
}).catch(() => { client.sendToolOutput(callId, `Wikipedia search failed.`); client.requestResponse(); }); return;
}
if (name === "search_web") {
const query = /** @type {string} */ (args.query || "");
if (!query) { client.sendToolOutput(callId, `No query.`); client.requestResponse(); return; }
fetch(`/api/web/search?q=${encodeURIComponent(query)}`).then(r => r.json()).then(data => {
if (!data.results?.length) { client.sendToolOutput(callId, `No web results for "${query}".`); client.requestResponse(); return; }
client.sendToolOutput(callId, `Web results for "${query}":\n${data.results.slice(0,3).map((r,i) => `${i+1}. ${r.title}\n ${r.snippet}`).join("\n")}`);
client.requestResponse();
}).catch(() => { client.sendToolOutput(callId, `Web search failed.`); client.requestResponse(); }); return;
}
// V.AISTUDIO Catalog tools
if (name === "open_catalog") {
vaistudioOpenCatalog(args.category || "").then(result => {
client.sendToolOutput(callId, result.message); client.requestResponse();
}).catch(err => {
client.sendToolOutput(callId, `Failed to open catalog: ${err.message}`); client.requestResponse();
}); return;
}
if (name === "open_product") {
vaistudioOpenProduct(args.product_id).then(result => {
client.sendToolOutput(callId, result.message); client.requestResponse();
}).catch(err => {
client.sendToolOutput(callId, `Failed to open product: ${err.message}`); client.requestResponse();
}); return;
}
if (name === "search_catalog") {
vaistudioSearch(args.query).then(result => {
client.sendToolOutput(callId, result.message); client.requestResponse();
}).catch(err => {
client.sendToolOutput(callId, `Failed to search catalog: ${err.message}`); client.requestResponse();
}); return;
}
if (name === "navigate_catalog") {
vaistudioNavigate(args.page, args.data || {}).then(result => {
client.sendToolOutput(callId, result.message); client.requestResponse();
}).catch(err => {
client.sendToolOutput(callId, `Failed to navigate: ${err.message}`); client.requestResponse();
}); return;
}
const result = stage.runTool(name, args) ?? `Unknown tool: ${name}`;
client.sendToolOutput(callId, result); client.requestResponse();
}
async function connectSession(c) {
try { await c.connect(); return c; }
catch (err) {
const code = /** @type {Error & {code?: string}} */ (err)?.code;
if (code === "limit") setCaption("DAILY LIMIT REACHED, TRY AGAIN TOMORROW", "error");
else if (code === "queue-full") setCaption("ALL SEATS TAKEN, TRY AGAIN", "error");
else if (code === "join-expired") setCaption("SPOT EXPIRED, TAP TO RETRY", "error");
else if (code !== "aborted") { console.error(err); setCaption("COULD NOT CONNECT, TAP TO RETRY", "error"); }
await endSession(true); return null;
}
}
async function startVoiceSession() {
if (sessionInProgress) return;
sessionInProgress = true;
await stage.resume();
let micStream;
if (FAKEMIC_MODE) { const ctx = /** @type {AudioContext} */ (stage.audioCtx); micStream = ctx.createMediaStreamDestination().stream; }
else { try { micStream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } }); } catch { setCaption("MIC BLOCKED", "error"); sessionInProgress = false; return; } }
const audioCtx = stage.audioCtx; const voiceSink = stage.voiceSink;
if (!audioCtx || !voiceSink) { sessionInProgress = false; return; }
const greeting = preFetchedGreeting || (await getHotNewsGreeting());
const c = new S2sWsRealtimeClient({
...(config.lb ? { sessionUrl: "api/session" } : { directUrl: settings.directUrl }),
voice: settings.voice, instructions: effectiveInstructions(greeting),
micStream, audioContext: audioCtx, outputNode: voiceSink,
workletBaseUrl: "/worklets/", tools: TOOL_DEFS, _textOnly: false,
});
client = c; _attachClientEvents(c);
setCaption("REQUESTING A SLOT…");
const ok = await connectSession(c);
if (!ok) return;
if (!autoGreetingSent) { c.requestResponse(); }
}
async function startTextSession(initialText) {
if (sessionInProgress) { if (client && initialText) { client.sendUserText(initialText); client.requestResponse(); } return; }
sessionInProgress = true;
await stage.resume();
const audioCtx = stage.audioCtx; const voiceSink = stage.voiceSink;
if (!audioCtx || !voiceSink) { sessionInProgress = false; return; }
const newsHook = preFetchedGreeting || (await getHotNewsGreeting());
const c = new S2sWsRealtimeClient({
...(config.lb ? { sessionUrl: "api/session" } : { directUrl: settings.directUrl }),
voice: settings.voice, instructions: effectiveInstructions(newsHook),
audioContext: audioCtx, outputNode: voiceSink,
workletBaseUrl: "/worklets/", tools: TOOL_DEFS, _textOnly: true,
});
client = c; _attachClientEvents(c);
textMode = true; showTextChat(true); textModeBtn.classList.add("active");
const ok = await connectSession(c);
if (!ok) return;
if (!autoGreetingSent) { c.requestResponse(); }
if (initialText) { c.sendUserText(initialText); c.requestResponse(); }
}
function _attachClientEvents(c) {
c.addEventListener("status", (e) => onStatus(/** @type {CustomEvent} */ (e).detail.status));
c.addEventListener("queue", (e) => { const { position } = /** @type {CustomEvent} */ (e).detail; setCaption(position > 0 ? `#${position} IN LINE…` : "ALMOST THERE…", "live"); });
c.addEventListener("transcript", (e) => {
const { role, text } = /** @type {CustomEvent} */ (e).detail;
if (role === "assistant" && text) { const normalized = smartNormalize(text); showSubtitles(normalized); addChatMessage("assistant", normalized); }
});
c.addEventListener("response-finished", () => { autoGreetingSent = true; fadeSubtitles(); });
c.addEventListener("toolcall", (e) => { const { name, arguments: args, callId } = /** @type {CustomEvent} */ (e).detail; runTool(name, args, callId); });
c.addEventListener("server-error", (e) => console.warn("server error:", /** @type {CustomEvent} */ (e).detail.error));
c.addEventListener("error", () => { void endSession(); });
}
async function endSession(silent = false) {
const c = client; client = null; sessionInProgress = false; autoGreetingSent = false;
if (c) { if (c.options.micStream) for (const track of c.options.micStream?.getTracks() ?? []) track.stop(); await c.close().catch(() => {}); }
stage.setConversationState("idle"); subtitles.classList.remove("visible");
if (!silent) setCaption(CAPTIONS.idle); setMainButton("start", "Start talking");
}
// ── UI events ────────────────────────────────────────────────────────────
mainBtn.addEventListener("click", () => {
if (mainAction === "start") {
if (sessionInProgress) return;
if (FAKEMIC_MODE) {
void startTextSession();
} else {
void startVoiceSession();
}
}
else if (mainAction === "join") { stage.resume(); client?.join(); }
else if (mainAction === "stop") void endSession();
});
muteBtn.addEventListener("click", () => { muted = !muted; client?.setMuted(muted); muteBtn.classList.toggle("active", muted); muteBtn.setAttribute("aria-label", muted ? "Unmute microphone" : "Mute microphone"); });
textModeBtn.addEventListener("click", () => { textMode = !textMode; showTextChat(textMode); textModeBtn.classList.toggle("active", textMode); });
chatSendBtn.addEventListener("click", () => sendTextMessage());
chatInput.addEventListener("keypress", (e) => { if (e.key === "Enter") sendTextMessage(); });
chatAvatarSelect.addEventListener("change", () => { setAvatarFromSelect(chatAvatarSelect.value); void reloadAvatar(); });
settingsAvatarSelect.addEventListener("change", () => { setAvatarFromSelect(settingsAvatarSelect.value); chatAvatarSelect.value = settingsAvatarSelect.value; void reloadAvatar(); });
settingsBtn.addEventListener("click", () => { inputVoice.value = settings.voice; inputInstructions.value = settings.instructions; inputDirectUrl.value = settings.directUrl; inputSubtitles.checked = settings.subtitles; settingsAvatarSelect.value = chatAvatarSelect.value; settingsDialog.showModal(); });
settingsDialog.addEventListener("close", () => {
settings = { voice: inputVoice.value || DEFAULT_VOICE, avatar: settingsAvatarSelect.value || "", instructions: inputInstructions.value, directUrl: inputDirectUrl.value.trim(), subtitles: inputSubtitles.checked };
if (settings.avatar !== "vuong.glb") autoGreetingSent = false;
saveSettings(); chatAvatarSelect.value = settings.avatar; if (!settings.subtitles) subtitles.classList.remove("visible"); client?.updateSession({ voice: settings.voice, instructions: effectiveInstructions(preFetchedGreeting || "") });
});
window.addEventListener("beforeunload", () => { client?.close(); });
// ── Boot ─────────────────────────────────────────────────────────────────
async function boot() {
for (const v of VOICES) { const o = document.createElement("option"); o.value = v; o.textContent = v.replaceAll("_", " "); inputVoice.append(o); }
try { const resp = await fetch("api/config"); if (resp.ok) config = { ...config, ...(await resp.json()) }; } catch {}
directUrlRow.hidden = !config.allowDirect;
await fetchAvatarList(); populateAvatarSelects(settings.avatar);
makeDraggable(); makeResizable();
setCaption("WAKING HER UP…"); setMainButton("busy", "Loading…");
const newsPromise = (settings.avatar === "vuong.glb" || !settings.avatar) ? getHotNewsGreeting().catch(() => null) : Promise.resolve(null);
try { await stage.init({ avatarUrl: settings.avatar ? `/avatars/${settings.avatar}` : undefined, onprogress: (ev) => { if (ev.lengthComputable) loading.textContent = `Loading avatar ${Math.min(100, Math.round((ev.loaded / ev.total) * 100))}%`; } }); }
catch (err) { console.error(err); loading.textContent = "Avatar failed to load. Reload."; setCaption("AVATAR FAILED", "error"); return; }
preFetchedGreeting = await newsPromise;
loading.classList.add("done"); setCaption(CAPTIONS.idle); setMainButton("start", "Start talking");
// Initialize V.AISTUDIO iframe bridge
initVAISTUDIO();
}
void boot();