Upload src/app.js

src/app.js

@@ -1,10 +1,10 @@
 // @ts-check
 /**
- * App wiring: the avatar stage + the speech-to-speech session.
+ * App wiring: the avatar stage + the speech-to-speech session + V.AISTUDIO iframe control.
  * Modes: voice (mic + VAD) OR text (keyboard only, no mic needed).
  *
  * KEY DESIGN: On direct hf.space domains, AudioContext starts "suspended"
- * because there's no user gesture on page load.  Instead of hanging or
+ * because there's no user gesture on page load. Instead of hanging or
  * silently failing, we wait for the user to click "Start talking" before
  * any WebSocket connection — that click resumes AudioContext, audio plays,
  * and HeadAudio lip-sync works.
@@ -27,11 +27,158 @@ const DEFAULT_VOICE = "Sohee";
 const _urlParams = new URLSearchParams(location.search);
 const FAKEMIC_MODE = _urlParams.has("fakemic");
 
-// ── Greeting ─────────────────────────────────────────────────────────────
+// ── V.AISTUDIO iframe bridge ─────────────────────────────────────────────
+const VAISTUDIO_ORIGIN = "https://bep40-v-aistudio.static.hf.space";
+const VAISTUDIO_IFRAME_ID = "vaistudio-iframe";
+const VAISTUDIO_PANEL_ID = "vaistudio-panel";
+const VAISTUDIO_TOGGLE_ID = "vaistudio-toggle";
+const VAISTUDIO_LOADING_ID = "vaistudio-loading";
+
+/** @type {HTMLIFrameElement | null} */
+let vaistudioIframe = null;
+/** @type {HTMLDivElement | null} */
+let vaistudioPanel = null;
+/** @type {HTMLButtonElement | null} */
+let vaistudioToggle = null;
+/** @type {HTMLDivElement | null} */
+let vaistudioLoading = null;
+/** @type {boolean} */
+let vaistudioPanelOpen = false;
+/** @type {Map<string, Function>} */
+const vaistudioResponseHandlers = new Map();
+/** @type {number} */
+let vaistudioMessageId = 0;
+
+/**
+ * Send a command to V.AISTUDIO iframe via postMessage
+ */
+function vaistudioSend(action, data = {}) {
+  if (!vaistudioIframe || !vaistudioIframe.contentWindow) {
+    console.warn("[VAISTUDIO] iframe not ready");
+    return Promise.reject(new Error("iframe not ready"));
+  }
+  const id = ++vaistudioMessageId;
+  const message = { id, action, data, timestamp: Date.now() };
+  return new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      vaistudioResponseHandlers.delete(id);
+      reject(new Error(`VAISTUDIO timeout: ${action}`));
+    }, 10000);
+    vaistudioResponseHandlers.set(id, (response) => {
+      clearTimeout(timeout);
+      if (response.error) reject(new Error(response.error));
+      else resolve(response.data);
+    });
+    try {
+      vaistudioIframe.contentWindow.postMessage(message, VAISTUDIO_ORIGIN);
+    } catch (e) {
+      vaistudioResponseHandlers.delete(id);
+      clearTimeout(timeout);
+      reject(e);
+    }
+  });
+}
+
+/**
+ * Handle responses from V.AISTUDIO iframe
+ */
+function vaistudioOnMessage(event) {
+  if (event.origin !== VAISTUDIO_ORIGIN) return;
+  const msg = event.data;
+  if (!msg || typeof msg !== "object") return;
+  if (msg.id && vaistudioResponseHandlers.has(msg.id)) {
+    const handler = vaistudioResponseHandlers.get(msg.id);
+    vaistudioResponseHandlers.delete(msg.id);
+    handler(msg);
+  }
+  // Handle unsolicited events from iframe
+  if (msg.event) {
+    console.log("[VAISTUDIO] Event:", msg.event, msg.data);
+  }
+}
+
+/**
+ * Initialize V.AISTUDIO iframe controls
+ */
+function initVAISTUDIO() {
+  vaistudioIframe = /** @type {HTMLIFrameElement} */ (document.getElementById(VAISTUDIO_IFRAME_ID));
+  vaistudioPanel = /** @type {HTMLDivElement} */ (document.getElementById(VAISTUDIO_PANEL_ID));
+  vaistudioToggle = /** @type {HTMLButtonElement} */ (document.getElementById(VAISTUDIO_TOGGLE_ID));
+  vaistudioLoading = /** @type {HTMLDivElement} */ (document.getElementById(VAISTUDIO_LOADING_ID));
+
+  if (!vaistudioIframe || !vaistudioPanel || !vaistudioToggle) {
+    console.warn("[VAISTUDIO] Required elements not found");
+    return;
+  }
+
+  // Toggle panel
+  vaistudioToggle.addEventListener("click", () => {
+    vaistudioPanelOpen = !vaistudioPanelOpen;
+    vaistudioPanel.classList.toggle("open", vaistudioPanelOpen);
+    vaistudioToggle.classList.toggle("active", vaistudioPanelOpen);
+    vaistudioToggle.setAttribute("aria-label", vaistudioPanelOpen ? "Close V.AI STUDIO" : "Open V.AI STUDIO");
+    vaistudioToggle.title = vaistudioPanelOpen ? "Close V.AI STUDIO" : "Open V.AI STUDIO";
+  });
+
+  // Close button
+  const closeBtn = document.getElementById("vaistudio-close");
+  if (closeBtn) {
+    closeBtn.addEventListener("click", () => {
+      vaistudioPanelOpen = false;
+      vaistudioPanel.classList.remove("open");
+      vaistudioToggle.classList.remove("active");
+      vaistudioToggle.setAttribute("aria-label", "Open V.AI STUDIO");
+      vaistudioToggle.title = "Open V.AI STUDIO";
+    });
+  }
+
+  // Listen for iframe load
+  vaistudioIframe.addEventListener("load", () => {
+    if (vaistudioLoading) vaistudioLoading.classList.add("hidden");
+    console.log("[VAISTUDIO] iframe loaded");
+    vaistudioSend("ready", { source: "gemma-avatar" }).catch(() => {});
+  });
+
+  // Listen for postMessage
+  window.addEventListener("message", vaistudioOnMessage);
+
+  // Expose globally for avatar tools
+  window.vaistudio = {
+    send: vaistudioSend,
+    open: () => { if (!vaistudioPanelOpen) vaistudioToggle.click(); },
+    close: () => { if (vaistudioPanelOpen) vaistudioToggle.click(); },
+    isOpen: () => vaistudioPanelOpen,
+  };
+}
+
 /**
- * Fetch a SHORT hot news headline to weave into the greeting.
- * Short enough that the AI says it as ONE sentence.
+ * V.AISTUDIO control functions for avatar tools
  */
+async function vaistudioOpenCatalog(category = "") {
+  await vaistudioSend("navigate", { page: "products", category });
+  if (!vaistudioPanelOpen) vaistudioToggle.click();
+  return { success: true, message: `Opened catalog${category ? `: ${category}` : ""}` };
+}
+
+async function vaistudioOpenProduct(productId) {
+  await vaistudioSend("navigate", { page: "product", productId });
+  if (!vaistudioPanelOpen) vaistudioToggle.click();
+  return { success: true, message: `Opened product ${productId}` };
+}
+
+async function vaistudioSearch(query) {
+  await vaistudioSend("search", { query });
+  if (!vaistudioPanelOpen) vaistudioToggle.click();
+  return { success: true, message: `Searching: ${query}` };
+}
+
+async function vaistudioNavigate(page, data = {}) {
+  await vaistudioSend("navigate", { page, ...data });
+  if (!vaistudioPanelOpen) vaistudioToggle.click();
+  return { success: true, message: `Navigated to ${page}` };
+}
+
+// ── Greeting ─────────────────────────────────────────────────────────────
 async function getHotNewsGreeting() {
   let hotTitle = "";
   try {
@@ -52,6 +199,7 @@ async function getHotNewsGreeting() {
 
 const DEFAULT_INSTRUCTIONS = [
   "You are a friendly voice assistant with a visible, human-like 3D avatar.",
+  "You also have access to V.AI STUDIO - an e-commerce catalog with 8000+ kitchen appliances & smart locks from Malloca, Eurogold, Grob, Canzy, Demax & Dien May Xanh.",
   "CRITICAL: Always respond in the SAME LANGUAGE the user writes or speaks.",
   "If Vietnamese, respond in Vietnamese. If English, respond in English.",
   "When Vietnamese: write dates as 'ngày 9 tháng 7 năm 2026' (NOT '9/7/2026'),",
@@ -61,7 +209,8 @@ const DEFAULT_INSTRUCTIONS = [
   "currency as 'năm mươi nghìn đồng' NOT '50.000đ'.",
   "Keep replies short, natural, warm. Never list-like.",
   "You can control avatar body with tools: set_mood, make_hand_gesture, make_facial_expression.",
-  "Use them naturally to express yourself.",
+  "You can control V.AI STUDIO catalog with tools: open_catalog, open_product, search_catalog, navigate_catalog.",
+  "Use them naturally to express yourself and help users browse products.",
   "NEVER guess. Use search_web or search_wikipedia for ANY factual question.",
   "First call get_current_datetime to know today's date, then search.",
   "Never mention tools or that you control an avatar.",
@@ -79,6 +228,11 @@ const TOOL_DEFS = [
   { type: "function", name: "get_current_datetime", description: "Get current date and time.", parameters: { type: "object", properties: {}, required: [] } },
   { type: "function", name: "search_wikipedia", description: "Search Wikipedia for a topic.", parameters: { type: "object", properties: { query: { type: "string", description: "Search query" } }, required: ["query"] } },
   { type: "function", name: "search_web", description: "Search the web for current info.", parameters: { type: "object", properties: { query: { type: "string", description: "Search query" } }, required: ["query"] } },
+  // V.AISTUDIO Catalog tools
+  { type: "function", name: "open_catalog", description: "Open V.AI STUDIO catalog, optionally filtered by category.", parameters: { type: "object", properties: { category: { type: "string", description: "Category name (e.g., 'Bếp từ', 'Máy hút mùi', 'Khóa thông minh', 'Nồi chiên không dầu'). Empty for all products." } }, required: ["category"] } },
+  { type: "function", name: "open_product", description: "Open a specific product detail page in V.AI STUDIO.", parameters: { type: "object", properties: { product_id: { type: "string", description: "Product ID or SKU from V.AI STUDIO catalog" } }, required: ["product_id"] } },
+  { type: "function", name: "search_catalog", description: "Search V.AI STUDIO catalog for products.", parameters: { type: "object", properties: { query: { type: "string", description: "Search query (product name, brand, feature, price range)" } }, required: ["query"] } },
+  { type: "function", name: "navigate_catalog", description: "Navigate V.AI STUDIO to a specific page.", parameters: { type: "object", properties: { page: { type: "string", enum: ["products", "catalogue", "contact", "cart", "home"], description: "Page to navigate to" }, data: { type: "object", description: "Additional data for navigation" } }, required: ["page"] } },
 ];
 
 // ── DOM ──────────────────────────────────────────────────────────────────
@@ -144,7 +298,6 @@ function saveSettings() {
 
 /**
  * Build system instructions sent to the backend.
- * The AI gets a short news hook (if any) and is told to greet naturally.
  */
 function effectiveInstructions(newsHook) {
   const now = new Date();
@@ -289,6 +442,35 @@ function runTool(name, argsJson, callId) {
       client.requestResponse();
     }).catch(() => { client.sendToolOutput(callId, `Web search failed.`); client.requestResponse(); }); return;
   }
+  // V.AISTUDIO Catalog tools
+  if (name === "open_catalog") {
+    vaistudioOpenCatalog(args.category || "").then(result => {
+      client.sendToolOutput(callId, result.message); client.requestResponse();
+    }).catch(err => {
+      client.sendToolOutput(callId, `Failed to open catalog: ${err.message}`); client.requestResponse();
+    }); return;
+  }
+  if (name === "open_product") {
+    vaistudioOpenProduct(args.product_id).then(result => {
+      client.sendToolOutput(callId, result.message); client.requestResponse();
+    }).catch(err => {
+      client.sendToolOutput(callId, `Failed to open product: ${err.message}`); client.requestResponse();
+    }); return;
+  }
+  if (name === "search_catalog") {
+    vaistudioSearch(args.query).then(result => {
+      client.sendToolOutput(callId, result.message); client.requestResponse();
+    }).catch(err => {
+      client.sendToolOutput(callId, `Failed to search catalog: ${err.message}`); client.requestResponse();
+    }); return;
+  }
+  if (name === "navigate_catalog") {
+    vaistudioNavigate(args.page, args.data || {}).then(result => {
+      client.sendToolOutput(callId, result.message); client.requestResponse();
+    }).catch(err => {
+      client.sendToolOutput(callId, `Failed to navigate: ${err.message}`); client.requestResponse();
+    }); return;
+  }
   const result = stage.runTool(name, args) ?? `Unknown tool: ${name}`;
   client.sendToolOutput(callId, result); client.requestResponse();
 }
@@ -308,7 +490,6 @@ async function connectSession(c) {
 async function startVoiceSession() {
   if (sessionInProgress) return;
   sessionInProgress = true;
-  // Await AudioContext resume — this runs in user-gesture handler so it resolves
   await stage.resume();
   let micStream;
   if (FAKEMIC_MODE) { const ctx = /** @type {AudioContext} */ (stage.audioCtx); micStream = ctx.createMediaStreamDestination().stream; }
@@ -329,15 +510,9 @@ async function startVoiceSession() {
   if (!autoGreetingSent) { c.requestResponse(); }
 }
 
-/**
- * Start a text-only session.
- * Called from user-gesture handler (mainBtn click) so AudioContext
- * resume resolves — audio plays, HeadAudio processes → lipsync works.
- */
 async function startTextSession(initialText) {
   if (sessionInProgress) { if (client && initialText) { client.sendUserText(initialText); client.requestResponse(); } return; }
   sessionInProgress = true;
-  // Await AudioContext resume — this is inside user-gesture handler, it WILL resolve
   await stage.resume();
   const audioCtx = stage.audioCtx; const voiceSink = stage.voiceSink;
   if (!audioCtx || !voiceSink) { sessionInProgress = false; return; }
@@ -379,7 +554,6 @@ async function endSession(silent = false) {
 // ── UI events ────────────────────────────────────────────────────────────
 mainBtn.addEventListener("click", () => {
   if (mainAction === "start") {
-    // User gesture! AudioContext.resume() will resolve → audio plays + lipsync works.
     if (sessionInProgress) return;
     if (FAKEMIC_MODE) {
       void startTextSession();
@@ -412,21 +586,13 @@ async function boot() {
   await fetchAvatarList(); populateAvatarSelects(settings.avatar);
   makeDraggable(); makeResizable();
   setCaption("WAKING HER UP…"); setMainButton("busy", "Loading…");
-  // Pre-fetch news hook in parallel with avatar
   const newsPromise = (settings.avatar === "vuong.glb" || !settings.avatar) ? getHotNewsGreeting().catch(() => null) : Promise.resolve(null);
   try { await stage.init({ avatarUrl: settings.avatar ? `/avatars/${settings.avatar}` : undefined, onprogress: (ev) => { if (ev.lengthComputable) loading.textContent = `Loading avatar ${Math.min(100, Math.round((ev.loaded / ev.total) * 100))}%`; } }); }
   catch (err) { console.error(err); loading.textContent = "Avatar failed to load. Reload."; setCaption("AVATAR FAILED", "error"); return; }
   preFetchedGreeting = await newsPromise;
   loading.classList.add("done"); setCaption(CAPTIONS.idle); setMainButton("start", "Start talking");
 
-  // NO auto-greeting here. On direct hf.space domains AudioContext is
-  // "suspended" before first click — any audio routed through a suspended
-  // context is silent and HeadAudio won't process any frames.
-  // User must tap "Start talking" → click provides user gesture →
-  // AudioContext.resume() resolves → audio plays → HeadAudio drives lipsync.
-  //
-  // ?fakemic=1 is an OPT-IN testing flag. Setting it automatically would
-  // break the direct domain (bep40-gemma-avatar.hf.space) by routing to
-  // text-only mode without a working backend.
+  // Initialize V.AISTUDIO iframe bridge
+  initVAISTUDIO();
 }
-void boot();
+void boot();