Spaces:
Paused
Paused
Upload src/app.js
Browse files- src/app.js +214 -467
src/app.js
CHANGED
|
@@ -1,75 +1,85 @@
|
|
| 1 |
// @ts-check
|
| 2 |
/**
|
| 3 |
* App wiring: the avatar stage + the speech-to-speech session.
|
| 4 |
-
*
|
| 5 |
-
*
|
|
|
|
|
|
|
|
|
|
| 6 |
*/
|
| 7 |
|
| 8 |
import { S2sWsRealtimeClient } from "./s2s/s2s-ws-client.js";
|
| 9 |
import { AvatarStage, AVATAR_MOODS, AVATAR_GESTURES } from "./avatar.js";
|
| 10 |
-
import { withSearchContext } from "./withSearchContext.js";
|
| 11 |
-
import { smartNormalize } from "./viNumberFix.js";
|
| 12 |
|
| 13 |
const VOICES = [
|
| 14 |
-
"Aiden",
|
| 15 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
];
|
| 17 |
const DEFAULT_VOICE = "Sohee";
|
| 18 |
|
| 19 |
-
// ── Vương avatar greeting (auto-updated World Cup info) ───────────────────
|
| 20 |
-
const getVietnameseGreeting = () => {
|
| 21 |
-
const now = new Date();
|
| 22 |
-
const dateStr = now.toLocaleDateString("vi-VN", {
|
| 23 |
-
weekday: "long", day: "numeric", month: "long", year: "numeric",
|
| 24 |
-
});
|
| 25 |
-
const timeStr = now.toLocaleTimeString("vi-VN", {
|
| 26 |
-
hour: "2-digit", minute: "2-digit",
|
| 27 |
-
});
|
| 28 |
-
// Base greeting - World Cup info will be fetched dynamically in boot
|
| 29 |
-
return `Xin chào, tôi tên là Vương đến từ V.AI STUDIO. Hôm nay là ${dateStr} lúc ${timeStr}. Bạn muốn nghe tin tức gì hôm nay? Hiện tại World Cup đang diễn ra sôi động, bạn muốn nghe không?`;
|
| 30 |
-
};
|
| 31 |
-
|
| 32 |
const DEFAULT_INSTRUCTIONS = [
|
| 33 |
"You are a friendly voice assistant with a visible, human-like 3D avatar: the user",
|
| 34 |
-
"sees you as a person on their screen.",
|
| 35 |
-
"
|
| 36 |
-
"If they write in Vietnamese, respond in Vietnamese. If they speak English, respond in English.",
|
| 37 |
-
"Never switch language mid-conversation.",
|
| 38 |
-
"When responding in Vietnamese: write dates as 'ngày 9 tháng 7 năm 2026' (NOT '9/7/2026'),",
|
| 39 |
-
"write years as 'năm hai nghìn không trăm hai mươi sáu' (NOT '2026'),",
|
| 40 |
-
"write numbers as words (e.g. 'ba mươi lăm' NOT '35'),",
|
| 41 |
-
"write percentages as 'mười lăm phần trăm' NOT '15%',",
|
| 42 |
-
"write currency as 'năm mươi nghìn đồng' NOT '50.000đ'.",
|
| 43 |
-
"This helps the text-to-speech engine read them correctly.",
|
| 44 |
-
"Keep replies short, natural and warm, never list-like.",
|
| 45 |
"You can control your avatar body with tools: set_mood changes your overall emotional",
|
| 46 |
"state, make_hand_gesture plays a hand gesture, make_facial_expression makes a quick",
|
| 47 |
"facial expression from a single face emoji. Use them naturally and sparingly to",
|
| 48 |
"express yourself: smile when greeting, shrug when unsure, thumbs up when agreeing.",
|
| 49 |
-
"CRITICAL: NEVER guess or say you do not know. You have search_web and search_wikipedia",
|
| 50 |
-
"tools that give you REAL-TIME information from the internet. For ANY question about",
|
| 51 |
-
"current events, sports, news, prices, weather, dates, or facts you are uncertain about:",
|
| 52 |
-
"FIRST call get_current_datetime to know today's date, THEN call search_web or",
|
| 53 |
-
"search_wikipedia with the exact query. Read the results and answer based on them.",
|
| 54 |
-
"This is MANDATORY. Never apologize for lacking information — just look it up.",
|
| 55 |
"Never mention the tools or that you are controlling an avatar.",
|
| 56 |
].join(" ");
|
| 57 |
|
| 58 |
const STORAGE_KEYS = {
|
| 59 |
voice: "avatar.voice",
|
| 60 |
-
avatar: "avatar.model",
|
| 61 |
instructions: "avatar.instructions",
|
| 62 |
directUrl: "avatar.directUrl",
|
| 63 |
subtitles: "avatar.subtitles",
|
| 64 |
};
|
| 65 |
|
|
|
|
| 66 |
const TOOL_DEFS = [
|
| 67 |
-
{
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
];
|
| 74 |
|
| 75 |
// ── DOM ──────────────────────────────────────────────────────────────────
|
|
@@ -78,7 +88,6 @@ const stageNode = $("#stage");
|
|
| 78 |
const mainBtn = /** @type {HTMLButtonElement} */ ($("#main-btn"));
|
| 79 |
const mainBtnLabel = $("#main-btn-label");
|
| 80 |
const muteBtn = /** @type {HTMLButtonElement} */ ($("#mute-btn"));
|
| 81 |
-
const textModeBtn = /** @type {HTMLButtonElement} */ ($("#text-mode-btn"));
|
| 82 |
const caption = $("#caption");
|
| 83 |
const subtitles = $("#subtitles");
|
| 84 |
const loading = $("#loading");
|
|
@@ -90,35 +99,21 @@ const inputDirectUrl = /** @type {HTMLInputElement} */ ($("#direct-url"));
|
|
| 90 |
const inputSubtitles = /** @type {HTMLInputElement} */ ($("#subtitles-toggle"));
|
| 91 |
const directUrlRow = $("#direct-url-row");
|
| 92 |
|
| 93 |
-
const textChat = /** @type {HTMLElement} */ ($("#text-chat"));
|
| 94 |
-
const chatHeader = /** @type {HTMLElement} */ ($("#chat-header"));
|
| 95 |
-
const chatMessages = /** @type {HTMLElement} */ ($("#chat-messages"));
|
| 96 |
-
const chatInput = /** @type {HTMLInputElement} */ ($("#chat-input"));
|
| 97 |
-
const chatSendBtn = /** @type {HTMLButtonElement} */ ($("#chat-send-btn"));
|
| 98 |
-
const chatCloseBtn = /** @type {HTMLButtonElement} */ ($("#chat-close-btn"));
|
| 99 |
-
const chatResizeHandle = /** @type {HTMLElement} */ ($("#chat-resize-handle"));
|
| 100 |
-
const chatAvatarSelect = /** @type {HTMLSelectElement} */ ($("#chat-avatar-select"));
|
| 101 |
-
const settingsAvatarSelect = /** @type {HTMLSelectElement} */ ($("#settings-avatar-select"));
|
| 102 |
-
|
| 103 |
// ── State ────────────────────────────────────────────────────────────────
|
| 104 |
const stage = new AvatarStage(stageNode);
|
| 105 |
/** @type {S2sWsRealtimeClient | null} */
|
| 106 |
let client = null;
|
| 107 |
let muted = false;
|
| 108 |
let subtitleTimer = 0;
|
| 109 |
-
let textMode = false;
|
| 110 |
/** @type {{ lb: boolean, allowDirect: boolean }} */
|
| 111 |
let config = { lb: false, allowDirect: true };
|
| 112 |
-
let avatarList = [];
|
| 113 |
-
let sessionInProgress = false;
|
| 114 |
-
let greetingSent = false;
|
| 115 |
|
| 116 |
function loadSettings() {
|
| 117 |
return {
|
| 118 |
voice: localStorage.getItem(STORAGE_KEYS.voice) || DEFAULT_VOICE,
|
| 119 |
-
avatar: localStorage.getItem(STORAGE_KEYS.avatar) || "vuong.glb", // Default: Vương
|
| 120 |
instructions: localStorage.getItem(STORAGE_KEYS.instructions) || "",
|
| 121 |
directUrl: localStorage.getItem(STORAGE_KEYS.directUrl) || "",
|
|
|
|
| 122 |
subtitles: localStorage.getItem(STORAGE_KEYS.subtitles) === "1",
|
| 123 |
};
|
| 124 |
}
|
|
@@ -126,277 +121,51 @@ let settings = loadSettings();
|
|
| 126 |
|
| 127 |
function saveSettings() {
|
| 128 |
localStorage.setItem(STORAGE_KEYS.voice, settings.voice);
|
| 129 |
-
localStorage.setItem(STORAGE_KEYS.avatar, settings.avatar);
|
| 130 |
localStorage.setItem(STORAGE_KEYS.instructions, settings.instructions);
|
| 131 |
localStorage.setItem(STORAGE_KEYS.directUrl, settings.directUrl);
|
| 132 |
localStorage.setItem(STORAGE_KEYS.subtitles, settings.subtitles ? "1" : "0");
|
| 133 |
}
|
| 134 |
|
|
|
|
| 135 |
function effectiveInstructions() {
|
| 136 |
-
const now = new Date();
|
| 137 |
-
const dateStr = now.toLocaleDateString("en-US", {
|
| 138 |
-
weekday: "long", year: "numeric", month: "long", day: "numeric",
|
| 139 |
-
});
|
| 140 |
-
const timeStr = now.toLocaleTimeString("en-US", {
|
| 141 |
-
hour: "2-digit", minute: "2-digit",
|
| 142 |
-
});
|
| 143 |
-
const dateLine = `Today is ${dateStr}. The current time is ${timeStr}.`;
|
| 144 |
const extra = settings.instructions.trim();
|
| 145 |
-
|
| 146 |
-
return extra ? `${base}\n\nAdditional instructions from the user:\n${extra}` : base;
|
| 147 |
-
}
|
| 148 |
-
|
| 149 |
-
// ── Avatar list ──────────────────────────────────────────────────────────
|
| 150 |
-
async function fetchAvatarList() {
|
| 151 |
-
try {
|
| 152 |
-
const resp = await fetch("/api/avatars");
|
| 153 |
-
if (resp.ok) {
|
| 154 |
-
const data = await resp.json();
|
| 155 |
-
avatarList = data.avatars || [];
|
| 156 |
-
}
|
| 157 |
-
} catch {}
|
| 158 |
-
}
|
| 159 |
-
|
| 160 |
-
function populateAvatarSelects(selectedName) {
|
| 161 |
-
for (const sel of [chatAvatarSelect, settingsAvatarSelect]) {
|
| 162 |
-
sel.innerHTML = "";
|
| 163 |
-
const def = document.createElement("option");
|
| 164 |
-
def.value = "";
|
| 165 |
-
def.textContent = "(Default - Brunette)";
|
| 166 |
-
sel.appendChild(def);
|
| 167 |
-
for (const name of avatarList) {
|
| 168 |
-
const opt = document.createElement("option");
|
| 169 |
-
opt.value = name;
|
| 170 |
-
// Vietnamese label for Vương
|
| 171 |
-
opt.textContent = name.replace(/\.glb$/i, "").replace(/_/g, " ") + (name.toLowerCase() === "vuong.glb" ? " 🎙️" : "");
|
| 172 |
-
sel.appendChild(opt);
|
| 173 |
-
}
|
| 174 |
-
if (selectedName && avatarList.includes(selectedName)) sel.value = selectedName;
|
| 175 |
-
}
|
| 176 |
-
}
|
| 177 |
-
|
| 178 |
-
function setAvatarFromSelect(value) {
|
| 179 |
-
settings.avatar = value || "";
|
| 180 |
-
saveSettings();
|
| 181 |
-
}
|
| 182 |
-
|
| 183 |
-
async function reloadAvatar() {
|
| 184 |
-
if (!stage.head) return;
|
| 185 |
-
loading.classList.remove("done");
|
| 186 |
-
loading.textContent = "Loading avatar...";
|
| 187 |
-
try {
|
| 188 |
-
await stage.init({
|
| 189 |
-
avatarUrl: settings.avatar ? `/avatars/${settings.avatar}` : undefined,
|
| 190 |
-
onprogress: (ev) => {
|
| 191 |
-
if (ev.lengthComputable) {
|
| 192 |
-
loading.textContent = `Loading avatar ${Math.min(100, Math.round((ev.loaded / ev.total) * 100))}%`;
|
| 193 |
-
}
|
| 194 |
-
},
|
| 195 |
-
});
|
| 196 |
-
} catch (err) { console.error(err); }
|
| 197 |
-
loading.classList.add("done");
|
| 198 |
-
}
|
| 199 |
-
|
| 200 |
-
// ── Clamp helper ─────────────────────────────────────────────────────────
|
| 201 |
-
function clampRect() {
|
| 202 |
-
const vw = window.innerWidth, vh = window.innerHeight;
|
| 203 |
-
const r = textChat.getBoundingClientRect();
|
| 204 |
-
let l = r.left, t = r.top;
|
| 205 |
-
const minTop = 10;
|
| 206 |
-
l = Math.max(10, Math.min(l, vw - r.width - 10));
|
| 207 |
-
t = Math.max(minTop, Math.min(t, vh - r.height - 10));
|
| 208 |
-
textChat.style.left = `${l}px`;
|
| 209 |
-
textChat.style.top = `${t}px`;
|
| 210 |
-
}
|
| 211 |
-
|
| 212 |
-
// ── Drag (mouse + touch) ─────────────────────────────────────────────────
|
| 213 |
-
function makeDraggable() {
|
| 214 |
-
let dragging = false;
|
| 215 |
-
let startX, startY, startLeft, startTop;
|
| 216 |
-
|
| 217 |
-
function getPos(e) {
|
| 218 |
-
const p = e.changedTouches ? e.changedTouches[0] : e;
|
| 219 |
-
return { x: p.clientX, y: p.clientY };
|
| 220 |
-
}
|
| 221 |
-
|
| 222 |
-
function onStart(e) {
|
| 223 |
-
if (e.target.closest("#chat-header-actions") || e.target.closest("#chat-avatar-select")) return;
|
| 224 |
-
const p = getPos(e);
|
| 225 |
-
dragging = true;
|
| 226 |
-
const rect = textChat.getBoundingClientRect();
|
| 227 |
-
startX = p.x; startY = p.y;
|
| 228 |
-
startLeft = rect.left; startTop = rect.top;
|
| 229 |
-
textChat.classList.add("dragging");
|
| 230 |
-
e.preventDefault();
|
| 231 |
-
}
|
| 232 |
-
|
| 233 |
-
function onMove(e) {
|
| 234 |
-
if (!dragging) return;
|
| 235 |
-
const p = getPos(e);
|
| 236 |
-
const dx = p.x - startX, dy = p.y - startY;
|
| 237 |
-
textChat.style.left = `${startLeft + dx}px`;
|
| 238 |
-
textChat.style.top = `${startTop + dy}px`;
|
| 239 |
-
textChat.style.right = "auto";
|
| 240 |
-
textChat.style.bottom = "auto";
|
| 241 |
-
textChat.style.width = "";
|
| 242 |
-
textChat.style.height = "";
|
| 243 |
-
e.preventDefault();
|
| 244 |
-
}
|
| 245 |
-
|
| 246 |
-
function onEnd() {
|
| 247 |
-
if (!dragging) return;
|
| 248 |
-
dragging = false;
|
| 249 |
-
textChat.classList.remove("dragging");
|
| 250 |
-
clampRect();
|
| 251 |
-
}
|
| 252 |
-
|
| 253 |
-
chatHeader.addEventListener("mousedown", onStart);
|
| 254 |
-
document.addEventListener("mousemove", onMove);
|
| 255 |
-
document.addEventListener("mouseup", onEnd);
|
| 256 |
-
chatHeader.addEventListener("touchstart", onStart, { passive: false });
|
| 257 |
-
document.addEventListener("touchmove", onMove, { passive: false });
|
| 258 |
-
document.addEventListener("touchend", onEnd);
|
| 259 |
-
}
|
| 260 |
-
|
| 261 |
-
// ── Resize (mouse + touch) ───────────────────────────────────────────────
|
| 262 |
-
function makeResizable() {
|
| 263 |
-
let resizing = false;
|
| 264 |
-
let startX, startY, startW, startH;
|
| 265 |
-
|
| 266 |
-
function getPos(e) {
|
| 267 |
-
const p = e.changedTouches ? e.changedTouches[0] : e;
|
| 268 |
-
return { x: p.clientX, y: p.clientY };
|
| 269 |
-
}
|
| 270 |
-
|
| 271 |
-
function onStart(e) {
|
| 272 |
-
resizing = true;
|
| 273 |
-
const rect = textChat.getBoundingClientRect();
|
| 274 |
-
const p = getPos(e);
|
| 275 |
-
startX = p.x; startY = p.y;
|
| 276 |
-
startW = rect.width; startH = rect.height;
|
| 277 |
-
textChat.classList.add("resizing");
|
| 278 |
-
e.preventDefault();
|
| 279 |
-
e.stopPropagation();
|
| 280 |
-
}
|
| 281 |
-
|
| 282 |
-
function onMove(e) {
|
| 283 |
-
if (!resizing) return;
|
| 284 |
-
const p = getPos(e);
|
| 285 |
-
const dw = p.x - startX, dh = p.y - startY;
|
| 286 |
-
const newW = Math.max(260, startW + dw);
|
| 287 |
-
const newH = Math.max(120, startH + dh);
|
| 288 |
-
textChat.style.width = `${newW}px`;
|
| 289 |
-
textChat.style.height = `${newH}px`;
|
| 290 |
-
textChat.style.right = "auto";
|
| 291 |
-
textChat.style.bottom = "auto";
|
| 292 |
-
e.preventDefault();
|
| 293 |
-
}
|
| 294 |
-
|
| 295 |
-
function onEnd() {
|
| 296 |
-
if (!resizing) return;
|
| 297 |
-
resizing = false;
|
| 298 |
-
textChat.classList.remove("resizing");
|
| 299 |
-
}
|
| 300 |
-
|
| 301 |
-
chatResizeHandle.addEventListener("mousedown", onStart);
|
| 302 |
-
document.addEventListener("mousemove", onMove);
|
| 303 |
-
document.addEventListener("mouseup", onEnd);
|
| 304 |
-
chatResizeHandle.addEventListener("touchstart", onStart, { passive: false });
|
| 305 |
-
document.addEventListener("touchmove", onMove, { passive: false });
|
| 306 |
-
document.addEventListener("touchend", onEnd);
|
| 307 |
}
|
| 308 |
|
| 309 |
// ── Captions / subtitles ─────────────────────────────────────────────────
|
|
|
|
| 310 |
function setCaption(text, kind = "") {
|
| 311 |
caption.textContent = text;
|
| 312 |
caption.className = kind;
|
| 313 |
}
|
|
|
|
|
|
|
| 314 |
function showSubtitles(text) {
|
| 315 |
if (!settings.subtitles) return;
|
| 316 |
clearTimeout(subtitleTimer);
|
| 317 |
subtitles.textContent = text;
|
| 318 |
subtitles.classList.add("visible");
|
| 319 |
}
|
|
|
|
| 320 |
function fadeSubtitles(delayMs = 2600) {
|
| 321 |
clearTimeout(subtitleTimer);
|
| 322 |
subtitleTimer = window.setTimeout(() => subtitles.classList.remove("visible"), delayMs);
|
| 323 |
}
|
| 324 |
|
| 325 |
-
// ── Text Chat UI ─────────────────────────────────────────────────────────
|
| 326 |
-
function addChatMessage(role, text) {
|
| 327 |
-
const msg = document.createElement("div");
|
| 328 |
-
msg.className = `chat-message ${role}`;
|
| 329 |
-
msg.textContent = text;
|
| 330 |
-
chatMessages.appendChild(msg);
|
| 331 |
-
chatMessages.scrollTop = chatMessages.scrollHeight;
|
| 332 |
-
}
|
| 333 |
-
|
| 334 |
-
function showTextChat(show) {
|
| 335 |
-
textChat.hidden = !show;
|
| 336 |
-
if (show) chatInput.focus();
|
| 337 |
-
}
|
| 338 |
-
|
| 339 |
-
// Fetch and update World Cup info for greeting
|
| 340 |
-
async function getWorldCupGreeting() {
|
| 341 |
-
try {
|
| 342 |
-
const searchResp = await fetch("/api/web/search?q=World+Cup+2026+latest+news");
|
| 343 |
-
if (searchResp.ok) {
|
| 344 |
-
const data = await searchResp.json();
|
| 345 |
-
if (data.results && data.results.length > 0) {
|
| 346 |
-
const latestNews = data.results[0].title || "";
|
| 347 |
-
return getVietnameseGreeting().replace("World Cup đang diễn ra sôi động", `World Cup đang có tin: ${latestNews}`);
|
| 348 |
-
}
|
| 349 |
-
}
|
| 350 |
-
} catch (e) {
|
| 351 |
-
console.warn("Could not fetch World Cup news:", e);
|
| 352 |
-
}
|
| 353 |
-
return getVietnameseGreeting();
|
| 354 |
-
}
|
| 355 |
-
|
| 356 |
-
function sendTextMessage() {
|
| 357 |
-
const text = chatInput.value.trim();
|
| 358 |
-
if (!text) return;
|
| 359 |
-
|
| 360 |
-
addChatMessage("user", text);
|
| 361 |
-
chatInput.value = "";
|
| 362 |
-
|
| 363 |
-
// Track if this is the first message to send greeting
|
| 364 |
-
if (!client || !sessionInProgress) {
|
| 365 |
-
void startTextSession(text);
|
| 366 |
-
return;
|
| 367 |
-
}
|
| 368 |
-
|
| 369 |
-
// Pre-process: auto-search web before sending to model
|
| 370 |
-
setCaption("LOOKING IT UP…");
|
| 371 |
-
void (async () => {
|
| 372 |
-
const enriched = await withSearchContext(text);
|
| 373 |
-
client.sendUserText(enriched);
|
| 374 |
-
client.requestResponse();
|
| 375 |
-
})();
|
| 376 |
-
}
|
| 377 |
-
|
| 378 |
-
// Close button
|
| 379 |
-
if (chatCloseBtn) {
|
| 380 |
-
chatCloseBtn.addEventListener("click", (e) => {
|
| 381 |
-
e.stopPropagation();
|
| 382 |
-
textMode = false;
|
| 383 |
-
showTextChat(false);
|
| 384 |
-
textModeBtn.classList.remove("active");
|
| 385 |
-
});
|
| 386 |
-
}
|
| 387 |
-
|
| 388 |
// ── Button ───────────────────────────────────────────────────────────────
|
|
|
|
| 389 |
let mainAction = "start";
|
|
|
|
|
|
|
| 390 |
function setMainButton(action, label) {
|
| 391 |
mainAction = action;
|
| 392 |
mainBtnLabel.textContent = label;
|
| 393 |
mainBtn.disabled = action === "busy";
|
| 394 |
mainBtn.classList.toggle("live", action === "stop");
|
| 395 |
muteBtn.hidden = action !== "stop";
|
| 396 |
-
textModeBtn.hidden = false;
|
| 397 |
}
|
| 398 |
|
| 399 |
-
// ── Status ──────────────────────────────────────────────────────
|
| 400 |
const CAPTIONS = {
|
| 401 |
idle: "TAP TO TALK",
|
| 402 |
"creating-session": "REQUESTING A SLOT…",
|
|
@@ -411,212 +180,151 @@ const CAPTIONS = {
|
|
| 411 |
error: "SOMETHING BROKE, TAP TO RETRY",
|
| 412 |
};
|
| 413 |
|
|
|
|
| 414 |
function onStatus(status) {
|
| 415 |
stage.setConversationState(status);
|
| 416 |
setCaption(CAPTIONS[status] ?? status, status === "error" ? "error" : status === "idle" || status === "closed" ? "" : "live");
|
|
|
|
| 417 |
switch (status) {
|
| 418 |
-
case "idle":
|
| 419 |
-
case "
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
case "
|
| 423 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 424 |
}
|
|
|
|
| 425 |
if (status === "user-speaking") {
|
| 426 |
subtitles.classList.remove("visible");
|
| 427 |
-
showTextChat(textMode);
|
| 428 |
}
|
| 429 |
}
|
| 430 |
|
| 431 |
-
// ── Tool
|
|
|
|
| 432 |
function runTool(name, argsJson, callId) {
|
| 433 |
if (!client) return;
|
|
|
|
| 434 |
let args = {};
|
| 435 |
-
try {
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
const result = `The current date and time is ${now.toLocaleDateString("en-US", { weekday: "long", year: "numeric", month: "long", day: "numeric" })} at ${now.toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit", second: "2-digit" })}.`;
|
| 440 |
-
client.sendToolOutput(callId, result);
|
| 441 |
-
client.requestResponse();
|
| 442 |
-
return;
|
| 443 |
-
}
|
| 444 |
-
|
| 445 |
-
if (name === "search_wikipedia") {
|
| 446 |
-
const query = /** @type {string} */ (args.query || "");
|
| 447 |
-
if (!query) { client.sendToolOutput(callId, "Please provide a search query."); client.requestResponse(); return; }
|
| 448 |
-
const wikiTitle = query.replace(/\s+/g, "_");
|
| 449 |
-
fetch(`/api/wiki/summary?title=${encodeURIComponent(wikiTitle)}`)
|
| 450 |
-
.then((r) => r.json())
|
| 451 |
-
.then((data) => {
|
| 452 |
-
if (data.extract) { client.sendToolOutput(callId, `From Wikipedia (${data.title}): ${data.extract}\nSource: ${data.url}`); client.requestResponse(); return; }
|
| 453 |
-
fetch(`/api/wiki/search?q=${encodeURIComponent(query)}`)
|
| 454 |
-
.then((r) => r.json())
|
| 455 |
-
.then((searchData) => {
|
| 456 |
-
if (!searchData.results || searchData.results.length === 0) { client.sendToolOutput(callId, `No Wikipedia results found for "${query}".`); client.requestResponse(); return; }
|
| 457 |
-
const first = searchData.results[0];
|
| 458 |
-
fetch(`/api/wiki/summary?title=${encodeURIComponent(first.title)}`)
|
| 459 |
-
.then((r) => r.json())
|
| 460 |
-
.then((summary) => { client.sendToolOutput(callId, summary.extract ? `From Wikipedia (${summary.title}): ${summary.extract}\nSource: ${summary.url}` : `Wikipedia results for "${query}":\n${searchData.results.map((r) => `• ${r.title}: ${r.snippet}`).join("\n")}`); client.requestResponse(); });
|
| 461 |
-
});
|
| 462 |
-
})
|
| 463 |
-
.catch(() => { client.sendToolOutput(callId, `Failed to search Wikipedia for "${query}".`); client.requestResponse(); });
|
| 464 |
-
return;
|
| 465 |
-
}
|
| 466 |
-
|
| 467 |
-
if (name === "search_web") {
|
| 468 |
-
const query = /** @type {string} */ (args.query || "");
|
| 469 |
-
if (!query) { client.sendToolOutput(callId, "Please provide a search query."); client.requestResponse(); return; }
|
| 470 |
-
fetch(`/api/web/search?q=${encodeURIComponent(query)}`)
|
| 471 |
-
.then((r) => r.json())
|
| 472 |
-
.then((data) => {
|
| 473 |
-
if (!data.results || data.results.length === 0) { client.sendToolOutput(callId, `No web results found for "${query}".`); client.requestResponse(); return; }
|
| 474 |
-
client.sendToolOutput(callId, `Web search results for "${query}":\n${data.results.slice(0, 3).map((r, i) => `${i + 1}. ${r.title}\n ${r.snippet}`).join("\n")}`);
|
| 475 |
-
client.requestResponse();
|
| 476 |
-
})
|
| 477 |
-
.catch(() => { client.sendToolOutput(callId, `Failed to search the web for "${query}".`); client.requestResponse(); });
|
| 478 |
-
return;
|
| 479 |
}
|
| 480 |
-
|
| 481 |
const result = stage.runTool(name, args) ?? `Unknown tool: ${name}`;
|
| 482 |
client.sendToolOutput(callId, result);
|
|
|
|
| 483 |
client.requestResponse();
|
| 484 |
}
|
| 485 |
|
| 486 |
-
// ──
|
| 487 |
-
async function
|
| 488 |
-
|
| 489 |
-
|
| 490 |
stage.resume();
|
| 491 |
|
| 492 |
let micStream;
|
| 493 |
if (new URLSearchParams(location.search).has("fakemic")) {
|
|
|
|
|
|
|
|
|
|
| 494 |
const ctx = /** @type {AudioContext} */ (stage.audioCtx);
|
| 495 |
micStream = ctx.createMediaStreamDestination().stream;
|
| 496 |
} else {
|
| 497 |
try {
|
| 498 |
-
micStream = await navigator.mediaDevices.getUserMedia({
|
|
|
|
|
|
|
| 499 |
} catch {
|
| 500 |
setCaption("MIC BLOCKED, ALLOW IT IN THE BROWSER AND RETRY", "error");
|
| 501 |
-
sessionInProgress = false;
|
| 502 |
return;
|
| 503 |
}
|
| 504 |
}
|
|
|
|
| 505 |
const audioCtx = stage.audioCtx;
|
| 506 |
const voiceSink = stage.voiceSink;
|
| 507 |
-
if (!audioCtx || !voiceSink)
|
| 508 |
|
| 509 |
const c = new S2sWsRealtimeClient({
|
| 510 |
...(config.lb ? { sessionUrl: "api/session" } : { directUrl: settings.directUrl }),
|
| 511 |
voice: settings.voice,
|
| 512 |
instructions: effectiveInstructions(),
|
| 513 |
-
micStream,
|
| 514 |
-
|
| 515 |
-
|
|
|
|
|
|
|
| 516 |
});
|
| 517 |
client = c;
|
| 518 |
-
_attachClientEvents(c);
|
| 519 |
|
| 520 |
-
|
| 521 |
-
await c.connect();
|
| 522 |
-
// Send greeting on first connect (Vương avatar intro)
|
| 523 |
-
if (!greetingSent && settings.avatar === "vuong.glb") {
|
| 524 |
-
const greeting = await getWorldCupGreeting();
|
| 525 |
-
c.sendUserText(greeting);
|
| 526 |
-
c.requestResponse();
|
| 527 |
-
greetingSent = true;
|
| 528 |
-
addChatMessage("assistant", greeting);
|
| 529 |
-
}
|
| 530 |
-
}
|
| 531 |
-
catch (err) {
|
| 532 |
-
const code = /** @type {Error & {code?: string}} */ (err)?.code;
|
| 533 |
-
if (code === "limit") setCaption("DAILY CONVERSATION LIMIT REACHED, TRY AGAIN TOMORROW", "error");
|
| 534 |
-
else if (code === "queue-full") setCaption("EVERY SEAT IS TAKEN, TRY AGAIN SHORTLY", "error");
|
| 535 |
-
else if (code === "join-expired") setCaption("YOUR SPOT EXPIRED, TAP TO TRY AGAIN", "error");
|
| 536 |
-
else if (code !== "aborted") { console.error(err); setCaption("COULD NOT CONNECT, TAP TO RETRY", "error"); }
|
| 537 |
-
await endSession(true);
|
| 538 |
-
}
|
| 539 |
-
}
|
| 540 |
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
return;
|
| 546 |
-
}
|
| 547 |
-
sessionInProgress = true;
|
| 548 |
-
stage.resume();
|
| 549 |
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
|
|
|
| 553 |
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
});
|
| 562 |
-
client = c;
|
| 563 |
-
_attachClientEvents(c);
|
| 564 |
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 568 |
|
| 569 |
try {
|
| 570 |
await c.connect();
|
| 571 |
-
// Send greeting on first connect (Vương avatar intro)
|
| 572 |
-
if (!greetingSent && settings.avatar === "vuong.glb") {
|
| 573 |
-
const greeting = await getWorldCupGreeting();
|
| 574 |
-
c.sendUserText(greeting);
|
| 575 |
-
c.requestResponse();
|
| 576 |
-
greetingSent = true;
|
| 577 |
-
addChatMessage("assistant", greeting);
|
| 578 |
-
}
|
| 579 |
-
if (initialText) {
|
| 580 |
-
const enriched = await withSearchContext(initialText);
|
| 581 |
-
c.sendUserText(enriched);
|
| 582 |
-
c.requestResponse();
|
| 583 |
-
}
|
| 584 |
} catch (err) {
|
| 585 |
const code = /** @type {Error & {code?: string}} */ (err)?.code;
|
| 586 |
-
if (code === "limit")
|
| 587 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 588 |
await endSession(true);
|
|
|
|
| 589 |
}
|
| 590 |
}
|
| 591 |
|
| 592 |
-
|
| 593 |
-
c.addEventListener("status", (e) => onStatus(/** @type {CustomEvent} */ (e).detail.status));
|
| 594 |
-
c.addEventListener("queue", (e) => { const { position } = /** @type {CustomEvent} */ (e).detail; setCaption(position > 0 ? `#${position} IN LINE…` : "ALMOST THERE…", "live"); });
|
| 595 |
-
|
| 596 |
-
// ── Transcript handler with Vietnamese number normalization ──────────
|
| 597 |
-
c.addEventListener("transcript", (e) => {
|
| 598 |
-
const { role, text } = /** @type {CustomEvent} */ (e).detail;
|
| 599 |
-
if (role === "assistant" && text) {
|
| 600 |
-
// Normalize numbers/dates for Vietnamese TTS display AND subtitles
|
| 601 |
-
const normalized = smartNormalize(text);
|
| 602 |
-
showSubtitles(normalized);
|
| 603 |
-
addChatMessage("assistant", normalized);
|
| 604 |
-
}
|
| 605 |
-
});
|
| 606 |
-
|
| 607 |
-
c.addEventListener("response-finished", () => fadeSubtitles());
|
| 608 |
-
c.addEventListener("toolcall", (e) => { const { name, arguments: args, callId } = /** @type {CustomEvent} */ (e).detail; runTool(name, args, callId); });
|
| 609 |
-
c.addEventListener("server-error", (e) => console.warn("server error:", /** @type {CustomEvent} */ (e).detail.error));
|
| 610 |
-
c.addEventListener("error", () => { void endSession(); });
|
| 611 |
-
}
|
| 612 |
-
|
| 613 |
async function endSession(silent = false) {
|
| 614 |
const c = client;
|
| 615 |
client = null;
|
| 616 |
-
sessionInProgress = false;
|
| 617 |
-
greetingSent = false; // Reset for next session
|
| 618 |
if (c) {
|
| 619 |
-
|
| 620 |
await c.close().catch(() => {});
|
| 621 |
}
|
| 622 |
stage.setConversationState("idle");
|
|
@@ -627,46 +335,85 @@ async function endSession(silent = false) {
|
|
| 627 |
|
| 628 |
// ── UI events ────────────────────────────────────────────────────────────
|
| 629 |
mainBtn.addEventListener("click", () => {
|
| 630 |
-
if (mainAction === "start") void
|
| 631 |
-
else if (mainAction === "join") {
|
| 632 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 633 |
});
|
| 634 |
-
muteBtn.addEventListener("click", () => { muted = !muted; client?.setMuted(muted); muteBtn.classList.toggle("active", muted); muteBtn.setAttribute("aria-label", muted ? "Unmute microphone" : "Mute microphone"); });
|
| 635 |
-
textModeBtn.addEventListener("click", () => { textMode = !textMode; showTextChat(textMode); textModeBtn.classList.toggle("active", textMode); });
|
| 636 |
-
chatSendBtn.addEventListener("click", () => sendTextMessage());
|
| 637 |
-
chatInput.addEventListener("keypress", (e) => { if (e.key === "Enter") sendTextMessage(); });
|
| 638 |
|
| 639 |
-
|
| 640 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 641 |
|
| 642 |
-
settingsBtn.addEventListener("click", () => { inputVoice.value = settings.voice; inputInstructions.value = settings.instructions; inputDirectUrl.value = settings.directUrl; inputSubtitles.checked = settings.subtitles; settingsAvatarSelect.value = chatAvatarSelect.value; settingsDialog.showModal(); });
|
| 643 |
settingsDialog.addEventListener("close", () => {
|
| 644 |
-
settings = {
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 648 |
});
|
| 649 |
|
| 650 |
-
window.addEventListener("beforeunload", () => {
|
|
|
|
|
|
|
| 651 |
|
| 652 |
// ── Boot ─────────────────────────────────────────────────────────────────
|
| 653 |
async function boot() {
|
| 654 |
-
for (const v of VOICES) {
|
| 655 |
-
|
| 656 |
-
|
|
|
|
|
|
|
|
|
|
| 657 |
|
| 658 |
-
|
| 659 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 660 |
|
| 661 |
-
setCaption("WAKING HER UP…");
|
|
|
|
| 662 |
try {
|
| 663 |
await stage.init({
|
| 664 |
-
avatarUrl: settings.avatar ? `/avatars/${settings.avatar}` : undefined,
|
| 665 |
onprogress: (ev) => {
|
| 666 |
-
if (ev.lengthComputable)
|
| 667 |
-
|
|
|
|
|
|
|
|
|
|
| 668 |
});
|
| 669 |
-
} catch (err) {
|
| 670 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 671 |
}
|
|
|
|
| 672 |
void boot();
|
|
|
|
| 1 |
// @ts-check
|
| 2 |
/**
|
| 3 |
* App wiring: the avatar stage + the speech-to-speech session.
|
| 4 |
+
*
|
| 5 |
+
* A session is one tap away: tap the button → mic → same-origin `/api/session`
|
| 6 |
+
* handshake → WebSocket to the granted compute → talk. The avatar carries all
|
| 7 |
+
* conversational state (listening, thinking, speaking) with its body; the
|
| 8 |
+
* caption under it is a quiet machine-voice echo of the same state.
|
| 9 |
*/
|
| 10 |
|
| 11 |
import { S2sWsRealtimeClient } from "./s2s/s2s-ws-client.js";
|
| 12 |
import { AvatarStage, AVATAR_MOODS, AVATAR_GESTURES } from "./avatar.js";
|
|
|
|
|
|
|
| 13 |
|
| 14 |
const VOICES = [
|
| 15 |
+
"Aiden",
|
| 16 |
+
"Ryan",
|
| 17 |
+
"Dylan",
|
| 18 |
+
"Eric",
|
| 19 |
+
"Ono_Anna",
|
| 20 |
+
"Serena",
|
| 21 |
+
"Sohee",
|
| 22 |
+
"Uncle_Fu",
|
| 23 |
+
"Vivian",
|
| 24 |
];
|
| 25 |
const DEFAULT_VOICE = "Sohee";
|
| 26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
const DEFAULT_INSTRUCTIONS = [
|
| 28 |
"You are a friendly voice assistant with a visible, human-like 3D avatar: the user",
|
| 29 |
+
"sees you as a person on their screen. This is a spoken conversation: keep replies",
|
| 30 |
+
"short, natural and warm, never list-like.",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
"You can control your avatar body with tools: set_mood changes your overall emotional",
|
| 32 |
"state, make_hand_gesture plays a hand gesture, make_facial_expression makes a quick",
|
| 33 |
"facial expression from a single face emoji. Use them naturally and sparingly to",
|
| 34 |
"express yourself: smile when greeting, shrug when unsure, thumbs up when agreeing.",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
"Never mention the tools or that you are controlling an avatar.",
|
| 36 |
].join(" ");
|
| 37 |
|
| 38 |
const STORAGE_KEYS = {
|
| 39 |
voice: "avatar.voice",
|
|
|
|
| 40 |
instructions: "avatar.instructions",
|
| 41 |
directUrl: "avatar.directUrl",
|
| 42 |
subtitles: "avatar.subtitles",
|
| 43 |
};
|
| 44 |
|
| 45 |
+
/** Function tools declared to the backend: the model plays the avatar. */
|
| 46 |
const TOOL_DEFS = [
|
| 47 |
+
{
|
| 48 |
+
type: "function",
|
| 49 |
+
name: "set_mood",
|
| 50 |
+
description: "Change your avatar's overall mood/emotional state.",
|
| 51 |
+
parameters: {
|
| 52 |
+
type: "object",
|
| 53 |
+
properties: {
|
| 54 |
+
mood: { type: "string", enum: AVATAR_MOODS, description: "Mood name." },
|
| 55 |
+
},
|
| 56 |
+
required: ["mood"],
|
| 57 |
+
},
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
type: "function",
|
| 61 |
+
name: "make_hand_gesture",
|
| 62 |
+
description: "Make a hand gesture with your avatar.",
|
| 63 |
+
parameters: {
|
| 64 |
+
type: "object",
|
| 65 |
+
properties: {
|
| 66 |
+
gesture: { type: "string", enum: AVATAR_GESTURES, description: "Gesture name." },
|
| 67 |
+
},
|
| 68 |
+
required: ["gesture"],
|
| 69 |
+
},
|
| 70 |
+
},
|
| 71 |
+
{
|
| 72 |
+
type: "function",
|
| 73 |
+
name: "make_facial_expression",
|
| 74 |
+
description: "Make a quick facial expression with your avatar, given as a single face emoji (e.g. 😊, 😮, 🤔).",
|
| 75 |
+
parameters: {
|
| 76 |
+
type: "object",
|
| 77 |
+
properties: {
|
| 78 |
+
emoji: { type: "string", description: "A single face emoji." },
|
| 79 |
+
},
|
| 80 |
+
required: ["emoji"],
|
| 81 |
+
},
|
| 82 |
+
},
|
| 83 |
];
|
| 84 |
|
| 85 |
// ── DOM ──────────────────────────────────────────────────────────────────
|
|
|
|
| 88 |
const mainBtn = /** @type {HTMLButtonElement} */ ($("#main-btn"));
|
| 89 |
const mainBtnLabel = $("#main-btn-label");
|
| 90 |
const muteBtn = /** @type {HTMLButtonElement} */ ($("#mute-btn"));
|
|
|
|
| 91 |
const caption = $("#caption");
|
| 92 |
const subtitles = $("#subtitles");
|
| 93 |
const loading = $("#loading");
|
|
|
|
| 99 |
const inputSubtitles = /** @type {HTMLInputElement} */ ($("#subtitles-toggle"));
|
| 100 |
const directUrlRow = $("#direct-url-row");
|
| 101 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
// ── State ────────────────────────────────────────────────────────────────
|
| 103 |
const stage = new AvatarStage(stageNode);
|
| 104 |
/** @type {S2sWsRealtimeClient | null} */
|
| 105 |
let client = null;
|
| 106 |
let muted = false;
|
| 107 |
let subtitleTimer = 0;
|
|
|
|
| 108 |
/** @type {{ lb: boolean, allowDirect: boolean }} */
|
| 109 |
let config = { lb: false, allowDirect: true };
|
|
|
|
|
|
|
|
|
|
| 110 |
|
| 111 |
function loadSettings() {
|
| 112 |
return {
|
| 113 |
voice: localStorage.getItem(STORAGE_KEYS.voice) || DEFAULT_VOICE,
|
|
|
|
| 114 |
instructions: localStorage.getItem(STORAGE_KEYS.instructions) || "",
|
| 115 |
directUrl: localStorage.getItem(STORAGE_KEYS.directUrl) || "",
|
| 116 |
+
// Off by default: the face already carries the conversation.
|
| 117 |
subtitles: localStorage.getItem(STORAGE_KEYS.subtitles) === "1",
|
| 118 |
};
|
| 119 |
}
|
|
|
|
| 121 |
|
| 122 |
function saveSettings() {
|
| 123 |
localStorage.setItem(STORAGE_KEYS.voice, settings.voice);
|
|
|
|
| 124 |
localStorage.setItem(STORAGE_KEYS.instructions, settings.instructions);
|
| 125 |
localStorage.setItem(STORAGE_KEYS.directUrl, settings.directUrl);
|
| 126 |
localStorage.setItem(STORAGE_KEYS.subtitles, settings.subtitles ? "1" : "0");
|
| 127 |
}
|
| 128 |
|
| 129 |
+
/** Persona + whatever extra guidance the user typed in Settings. */
|
| 130 |
function effectiveInstructions() {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
const extra = settings.instructions.trim();
|
| 132 |
+
return extra ? `${DEFAULT_INSTRUCTIONS}\n\nAdditional instructions from the user:\n${extra}` : DEFAULT_INSTRUCTIONS;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
}
|
| 134 |
|
| 135 |
// ── Captions / subtitles ─────────────────────────────────────────────────
|
| 136 |
+
/** @param {string} text @param {""|"live"|"error"} [kind] */
|
| 137 |
function setCaption(text, kind = "") {
|
| 138 |
caption.textContent = text;
|
| 139 |
caption.className = kind;
|
| 140 |
}
|
| 141 |
+
|
| 142 |
+
/** @param {string} text */
|
| 143 |
function showSubtitles(text) {
|
| 144 |
if (!settings.subtitles) return;
|
| 145 |
clearTimeout(subtitleTimer);
|
| 146 |
subtitles.textContent = text;
|
| 147 |
subtitles.classList.add("visible");
|
| 148 |
}
|
| 149 |
+
|
| 150 |
function fadeSubtitles(delayMs = 2600) {
|
| 151 |
clearTimeout(subtitleTimer);
|
| 152 |
subtitleTimer = window.setTimeout(() => subtitles.classList.remove("visible"), delayMs);
|
| 153 |
}
|
| 154 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 155 |
// ── Button ───────────────────────────────────────────────────────────────
|
| 156 |
+
/** @type {"start" | "join" | "stop" | "busy"} */
|
| 157 |
let mainAction = "start";
|
| 158 |
+
|
| 159 |
+
/** @param {"start" | "join" | "stop" | "busy"} action @param {string} label */
|
| 160 |
function setMainButton(action, label) {
|
| 161 |
mainAction = action;
|
| 162 |
mainBtnLabel.textContent = label;
|
| 163 |
mainBtn.disabled = action === "busy";
|
| 164 |
mainBtn.classList.toggle("live", action === "stop");
|
| 165 |
muteBtn.hidden = action !== "stop";
|
|
|
|
| 166 |
}
|
| 167 |
|
| 168 |
+
// ── Status handling ──────────────────────────────────────────────────────
|
| 169 |
const CAPTIONS = {
|
| 170 |
idle: "TAP TO TALK",
|
| 171 |
"creating-session": "REQUESTING A SLOT…",
|
|
|
|
| 180 |
error: "SOMETHING BROKE, TAP TO RETRY",
|
| 181 |
};
|
| 182 |
|
| 183 |
+
/** @param {string} status */
|
| 184 |
function onStatus(status) {
|
| 185 |
stage.setConversationState(status);
|
| 186 |
setCaption(CAPTIONS[status] ?? status, status === "error" ? "error" : status === "idle" || status === "closed" ? "" : "live");
|
| 187 |
+
|
| 188 |
switch (status) {
|
| 189 |
+
case "idle":
|
| 190 |
+
case "closed":
|
| 191 |
+
setMainButton("start", "Start talking");
|
| 192 |
+
break;
|
| 193 |
+
case "error":
|
| 194 |
+
setMainButton("start", "Retry");
|
| 195 |
+
break;
|
| 196 |
+
case "creating-session":
|
| 197 |
+
case "connecting":
|
| 198 |
+
setMainButton("busy", "Connecting…");
|
| 199 |
+
break;
|
| 200 |
+
case "queued":
|
| 201 |
+
setMainButton("stop", "Leave queue");
|
| 202 |
+
break;
|
| 203 |
+
case "your-turn":
|
| 204 |
+
setMainButton("join", "Join now");
|
| 205 |
+
break;
|
| 206 |
+
default:
|
| 207 |
+
// connected / user-speaking / processing / ai-speaking
|
| 208 |
+
setMainButton("stop", "End conversation");
|
| 209 |
+
break;
|
| 210 |
}
|
| 211 |
+
|
| 212 |
if (status === "user-speaking") {
|
| 213 |
subtitles.classList.remove("visible");
|
|
|
|
| 214 |
}
|
| 215 |
}
|
| 216 |
|
| 217 |
+
// ── Tool executor ────────────────────────────────────────────────────────
|
| 218 |
+
/** @param {string} name @param {string} argsJson @param {string} callId */
|
| 219 |
function runTool(name, argsJson, callId) {
|
| 220 |
if (!client) return;
|
| 221 |
+
/** @type {Record<string, unknown>} */
|
| 222 |
let args = {};
|
| 223 |
+
try {
|
| 224 |
+
args = JSON.parse(argsJson || "{}");
|
| 225 |
+
} catch {
|
| 226 |
+
// keep {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 227 |
}
|
|
|
|
| 228 |
const result = stage.runTool(name, args) ?? `Unknown tool: ${name}`;
|
| 229 |
client.sendToolOutput(callId, result);
|
| 230 |
+
// The turn continues after a tool call only when we ask for the follow-up.
|
| 231 |
client.requestResponse();
|
| 232 |
}
|
| 233 |
|
| 234 |
+
// ── Session lifecycle ────────────────────────────────────────────────────
|
| 235 |
+
async function startSession() {
|
| 236 |
+
// Everything audible hangs off the avatar's AudioContext; resume it inside
|
| 237 |
+
// the tap gesture or iOS keeps it suspended (silent).
|
| 238 |
stage.resume();
|
| 239 |
|
| 240 |
let micStream;
|
| 241 |
if (new URLSearchParams(location.search).has("fakemic")) {
|
| 242 |
+
// Dev/testing hook: a silent synthetic mic, so the session can be driven
|
| 243 |
+
// end-to-end (handshake, WS, TTS playback, lip-sync) without a real mic
|
| 244 |
+
// or a native permission prompt.
|
| 245 |
const ctx = /** @type {AudioContext} */ (stage.audioCtx);
|
| 246 |
micStream = ctx.createMediaStreamDestination().stream;
|
| 247 |
} else {
|
| 248 |
try {
|
| 249 |
+
micStream = await navigator.mediaDevices.getUserMedia({
|
| 250 |
+
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
| 251 |
+
});
|
| 252 |
} catch {
|
| 253 |
setCaption("MIC BLOCKED, ALLOW IT IN THE BROWSER AND RETRY", "error");
|
|
|
|
| 254 |
return;
|
| 255 |
}
|
| 256 |
}
|
| 257 |
+
|
| 258 |
const audioCtx = stage.audioCtx;
|
| 259 |
const voiceSink = stage.voiceSink;
|
| 260 |
+
if (!audioCtx || !voiceSink) return;
|
| 261 |
|
| 262 |
const c = new S2sWsRealtimeClient({
|
| 263 |
...(config.lb ? { sessionUrl: "api/session" } : { directUrl: settings.directUrl }),
|
| 264 |
voice: settings.voice,
|
| 265 |
instructions: effectiveInstructions(),
|
| 266 |
+
micStream,
|
| 267 |
+
audioContext: audioCtx,
|
| 268 |
+
outputNode: voiceSink,
|
| 269 |
+
workletBaseUrl: "/worklets/",
|
| 270 |
+
tools: TOOL_DEFS,
|
| 271 |
});
|
| 272 |
client = c;
|
|
|
|
| 273 |
|
| 274 |
+
c.addEventListener("status", (e) => onStatus(/** @type {CustomEvent} */ (e).detail.status));
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 275 |
|
| 276 |
+
c.addEventListener("queue", (e) => {
|
| 277 |
+
const { position } = /** @type {CustomEvent} */ (e).detail;
|
| 278 |
+
setCaption(position > 0 ? `#${position} IN LINE…` : "ALMOST THERE…", "live");
|
| 279 |
+
});
|
|
|
|
|
|
|
|
|
|
|
|
|
| 280 |
|
| 281 |
+
c.addEventListener("transcript", (e) => {
|
| 282 |
+
const { role, text } = /** @type {CustomEvent} */ (e).detail;
|
| 283 |
+
if (role === "assistant" && text) showSubtitles(text);
|
| 284 |
+
});
|
| 285 |
|
| 286 |
+
c.addEventListener("response-finished", () => {
|
| 287 |
+
fadeSubtitles();
|
| 288 |
+
});
|
| 289 |
+
|
| 290 |
+
c.addEventListener("toolcall", (e) => {
|
| 291 |
+
const { name, arguments: args, callId } = /** @type {CustomEvent} */ (e).detail;
|
| 292 |
+
runTool(name, args, callId);
|
| 293 |
});
|
|
|
|
|
|
|
| 294 |
|
| 295 |
+
c.addEventListener("server-error", (e) => {
|
| 296 |
+
console.warn("server error:", /** @type {CustomEvent} */ (e).detail.error);
|
| 297 |
+
});
|
| 298 |
+
|
| 299 |
+
c.addEventListener("error", () => {
|
| 300 |
+
void endSession();
|
| 301 |
+
});
|
| 302 |
|
| 303 |
try {
|
| 304 |
await c.connect();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 305 |
} catch (err) {
|
| 306 |
const code = /** @type {Error & {code?: string}} */ (err)?.code;
|
| 307 |
+
if (code === "limit") {
|
| 308 |
+
setCaption("DAILY CONVERSATION LIMIT REACHED, TRY AGAIN TOMORROW", "error");
|
| 309 |
+
} else if (code === "queue-full") {
|
| 310 |
+
setCaption("EVERY SEAT IS TAKEN, TRY AGAIN SHORTLY", "error");
|
| 311 |
+
} else if (code === "join-expired") {
|
| 312 |
+
setCaption("YOUR SPOT EXPIRED, TAP TO TRY AGAIN", "error");
|
| 313 |
+
} else if (code !== "aborted") {
|
| 314 |
+
console.error(err);
|
| 315 |
+
setCaption("COULD NOT CONNECT, TAP TO RETRY", "error");
|
| 316 |
+
}
|
| 317 |
await endSession(true);
|
| 318 |
+
return;
|
| 319 |
}
|
| 320 |
}
|
| 321 |
|
| 322 |
+
/** @param {boolean} [silent] Keep the current caption (e.g. an error). */
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 323 |
async function endSession(silent = false) {
|
| 324 |
const c = client;
|
| 325 |
client = null;
|
|
|
|
|
|
|
| 326 |
if (c) {
|
| 327 |
+
for (const track of c.options.micStream?.getTracks() ?? []) track.stop();
|
| 328 |
await c.close().catch(() => {});
|
| 329 |
}
|
| 330 |
stage.setConversationState("idle");
|
|
|
|
| 335 |
|
| 336 |
// ── UI events ────────────────────────────────────────────────────────────
|
| 337 |
mainBtn.addEventListener("click", () => {
|
| 338 |
+
if (mainAction === "start") void startSession();
|
| 339 |
+
else if (mainAction === "join") {
|
| 340 |
+
stage.resume(); // fresh gesture: re-arm audio before dialing
|
| 341 |
+
client?.join();
|
| 342 |
+
} else if (mainAction === "stop") void endSession();
|
| 343 |
+
});
|
| 344 |
+
|
| 345 |
+
muteBtn.addEventListener("click", () => {
|
| 346 |
+
muted = !muted;
|
| 347 |
+
client?.setMuted(muted);
|
| 348 |
+
muteBtn.classList.toggle("active", muted);
|
| 349 |
+
muteBtn.setAttribute("aria-label", muted ? "Unmute microphone" : "Mute microphone");
|
| 350 |
});
|
|
|
|
|
|
|
|
|
|
|
|
|
| 351 |
|
| 352 |
+
settingsBtn.addEventListener("click", () => {
|
| 353 |
+
inputVoice.value = settings.voice;
|
| 354 |
+
inputInstructions.value = settings.instructions;
|
| 355 |
+
inputDirectUrl.value = settings.directUrl;
|
| 356 |
+
inputSubtitles.checked = settings.subtitles;
|
| 357 |
+
settingsDialog.showModal();
|
| 358 |
+
});
|
| 359 |
|
|
|
|
| 360 |
settingsDialog.addEventListener("close", () => {
|
| 361 |
+
settings = {
|
| 362 |
+
voice: inputVoice.value || DEFAULT_VOICE,
|
| 363 |
+
instructions: inputInstructions.value,
|
| 364 |
+
directUrl: inputDirectUrl.value.trim(),
|
| 365 |
+
subtitles: inputSubtitles.checked,
|
| 366 |
+
};
|
| 367 |
+
saveSettings();
|
| 368 |
+
if (!settings.subtitles) subtitles.classList.remove("visible");
|
| 369 |
+
// Voice/instructions apply live to an ongoing session.
|
| 370 |
+
client?.updateSession({ voice: settings.voice, instructions: effectiveInstructions() });
|
| 371 |
});
|
| 372 |
|
| 373 |
+
window.addEventListener("beforeunload", () => {
|
| 374 |
+
client?.close();
|
| 375 |
+
});
|
| 376 |
|
| 377 |
// ── Boot ─────────────────────────────────────────────────────────────────
|
| 378 |
async function boot() {
|
| 379 |
+
for (const v of VOICES) {
|
| 380 |
+
const o = document.createElement("option");
|
| 381 |
+
o.value = v;
|
| 382 |
+
o.textContent = v.replaceAll("_", " ");
|
| 383 |
+
inputVoice.append(o);
|
| 384 |
+
}
|
| 385 |
|
| 386 |
+
try {
|
| 387 |
+
const resp = await fetch("api/config");
|
| 388 |
+
if (resp.ok) config = { ...config, ...(await resp.json()) };
|
| 389 |
+
} catch {
|
| 390 |
+
// defaults keep direct mode available
|
| 391 |
+
}
|
| 392 |
+
directUrlRow.hidden = !config.allowDirect;
|
| 393 |
|
| 394 |
+
setCaption("WAKING HER UP…");
|
| 395 |
+
setMainButton("busy", "Loading…");
|
| 396 |
try {
|
| 397 |
await stage.init({
|
|
|
|
| 398 |
onprogress: (ev) => {
|
| 399 |
+
if (ev.lengthComputable) {
|
| 400 |
+
const pct = Math.min(100, Math.round((ev.loaded / ev.total) * 100));
|
| 401 |
+
loading.textContent = `Loading avatar ${pct}%`;
|
| 402 |
+
}
|
| 403 |
+
},
|
| 404 |
});
|
| 405 |
+
} catch (err) {
|
| 406 |
+
console.error(err);
|
| 407 |
+
loading.textContent = "The avatar failed to load. Check the console and reload.";
|
| 408 |
+
setCaption("AVATAR FAILED TO LOAD", "error");
|
| 409 |
+
return;
|
| 410 |
+
}
|
| 411 |
+
loading.classList.add("done");
|
| 412 |
+
setCaption(CAPTIONS.idle);
|
| 413 |
+
setMainButton("start", "Start talking");
|
| 414 |
+
|
| 415 |
+
// Debug handles
|
| 416 |
+
Object.assign(window, { stage, getClient: () => client });
|
| 417 |
}
|
| 418 |
+
|
| 419 |
void boot();
|