Talk it through
Ask questions in your own words. Gemini Live listens, understands and responds with a natural teaching voice.
const app = document.querySelector("#app");
const toast = document.querySelector("#toast");
const API = "/api/v1";
function showToast(message) {
toast.textContent = message;
toast.classList.add("show");
clearTimeout(showToast.timer);
showToast.timer = setTimeout(() => toast.classList.remove("show"), 3400);
}
function escapeHtml(value = "") {
return String(value).replace(/[&<>"']/g, (char) => ({
"&": "&", "<": "<", ">": ">", '"': """, "'": "'",
})[char]);
}
function lessonId() {
const match = location.pathname.match(/^\/lesson\/([^/]+)/);
return match && match[1] !== "new" ? match[1] : null;
}
async function fetchLesson(id) {
const response = await fetch(`${API}/sessions/${id}`);
if (!response.ok) throw new Error("Lesson not found");
return response.json();
}
function setActiveNavigation() {
document.querySelectorAll("[data-nav]").forEach((link) => {
link.classList.toggle("active", link.dataset.nav === location.pathname);
});
}
function landingPage() {
return `
Gargi listens, explains, draws and adapts in real time—turning every question into a lesson made just for you. Speak naturally, watch ideas take shape on the whiteboard, and check your understanding without leaving the lesson. Ask questions in your own words. Gemini Live listens, understands and responds with a natural teaching voice. Gargi turns explanations into diagrams, equations and step-by-step whiteboard notes. Quick adaptive quizzes uncover misconceptions and shape the next explanation around you. Give Gargi a topic and a little context. Your classroom will be ready in seconds.Learn anything.
Understand it deeply.Teaching that meets you
where you are.Talk it through
See the idea
Make it stick
What are you curious about?
Start the conversation and Gargi will build diagrams, equations and notes around your questions.
${escapeHtml(source)}`;
return `Tap to ask Gargi a question
Tap again when you finish speakingThree quick questions based on your latest conversation.
${escapeHtml(lesson.topic)} · ${Math.floor(lesson.messages.length / 2)} conversation turns
${escapeHtml(latest?.summary || lesson.objective || "Start a conversation to build your lesson summary.")}
Gargi will track and clarify misconceptions as your lesson develops.
"}${escapeHtml(latest?.suggested_follow_up || "Continue the lesson and ask for a practical example.")}
Keep exploring →Level: ${escapeHtml(lesson.learner_level)}
Language: ${escapeHtml(lesson.language)}
Status: ${escapeHtml(lesson.status)}
${escapeHtml(result.explanation)}
${escapeHtml(result.explanation)}
`; board.querySelector("#inline-review-board").addEventListener("click", showWhiteboard); board.querySelector("#inline-next-question").addEventListener("click", () => { if (index === currentQuiz.length - 1) { showToast(`Quiz complete: ${correct}/${currentQuiz.length} correct.`); showWhiteboard(); } else { index++; showQuestion(); } }); }); } showQuestion(); } function setAvatarMouth(level = 0) { if (!live2dModel?.internalModel?.coreModel) return; try { const coreModel = live2dModel.internalModel.coreModel; const value = Math.max(0, Math.min(1, level)); if (typeof coreModel.setParameterValueById === "function") { coreModel.setParameterValueById("ParamMouthOpenY", value); } } catch {} } function stopFallbackLipSync() { clearInterval(fallbackLipSyncTimer); fallbackLipSyncTimer = null; panel.style.setProperty("--talk-level", "0"); setAvatarMouth(0); } function startFallbackLipSync() { clearInterval(fallbackLipSyncTimer); fallbackLipSyncTimer = setInterval(() => { const level = 0.18 + Math.random() * 0.62; panel.style.setProperty("--talk-level", level.toFixed(2)); setAvatarMouth(level); }, 90); } async function initializeLive2DAvatar() { if (!avatarCanvas || !avatarMount || !window.PIXI?.live2d?.Live2DModel) { if (avatarLabel) avatarLabel.textContent = "Live2D runtime unavailable"; return; } try { live2dApp = new PIXI.Application({ view: avatarCanvas, resizeTo: avatarMount, backgroundAlpha: 0, antialias: true, autoDensity: true, }); live2dModel = await PIXI.live2d.Live2DModel.from("/static/vendor/live2d/haru/haru_greeter_t03.model3.json", { autoInteract: true, }); live2dModel.anchor.set(0.5, 0.52); live2dApp.stage.addChild(live2dModel); const fitModel = () => { const width = live2dApp.renderer.width; const height = live2dApp.renderer.height; if (!width || !height) return; live2dModel.scale.set(1); live2dModel.position.set(width / 2, height * 0.78); const scale = Math.min(width / live2dModel.width, height / live2dModel.height) * 1.68; live2dModel.scale.set(scale); }; fitModel(); window.addEventListener("resize", fitModel); avatarFallback?.classList.add("hidden"); avatarMount.classList.add("live2d-ready"); avatarLabel.textContent = "Haru Live2D teacher ready"; live2dModel.on("hit", () => live2dModel.motion("Tap")); } catch (error) { console.warn("Live2D avatar failed to load", error); avatarLabel.textContent = "Live2D fallback active"; } } async function ensurePlaybackContext() { playbackContext ||= new AudioContext({sampleRate: 24000}); if (playbackContext.state === "suspended") await playbackContext.resume(); } async function playPcm(buffer) { await ensurePlaybackContext(); heardLiveAudioThisTurn = true; window.speechSynthesis?.cancel(); clearTimeout(fallbackSpeechTimer); fallbackSpeechBuffer = ""; stopFallbackLipSync(); const pcm = new Int16Array(buffer); const samples = new Float32Array(pcm.length); let sum = 0; for (let index = 0; index < pcm.length; index++) { samples[index] = pcm[index] / 32768; sum += samples[index] * samples[index]; } const rms = Math.sqrt(sum / Math.max(samples.length, 1)); const talkLevel = Math.min(1, rms * 8); panel.style.setProperty("--talk-level", talkLevel.toFixed(2)); setAvatarMouth(talkLevel); const audio = playbackContext.createBuffer(1, samples.length, 24000); audio.copyToChannel(samples, 0); const source = playbackContext.createBufferSource(); source.buffer = audio; source.connect(playbackContext.destination); nextPlayTime = Math.max(nextPlayTime, playbackContext.currentTime + 0.04); source.start(nextPlayTime); nextPlayTime += audio.duration; playbackSources.add(source); activeSources = playbackSources.size; setState("speaking", "Gargi is speaking"); source.onended = () => { playbackSources.delete(source); activeSources = playbackSources.size; if (activeSources <= 0 && !recording) { panel.style.setProperty("--talk-level", "0"); setAvatarMouth(0); setState("ready", "Gargi is ready"); } }; } async function startMicrophone() { if (startingMicrophone || recording) return; if (!socketReady) return showToast("The live classroom is still connecting."); if (!navigator.mediaDevices?.getUserMedia) return showToast("This browser does not support microphone capture."); startingMicrophone = true; try { await ensurePlaybackContext(); stopPlayback(false); mediaStream = await navigator.mediaDevices.getUserMedia({ audio: {channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true}, }); captureContext = new AudioContext(); await captureContext.audioWorklet.addModule("/static/mic-processor.js"); micSource = captureContext.createMediaStreamSource(mediaStream); micNode = new AudioWorkletNode(captureContext, "gargi-mic-processor"); const silentGain = captureContext.createGain(); silentGain.gain.value = 0; micNode.port.onmessage = (event) => { if (recording && socket?.readyState === WebSocket.OPEN) socket.send(event.data); }; micSource.connect(micNode); micNode.connect(silentGain).connect(captureContext.destination); heardLiveAudioThisTurn = false; fallbackSpeechText = ""; fallbackSpeechBuffer = ""; clearTimeout(fallbackSpeechTimer); recording = true; micButton.classList.add("recording"); setState("listening", "Listening"); } catch (error) { showToast(error.name === "NotAllowedError" ? "Microphone permission was denied." : "The microphone could not start."); await stopMicrophone(false); } finally { startingMicrophone = false; } } function stopPlayback(updateState = true) { window.speechSynthesis?.cancel(); clearTimeout(fallbackSpeechTimer); fallbackSpeechBuffer = ""; stopFallbackLipSync(); playbackSources.forEach((source) => { source.onended = null; try { source.stop(); } catch {} try { source.disconnect(); } catch {} }); playbackSources.clear(); activeSources = 0; nextPlayTime = playbackContext?.currentTime || 0; panel.style.setProperty("--talk-level", "0"); setAvatarMouth(0); if (updateState && !recording) setState("ready", "Gargi is ready"); } function speakBrowserFallback(text, cancelExisting = false) { if (!window.speechSynthesis || !text.trim()) return; if (cancelExisting) window.speechSynthesis.cancel(); const utterance = new SpeechSynthesisUtterance(text.trim()); utterance.lang = "en-US"; utterance.rate = 0.95; utterance.pitch = 1.08; const voices = window.speechSynthesis.getVoices(); utterance.voice = voices.find((voice) => /Jenny|Aria|Zira|Samantha|Female/i.test(voice.name)) || voices.find((voice) => voice.lang?.startsWith("en")) || null; utterance.onstart = () => { startFallbackLipSync(); setState("speaking", "Gargi is speaking"); }; utterance.onend = () => { if (!window.speechSynthesis.speaking) stopFallbackLipSync(); if (!recording) setState("ready", "Ask a follow-up"); }; utterance.onerror = stopFallbackLipSync; window.speechSynthesis.speak(utterance); } function flushFallbackSpeech(force = false) { if (heardLiveAudioThisTurn || !fallbackSpeechBuffer.trim()) return; const source = fallbackSpeechBuffer.trimStart(); let textToSpeak = ""; let remaining = source; if (force) { textToSpeak = source; remaining = ""; } else { const sentenceEnd = Math.max(source.lastIndexOf("."), source.lastIndexOf("?"), source.lastIndexOf("!")); if (sentenceEnd >= 0) { textToSpeak = source.slice(0, sentenceEnd + 1); remaining = source.slice(sentenceEnd + 1); } else if (source.length > 130) { const splitAt = Math.max(source.lastIndexOf(","), source.lastIndexOf(" ")); textToSpeak = source.slice(0, splitAt > 30 ? splitAt : source.length); remaining = source.slice(textToSpeak.length); } } fallbackSpeechBuffer = remaining.trimStart(); if (textToSpeak.trim()) speakBrowserFallback(textToSpeak, false); } function scheduleRealtimeFallbackSpeech() { if (heardLiveAudioThisTurn) return; clearTimeout(fallbackSpeechTimer); const delay = /[.!?]\s*$/.test(fallbackSpeechBuffer.trim()) ? 80 : 520; fallbackSpeechTimer = setTimeout(() => flushFallbackSpeech(false), delay); } async function stopMicrophone(sendEnd = true) { if (!recording && !mediaStream && !captureContext) return; const contextToClose = captureContext; recording = false; micButton.classList.remove("recording"); if (sendEnd && socket?.readyState === WebSocket.OPEN) { socket.send(JSON.stringify({type: "audio_end"})); setState("thinking", "Gargi is thinking"); } else { setState("ready", "Gargi is ready"); } if (micNode?.port) micNode.port.onmessage = null; micNode?.disconnect(); micSource?.disconnect(); mediaStream?.getTracks().forEach((track) => track.stop()); mediaStream = captureContext = micNode = micSource = null; if (contextToClose && contextToClose.state !== "closed") { contextToClose.close().catch(() => {}); } } socket = new WebSocket(`${protocol}//${location.host}${API}/sessions/${id}/live`); socket.binaryType = "arraybuffer"; socket.onopen = () => setState("connecting", "Connecting"); socket.onmessage = async (event) => { if (event.data instanceof ArrayBuffer) { if (recording) return; return playPcm(event.data); } const message = JSON.parse(event.data); if (message.type === "ready") { socketReady = true; setState("ready", "Gargi is ready"); } else if (message.type === "input_transcription") { const text = message.text.trim(); if (pendingTyped && text.toLowerCase() === pendingTyped.toLowerCase()) pendingTyped = ""; else if (text) appendMessage("student", text, true); } else if (message.type === "output_transcription") { fallbackSpeechText += message.text; fallbackSpeechBuffer += message.text; scheduleRealtimeFallbackSpeech(); appendMessage("teacher", message.text, true); if (!recording) setState("speaking", "Gargi is speaking"); } else if (message.type === "lesson_payload") { currentBoardBlocks = message.whiteboard_blocks || []; currentQuiz = message.quiz || []; board.innerHTML = renderBoard(message.whiteboard_blocks); boardStatus.textContent = "Updated just now"; quizLink.classList.remove("hidden"); showToast(message.fallback ? (message.notice || "Fallback whiteboard and quiz are ready.") : "Whiteboard and quiz are ready."); } else if (message.type === "interrupted") { stopPlayback(false); setState(recording ? "listening" : "ready", recording ? "Listening" : "Gargi is ready"); } else if (message.type === "turn_complete") { if (!heardLiveAudioThisTurn && fallbackSpeechText.trim()) { clearTimeout(fallbackSpeechTimer); flushFallbackSpeech(true); } heardLiveAudioThisTurn = false; fallbackSpeechText = ""; fallbackSpeechBuffer = ""; if (activeSources <= 0) setState("ready", "Ask a follow-up"); } else if (message.type === "error") { const retry = message.retry_delay ? ` Try again in ${message.retry_delay}.` : ""; showToast(`${message.message || "The live classroom hit a problem."}${retry}`); setState("error", "Connection problem"); } }; socket.onclose = () => { socketReady = false; setState("error", "Voice disconnected"); }; socket.onerror = () => showToast("Could not connect to the live classroom."); initializeLive2DAvatar(); quizLink?.addEventListener("click", (event) => { event.preventDefault(); if (quizLink.textContent === "Whiteboard") showWhiteboard(); else showInlineQuiz(); }); micButton.addEventListener("click", () => recording ? stopMicrophone(true) : startMicrophone()); form.addEventListener("submit", async (event) => { event.preventDefault(); const input = document.querySelector("#ask-input"); const text = input.value.trim(); if (!text) return; if (!socketReady) return showToast("The live classroom is still connecting."); await ensurePlaybackContext(); heardLiveAudioThisTurn = false; fallbackSpeechText = ""; fallbackSpeechBuffer = ""; clearTimeout(fallbackSpeechTimer); pendingTyped = text; appendMessage("student", text); socket.send(JSON.stringify({type: "text", text})); input.value = ""; setState("thinking", "Gargi is thinking"); }); window.addEventListener("beforeunload", () => { if (socket?.readyState === WebSocket.OPEN) socket.send(JSON.stringify({type: "close"})); stopPlayback(false); stopFallbackLipSync(); live2dApp?.destroy(true); mediaStream?.getTracks().forEach((track) => track.stop()); }); } render();