const $ = (id) => document.getElementById(id); const cam = $("cam"), canvas = $("canvas"), statusEl = $("status"), answerEl = $("answer"), hintEl = $("hint"), langBtn = $("lang"), player = $("player"), stage = $("stage"), btnAsk = $("btn-ask"), btnDescribe = $("btn-describe"), btnLive = $("btn-live"), a11yBtn = $("a11y"); const intro = $("intro"), introBodyEl = $("intro-body"), introHowEl = $("intro-how"), introCloseBtn = $("intro-close"), introVideo = $("intro-video"), introEn = $("intro-en"), introPt = $("intro-pt"), introReplay = $("intro-replay"), introHintEl = $("intro-hint"); function vibrate(ms) { try { navigator.vibrate && navigator.vibrate(ms); } catch (e) {} } // ---- language (auto-detected from the browser; switchable by voice or button) ---- let lang = (navigator.language || "en").toLowerCase().startsWith("pt") ? "pt" : "en"; const T = { en: { idle: "Iris", listening: "Listening…", thinking: "Thinking…", hint: "Tap: describe · Hold: ask · Double-tap: live", camErr: "Camera blocked. Allow access.", err: "Something went wrong", langListen: "Listening… say your language", welcome: "Welcome to Iris. Tap the screen to describe what is in front of you. Hold to ask a question. Double-tap to turn live mode on or off, which announces new things around you.", liveOn: "Live mode on.", liveOff: "Live mode off.", confLang: "English selected.", introBody: "This is Iris, your eyes by voice. It reads your money, your bills, and your medicine, out loud.", introHow: "Tap to describe · Hold to ask · Double-tap (or say \"live mode\") for live mode", introSpeak: "This is Iris, your eyes by voice. It reads your money, bills, and medicine out loud. To use it: tap the screen to describe what is in front of you, hold and speak to ask a question, and double-tap, or say live mode, to turn live mode on or off. To close this, say close, or tap the big button.", introClose: "Close", introVideo: "Watch the demo", introHint: "Say \"close\" or tap the button", bAsk: "Ask", bDescribe: "Describe", bLive: "Live" }, pt: { idle: "Iris", listening: "Ouvindo…", thinking: "Pensando…", hint: "Toque: descrever · Segurar: perguntar · Toque duplo: ao vivo", camErr: "Câmera bloqueada. Permita o acesso.", err: "Algo deu errado", langListen: "Ouvindo… diga seu idioma", welcome: "Bem-vindo ao Iris. Toque na tela para descrever o que está à sua frente. Segure para fazer uma pergunta. Toque duas vezes para ligar ou desligar o modo ao vivo, que avisa o que aparece de novo à sua volta.", liveOn: "Modo ao vivo ligado.", liveOff: "Modo ao vivo desligado.", confLang: "Português selecionado.", introBody: "Este é o Iris, os seus olhos por voz. Ele lê o seu dinheiro, suas contas e seus remédios, em voz alta.", introHow: "Toque para descrever · Segure para perguntar · Toque duplo (ou diga \"modo ao vivo\") para o modo ao vivo", introSpeak: "Este é o Iris, os seus olhos por voz. Ele lê o seu dinheiro, suas contas e seus remédios em voz alta. Para usar: toque na tela para descrever o que está à sua frente, segure e fale para fazer uma pergunta, e toque duas vezes, ou diga modo ao vivo, para ligar ou desligar o modo ao vivo. Para fechar, diga fechar, ou toque no botão grande.", introClose: "Fechar", introVideo: "Ver o vídeo", introHint: "Diga \"fechar\" ou toque no botão", bAsk: "Perguntar", bDescribe: "Descrever", bLive: "Ao vivo" }, }; const t = (k) => T[lang][k]; const LANG_PROMPT = "Segure e diga seu idioma · Hold and say your language"; function setState(s, msg) { document.body.dataset.state = s || ""; if (msg !== undefined) statusEl.textContent = (liveOn ? "● " : "") + msg; } // UI speech via the browser voice (instructions/confirmations) function speak(text) { try { speechSynthesis.cancel(); const u = new SpeechSynthesisUtterance(text); u.lang = lang === "pt" ? "pt-BR" : "en-US"; speechSynthesis.speak(u); } catch (e) { console.error("speak:", e); } } // ---- state ---- let mode = "intro"; // "intro" while the welcome popup is open, then "normal" let busy = false; let liveOn = false, liveTimer = null; let holding = false, recording = false, holdTimer = null; const HOLD_MS = 350; langBtn.onclick = (e) => { e.stopPropagation(); lang = lang === "en" ? "pt" : "en"; document.documentElement.lang = lang; langBtn.textContent = lang.toUpperCase(); hintEl.textContent = t("hint"); updateLabels(); if (!busy) setState("", t("idle")); }; // ---- unlock audio (autoplay) on the first tap ---- const SILENT = "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA="; let audioUnlocked = false; function unlockAudio() { if (audioUnlocked) return; audioUnlocked = true; player.src = SILENT; player.play().catch(() => {}); } // ---- live camera ---- async function startCamera() { for (const c of [{ video: { facingMode: { ideal: "environment" } }, audio: false }, { video: true, audio: false }]) { try { cam.srcObject = await navigator.mediaDevices.getUserMedia(c); return; } catch (e) {} } setState("", t("camErr")); } function grabFrame() { const w = cam.videoWidth, h = cam.videoHeight; if (!w || !h) return Promise.resolve(null); canvas.width = w; canvas.height = h; canvas.getContext("2d").drawImage(cam, 0, 0, w, h); return new Promise((res) => canvas.toBlob(res, "image/jpeg", 0.85)); } // ---- microphone ---- let micStream = null, chunks = [], recorder = null; async function startRec() { try { if (!micStream) micStream = await navigator.mediaDevices.getUserMedia({ audio: true }); } catch (e) { return false; } chunks = []; recorder = new MediaRecorder(micStream); recorder.ondataavailable = (e) => { if (e.data.size) chunks.push(e.data); }; recorder.start(); return true; } function stopRec() { return new Promise((res) => { if (!recorder || recorder.state === "inactive") return res(null); recorder.onstop = () => res(chunks.length ? new Blob(chunks, { type: recorder.mimeType || "audio/webm" }) : null); recorder.stop(); }); } // ---- backend (gradio client loaded lazily, so the UI never waits on the CDN) ---- let Client, handle_file, client = null, clientReady = null; function ensureClient() { if (!clientReady) { clientReady = (async () => { const mod = await import("https://esm.sh/@gradio/client"); Client = mod.Client; handle_file = mod.handle_file; client = await Client.connect(window.location.origin); console.log("Iris connected"); })().catch((e) => { console.error("connect:", e); clientReady = null; }); } return clientReady; } // warm the model while the welcome popup plays, so the first real description is fast. // Once per session (so a reload doesn't burn ZeroGPU quota), best-effort, never blocks the UI. let warmedSession = false; try { warmedSession = sessionStorage.getItem("iris_warmed") === "1"; } catch (e) {} async function warmup() { if (warmedSession) return; warmedSession = true; try { sessionStorage.setItem("iris_warmed", "1"); } catch (e) {} try { await ensureClient(); if (!client) { warmedSession = false; return; } // let it retry if the connection failed const c = document.createElement("canvas"); c.width = 32; c.height = 32; const blob = await new Promise((r) => c.toBlob(r, "image/jpeg", 0.6)); if (blob) client.predict("/describe", { image: handle_file(blob), lang }).catch(() => {}); // warm the VLM on the GPU; discard the result // also warm speech-to-text (Whisper, CPU) so the first "hold to ask" is fast too try { const wav = await (await fetch(SILENT)).blob(); client.predict("/detect_lang", { audio: handle_file(wav) }).catch(() => {}); } catch (e) {} } catch (e) { warmedSession = false; } } async function send(frame, audio, qtext = "") { if (busy) return; busy = true; answerEl.textContent = ""; setState("thinking", t("thinking")); try { await ensureClient(); if (!client) throw new Error("client not ready"); const payload = { image: handle_file(frame), lang }; if (audio) payload.audio = handle_file(audio); if (qtext) payload.qtext = qtext; const result = await client.predict("/describe", payload); const out = Array.isArray(result.data) ? result.data[0] : result.data; console.log("Iris result:", out); if (out && out.command) { busy = false; handleCommand(out.command); return; } answerEl.textContent = (out && out.answer) || ""; setState("speaking", ""); const a = out && out.audio; let url = a && a.url; if (!url && a && a.path) url = window.location.origin + "/gradio_api/file=" + a.path; if (url) { pauseSR(); player.src = url; try { await player.play(); } catch (err) { console.error("play:", err); } } else { resetSoon(); } } catch (e) { console.error("describe:", e); setState("", t("err")); resetSoon(); } finally { busy = false; } } function resetSoon() { setTimeout(() => { resumeSR(); if (!busy) setState("", t("idle")); }, 600); } player.addEventListener("ended", () => { setTimeout(resumeSR, 700); if (!busy) setState("", t("idle")); }); // ---- welcome popup (shown on every open): pitch read aloud, closeable by voice/tap ---- const DEMO_VIDEO_URL = "https://youtu.be/h4AJOWuDCVc"; // demo video let introOpen = true, autoCloseTimer = null, introRecog = null, introSpokenSession = false; try { introSpokenSession = sessionStorage.getItem("iris_introSpoken") === "1"; } catch (e) {} const CLOSE_WORDS = /\b(close|fechar|ok|okay|start|begin|continue|continuar|comecar|entendi|pode fechar|got it)\b/i; function renderIntro() { introBodyEl.textContent = t("introBody"); introHowEl.textContent = t("introHow"); introCloseBtn.textContent = t("introClose"); introVideo.textContent = t("introVideo"); introVideo.href = DEMO_VIDEO_URL; introHintEl.textContent = t("introHint"); introEn.setAttribute("aria-pressed", lang === "en" ? "true" : "false"); introPt.setAttribute("aria-pressed", lang === "pt" ? "true" : "false"); } function showIntro() { mode = "intro"; renderIntro(); intro.hidden = false; try { introCloseBtn.focus(); } catch (e) {} speakIntro(false); // narrate on load where the browser allows (desktop) intro.addEventListener("pointerdown", onIntroFirstTap); // mobile: first tap unlocks audio + narrates document.addEventListener("keydown", onIntroKey); warmup(); // warm the model while the user listens (first description is then fast) } function onIntroFirstTap(e) { unlockAudio(); if (e.target.closest("button, a")) return; // controls handle themselves if (!introSpokenSession) speakIntro(false); } function onIntroKey(e) { if (!introOpen) return; if (e.key === "Escape") { closeIntro(); return; } if (e.key === "Tab") { // simple focus trap inside the card const f = intro.querySelectorAll("button, a[href]"); if (!f.length) return; const first = f[0], last = f[f.length - 1]; if (e.shiftKey && document.activeElement === first) { e.preventDefault(); last.focus(); } else if (!e.shiftKey && document.activeElement === last) { e.preventDefault(); first.focus(); } } } function speakIntro(force) { if (!introOpen) return; if (introSpokenSession && !force) return; // don't re-narrate on a same-session reload unlockAudio(); try { speechSynthesis.cancel(); const u = new SpeechSynthesisUtterance(t("introSpeak")); u.lang = lang === "pt" ? "pt-BR" : "en-US"; u.onstart = () => { introSpokenSession = true; try { sessionStorage.setItem("iris_introSpoken", "1"); } catch (e) {} armCloseListener(); }; u.onend = () => { if (introOpen) scheduleAutoClose(); }; speechSynthesis.speak(u); } catch (e) { /* will retry on the first tap */ } } function scheduleAutoClose() { cancelAutoClose(); autoCloseTimer = setTimeout(() => { if (introOpen) closeIntro(); }, 2500); // hands-free close after the narration } function cancelAutoClose() { if (autoCloseTimer) { clearTimeout(autoCloseTimer); autoCloseTimer = null; } } function armCloseListener() { if (introRecog) return; const SR = window.SpeechRecognition || window.webkitSpeechRecognition; if (!SR) return; introRecog = new SR(); introRecog.continuous = true; introRecog.interimResults = false; introRecog.lang = lang === "pt" ? "pt-BR" : "en-US"; introRecog.onresult = (e) => { const raw = (e.results[e.results.length - 1][0].transcript || ""); const txt = raw.normalize("NFD").replace(/[̀-ͯ]/g, ""); // strip accents if (CLOSE_WORDS.test(txt)) closeIntro(); }; introRecog.onend = () => { if (introOpen && introRecog) { try { introRecog.start(); } catch (e) {} } }; introRecog.onerror = () => {}; try { introRecog.start(); } catch (e) {} } function closeIntro() { if (!introOpen) return; introOpen = false; cancelAutoClose(); if (introRecog) { try { introRecog.stop(); } catch (e) {} introRecog = null; } try { speechSynthesis.cancel(); } catch (e) {} intro.hidden = true; document.removeEventListener("keydown", onIntroKey); mode = "normal"; try { btnDescribe.focus(); } catch (e) {} setState("", t("idle")); startCamera(); // start the app now (pitch first, camera permission after) } function setIntroLang(l) { cancelAutoClose(); if (l !== lang) { lang = l; document.documentElement.lang = lang; langBtn.textContent = lang.toUpperCase(); hintEl.textContent = t("hint"); updateLabels(); } renderIntro(); speakIntro(true); // replay in the chosen language } introCloseBtn.addEventListener("click", (e) => { e.stopPropagation(); closeIntro(); }); introVideo.addEventListener("click", (e) => { e.stopPropagation(); cancelAutoClose(); }); // let them watch (opens a new tab) introReplay.addEventListener("click", (e) => { e.stopPropagation(); cancelAutoClose(); speakIntro(true); }); introEn.addEventListener("click", (e) => { e.stopPropagation(); setIntroLang("en"); }); introPt.addEventListener("click", (e) => { e.stopPropagation(); setIntroLang("pt"); }); // ---- live mode (toggled by voice command or double-tap) ---- function handleCommand(cmd) { if (cmd === "live_on") setLive(true); else if (cmd === "live_off") setLive(false); } const LIVE_INTERVAL = 3500; // check the scene every 3.5s const LIVE_THRESHOLD = 15; // mean per-pixel change (0-255) to trigger the VLM const LIVE_COOLDOWN = 5000; // quiet window after an auto-alert -> gap for the user to ask let lastSig = null, history = []; // recent descriptions (anti-repetition) let suppressLiveUntil = 0; // pause auto-alerts after an alert OR a user question (their question wins) // in-browser object detection gates the live mode (semantic change, not pixels). // Falls back to the cheap pixel-diff if the detector can't load. let detector = null, detectorTried = false; let seenClasses = new Map(); // class name -> last tick index it was seen let tickN = 0; const CLASS_TTL = 4; // ticks a class is remembered before it can re-trigger const PERSON_GONE = 4; // ticks a person must be absent before a re-entry counts as new (~14s) async function loadDetector() { if (detector || detectorTried || !window.cocoSsd) return; detectorTried = true; try { detector = await window.cocoSsd.load({ base: "lite_mobilenet_v2" }); console.log("Iris: object detector ready"); } catch (e) { console.warn("detector load failed, using pixel-diff:", e); } } function setLive(on) { liveOn = on; btnLive.setAttribute("aria-pressed", on ? "true" : "false"); btnLive.classList.toggle("on", on); vibrate(on ? [20, 40, 20] : 20); speak(on ? t("liveOn") : t("liveOff")); setState("", t("idle")); clearInterval(liveTimer); liveTimer = null; lastSig = null; history = []; seenClasses = new Map(); if (on) { loadDetector(); liveTimer = setInterval(liveTick, LIVE_INTERVAL); startListening(); // baseline: describe the scene once shortly after turning on setTimeout(async () => { if (liveOn && !busy && player.paused) { const f = await grabFrame(); if (f) sendWatch(f); } }, 900); } else stopListening(); } // ---- hands-free continuous listening (live mode) via Web Speech API ---- let recog = null, listening = false, srPaused = false; function setupRecognition() { const SR = window.SpeechRecognition || window.webkitSpeechRecognition; if (!SR) { console.warn("SpeechRecognition not supported in this browser"); return null; } const r = new SR(); r.continuous = true; r.interimResults = false; r.onresult = (e) => { if (srPaused || busy) return; const txt = e.results[e.results.length - 1][0].transcript.trim(); if (txt) onVoice(txt); }; r.onend = () => { if (listening && !srPaused) { try { r.start(); } catch (e) {} } }; // keep alive r.onerror = (e) => { console.log("SR:", e.error); }; return r; } function startListening() { if (!recog) recog = setupRecognition(); if (!recog || listening) return; listening = true; recog.lang = lang === "pt" ? "pt-BR" : "en-US"; try { recog.start(); } catch (e) {} } function stopListening() { listening = false; if (recog) { try { recog.stop(); } catch (e) {} } } // stop the recognizer WHILE Iris speaks (so it never transcribes its own voice) function pauseSR() { srPaused = true; if (recog) { try { recog.stop(); } catch (e) {} } } function resumeSR() { srPaused = false; if (listening && recog) { try { recog.start(); } catch (e) {} } } async function onVoice(txt, tries = 0) { suppressLiveUntil = performance.now() + 12000; // the user's question wins: mute auto-alerts for a while if (busy) { if (tries < 3) setTimeout(() => onVoice(txt, tries + 1), 700); return; } // retry if a watch is in flight const frame = await grabFrame(); if (frame) send(frame, null, txt); // text question (from browser speech recognition) } // cheap visual signature (32x32 gray) to detect change without calling the model function frameSignature() { if (!cam.videoWidth) return null; const c = document.createElement("canvas"); c.width = 32; c.height = 32; const ctx = c.getContext("2d"); ctx.drawImage(cam, 0, 0, 32, 32); const d = ctx.getImageData(0, 0, 32, 32).data; const g = new Uint8Array(1024); for (let i = 0; i < 1024; i++) g[i] = (d[i * 4] + d[i * 4 + 1] + d[i * 4 + 2]) / 3; return g; } function changeAmount(a, b) { if (!a || !b) return 999; let s = 0; for (let i = 0; i < a.length; i++) s += Math.abs(a[i] - b[i]); return s / a.length; } async function liveTick() { if (busy || mode !== "normal" || !liveOn || !player.paused) return; if (performance.now() < suppressLiveUntil) return; // user just asked / we just spoke -> stay quiet, let them talk if (!detector || !cam.videoWidth) return; // alerts are person-arrival only -> need the detector tickN++; // Live alerts fire ONLY when a NEW person enters (per the spec: rare ambient alerts, not narration). // Re-describing objects/movement makes the small VLM hallucinate and repeat; objects/text/colors are // answered on demand (the user asks), which is accurate. So we gate strictly on a fresh "person". let personNew = false; try { const preds = await detector.detect(cam, 5); let hasPerson = false; for (const p of preds) if (p.class === "person" && p.score >= 0.6) hasPerson = true; if (hasPerson) { const last = seenClasses.get("person"); if (last === undefined || tickN - last > PERSON_GONE) personNew = true; // absent a while -> a real arrival seenClasses.set("person", tickN); } } catch (e) { return; } if (personNew) { suppressLiveUntil = performance.now() + LIVE_COOLDOWN; // one alert, then a quiet gap to ask const frame = await grabFrame(); if (frame) sendWatch(frame, "person"); } } // word-set similarity (accent-insensitive) — the model often re-emits a near-identical // line, so we drop it client-side instead of trusting the "do not repeat" prompt. function words(s) { return (s || "").toLowerCase().normalize("NFD").replace(/[̀-ͯ]/g, "") .replace(/[^a-z0-9 ]/g, " ").split(/\s+/).filter((w) => w.length > 2); // NFD + strip diacritics } function tooSimilar(a, b) { const A = new Set(words(a)), B = new Set(words(b)); if (!A.size || !B.size) return false; let inter = 0; for (const w of A) if (B.has(w)) inter++; return inter / (A.size + B.size - inter) >= 0.5; // Jaccard >= 0.5 -> basically the same alert } async function sendWatch(frame, hint = "") { if (busy) return; busy = true; try { await ensureClient(); if (!client) return; const result = await client.predict("/watch", { image: handle_file(frame), prev: history.join(" · "), lang, hint }); const out = Array.isArray(result.data) ? result.data[0] : result.data; if (out && out.speak && out.answer && !history.some((h) => tooSimilar(h, out.answer))) { history.push(out.answer); if (history.length > 5) history.shift(); answerEl.textContent = out.answer; setState("speaking", ""); const a = out.audio; let url = a && a.url; if (!url && a && a.path) url = window.location.origin + "/gradio_api/file=" + a.path; if (url) { pauseSR(); player.src = url; try { await player.play(); } catch (e) { console.error("play:", e); } } } } catch (e) { console.error("watch:", e); } finally { busy = false; } } // ---- interaction ---- stage.addEventListener("pointerdown", async () => { if (mode !== "normal") return; // welcome popup is open if (liveOn) { speechSynthesis.cancel(); player.pause(); } if (busy) return; unlockAudio(); holding = true; stage.classList.add("armed"); holdTimer = setTimeout(async () => { recording = await startRec(); if (recording) setState("listening", t("listening")); }, HOLD_MS); }); let lastTapTime = 0, tapTimer = null; const DOUBLE_MS = 320; // double-tap window async function endPress() { if (mode !== "normal") return; // welcome popup is open if (!holding) return; holding = false; stage.classList.remove("armed"); clearTimeout(holdTimer); if (recording) { // held and spoke = a question recording = false; const frame = await grabFrame(); const audio = await stopRec(); if (frame) send(frame, audio); else setState("", t("camErr")); return; } // quick tap: double-tap toggles live mode; single tap describes const now = performance.now(); if (now - lastTapTime < DOUBLE_MS) { lastTapTime = 0; if (tapTimer) { clearTimeout(tapTimer); tapTimer = null; } setLive(!liveOn); } else { lastTapTime = now; if (tapTimer) clearTimeout(tapTimer); tapTimer = setTimeout(async () => { tapTimer = null; const frame = await grabFrame(); if (frame) send(frame, null); else setState("", t("camErr")); }, DOUBLE_MS); } } stage.addEventListener("pointerup", endPress); stage.addEventListener("pointercancel", endPress); // ---- explicit buttons (low vision / keyboard / screen reader) ---- btnDescribe.addEventListener("click", async (e) => { e.stopPropagation(); if (mode !== "normal" || busy) return; unlockAudio(); vibrate(10); const frame = await grabFrame(); if (frame) send(frame, null); else setState("", t("camErr")); }); let askRecBtn = false; btnAsk.addEventListener("pointerdown", async (e) => { e.stopPropagation(); if (mode !== "normal" || busy) return; unlockAudio(); vibrate(10); askRecBtn = await startRec(); if (askRecBtn) setState("listening", t("listening")); }); async function btnAskEnd(e) { if (e) e.stopPropagation(); if (!askRecBtn) return; askRecBtn = false; const frame = await grabFrame(); const audio = await stopRec(); if (frame) send(frame, audio); else setState("", t("camErr")); } btnAsk.addEventListener("pointerup", btnAskEnd); btnAsk.addEventListener("pointerleave", btnAskEnd); btnAsk.addEventListener("pointercancel", btnAskEnd); btnLive.addEventListener("click", (e) => { e.stopPropagation(); if (mode !== "normal") return; unlockAudio(); setLive(!liveOn); }); // ---- accessibility: max contrast + larger text (persisted) ---- let boost = false; try { boost = localStorage.getItem("iris_a11y") === "1"; } catch (e) {} document.body.classList.toggle("a11y-boost", boost); a11yBtn.setAttribute("aria-pressed", boost ? "true" : "false"); a11yBtn.addEventListener("click", (e) => { e.stopPropagation(); boost = !boost; document.body.classList.toggle("a11y-boost", boost); a11yBtn.setAttribute("aria-pressed", boost ? "true" : "false"); try { localStorage.setItem("iris_a11y", boost ? "1" : "0"); } catch (e) {} vibrate(10); }); // button labels per language function updateLabels() { btnAsk.querySelector(".ctl-lbl").textContent = t("bAsk"); btnDescribe.querySelector(".ctl-lbl").textContent = t("bDescribe"); btnLive.querySelector(".ctl-lbl").textContent = t("bLive"); } // ---- boot ---- (async () => { document.documentElement.lang = lang; langBtn.textContent = lang.toUpperCase(); updateLabels(); hintEl.textContent = t("hint"); setState("", t("idle")); showIntro(); // welcome popup on every open (the camera starts when it closes) loadDetector(); // preload the object detector in the background (non-blocking) ensureClient(); // connect in the background; the UI never waits on it })();