live mode: in-browser object-detection gate + browser-language onboarding

- COCO-SSD (TF.js) gates the live mode: the VLM only wakes when a NEW object/person
class appears (semantic change, not pixels); per-class TTL avoids re-announcing;
graceful fallback to the pixel-diff if the detector can't load; scene baseline on start
- onboarding: default to the browser language, speak the welcome on first tap and offer
to switch by voice (listens 5s); 'tap to begin' prompt (screen-reader announced)
- detector model runs in the browser (~5MB) — does not count toward <=32B / Tiny Titan

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

frontend/app.js

@@ -19,6 +19,8 @@ const T = {
         langListen: "Listening… say your language",
         welcome: "Welcome to Iris. Tap the screen to describe what is in front of you. Hold to ask a question. Double-tap to turn live mode on or off, which announces new things around you.",
         liveOn: "Live mode on.", liveOff: "Live mode off.", confLang: "English selected.",
+        startTap: "Iris. Tap the screen to begin.",
+        welcomeAsk: "Welcome to Iris, in English. Tap the screen to describe what is in front of you, hold to ask a question, double-tap for live mode. To use Portuguese instead, say 'português' now.",
         bAsk: "Ask", bDescribe: "Describe", bLive: "Live" },
   pt: { idle: "Iris", listening: "Ouvindo…", thinking: "Pensando…",
         hint: "Toque: descrever · Segurar: perguntar · Toque duplo: ao vivo",
@@ -26,6 +28,8 @@ const T = {
         langListen: "Ouvindo… diga seu idioma",
         welcome: "Bem-vindo ao Iris. Toque na tela para descrever o que está à sua frente. Segure para fazer uma pergunta. Toque duas vezes para ligar ou desligar o modo ao vivo, que avisa o que aparece de novo à sua volta.",
         liveOn: "Modo ao vivo ligado.", liveOff: "Modo ao vivo desligado.", confLang: "Português selecionado.",
+        startTap: "Iris. Toque na tela para começar.",
+        welcomeAsk: "Bem-vindo ao Iris, em português. Toque na tela para descrever o que está à sua frente, segure para perguntar, e toque duas vezes para o modo ao vivo. Para usar em inglês, diga 'inglês' agora.",
         bAsk: "Perguntar", bDescribe: "Descrever", bLive: "Ao vivo" },
 };
 const t = (k) => T[lang][k];
@@ -61,7 +65,7 @@ langBtn.onclick = (e) => {
   langBtn.textContent = lang.toUpperCase();
   hintEl.textContent = onboarded ? t("hint") : "";
   updateLabels();
-  if (!busy) setState("", onboarded ? t("idle") : LANG_PROMPT);
+  if (!busy) setState("", onboarded ? t("idle") : t("startTap"));
 };
 
 // ---- unlock audio (autoplay) on the first tap ----
@@ -146,25 +150,52 @@ async function send(frame, audio, qtext = "") {
 function resetSoon() { setTimeout(() => { srMuted = false; if (!busy) setState("", t("idle")); }, 600); }
 player.addEventListener("ended", () => { srMuted = false; if (!busy) setState("", t("idle")); });
 
-// ---- choose language by voice (onboarding) ----
-async function chooseLang(audio) {
-  if (!audio || !client) { setState("", LANG_PROMPT); return; }
-  busy = true; setState("thinking", t("thinking"));
-  try {
-    const result = await client.predict("/detect_lang", { audio: handle_file(audio) });
-    const out = Array.isArray(result.data) ? result.data[0] : result.data;
-    lang = (out && out.lang === "en") ? "en" : "pt";
+// ---- first-run onboarding: default to the browser language, speak it, offer to switch ----
+function onboard() {
+  onboarded = true; mode = "normal";
+  try { localStorage.setItem("iris_onboarded", "1"); } catch (e) {}
+  hintEl.textContent = t("hint");
+  updateLabels();
+  const u = new SpeechSynthesisUtterance(t("welcomeAsk"));
+  u.lang = lang === "pt" ? "pt-BR" : "en-US";
+  u.onend = listenForLangChoice;          // after the welcome, listen for a language switch
+  setState("speaking", t("idle"));
+  try { speechSynthesis.cancel(); speechSynthesis.speak(u); }
+  catch (e) { listenForLangChoice(); }
+}
+
+function listenForLangChoice() {
+  const SR = window.SpeechRecognition || window.webkitSpeechRecognition;
+  if (!SR) { setState("", t("idle")); return; }
+  const r = new SR();
+  r.lang = lang === "pt" ? "pt-BR" : "en-US";
+  r.continuous = false; r.interimResults = false;
+  let handled = false;
+  const finish = () => { if (!handled) { handled = true; setState("", t("idle")); } };
+  const to = setTimeout(() => { try { r.stop(); } catch (e) {} finish(); }, 5000);
+  r.onresult = (e) => {
+    handled = true; clearTimeout(to);
+    const txt = (e.results[0][0].transcript || "").toLowerCase();
+    if (txt.includes("ingl") || txt.includes("english")) switchLang("en");
+    else if (txt.includes("portug") || txt.includes("brasil")) switchLang("pt");
+    else setState("", t("idle"));
+    try { r.stop(); } catch (e) {}
+  };
+  r.onerror = () => { clearTimeout(to); finish(); };
+  setState("listening", t("listening"));
+  try { r.start(); } catch (e) { finish(); }
+}
+
+function switchLang(newLang) {
+  if (newLang !== lang) {
+    lang = newLang;
     document.documentElement.lang = lang;
     langBtn.textContent = lang.toUpperCase();
-    onboarded = true; mode = "normal";
-    try { localStorage.setItem("iris_onboarded", "1"); } catch (e) {}
     hintEl.textContent = t("hint");
-    setState("", t("idle"));
-    speak(t("confLang") + " " + t("welcome"));
-  } catch (e) {
-    console.error("detect_lang:", e);
-    setState("", LANG_PROMPT);
-  } finally { busy = false; }
+    updateLabels();
+  }
+  setState("", t("idle"));
+  speak(t("confLang") + " " + t("welcome"));
 }
 
 // ---- live mode (toggled by voice command or double-tap) ----
@@ -176,6 +207,22 @@ const LIVE_INTERVAL = 2500;   // check the scene every 2.5s
 const LIVE_THRESHOLD = 14;    // mean per-pixel change (0-255) to trigger the VLM
 let lastSig = null, history = [];   // recent descriptions (anti-repetition)
 
+// in-browser object detection gates the live mode (semantic change, not pixels).
+// Falls back to the cheap pixel-diff if the detector can't load.
+let detector = null, detectorTried = false;
+let seenClasses = new Map();   // class name -> last tick index it was seen
+let tickN = 0;
+const CLASS_TTL = 4;           // ticks a class is remembered before it can re-trigger
+
+async function loadDetector() {
+  if (detector || detectorTried || !window.cocoSsd) return;
+  detectorTried = true;
+  try {
+    detector = await window.cocoSsd.load({ base: "lite_mobilenet_v2" });
+    console.log("Iris: object detector ready");
+  } catch (e) { console.warn("detector load failed, using pixel-diff:", e); }
+}
+
 function setLive(on) {
   liveOn = on;
   btnLive.setAttribute("aria-pressed", on ? "true" : "false");
@@ -184,9 +231,16 @@ function setLive(on) {
   speak(on ? t("liveOn") : t("liveOff"));
   setState("", t("idle"));
   clearInterval(liveTimer); liveTimer = null;
-  lastSig = null; history = [];
-  if (on) { liveTimer = setInterval(liveTick, LIVE_INTERVAL); startListening(); }
-  else stopListening();
+  lastSig = null; history = []; seenClasses = new Map();
+  if (on) {
+    loadDetector();
+    liveTimer = setInterval(liveTick, LIVE_INTERVAL);
+    startListening();
+    // baseline: describe the scene once shortly after turning on
+    setTimeout(async () => {
+      if (liveOn && !busy && player.paused) { const f = await grabFrame(); if (f) sendWatch(f); }
+    }, 900);
+  } else stopListening();
 }
 
 // ---- hands-free continuous listening (live mode) via Web Speech API ----
@@ -243,6 +297,30 @@ function changeAmount(a, b) {
 
 async function liveTick() {
   if (busy || mode !== "normal" || !liveOn || !player.paused) return;
+  tickN++;
+
+  // semantic gate: only wake the VLM when a NEW object/person class appears
+  if (detector && cam.videoWidth) {
+    let preds = null;
+    try { preds = await detector.detect(cam, 10); } catch (e) {}
+    if (preds) {
+      let novel = false;
+      for (const p of preds) {
+        if (p.score < 0.55) continue;
+        const last = seenClasses.get(p.class);
+        if (last === undefined || tickN - last > CLASS_TTL) novel = true;
+        seenClasses.set(p.class, tickN);
+      }
+      for (const [c, n] of seenClasses) if (tickN - n > CLASS_TTL) seenClasses.delete(c);
+      if (novel) {
+        const frame = await grabFrame();
+        if (frame) sendWatch(frame);
+      }
+      return;
+    }
+  }
+
+  // fallback: cheap pixel-diff gate
   const sig = frameSignature();
   if (!sig) return;
   if (lastSig && changeAmount(lastSig, sig) < LIVE_THRESHOLD) return; // static scene -> stay quiet
@@ -273,10 +351,8 @@ async function sendWatch(frame) {
 // ---- interaction ----
 stage.addEventListener("pointerdown", async () => {
   if (mode === "onboarding") {
-    if (busy) return;
     unlockAudio();
-    recording = await startRec();
-    if (recording) setState("listening", t("langListen"));
+    onboard();             // speak welcome in the browser language + offer to switch
     return;
   }
   if (liveOn) { speechSynthesis.cancel(); player.pause(); }
@@ -294,12 +370,7 @@ let lastTapTime = 0, tapTimer = null;
 const DOUBLE_MS = 320;   // double-tap window
 
 async function endPress() {
-  if (mode === "onboarding") {
-    if (!recording) return;
-    recording = false;
-    const audio = await stopRec();
-    return chooseLang(audio);
-  }
+  if (mode === "onboarding") return;   // onboarding is handled on pointerdown
   if (!holding) return;
   holding = false;
   stage.classList.remove("armed");
@@ -394,8 +465,9 @@ function updateLabels() {
   langBtn.textContent = lang.toUpperCase();
   updateLabels();
   hintEl.textContent = onboarded ? t("hint") : "";
-  setState("", onboarded ? t("idle") : LANG_PROMPT);
+  setState("", onboarded ? t("idle") : t("startTap"));
   await startCamera();
-  try { client = await Client.connect(window.location.origin); console.log("Iris conectado"); }
+  loadDetector();   // preload the object detector in the background (non-blocking)
+  try { client = await Client.connect(window.location.origin); console.log("Iris connected"); }
   catch (e) { console.error("connect:", e); setState("", t("err")); }
 })();

frontend/index.html

@@ -54,6 +54,10 @@
   </nav>
 
   <audio id="player" playsinline></audio>
+
+  <!-- in-browser object detection (gates live mode); falls back to pixel-diff if unavailable -->
+  <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@4"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/coco-ssd@2"></script>
   <script type="module" src="/static/app.js"></script>
 </body>
 </html>