live mode: stop self-hearing, prioritize user questions, kill repetition/hallucination

- pause speech recognition while Iris speaks (no more transcribing its own TTS)
- a spoken question mutes ambient alerts for 12s (the question always wins)
- 5s cooldown after each alert -> a listening gap so questions can land
- client-side near-duplicate dedup (Jaccard >= 0.5) drops repeated alerts
- live alerts fire ONLY on a newly-arrived person; objects/text/colors are
answered on demand (accurate), not auto-narrated (the small VLM hallucinates)
- add run.sh local launcher (frees the port, warmup on)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app.py

@@ -96,18 +96,18 @@ def describe(image: FileData, audio: FileData | None = None, lang: str = "pt",
 
 # Live mode: describe ONLY new/relevant things, sparingly (avoid verbosity).
 WATCH_PT = (
-    "Você observa o ambiente para uma pessoa cega, com PARCIMÔNIA. "
-    "Coisas JÁ DITAS recentemente (não repita nenhuma): \"{prev}\". "
-    "Olhe a cena AGORA. Se algo NOVO e relevante apareceu — que ainda NÃO foi dito "
-    "(uma pessoa, um objeto trazido, um texto, um obstáculo próximo) — diga em UMA "
-    "frase curta o que é. Se nada de novo relevante apareceu, responda APENAS: NADA."
+    "Você observa o ambiente para uma pessoa cega. "
+    "Coisas já ditas (não repita): \"{prev}\". "
+    "Olhe a cena AGORA. Se apareceu QUALQUER objeto ou pessoa novo — mesmo pequeno "
+    "(óculos, celular, copo, papel, comida) — diga em UMA frase curta o que é. "
+    "Só responda NADA se a cena estiver basicamente igual ao que já foi dito."
 )
 WATCH_EN = (
-    "You watch the environment for a blind person, SPARINGLY. "
-    "Already said recently (do not repeat any): \"{prev}\". "
-    "Look at the scene NOW. If something NEW and relevant appeared — not already said "
-    "(a person, an object, text, a nearby obstacle) — say it in ONE short sentence. "
-    "If nothing new and relevant appeared, reply ONLY the word: NONE."
+    "You watch the environment for a blind person. "
+    "Already said (do not repeat): \"{prev}\". "
+    "Look at the scene NOW. If ANY new object or person appeared — even small "
+    "(glasses, a phone, a cup, paper, food) — say what it is in ONE short sentence. "
+    "Only reply NONE if the scene is basically the same as what was already said."
 )
 _QUIET = {"nada", "none", "nenhum", "nothing", "sem novidade", "no change"}
 
@@ -118,15 +118,29 @@ def is_quiet(answer: str) -> bool:
     return (not low) or low in _QUIET
 
 
+HINT_PT = ("Uma pessoa entrou no campo de visão. Em UMA frase de no máximo 12 palavras, "
+           "avise de forma útil para um cego (ex.: 'Há alguém à sua frente.'). Descreva só o "
+           "que você vê com CERTEZA; não invente; não repita o que já foi dito: \"{prev}\".")
+HINT_EN = ("A person entered the field of view. In ONE sentence of at most 12 words, alert a "
+           "blind person usefully (e.g. 'Someone is in front of you.'). Describe only what you "
+           "see for CERTAIN; do not invent; do not repeat what was already said: \"{prev}\".")
+
+
 @app.api(name="watch")
-def watch(image: FileData, prev: str = "", lang: str = "pt") -> dict:
-    """Live mode. First call (no history) describes the scene as a baseline;
-    after that, speak ONLY when something new/relevant entered (else stay quiet)."""
+def watch(image: FileData, prev: str = "", lang: str = "pt", hint: str = "") -> dict:
+    """Live mode. First call (no history) describes the scene as a baseline. When the
+    in-browser detector flags a new object (`hint`), describe it (trust the gate).
+    Otherwise (pixel-diff fallback) speak ONLY if something new entered, else stay quiet."""
     t0 = time.time()
     prev = (prev or "").strip()
+    hint = (hint or "").strip()
     if not prev:
         # baseline: describe the current scene so the user hears the surroundings
         answer = vlm.describe(_path(image), lang=lang)
+    elif hint:
+        # the detector already decided something new is here -> just describe it
+        tmpl = HINT_EN if lang == "en" else HINT_PT
+        answer = vlm.describe(_path(image), lang=lang, system=tmpl.format(hint=hint, prev=prev))
     else:
         tmpl = WATCH_EN if lang == "en" else WATCH_PT
         sys = tmpl.format(prev=prev)

frontend/app.js

@@ -135,7 +135,7 @@ async function send(frame, audio, qtext = "") {
     let url = a && a.url;
     if (!url && a && a.path) url = window.location.origin + "/gradio_api/file=" + a.path;
     if (url) {
-      srMuted = true;
+      pauseSR();
       player.src = url;
       try { await player.play(); } catch (err) { console.error("play:", err); }
     } else { resetSoon(); }
@@ -147,21 +147,26 @@ async function send(frame, audio, qtext = "") {
     busy = false;
   }
 }
-function resetSoon() { setTimeout(() => { srMuted = false; if (!busy) setState("", t("idle")); }, 600); }
-player.addEventListener("ended", () => { srMuted = false; if (!busy) setState("", t("idle")); });
+function resetSoon() { setTimeout(() => { resumeSR(); if (!busy) setState("", t("idle")); }, 600); }
+player.addEventListener("ended", () => { setTimeout(resumeSR, 700); if (!busy) setState("", t("idle")); });
 
 // ---- first-run onboarding: default to the browser language, speak it, offer to switch ----
 function onboard() {
-  onboarded = true; mode = "normal";
-  try { localStorage.setItem("iris_onboarded", "1"); } catch (e) {}
-  hintEl.textContent = t("hint");
-  updateLabels();
+  if (onboarded) return;
   const u = new SpeechSynthesisUtterance(t("welcomeAsk"));
   u.lang = lang === "pt" ? "pt-BR" : "en-US";
+  // mark onboarded only when speech actually STARTS — so if the browser blocks
+  // autoplay on load, the mode stays "onboarding" and the first tap triggers it.
+  u.onstart = () => {
+    onboarded = true; mode = "normal";
+    try { localStorage.setItem("iris_onboarded", "1"); } catch (e) {}
+    hintEl.textContent = t("hint");
+    updateLabels();
+    setState("speaking", t("idle"));
+  };
   u.onend = listenForLangChoice;          // after the welcome, listen for a language switch
-  setState("speaking", t("idle"));
   try { speechSynthesis.cancel(); speechSynthesis.speak(u); }
-  catch (e) { listenForLangChoice(); }
+  catch (e) { /* will retry on the first tap */ }
 }
 
 function listenForLangChoice() {
@@ -203,9 +208,11 @@ function handleCommand(cmd) {
   if (cmd === "live_on") setLive(true);
   else if (cmd === "live_off") setLive(false);
 }
-const LIVE_INTERVAL = 2500;   // check the scene every 2.5s
-const LIVE_THRESHOLD = 14;    // mean per-pixel change (0-255) to trigger the VLM
+const LIVE_INTERVAL = 3500;   // check the scene every 3.5s
+const LIVE_THRESHOLD = 15;    // mean per-pixel change (0-255) to trigger the VLM
+const LIVE_COOLDOWN = 5000;   // quiet window after an auto-alert -> gap for the user to ask
 let lastSig = null, history = [];   // recent descriptions (anti-repetition)
+let suppressLiveUntil = 0;    // pause auto-alerts after an alert OR a user question (their question wins)
 
 // in-browser object detection gates the live mode (semantic change, not pixels).
 // Falls back to the cheap pixel-diff if the detector can't load.
@@ -213,6 +220,7 @@ let detector = null, detectorTried = false;
 let seenClasses = new Map();   // class name -> last tick index it was seen
 let tickN = 0;
 const CLASS_TTL = 4;           // ticks a class is remembered before it can re-trigger
+const PERSON_GONE = 4;         // ticks a person must be absent before a re-entry counts as new (~14s)
 
 async function loadDetector() {
   if (detector || detectorTried || !window.cocoSsd) return;
@@ -244,7 +252,7 @@ function setLive(on) {
 }
 
 // ---- hands-free continuous listening (live mode) via Web Speech API ----
-let recog = null, listening = false, srMuted = false;
+let recog = null, listening = false, srPaused = false;
 function setupRecognition() {
   const SR = window.SpeechRecognition || window.webkitSpeechRecognition;
   if (!SR) { console.warn("SpeechRecognition not supported in this browser"); return null; }
@@ -252,11 +260,11 @@ function setupRecognition() {
   r.continuous = true;
   r.interimResults = false;
   r.onresult = (e) => {
-    if (srMuted || busy) return;
+    if (srPaused || busy) return;
     const txt = e.results[e.results.length - 1][0].transcript.trim();
     if (txt) onVoice(txt);
   };
-  r.onend = () => { if (listening) { try { r.start(); } catch (e) {} } };  // keep alive
+  r.onend = () => { if (listening && !srPaused) { try { r.start(); } catch (e) {} } };  // keep alive
   r.onerror = (e) => { console.log("SR:", e.error); };
   return r;
 }
@@ -271,8 +279,12 @@ function stopListening() {
   listening = false;
   if (recog) { try { recog.stop(); } catch (e) {} }
 }
-async function onVoice(txt) {
-  if (busy) return;
+// stop the recognizer WHILE Iris speaks (so it never transcribes its own voice)
+function pauseSR() { srPaused = true; if (recog) { try { recog.stop(); } catch (e) {} } }
+function resumeSR() { srPaused = false; if (listening && recog) { try { recog.start(); } catch (e) {} } }
+async function onVoice(txt, tries = 0) {
+  suppressLiveUntil = performance.now() + 12000;   // the user's question wins: mute auto-alerts for a while
+  if (busy) { if (tries < 3) setTimeout(() => onVoice(txt, tries + 1), 700); return; }  // retry if a watch is in flight
   const frame = await grabFrame();
   if (frame) send(frame, null, txt);   // text question (from browser speech recognition)
 }
@@ -297,52 +309,59 @@ function changeAmount(a, b) {
 
 async function liveTick() {
   if (busy || mode !== "normal" || !liveOn || !player.paused) return;
+  if (performance.now() < suppressLiveUntil) return;   // user just asked / we just spoke -> stay quiet, let them talk
+  if (!detector || !cam.videoWidth) return;            // alerts are person-arrival only -> need the detector
   tickN++;
 
-  // semantic gate: only wake the VLM when a NEW object/person class appears
-  if (detector && cam.videoWidth) {
-    let preds = null;
-    try { preds = await detector.detect(cam, 10); } catch (e) {}
-    if (preds) {
-      let novel = false;
-      for (const p of preds) {
-        if (p.score < 0.55) continue;
-        const last = seenClasses.get(p.class);
-        if (last === undefined || tickN - last > CLASS_TTL) novel = true;
-        seenClasses.set(p.class, tickN);
-      }
-      for (const [c, n] of seenClasses) if (tickN - n > CLASS_TTL) seenClasses.delete(c);
-      if (novel) {
-        const frame = await grabFrame();
-        if (frame) sendWatch(frame);
-      }
-      return;
+  // Live alerts fire ONLY when a NEW person enters (per the spec: rare ambient alerts, not narration).
+  // Re-describing objects/movement makes the small VLM hallucinate and repeat; objects/text/colors are
+  // answered on demand (the user asks), which is accurate. So we gate strictly on a fresh "person".
+  let personNew = false;
+  try {
+    const preds = await detector.detect(cam, 5);
+    let hasPerson = false;
+    for (const p of preds) if (p.class === "person" && p.score >= 0.6) hasPerson = true;
+    if (hasPerson) {
+      const last = seenClasses.get("person");
+      if (last === undefined || tickN - last > PERSON_GONE) personNew = true;   // absent a while -> a real arrival
+      seenClasses.set("person", tickN);
     }
+  } catch (e) { return; }
+
+  if (personNew) {
+    suppressLiveUntil = performance.now() + LIVE_COOLDOWN;   // one alert, then a quiet gap to ask
+    const frame = await grabFrame();
+    if (frame) sendWatch(frame, "person");
   }
+}
 
-  // fallback: cheap pixel-diff gate
-  const sig = frameSignature();
-  if (!sig) return;
-  if (lastSig && changeAmount(lastSig, sig) < LIVE_THRESHOLD) return; // static scene -> stay quiet
-  lastSig = sig;
-  const frame = await grabFrame();
-  if (frame) sendWatch(frame);
+// word-set similarity (accent-insensitive) — the model often re-emits a near-identical
+// line, so we drop it client-side instead of trusting the "do not repeat" prompt.
+function words(s) {
+  return (s || "").toLowerCase().normalize("NFD").replace(/[̀-ͯ]/g, "")
+    .replace(/[^a-z0-9 ]/g, " ").split(/\s+/).filter((w) => w.length > 2);   // NFD + strip diacritics
+}
+function tooSimilar(a, b) {
+  const A = new Set(words(a)), B = new Set(words(b));
+  if (!A.size || !B.size) return false;
+  let inter = 0; for (const w of A) if (B.has(w)) inter++;
+  return inter / (A.size + B.size - inter) >= 0.5;   // Jaccard >= 0.5 -> basically the same alert
 }
 
-async function sendWatch(frame) {
+async function sendWatch(frame, hint = "") {
   if (busy) return;
   busy = true;
   try {
-    const result = await client.predict("/watch", { image: handle_file(frame), prev: history.join(" · "), lang });
+    const result = await client.predict("/watch", { image: handle_file(frame), prev: history.join(" · "), lang, hint });
     const out = Array.isArray(result.data) ? result.data[0] : result.data;
-    if (out && out.speak) {
-      if (out.answer) { history.push(out.answer); if (history.length > 5) history.shift(); }
-      answerEl.textContent = out.answer || "";
+    if (out && out.speak && out.answer && !history.some((h) => tooSimilar(h, out.answer))) {
+      history.push(out.answer); if (history.length > 5) history.shift();
+      answerEl.textContent = out.answer;
       setState("speaking", "");
       const a = out.audio;
       let url = a && a.url;
       if (!url && a && a.path) url = window.location.origin + "/gradio_api/file=" + a.path;
-      if (url) { srMuted = true; player.src = url; try { await player.play(); } catch (e) { console.error("play:", e); } }
+      if (url) { pauseSR(); player.src = url; try { await player.play(); } catch (e) { console.error("play:", e); } }
     }
   } catch (e) { console.error("watch:", e); }
   finally { busy = false; }
@@ -466,6 +485,7 @@ function updateLabels() {
   updateLabels();
   hintEl.textContent = onboarded ? t("hint") : "";
   setState("", onboarded ? t("idle") : t("startTap"));
+  if (!onboarded) onboard();   // try to speak the welcome on load (works where the browser allows it)
   await startCamera();
   loadDetector();   // preload the object detector in the background (non-blocking)
   try { client = await Client.connect(window.location.origin); console.log("Iris connected"); }