File size: 25,625 Bytes
26dae50
 
 
df6b3ac
 
 
1f8aca6
 
 
dc59186
df6b3ac
 
 
f7a11cd
df6b3ac
26dae50
 
 
df6b3ac
97358c0
df6b3ac
 
 
1f8aca6
 
 
dc59186
df6b3ac
26dae50
df6b3ac
97358c0
df6b3ac
 
 
1f8aca6
 
 
dc59186
df6b3ac
26dae50
 
df6b3ac
26dae50
 
 
df6b3ac
 
f7a11cd
df6b3ac
 
 
 
 
 
 
26dae50
 
f7a11cd
dc59186
df6b3ac
 
 
 
 
26dae50
 
 
 
df6b3ac
dc59186
df6b3ac
dc59186
26dae50
 
f7a11cd
df6b3ac
 
 
 
 
 
 
 
 
f7a11cd
26dae50
df6b3ac
26dae50
df6b3ac
26dae50
 
 
 
 
 
 
 
 
 
 
f7a11cd
df6b3ac
26dae50
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
250c08a
 
 
 
 
 
 
 
 
 
 
 
 
26dae50
8f5fad5
 
 
 
 
 
 
 
 
 
 
 
 
bf47020
 
 
8f5fad5
 
 
df6b3ac
26dae50
 
 
 
 
250c08a
 
df6b3ac
26dae50
df6b3ac
26dae50
 
df6b3ac
 
26dae50
 
df6b3ac
 
 
 
19657cc
df6b3ac
 
 
26dae50
df6b3ac
26dae50
 
 
 
 
 
19657cc
 
26dae50
dc59186
214efbc
dc59186
 
 
 
 
 
1f8aca6
dc59186
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8f5fad5
dc59186
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1f8aca6
dc59186
 
 
 
 
 
 
 
 
 
 
 
 
 
2bdcfe3
dc59186
2bdcfe3
dc59186
 
2bdcfe3
dc59186
 
 
 
 
 
 
 
2bdcfe3
dc59186
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2bdcfe3
dc59186
df6b3ac
 
dc59186
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f7a11cd
df6b3ac
 
 
 
19657cc
 
 
f7a11cd
19657cc
df6b3ac
2bdcfe3
 
 
 
 
 
19657cc
2bdcfe3
 
 
 
 
 
 
 
 
 
df6b3ac
 
 
 
 
 
 
 
2bdcfe3
 
 
 
 
 
 
 
 
 
df6b3ac
 
 
19657cc
df6b3ac
 
f7a11cd
df6b3ac
 
 
 
19657cc
df6b3ac
 
 
19657cc
df6b3ac
 
 
 
 
 
 
 
 
 
 
 
 
 
19657cc
 
 
 
 
 
df6b3ac
f7a11cd
df6b3ac
 
f7a11cd
df6b3ac
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19657cc
 
2bdcfe3
 
19657cc
 
 
 
 
 
 
 
 
 
 
 
2bdcfe3
19657cc
 
 
 
 
 
2bdcfe3
19657cc
2bdcfe3
19657cc
 
 
 
 
 
 
 
 
 
 
df6b3ac
26dae50
19657cc
df6b3ac
 
 
250c08a
 
19657cc
df6b3ac
19657cc
 
 
df6b3ac
 
 
 
19657cc
df6b3ac
 
 
 
 
f7a11cd
df6b3ac
dc59186
df6b3ac
26dae50
df6b3ac
26dae50
 
 
 
 
 
 
 
df6b3ac
f7a11cd
df6b3ac
26dae50
dc59186
26dae50
 
 
 
df6b3ac
f7a11cd
df6b3ac
 
 
 
 
 
 
f7a11cd
df6b3ac
 
 
 
 
 
 
 
 
 
 
 
 
 
26dae50
 
 
 
f7a11cd
df6b3ac
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f7a11cd
df6b3ac
 
 
 
 
 
 
 
 
 
 
 
 
f7a11cd
df6b3ac
 
 
 
 
 
26dae50
 
df6b3ac
26dae50
df6b3ac
dc59186
 
 
2bdcfe3
250c08a
26dae50
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
const $ = (id) => document.getElementById(id);
const cam = $("cam"), canvas = $("canvas"), statusEl = $("status"),
      answerEl = $("answer"), hintEl = $("hint"), langBtn = $("lang"),
      player = $("player"), stage = $("stage"),
      btnAsk = $("btn-ask"), btnDescribe = $("btn-describe"),
      btnLive = $("btn-live"), a11yBtn = $("a11y");
const intro = $("intro"), introBodyEl = $("intro-body"), introHowEl = $("intro-how"),
      introCloseBtn = $("intro-close"), introVideo = $("intro-video"),
      introEn = $("intro-en"), introPt = $("intro-pt"),
      introReplay = $("intro-replay"), introHintEl = $("intro-hint");

function vibrate(ms) { try { navigator.vibrate && navigator.vibrate(ms); } catch (e) {} }

// ---- language (auto-detected from the browser; switchable by voice or button) ----
let lang = (navigator.language || "en").toLowerCase().startsWith("pt") ? "pt" : "en";

const T = {
  en: { idle: "Iris", listening: "Listening…", thinking: "Thinking…",
        hint: "Tap: describe · Hold: ask · Double-tap: live",
        camErr: "Camera blocked. Allow access.", err: "Something went wrong",
        langListen: "Listening… say your language",
        welcome: "Welcome to Iris. Tap the screen to describe what is in front of you. Hold to ask a question. Double-tap to turn live mode on or off, which announces new things around you.",
        liveOn: "Live mode on.", liveOff: "Live mode off.", confLang: "English selected.",
        introBody: "This is Iris, your eyes by voice. It reads your money, your bills, and your medicine, out loud.",
        introHow: "Tap to describe · Hold to ask · Double-tap (or say \"live mode\") for live mode",
        introSpeak: "This is Iris, your eyes by voice. It reads your money, bills, and medicine out loud. To use it: tap the screen to describe what is in front of you, hold and speak to ask a question, and double-tap, or say live mode, to turn live mode on or off. To close this, say close, or tap the big button.",
        introClose: "Close", introVideo: "Watch the demo", introHint: "Say \"close\" or tap the button",
        bAsk: "Ask", bDescribe: "Describe", bLive: "Live" },
  pt: { idle: "Iris", listening: "Ouvindo…", thinking: "Pensando…",
        hint: "Toque: descrever · Segurar: perguntar · Toque duplo: ao vivo",
        camErr: "Câmera bloqueada. Permita o acesso.", err: "Algo deu errado",
        langListen: "Ouvindo… diga seu idioma",
        welcome: "Bem-vindo ao Iris. Toque na tela para descrever o que está à sua frente. Segure para fazer uma pergunta. Toque duas vezes para ligar ou desligar o modo ao vivo, que avisa o que aparece de novo à sua volta.",
        liveOn: "Modo ao vivo ligado.", liveOff: "Modo ao vivo desligado.", confLang: "Português selecionado.",
        introBody: "Este é o Iris, os seus olhos por voz. Ele lê o seu dinheiro, suas contas e seus remédios, em voz alta.",
        introHow: "Toque para descrever · Segure para perguntar · Toque duplo (ou diga \"modo ao vivo\") para o modo ao vivo",
        introSpeak: "Este é o Iris, os seus olhos por voz. Ele lê o seu dinheiro, suas contas e seus remédios em voz alta. Para usar: toque na tela para descrever o que está à sua frente, segure e fale para fazer uma pergunta, e toque duas vezes, ou diga modo ao vivo, para ligar ou desligar o modo ao vivo. Para fechar, diga fechar, ou toque no botão grande.",
        introClose: "Fechar", introVideo: "Ver o vídeo", introHint: "Diga \"fechar\" ou toque no botão",
        bAsk: "Perguntar", bDescribe: "Descrever", bLive: "Ao vivo" },
};
const t = (k) => T[lang][k];
const LANG_PROMPT = "Segure e diga seu idioma · Hold and say your language";

function setState(s, msg) {
  document.body.dataset.state = s || "";
  if (msg !== undefined) statusEl.textContent = (liveOn ? "● " : "") + msg;
}
// UI speech via the browser voice (instructions/confirmations)
function speak(text) {
  try {
    speechSynthesis.cancel();
    const u = new SpeechSynthesisUtterance(text);
    u.lang = lang === "pt" ? "pt-BR" : "en-US";
    speechSynthesis.speak(u);
  } catch (e) { console.error("speak:", e); }
}

// ---- state ----
let mode = "intro";   // "intro" while the welcome popup is open, then "normal"
let busy = false;
let liveOn = false, liveTimer = null;
let holding = false, recording = false, holdTimer = null;
const HOLD_MS = 350;

langBtn.onclick = (e) => {
  e.stopPropagation();
  lang = lang === "en" ? "pt" : "en";
  document.documentElement.lang = lang;
  langBtn.textContent = lang.toUpperCase();
  hintEl.textContent = t("hint");
  updateLabels();
  if (!busy) setState("", t("idle"));
};

// ---- unlock audio (autoplay) on the first tap ----
const SILENT = "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=";
let audioUnlocked = false;
function unlockAudio() {
  if (audioUnlocked) return;
  audioUnlocked = true;
  player.src = SILENT;
  player.play().catch(() => {});
}

// ---- live camera ----
async function startCamera() {
  for (const c of [{ video: { facingMode: { ideal: "environment" } }, audio: false }, { video: true, audio: false }]) {
    try { cam.srcObject = await navigator.mediaDevices.getUserMedia(c); return; }
    catch (e) {}
  }
  setState("", t("camErr"));
}
function grabFrame() {
  const w = cam.videoWidth, h = cam.videoHeight;
  if (!w || !h) return Promise.resolve(null);
  canvas.width = w; canvas.height = h;
  canvas.getContext("2d").drawImage(cam, 0, 0, w, h);
  return new Promise((res) => canvas.toBlob(res, "image/jpeg", 0.85));
}

// ---- microphone ----
let micStream = null, chunks = [], recorder = null;
async function startRec() {
  try { if (!micStream) micStream = await navigator.mediaDevices.getUserMedia({ audio: true }); }
  catch (e) { return false; }
  chunks = [];
  recorder = new MediaRecorder(micStream);
  recorder.ondataavailable = (e) => { if (e.data.size) chunks.push(e.data); };
  recorder.start();
  return true;
}
function stopRec() {
  return new Promise((res) => {
    if (!recorder || recorder.state === "inactive") return res(null);
    recorder.onstop = () => res(chunks.length ? new Blob(chunks, { type: recorder.mimeType || "audio/webm" }) : null);
    recorder.stop();
  });
}

// ---- backend (gradio client loaded lazily, so the UI never waits on the CDN) ----
let Client, handle_file, client = null, clientReady = null;
function ensureClient() {
  if (!clientReady) {
    clientReady = (async () => {
      const mod = await import("https://esm.sh/@gradio/client");
      Client = mod.Client; handle_file = mod.handle_file;
      client = await Client.connect(window.location.origin);
      console.log("Iris connected");
    })().catch((e) => { console.error("connect:", e); clientReady = null; });
  }
  return clientReady;
}

// warm the model while the welcome popup plays, so the first real description is fast.
// Once per session (so a reload doesn't burn ZeroGPU quota), best-effort, never blocks the UI.
let warmedSession = false;
try { warmedSession = sessionStorage.getItem("iris_warmed") === "1"; } catch (e) {}
async function warmup() {
  if (warmedSession) return;
  warmedSession = true;
  try { sessionStorage.setItem("iris_warmed", "1"); } catch (e) {}
  try {
    await ensureClient();
    if (!client) { warmedSession = false; return; }   // let it retry if the connection failed
    const c = document.createElement("canvas"); c.width = 32; c.height = 32;
    const blob = await new Promise((r) => c.toBlob(r, "image/jpeg", 0.6));
    if (blob) client.predict("/describe", { image: handle_file(blob), lang }).catch(() => {});  // warm the VLM on the GPU; discard the result
    // also warm speech-to-text (Whisper, CPU) so the first "hold to ask" is fast too
    try { const wav = await (await fetch(SILENT)).blob(); client.predict("/detect_lang", { audio: handle_file(wav) }).catch(() => {}); } catch (e) {}
  } catch (e) { warmedSession = false; }
}

async function send(frame, audio, qtext = "") {
  if (busy) return;
  busy = true;
  answerEl.textContent = "";
  setState("thinking", t("thinking"));
  try {
    await ensureClient();
    if (!client) throw new Error("client not ready");
    const payload = { image: handle_file(frame), lang };
    if (audio) payload.audio = handle_file(audio);
    if (qtext) payload.qtext = qtext;
    const result = await client.predict("/describe", payload);
    const out = Array.isArray(result.data) ? result.data[0] : result.data;
    console.log("Iris result:", out);
    if (out && out.command) { busy = false; handleCommand(out.command); return; }
    answerEl.textContent = (out && out.answer) || "";
    setState("speaking", "");
    const a = out && out.audio;
    let url = a && a.url;
    if (!url && a && a.path) url = window.location.origin + "/gradio_api/file=" + a.path;
    if (url) {
      pauseSR();
      player.src = url;
      try { await player.play(); } catch (err) { console.error("play:", err); }
    } else { resetSoon(); }
  } catch (e) {
    console.error("describe:", e);
    setState("", t("err"));
    resetSoon();
  } finally {
    busy = false;
  }
}
function resetSoon() { setTimeout(() => { resumeSR(); if (!busy) setState("", t("idle")); }, 600); }
player.addEventListener("ended", () => { setTimeout(resumeSR, 700); if (!busy) setState("", t("idle")); });

// ---- welcome popup (shown on every open): pitch read aloud, closeable by voice/tap ----
const DEMO_VIDEO_URL = "https://youtu.be/h4AJOWuDCVc";   // demo video
let introOpen = true, autoCloseTimer = null, introRecog = null, introSpokenSession = false;
try { introSpokenSession = sessionStorage.getItem("iris_introSpoken") === "1"; } catch (e) {}
const CLOSE_WORDS = /\b(close|fechar|ok|okay|start|begin|continue|continuar|comecar|entendi|pode fechar|got it)\b/i;

function renderIntro() {
  introBodyEl.textContent = t("introBody");
  introHowEl.textContent = t("introHow");
  introCloseBtn.textContent = t("introClose");
  introVideo.textContent = t("introVideo");
  introVideo.href = DEMO_VIDEO_URL;
  introHintEl.textContent = t("introHint");
  introEn.setAttribute("aria-pressed", lang === "en" ? "true" : "false");
  introPt.setAttribute("aria-pressed", lang === "pt" ? "true" : "false");
}

function showIntro() {
  mode = "intro";
  renderIntro();
  intro.hidden = false;
  try { introCloseBtn.focus(); } catch (e) {}
  speakIntro(false);                       // narrate on load where the browser allows (desktop)
  intro.addEventListener("pointerdown", onIntroFirstTap);   // mobile: first tap unlocks audio + narrates
  document.addEventListener("keydown", onIntroKey);
  warmup();                                // warm the model while the user listens (first description is then fast)
}

function onIntroFirstTap(e) {
  unlockAudio();
  if (e.target.closest("button, a")) return;   // controls handle themselves
  if (!introSpokenSession) speakIntro(false);
}

function onIntroKey(e) {
  if (!introOpen) return;
  if (e.key === "Escape") { closeIntro(); return; }
  if (e.key === "Tab") {                        // simple focus trap inside the card
    const f = intro.querySelectorAll("button, a[href]");
    if (!f.length) return;
    const first = f[0], last = f[f.length - 1];
    if (e.shiftKey && document.activeElement === first) { e.preventDefault(); last.focus(); }
    else if (!e.shiftKey && document.activeElement === last) { e.preventDefault(); first.focus(); }
  }
}

function speakIntro(force) {
  if (!introOpen) return;
  if (introSpokenSession && !force) return;     // don't re-narrate on a same-session reload
  unlockAudio();
  try {
    speechSynthesis.cancel();
    const u = new SpeechSynthesisUtterance(t("introSpeak"));
    u.lang = lang === "pt" ? "pt-BR" : "en-US";
    u.onstart = () => {
      introSpokenSession = true;
      try { sessionStorage.setItem("iris_introSpoken", "1"); } catch (e) {}
      armCloseListener();
    };
    u.onend = () => { if (introOpen) scheduleAutoClose(); };
    speechSynthesis.speak(u);
  } catch (e) { /* will retry on the first tap */ }
}

function scheduleAutoClose() {
  cancelAutoClose();
  autoCloseTimer = setTimeout(() => { if (introOpen) closeIntro(); }, 2500);   // hands-free close after the narration
}
function cancelAutoClose() { if (autoCloseTimer) { clearTimeout(autoCloseTimer); autoCloseTimer = null; } }

function armCloseListener() {
  if (introRecog) return;
  const SR = window.SpeechRecognition || window.webkitSpeechRecognition;
  if (!SR) return;
  introRecog = new SR();
  introRecog.continuous = true; introRecog.interimResults = false;
  introRecog.lang = lang === "pt" ? "pt-BR" : "en-US";
  introRecog.onresult = (e) => {
    const raw = (e.results[e.results.length - 1][0].transcript || "");
    const txt = raw.normalize("NFD").replace(/[̀-ͯ]/g, "");   // strip accents
    if (CLOSE_WORDS.test(txt)) closeIntro();
  };
  introRecog.onend = () => { if (introOpen && introRecog) { try { introRecog.start(); } catch (e) {} } };
  introRecog.onerror = () => {};
  try { introRecog.start(); } catch (e) {}
}

function closeIntro() {
  if (!introOpen) return;
  introOpen = false;
  cancelAutoClose();
  if (introRecog) { try { introRecog.stop(); } catch (e) {} introRecog = null; }
  try { speechSynthesis.cancel(); } catch (e) {}
  intro.hidden = true;
  document.removeEventListener("keydown", onIntroKey);
  mode = "normal";
  try { btnDescribe.focus(); } catch (e) {}
  setState("", t("idle"));
  startCamera();   // start the app now (pitch first, camera permission after)
}

function setIntroLang(l) {
  cancelAutoClose();
  if (l !== lang) {
    lang = l; document.documentElement.lang = lang;
    langBtn.textContent = lang.toUpperCase(); hintEl.textContent = t("hint"); updateLabels();
  }
  renderIntro();
  speakIntro(true);   // replay in the chosen language
}

introCloseBtn.addEventListener("click", (e) => { e.stopPropagation(); closeIntro(); });
introVideo.addEventListener("click", (e) => { e.stopPropagation(); cancelAutoClose(); });   // let them watch (opens a new tab)
introReplay.addEventListener("click", (e) => { e.stopPropagation(); cancelAutoClose(); speakIntro(true); });
introEn.addEventListener("click", (e) => { e.stopPropagation(); setIntroLang("en"); });
introPt.addEventListener("click", (e) => { e.stopPropagation(); setIntroLang("pt"); });

// ---- live mode (toggled by voice command or double-tap) ----
function handleCommand(cmd) {
  if (cmd === "live_on") setLive(true);
  else if (cmd === "live_off") setLive(false);
}
const LIVE_INTERVAL = 3500;   // check the scene every 3.5s
const LIVE_THRESHOLD = 15;    // mean per-pixel change (0-255) to trigger the VLM
const LIVE_COOLDOWN = 5000;   // quiet window after an auto-alert -> gap for the user to ask
let lastSig = null, history = [];   // recent descriptions (anti-repetition)
let suppressLiveUntil = 0;    // pause auto-alerts after an alert OR a user question (their question wins)

// in-browser object detection gates the live mode (semantic change, not pixels).
// Falls back to the cheap pixel-diff if the detector can't load.
let detector = null, detectorTried = false;
let seenClasses = new Map();   // class name -> last tick index it was seen
let tickN = 0;
const CLASS_TTL = 4;           // ticks a class is remembered before it can re-trigger
const PERSON_GONE = 4;         // ticks a person must be absent before a re-entry counts as new (~14s)

async function loadDetector() {
  if (detector || detectorTried || !window.cocoSsd) return;
  detectorTried = true;
  try {
    detector = await window.cocoSsd.load({ base: "lite_mobilenet_v2" });
    console.log("Iris: object detector ready");
  } catch (e) { console.warn("detector load failed, using pixel-diff:", e); }
}

function setLive(on) {
  liveOn = on;
  btnLive.setAttribute("aria-pressed", on ? "true" : "false");
  btnLive.classList.toggle("on", on);
  vibrate(on ? [20, 40, 20] : 20);
  speak(on ? t("liveOn") : t("liveOff"));
  setState("", t("idle"));
  clearInterval(liveTimer); liveTimer = null;
  lastSig = null; history = []; seenClasses = new Map();
  if (on) {
    loadDetector();
    liveTimer = setInterval(liveTick, LIVE_INTERVAL);
    startListening();
    // baseline: describe the scene once shortly after turning on
    setTimeout(async () => {
      if (liveOn && !busy && player.paused) { const f = await grabFrame(); if (f) sendWatch(f); }
    }, 900);
  } else stopListening();
}

// ---- hands-free continuous listening (live mode) via Web Speech API ----
let recog = null, listening = false, srPaused = false;
function setupRecognition() {
  const SR = window.SpeechRecognition || window.webkitSpeechRecognition;
  if (!SR) { console.warn("SpeechRecognition not supported in this browser"); return null; }
  const r = new SR();
  r.continuous = true;
  r.interimResults = false;
  r.onresult = (e) => {
    if (srPaused || busy) return;
    const txt = e.results[e.results.length - 1][0].transcript.trim();
    if (txt) onVoice(txt);
  };
  r.onend = () => { if (listening && !srPaused) { try { r.start(); } catch (e) {} } };  // keep alive
  r.onerror = (e) => { console.log("SR:", e.error); };
  return r;
}
function startListening() {
  if (!recog) recog = setupRecognition();
  if (!recog || listening) return;
  listening = true;
  recog.lang = lang === "pt" ? "pt-BR" : "en-US";
  try { recog.start(); } catch (e) {}
}
function stopListening() {
  listening = false;
  if (recog) { try { recog.stop(); } catch (e) {} }
}
// stop the recognizer WHILE Iris speaks (so it never transcribes its own voice)
function pauseSR() { srPaused = true; if (recog) { try { recog.stop(); } catch (e) {} } }
function resumeSR() { srPaused = false; if (listening && recog) { try { recog.start(); } catch (e) {} } }
async function onVoice(txt, tries = 0) {
  suppressLiveUntil = performance.now() + 12000;   // the user's question wins: mute auto-alerts for a while
  if (busy) { if (tries < 3) setTimeout(() => onVoice(txt, tries + 1), 700); return; }  // retry if a watch is in flight
  const frame = await grabFrame();
  if (frame) send(frame, null, txt);   // text question (from browser speech recognition)
}

// cheap visual signature (32x32 gray) to detect change without calling the model
function frameSignature() {
  if (!cam.videoWidth) return null;
  const c = document.createElement("canvas"); c.width = 32; c.height = 32;
  const ctx = c.getContext("2d");
  ctx.drawImage(cam, 0, 0, 32, 32);
  const d = ctx.getImageData(0, 0, 32, 32).data;
  const g = new Uint8Array(1024);
  for (let i = 0; i < 1024; i++) g[i] = (d[i * 4] + d[i * 4 + 1] + d[i * 4 + 2]) / 3;
  return g;
}
function changeAmount(a, b) {
  if (!a || !b) return 999;
  let s = 0;
  for (let i = 0; i < a.length; i++) s += Math.abs(a[i] - b[i]);
  return s / a.length;
}

async function liveTick() {
  if (busy || mode !== "normal" || !liveOn || !player.paused) return;
  if (performance.now() < suppressLiveUntil) return;   // user just asked / we just spoke -> stay quiet, let them talk
  if (!detector || !cam.videoWidth) return;            // alerts are person-arrival only -> need the detector
  tickN++;

  // Live alerts fire ONLY when a NEW person enters (per the spec: rare ambient alerts, not narration).
  // Re-describing objects/movement makes the small VLM hallucinate and repeat; objects/text/colors are
  // answered on demand (the user asks), which is accurate. So we gate strictly on a fresh "person".
  let personNew = false;
  try {
    const preds = await detector.detect(cam, 5);
    let hasPerson = false;
    for (const p of preds) if (p.class === "person" && p.score >= 0.6) hasPerson = true;
    if (hasPerson) {
      const last = seenClasses.get("person");
      if (last === undefined || tickN - last > PERSON_GONE) personNew = true;   // absent a while -> a real arrival
      seenClasses.set("person", tickN);
    }
  } catch (e) { return; }

  if (personNew) {
    suppressLiveUntil = performance.now() + LIVE_COOLDOWN;   // one alert, then a quiet gap to ask
    const frame = await grabFrame();
    if (frame) sendWatch(frame, "person");
  }
}

// word-set similarity (accent-insensitive) — the model often re-emits a near-identical
// line, so we drop it client-side instead of trusting the "do not repeat" prompt.
function words(s) {
  return (s || "").toLowerCase().normalize("NFD").replace(/[̀-ͯ]/g, "")
    .replace(/[^a-z0-9 ]/g, " ").split(/\s+/).filter((w) => w.length > 2);   // NFD + strip diacritics
}
function tooSimilar(a, b) {
  const A = new Set(words(a)), B = new Set(words(b));
  if (!A.size || !B.size) return false;
  let inter = 0; for (const w of A) if (B.has(w)) inter++;
  return inter / (A.size + B.size - inter) >= 0.5;   // Jaccard >= 0.5 -> basically the same alert
}

async function sendWatch(frame, hint = "") {
  if (busy) return;
  busy = true;
  try {
    await ensureClient();
    if (!client) return;
    const result = await client.predict("/watch", { image: handle_file(frame), prev: history.join(" · "), lang, hint });
    const out = Array.isArray(result.data) ? result.data[0] : result.data;
    if (out && out.speak && out.answer && !history.some((h) => tooSimilar(h, out.answer))) {
      history.push(out.answer); if (history.length > 5) history.shift();
      answerEl.textContent = out.answer;
      setState("speaking", "");
      const a = out.audio;
      let url = a && a.url;
      if (!url && a && a.path) url = window.location.origin + "/gradio_api/file=" + a.path;
      if (url) { pauseSR(); player.src = url; try { await player.play(); } catch (e) { console.error("play:", e); } }
    }
  } catch (e) { console.error("watch:", e); }
  finally { busy = false; }
}

// ---- interaction ----
stage.addEventListener("pointerdown", async () => {
  if (mode !== "normal") return;   // welcome popup is open
  if (liveOn) { speechSynthesis.cancel(); player.pause(); }
  if (busy) return;
  unlockAudio();
  holding = true;
  stage.classList.add("armed");
  holdTimer = setTimeout(async () => {
    recording = await startRec();
    if (recording) setState("listening", t("listening"));
  }, HOLD_MS);
});

let lastTapTime = 0, tapTimer = null;
const DOUBLE_MS = 320;   // double-tap window

async function endPress() {
  if (mode !== "normal") return;   // welcome popup is open
  if (!holding) return;
  holding = false;
  stage.classList.remove("armed");
  clearTimeout(holdTimer);

  if (recording) {                       // held and spoke = a question
    recording = false;
    const frame = await grabFrame();
    const audio = await stopRec();
    if (frame) send(frame, audio); else setState("", t("camErr"));
    return;
  }

  // quick tap: double-tap toggles live mode; single tap describes
  const now = performance.now();
  if (now - lastTapTime < DOUBLE_MS) {
    lastTapTime = 0;
    if (tapTimer) { clearTimeout(tapTimer); tapTimer = null; }
    setLive(!liveOn);
  } else {
    lastTapTime = now;
    if (tapTimer) clearTimeout(tapTimer);
    tapTimer = setTimeout(async () => {
      tapTimer = null;
      const frame = await grabFrame();
      if (frame) send(frame, null); else setState("", t("camErr"));
    }, DOUBLE_MS);
  }
}
stage.addEventListener("pointerup", endPress);
stage.addEventListener("pointercancel", endPress);

// ---- explicit buttons (low vision / keyboard / screen reader) ----
btnDescribe.addEventListener("click", async (e) => {
  e.stopPropagation();
  if (mode !== "normal" || busy) return;
  unlockAudio(); vibrate(10);
  const frame = await grabFrame();
  if (frame) send(frame, null); else setState("", t("camErr"));
});

let askRecBtn = false;
btnAsk.addEventListener("pointerdown", async (e) => {
  e.stopPropagation();
  if (mode !== "normal" || busy) return;
  unlockAudio(); vibrate(10);
  askRecBtn = await startRec();
  if (askRecBtn) setState("listening", t("listening"));
});
async function btnAskEnd(e) {
  if (e) e.stopPropagation();
  if (!askRecBtn) return; askRecBtn = false;
  const frame = await grabFrame();
  const audio = await stopRec();
  if (frame) send(frame, audio); else setState("", t("camErr"));
}
btnAsk.addEventListener("pointerup", btnAskEnd);
btnAsk.addEventListener("pointerleave", btnAskEnd);
btnAsk.addEventListener("pointercancel", btnAskEnd);

btnLive.addEventListener("click", (e) => {
  e.stopPropagation();
  if (mode !== "normal") return;
  unlockAudio();
  setLive(!liveOn);
});

// ---- accessibility: max contrast + larger text (persisted) ----
let boost = false;
try { boost = localStorage.getItem("iris_a11y") === "1"; } catch (e) {}
document.body.classList.toggle("a11y-boost", boost);
a11yBtn.setAttribute("aria-pressed", boost ? "true" : "false");
a11yBtn.addEventListener("click", (e) => {
  e.stopPropagation();
  boost = !boost;
  document.body.classList.toggle("a11y-boost", boost);
  a11yBtn.setAttribute("aria-pressed", boost ? "true" : "false");
  try { localStorage.setItem("iris_a11y", boost ? "1" : "0"); } catch (e) {}
  vibrate(10);
});

// button labels per language
function updateLabels() {
  btnAsk.querySelector(".ctl-lbl").textContent = t("bAsk");
  btnDescribe.querySelector(".ctl-lbl").textContent = t("bDescribe");
  btnLive.querySelector(".ctl-lbl").textContent = t("bLive");
}

// ---- boot ----
(async () => {
  document.documentElement.lang = lang;
  langBtn.textContent = lang.toUpperCase();
  updateLabels();
  hintEl.textContent = t("hint");
  setState("", t("idle"));
  showIntro();      // welcome popup on every open (the camera starts when it closes)
  loadDetector();   // preload the object detector in the background (non-blocking)
  ensureClient();   // connect in the background; the UI never waits on it
})();