chore: deploy private lightloom build

app.py

@@ -145,7 +145,7 @@ def health() -> dict[str, Any]:
         "privacy_mode": os.getenv("LIGHTLOOM_PRIVACY_MODE", "1") == "1",
         "cpu_count": os.cpu_count(),
         "director_backend": CONFIG.director_backend,
-        "build": "clean-scenes-1",
+        "build": "clean-asr-1",
     }
 
 

frontend/js/asr-worker.js

@@ -0,0 +1,47 @@
+/* frontend/js/asr-worker.js — in-browser streaming ASR (Whisper) via transformers.js.
+ * Runs Whisper-base.en ENTIRELY in the user's browser (WebGPU, WASM fallback) so the
+ * live transcript appears AS you speak — instant, private, OFF THE GRID (no cloud ASR),
+ * and it frees the Space GPU. Loaded as a module Worker so transcription never blocks
+ * the scroll animation. Purely additive: the server still transcribes for painting.
+ */
+import { pipeline, env } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.5";
+
+env.allowLocalModels = false;
+
+const MODEL = "onnx-community/whisper-base.en";
+let transcriber = null;
+let loadPromise = null;
+let backend = null;
+
+async function load() {
+  if (transcriber) return transcriber;
+  if (!loadPromise) {
+    loadPromise = (async () => {
+      try {
+        transcriber = await pipeline("automatic-speech-recognition", MODEL, { device: "webgpu", dtype: "fp32" });
+        backend = "webgpu";
+      } catch (e) {
+        transcriber = await pipeline("automatic-speech-recognition", MODEL, { device: "wasm" });
+        backend = "wasm";
+      }
+      return transcriber;
+    })();
+  }
+  return loadPromise;
+}
+
+self.onmessage = async (e) => {
+  const { id, type, audio } = e.data || {};
+  try {
+    if (type === "load") {
+      await load();
+      self.postMessage({ type: "ready", backend });
+    } else if (type === "transcribe") {
+      const t = await load();
+      const out = await t(audio, { language: "en", task: "transcribe", return_timestamps: false });
+      self.postMessage({ id, type: "result", text: (out && out.text ? out.text : "").trim() });
+    }
+  } catch (err) {
+    self.postMessage({ id, type: "error", error: String((err && err.message) || err).slice(0, 200) });
+  }
+};

frontend/js/asr.js

@@ -0,0 +1,61 @@
+/* frontend/js/asr.js — LL.asr: manager for the in-browser streaming Whisper worker.
+ *
+ * LL.asr.init()                 -> Promise<bool>  (true if the worker + model loaded)
+ * LL.asr.ready                  -> bool
+ * LL.asr.transcribe(float32)    -> Promise<string>  (16 kHz mono audio -> text)
+ *
+ * Purely additive + best-effort: init() resolves false (never throws) if WebGPU/WASM
+ * is unavailable, so the caller silently falls back to the existing server-side ASR.
+ */
+"use strict";
+
+const LL = (window.LL = window.LL || {});
+
+const asr = {
+  _worker: null,
+  _ready: false,
+  _readyPromise: null,
+  _seq: 0,
+  _pending: new Map(),
+
+  /** Spawn the worker and load the model. Resolves true on success, false on any failure
+   *  (no WebGPU, no module-worker, model download blocked, 60 s timeout). Never throws. */
+  init() {
+    if (this._readyPromise) return this._readyPromise;
+    this._readyPromise = new Promise((resolve) => {
+      let settled = false;
+      const done = (v) => { if (!settled) { settled = true; resolve(v); } };
+      try {
+        this._worker = new Worker(new URL("./asr-worker.js", import.meta.url), { type: "module" });
+        this._worker.onmessage = (e) => {
+          const d = e.data || {};
+          if (d.type === "ready") { this._ready = true; this.backend = d.backend; done(true); }
+          else if (d.type === "result" || d.type === "error") {
+            const p = this._pending.get(d.id);
+            if (p) { this._pending.delete(d.id); d.type === "result" ? p.resolve(d.text) : p.reject(d.error); }
+          }
+        };
+        this._worker.onerror = () => done(false);
+        this._worker.postMessage({ type: "load" });
+        setTimeout(() => done(false), 60000); // model didn't load in time -> fall back
+      } catch (_) { done(false); }
+    });
+    return this._readyPromise;
+  },
+
+  get ready() { return this._ready; },
+
+  /** Transcribe a 16 kHz mono Float32Array. The buffer is COPIED (caller keeps theirs). */
+  transcribe(audioFloat32) {
+    if (!this._ready || !this._worker) return Promise.reject("asr_not_ready");
+    const copy = audioFloat32.slice();
+    const id = ++this._seq;
+    return new Promise((resolve, reject) => {
+      this._pending.set(id, { resolve, reject });
+      this._worker.postMessage({ id, type: "transcribe", audio: copy }, [copy.buffer]);
+    });
+  },
+};
+
+LL.asr = asr;
+export default asr;

frontend/js/controller.js

@@ -10,6 +10,7 @@
 import "./api.js";
 import "./recorder.js";
 import "./stage-scroll.js";
+import "./asr.js";
 // (subtitles.js is intentionally not used: captions render in #transcript, which
 // carries aria-live="polite" for screen readers.)
 
@@ -50,6 +51,9 @@ const controller = {
     this._startAmbient(); // a pre-rendered scroll flows behind the intro + during warm-up
     this._loadLedger();   // keep the About param count honest + current from /health
     this._prewarm();      // load the GPU models DURING the intro so the first phrase is ~instant
+    // best-effort in-browser Whisper for an INSTANT live transcript (off-grid, frees the
+    // GPU). If WebGPU/WASM is unavailable it silently stays off and the server ASR is used.
+    if (LL.asr && LL.asr.init) LL.asr.init().then((ok) => { this._asrReady = ok; }).catch(() => {});
     return this;
   },
 
@@ -150,6 +154,8 @@ const controller = {
       },
       // LIVE narration: each VAD-cut phrase is painted into the SAME scroll as it's spoken.
       onSegment: (wavB64) => this._onLiveSegment(wavB64),
+      // INSTANT live transcript via in-browser Whisper (additive; the server still paints).
+      onInterim: (float32) => this._onInterim(float32),
       onTranscript: (text, error) => {
         const clean = (text || "").trim();
         if (clean) this.startWorld(clean);
@@ -249,6 +255,19 @@ const controller = {
     }
   },
 
+  /** In-browser ASR interim -> INSTANT live transcript ("words as you speak"). Display only;
+   *  the server-transcribed phrase still drives painting. Skips while a transcription is in
+   *  flight so it never backs up behind the audio. */
+  _onInterim(float32) {
+    if (!this._liveOn || !this._asrReady || this._interimBusy || !LL.asr || !LL.asr.ready) return;
+    this._orch("asr", true);
+    this._interimBusy = true;
+    LL.asr.transcribe(float32)
+      .then((text) => { const s = (text || "").trim(); if (s && this._liveOn) this._transcript(s); })
+      .catch(() => {})
+      .finally(() => { this._interimBusy = false; });
+  },
+
   _onEvent(ev) {
     if (!ev || typeof ev !== "object") return;
     switch (ev.stage) {

frontend/js/recorder.js

@@ -40,7 +40,7 @@ const recorder = {
   _chunks: [],
   _inRate: 48000,
   _capTimer: null,
-  _cb: { onState: null, onTranscript: null, onSegment: null, getLang: null },
+  _cb: { onState: null, onTranscript: null, onSegment: null, onInterim: null, getLang: null },
   // live-segmentation state (only used when onSegment is wired)
   _live: false,
   _seg: [],
@@ -49,11 +49,13 @@ const recorder = {
   _silMs: 0,
   _nf: 0.004, // running noise floor (adaptive VAD)
   _pk: 0.02, // running speech peak (adaptive VAD)
+  _lastInterimAt: 0, // for in-browser live transcript pacing
 
   init(opts = {}) {
     this._cb.onState = typeof opts.onState === "function" ? opts.onState : null;
     this._cb.onTranscript = typeof opts.onTranscript === "function" ? opts.onTranscript : null;
     this._cb.onSegment = typeof opts.onSegment === "function" ? opts.onSegment : null;
+    this._cb.onInterim = typeof opts.onInterim === "function" ? opts.onInterim : null;
     this._cb.getLang = typeof opts.getLang === "function" ? opts.getLang : null;
     return this;
   },
@@ -117,6 +119,16 @@ const recorder = {
       this._segMs += ms;
       if (voicedFrame) { this._voicedMs += ms; this._silMs = 0; }
       else { this._silMs += ms; }
+      // in-browser LIVE TRANSCRIPT: every ~1.1s of an active segment, hand the current
+      // audio to the client-side ASR for an instant "words as you speak" caption. Purely
+      // additive — the server still transcribes the finished segment for PAINTING.
+      if (this._cb.onInterim && this._voicedMs >= 350) {
+        const nowMs = typeof performance !== "undefined" ? performance.now() : Date.now();
+        if (nowMs - this._lastInterimAt > 1100) {
+          this._lastInterimAt = nowMs;
+          try { this._cb.onInterim(this._segTo16k()); } catch (_) {}
+        }
+      }
       const enough = this._voicedMs >= VAD_MIN_VOICED_MS;
       const naturalPause = this._silMs >= VAD_SIL_MS && enough;
       const microPause = this._segMs >= VAD_SOFT_MS && !voicedFrame && enough; // cut in a word gap
@@ -140,6 +152,17 @@ const recorder = {
     this._silMs = 0;
   },
 
+  /** Merge the current live segment's Float32 chunks and resample to 16 kHz mono (a COPY,
+   *  so the live buffer keeps growing) — for the in-browser ASR interim transcript. */
+  _segTo16k() {
+    let length = 0;
+    for (const c of this._seg) length += c.length;
+    const merged = new Float32Array(length);
+    let off = 0;
+    for (const c of this._seg) { merged.set(c, off); off += c.length; }
+    return this._resample(merged, this._inRate, TARGET_SR);
+  },
+
   /** Encode the buffered phrase and hand it to onSegment, then start a fresh one.
    *  Runs WHILE recording continues — this is what makes the world build as you talk. */
   _cutLiveSegment() {

scripts/asr-feasibility.cjs

@@ -0,0 +1,12 @@
+const { chromium } = require("playwright");
+(async () => {
+  const browser = await chromium.launch({ headless: false, args: ["--enable-unsafe-webgpu", "--enable-features=Vulkan", "--use-gl=angle", "--use-angle=swiftshader", "--ignore-gpu-blocklist"] });
+  const page = await browser.newPage();
+  page.on("console", (m) => console.log("PAGE:", m.text().slice(0, 200)));
+  await page.goto("http://127.0.0.1:8842/scripts/asr-feasibility.html", { waitUntil: "domcontentloaded" });
+  for (let i = 0; i < 150; i++) { const done = await page.evaluate(() => window.__done).catch(() => false); if (done) break; await page.waitForTimeout(2000); }
+  const r = await page.evaluate(() => ({ log: window.__log, device: window.__device, text: window.__text, rt: window.__rt, err: window.__err }));
+  console.log("=== RESULT ===" + (r.log || ""));
+  console.log("DEVICE:", r.device, "| RTfactor:", r.rt, "| ERR:", r.err || "none");
+  await browser.close();
+})();

scripts/asr-feasibility.html

@@ -0,0 +1,41 @@
+<!doctype html><html><head><meta charset="utf-8"><title>ASR feasibility</title></head>
+<body><pre id="log">loading…</pre>
+<script type="module">
+const log = (m) => { document.getElementById("log").textContent += "\n" + m; window.__log = (window.__log||"") + "\n" + m; };
+window.__done = false;
+try {
+  const t0 = performance.now();
+  const { pipeline, env } = await import("https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.5");
+  env.allowLocalModels = false;
+  log("transformers.js loaded");
+  // try WebGPU, fall back to wasm
+  let device = "webgpu";
+  let transcriber;
+  try {
+    transcriber = await pipeline("automatic-speech-recognition", "onnx-community/whisper-base.en", { device: "webgpu", dtype: "fp32" });
+    log("model loaded on WEBGPU in " + Math.round(performance.now()-t0) + "ms");
+  } catch (e) {
+    device = "wasm";
+    log("webgpu failed (" + String(e).slice(0,80) + "), trying wasm…");
+    transcriber = await pipeline("automatic-speech-recognition", "onnx-community/whisper-base.en", { device: "wasm" });
+    log("model loaded on WASM in " + Math.round(performance.now()-t0) + "ms");
+  }
+  // fetch + decode the test wav to 16k mono Float32
+  const resp = await fetch("/.shots/cont_en.wav");
+  const buf = await resp.arrayBuffer();
+  const ac = new (window.AudioContext||window.webkitAudioContext)({ sampleRate: 16000 });
+  const decoded = await ac.decodeAudioData(buf);
+  const audio = decoded.getChannelData(0);
+  log("audio decoded: " + (audio.length/16000).toFixed(1) + "s @ " + decoded.sampleRate + "Hz");
+  const t1 = performance.now();
+  const out = await transcriber(audio, { language: "en", task: "transcribe", return_timestamps: false });
+  const dt = performance.now()-t1;
+  log("TRANSCRIBED in " + Math.round(dt) + "ms (" + (dt/(audio.length/16000)).toFixed(2) + "x realtime, " + device + ")");
+  log("TEXT: " + (out.text||"").trim());
+  window.__device = device; window.__text = (out.text||"").trim(); window.__rt = dt/(audio.length/16000);
+} catch (e) {
+  log("FATAL: " + String(e).slice(0,300));
+  window.__err = String(e).slice(0,300);
+}
+window.__done = true;
+</script></body></html>