chore: deploy private lightloom build

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+2406.09394v4.pdf filter=lfs diff=lfs merge=lfs -text

2406.09394v4.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f4b043583868ed52c9cfaf952ecc0eb8a9e4434d590a02ae7e46da98679fd4a
+size 28467487

app.py

@@ -144,7 +144,7 @@ def health() -> dict[str, Any]:
         "privacy_mode": os.getenv("LIGHTLOOM_PRIVACY_MODE", "1") == "1",
         "cpu_count": os.cpu_count(),
         "director_backend": CONFIG.director_backend,
-        "build": "showcase-2",
+        "build": "voice-1",
     }
 
 
@@ -225,6 +225,43 @@ def asr_probe_endpoint() -> dict[str, Any]:
     return {"ok": True, "data": _asr_probe_on_gpu()}
 
 
+# --- VOICE input (the "speak" in "speak, and watch your story become a world"):
+# Cohere Transcribe turns the narrator's mic audio into text, which then drives the
+# same panorama pipeline. The browser captures a 16 kHz mono PCM WAV and sends it
+# base64-encoded; read_wav decodes it with the stdlib (no ffmpeg needed). ---
+@spaces.GPU(duration=60)
+def _transcribe_on_gpu(wav_path: str, lang: str) -> dict[str, Any]:
+    from lightloom.audio_in.asr import load_asr, read_wav, transcribe
+
+    audio, sr = read_wav(wav_path)
+    processor, model = load_asr()
+    text = transcribe(processor, model, audio, sampling_rate=sr, language=lang)
+    return {"text": text}
+
+
+@app.api(name="transcribe", concurrency_id="gpu", concurrency_limit=1, stream_every=0.05)
+def transcribe_api(audio_b64: str = "", lang: str = "en") -> dict:
+    import base64
+    import tempfile
+
+    yield {"stage": "transcribing"}
+    try:
+        payload = audio_b64.split(",")[-1] if audio_b64 else ""
+        raw = base64.b64decode(payload) if payload else b""
+        if len(raw) < 64:  # nothing meaningful captured
+            yield {"stage": "error", "error": "empty_audio"}
+            return
+        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as handle:
+            handle.write(raw)
+            path = handle.name
+        result = _transcribe_on_gpu(path, lang)
+        yield {"stage": "transcript", "text": (result.get("text") or "").strip()}
+    except Exception as exc:  # noqa: BLE001 - quota/decoding guard.
+        message = str(exc).lower()
+        stage = "quota_exceeded" if any(t in message for t in ("gpu", "quota", "zerogpu", "exceeded")) else "error"
+        yield {"stage": stage, "error": str(exc)[:200]}
+
+
 # --- Panorama (Voice-Scroll-in-3D) verification: klein-base + the 360 ERP outpaint
 # LoRA produce a 2:1 equirectangular panorama via Flux2 inpaint. Code-only (diffusers
 # already ships Flux2KleinInpaintPipeline + load_lora_weights). ---

frontend/index.html

@@ -474,6 +474,8 @@ Years later, the same road lay buried under snow and silence.</textarea>
       controller). Same-origin /frontend paths.
     -->
     <script type="module" src="/frontend/js/api.js"></script>
+    <!-- Voice input (LL.recorder): mic -> 16 kHz WAV -> Cohere Transcribe -> recital. -->
+    <script type="module" src="/frontend/js/recorder.js"></script>
     <!--
       stage-gl.js (LL.stage): a WebGL2 beat renderer kept ONLY as a passive
       fallback — it never grabs #stage-canvas unless the controller calls

frontend/js/api.js

@@ -73,6 +73,42 @@ const api = {
     return this._streamEndpoint("/panorama", text, lang, onEvent);
   },
 
+  /**
+   * Transcribe a base64 WAV (the narrator's mic audio) via Cohere Transcribe.
+   * Resolves with {text, error}. Used by LL.recorder; the transcript then drives
+   * streamPanorama just like a typed recital.
+   *
+   * @param {string} audioB64  data:audio/wav;base64,... (or bare base64)
+   * @param {string} lang      ISO code ("en"/"es"/...); default "en"
+   * @param {(event:object)=>void} [onEvent]  optional per-stage callback
+   * @returns {Promise<{text:string, error:?string}>}
+   */
+  async transcribe(audioB64, lang = "en", onEvent) {
+    const emit = typeof onEvent === "function" ? onEvent : () => {};
+    let client;
+    try {
+      client = await this.connect();
+    } catch (err) {
+      return { text: "", error: `connect_failed: ${shortErr(err)}` };
+    }
+    let text = "";
+    let error = null;
+    try {
+      const job = client.submit("/transcribe", { audio_b64: audioB64, lang: lang || "en" });
+      for await (const msg of job) {
+        if (!msg || msg.type !== "data") continue;
+        const ev = asEvent(msg.data);
+        if (!ev || typeof ev !== "object") continue;
+        emit(ev);
+        if (ev.stage === "transcript") text = ev.text || "";
+        if (ev.stage === "error" || ev.stage === "quota_exceeded") error = ev.error || ev.stage;
+      }
+    } catch (err) {
+      error = `stream_failed: ${shortErr(err)}`;
+    }
+    return { text, error };
+  },
+
   /** Shared streaming core for the @app.api endpoints (recital + panorama). */
   async _streamEndpoint(endpoint, text, lang = "en", onEvent) {
     const emit = typeof onEvent === "function" ? onEvent : () => {};

frontend/js/recorder.js

@@ -0,0 +1,202 @@
+// frontend/js/recorder.js — LL.recorder: capture the narrator's voice and turn it
+// into text via Cohere Transcribe (the "speak" in "speak, and watch your story
+// become a world"). We capture raw PCM through the Web Audio API and encode a
+// 16 kHz mono WAV in-browser (no MediaRecorder/webm, no server-side ffmpeg), then
+// hand the base64 WAV to LL.api.transcribe — the transcript drives the same
+// panorama pipeline as the typed recital.
+//
+// Public surface:
+//   LL.recorder.init({ onState, onTranscript, getLang })
+//   LL.recorder.toggle()   -> start if idle, stop+transcribe if recording
+//   LL.recorder.start() / .stop() / .cancel()
+//   LL.recorder.state      -> "idle" | "recording" | "transcribing"
+//
+// Vanilla ES module, no deps.
+"use strict";
+
+const LL = (window.LL = window.LL || {});
+const TARGET_SR = 16000; // Cohere Transcribe expects 16 kHz mono
+const MAX_SECONDS = 30; // safety cap so a forgotten mic can't record forever
+
+const recorder = {
+  state: "idle",
+  _ctx: null,
+  _stream: null,
+  _source: null,
+  _node: null,
+  _chunks: [],
+  _inRate: 48000,
+  _capTimer: null,
+  _cb: { onState: null, onTranscript: null, getLang: null },
+
+  init(opts = {}) {
+    this._cb.onState = typeof opts.onState === "function" ? opts.onState : null;
+    this._cb.onTranscript = typeof opts.onTranscript === "function" ? opts.onTranscript : null;
+    this._cb.getLang = typeof opts.getLang === "function" ? opts.getLang : null;
+    return this;
+  },
+
+  /** Browser support gate the controller can check before offering the mic. */
+  get supported() {
+    return !!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia && (window.AudioContext || window.webkitAudioContext));
+  },
+
+  toggle() {
+    if (this.state === "recording") return this.stop();
+    if (this.state === "idle") return this.start();
+    // "transcribing" — ignore re-entry.
+  },
+
+  async start() {
+    if (this.state !== "idle") return;
+    if (!this.supported) {
+      this._setState("idle");
+      if (this._cb.onTranscript) this._cb.onTranscript("", "unsupported");
+      return;
+    }
+    try {
+      this._stream = await navigator.mediaDevices.getUserMedia({
+        audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
+      });
+    } catch (err) {
+      this._setState("idle");
+      if (this._cb.onTranscript) this._cb.onTranscript("", "mic_denied");
+      return;
+    }
+    const AC = window.AudioContext || window.webkitAudioContext;
+    this._ctx = new AC();
+    this._inRate = this._ctx.sampleRate || 48000;
+    this._source = this._ctx.createMediaStreamSource(this._stream);
+    // ScriptProcessorNode is deprecated but universally available (incl. headless
+    // Chromium) and ample for short narration; AudioWorklet would be overkill here.
+    this._node = this._ctx.createScriptProcessor(4096, 1, 1);
+    this._chunks = [];
+    this._node.onaudioprocess = (e) => {
+      const ch = e.inputBuffer.getChannelData(0);
+      this._chunks.push(new Float32Array(ch)); // copy: the buffer is reused
+    };
+    this._source.connect(this._node);
+    this._node.connect(this._ctx.destination);
+    this._setState("recording");
+    this._capTimer = setTimeout(() => {
+      if (this.state === "recording") this.stop();
+    }, MAX_SECONDS * 1000);
+  },
+
+  async stop() {
+    if (this.state !== "recording") return;
+    if (this._capTimer) {
+      clearTimeout(this._capTimer);
+      this._capTimer = null;
+    }
+    this._teardownGraph();
+    this._setState("transcribing");
+
+    const wavB64 = this._encodeWavBase64(this._chunks, this._inRate);
+    this._chunks = [];
+    const lang = (this._cb.getLang && this._cb.getLang()) || "en";
+
+    let text = "";
+    let error = null;
+    try {
+      if (LL.api && typeof LL.api.transcribe === "function") {
+        const res = await LL.api.transcribe(wavB64, lang);
+        text = (res && res.text) || "";
+        error = res && res.error;
+      } else {
+        error = "no_api";
+      }
+    } catch (err) {
+      error = String((err && err.message) || err).slice(0, 120);
+    }
+    this._setState("idle");
+    if (this._cb.onTranscript) this._cb.onTranscript(text, error);
+  },
+
+  cancel() {
+    if (this._capTimer) {
+      clearTimeout(this._capTimer);
+      this._capTimer = null;
+    }
+    this._teardownGraph();
+    this._chunks = [];
+    this._setState("idle");
+  },
+
+  _teardownGraph() {
+    try { if (this._node) this._node.disconnect(); } catch (_) {}
+    try { if (this._source) this._source.disconnect(); } catch (_) {}
+    try { if (this._stream) this._stream.getTracks().forEach((t) => t.stop()); } catch (_) {}
+    try { if (this._ctx && this._ctx.state !== "closed") this._ctx.close(); } catch (_) {}
+    this._node = this._source = this._stream = this._ctx = null;
+  },
+
+  _setState(s) {
+    this.state = s;
+    if (this._cb.onState) {
+      try { this._cb.onState(s); } catch (_) {}
+    }
+  },
+
+  // ---- encode captured Float32 chunks -> 16 kHz mono 16-bit PCM WAV (base64) ----
+  _encodeWavBase64(chunks, inRate) {
+    let length = 0;
+    for (const c of chunks) length += c.length;
+    const merged = new Float32Array(length);
+    let off = 0;
+    for (const c of chunks) {
+      merged.set(c, off);
+      off += c.length;
+    }
+    const pcm = this._resample(merged, inRate, TARGET_SR);
+    const buffer = new ArrayBuffer(44 + pcm.length * 2);
+    const view = new DataView(buffer);
+    const writeStr = (o, s) => { for (let i = 0; i < s.length; i++) view.setUint8(o + i, s.charCodeAt(i)); };
+    const dataLen = pcm.length * 2;
+    writeStr(0, "RIFF");
+    view.setUint32(4, 36 + dataLen, true);
+    writeStr(8, "WAVE");
+    writeStr(12, "fmt ");
+    view.setUint32(16, 16, true); // PCM chunk size
+    view.setUint16(20, 1, true); // PCM
+    view.setUint16(22, 1, true); // mono
+    view.setUint32(24, TARGET_SR, true);
+    view.setUint32(28, TARGET_SR * 2, true); // byte rate
+    view.setUint16(32, 2, true); // block align
+    view.setUint16(34, 16, true); // bits per sample
+    writeStr(36, "data");
+    view.setUint32(40, dataLen, true);
+    let p = 44;
+    for (let i = 0; i < pcm.length; i++, p += 2) {
+      let s = Math.max(-1, Math.min(1, pcm[i]));
+      view.setInt16(p, s < 0 ? s * 0x8000 : s * 0x7fff, true);
+    }
+    // ArrayBuffer -> base64 in chunks (avoid call-stack limits on big buffers).
+    const bytes = new Uint8Array(buffer);
+    let bin = "";
+    const CH = 0x8000;
+    for (let i = 0; i < bytes.length; i += CH) {
+      bin += String.fromCharCode.apply(null, bytes.subarray(i, i + CH));
+    }
+    return "data:audio/wav;base64," + btoa(bin);
+  },
+
+  _resample(input, inRate, outRate) {
+    if (inRate === outRate) return input;
+    const ratio = inRate / outRate;
+    const outLen = Math.round(input.length / ratio);
+    const out = new Float32Array(outLen);
+    for (let i = 0; i < outLen; i++) {
+      const idx = i * ratio;
+      const i0 = Math.floor(idx);
+      const i1 = Math.min(i0 + 1, input.length - 1);
+      const frac = idx - i0;
+      out[i] = input[i0] * (1 - frac) + input[i1] * frac; // linear interpolation
+    }
+    return out;
+  },
+};
+
+LL.recorder = recorder;
+
+export default LL.recorder;

frontend/js/stage.js

@@ -29,6 +29,10 @@ const COPY = {
   painting: { es: "Pintando el plano…", en: "Painting the shot…" },
   fogged: { es: "El plano se veló; seguimos rodando.", en: "The shot got fogged; rolling on." },
   empty: { es: "Pega un poema antes de rodar.", en: "Paste a poem before rolling." },
+  listening: { es: "Te escucho… habla y pulsa para parar.", en: "Listening… speak, then click to stop." },
+  transcribing: { es: "Transcribiendo tu voz…", en: "Transcribing your voice…" },
+  micDenied: { es: "No pude usar el micrófono. Escribe el poema y pulsa Rodar.", en: "Couldn't use the mic. Type your poem and hit Roll." },
+  noSpeech: { es: "No escuché nada. Inténtalo otra vez o escribe el poema.", en: "I didn't catch that. Try again or type the poem." },
   connect: {
     es: "No se pudo abrir la sala. Reintenta en un momento.",
     en: "Couldn't open the theater. Try again in a moment.",
@@ -82,6 +86,7 @@ const controller = {
     this._wireControls();
     this._wireLangToggle();
     this._initStageRenderer();
+    this._initRecorder();
     this._applyLang(this.lang); // paint all data-es/data-en nodes once
     this.show("lobby");
     return this;
@@ -107,10 +112,16 @@ const controller = {
     on("btn-recital", "click", () => this.startRecital());
     on("btn-recital-go", "click", () => this.startRecital());
     on("btn-narrate", "click", () => {
-      // Live narration (mic) is a sibling module; if it isn't present, fall back
-      // to the recital flow so the button is never dead.
-      if (LL.recorder && typeof LL.recorder.start === "function") LL.recorder.start();
-      else this.startRecital();
+      // Live narration: capture the mic, transcribe (Cohere), then film the world.
+      // If the recorder isn't available/supported, fall back to the typed recital
+      // so the button is never dead.
+      if (LL.recorder && LL.recorder.supported && typeof LL.recorder.toggle === "function") {
+        LL.recorder.toggle();
+      } else {
+        const ta = document.getElementById("recital-text");
+        if (ta && ta.value.trim()) this.startRecital();
+        else { this.toast(t(COPY.micDenied, this.lang)); if (ta) ta.focus(); }
+      }
     });
     on("btn-showcase", "click", () => this.startShowcase());
   },
@@ -168,6 +179,38 @@ const controller = {
     return LL.pano || LL.stage || null;
   },
 
+  /** Wire the voice recorder (LL.recorder): mic -> Cohere Transcribe -> the
+   *  transcript fills the recital and films the world. Mic state drives toasts +
+   *  a recording class on the Narrate button. No-op if the recorder is absent. */
+  _initRecorder() {
+    if (!LL.recorder || typeof LL.recorder.init !== "function") return;
+    LL.recorder.init({
+      getLang: () => this.lang,
+      onState: (s) => {
+        const btn = document.getElementById("btn-narrate");
+        if (btn) {
+          btn.classList.toggle("is-recording", s === "recording");
+          btn.classList.toggle("is-busy", s === "transcribing");
+          btn.setAttribute("aria-pressed", String(s === "recording"));
+        }
+        if (s === "recording") this.toast(t(COPY.listening, this.lang));
+        else if (s === "transcribing") this.toast(t(COPY.transcribing, this.lang));
+      },
+      onTranscript: (text, error) => {
+        const ta = document.getElementById("recital-text");
+        const clean = (text || "").trim();
+        if (clean) {
+          if (ta) ta.value = clean;
+          this.startRecital();
+        } else {
+          const denied = error === "mic_denied" || error === "unsupported";
+          this.toast(t(denied ? COPY.micDenied : COPY.noSpeech, this.lang));
+          if (ta) ta.focus();
+        }
+      },
+    });
+  },
+
   // ---- the main flow: start a recital ---------------------------------------
 
   /**

requirements.txt

@@ -12,3 +12,7 @@ diffusers @ git+https://github.com/huggingface/diffusers.git@784fa62652fb2719d41
 peft>=0.14
 llama-cpp-python==0.3.28
 spaces==0.50.4
+# Cohere Transcribe's processor (trust_remote_code) imports librosa at load; soundfile
+# lets us decode the browser's PCM WAV without ffmpeg. Verified conflict-free locally.
+librosa>=0.10
+soundfile>=0.12

scripts/shoot-voice.cjs

@@ -0,0 +1,96 @@
+// Validate the voice wiring locally: fake mic -> LL.recorder captures + encodes a
+// WAV -> (stubbed) LL.api.transcribe -> transcript fills the recital -> (stubbed)
+// streamPanorama reveals the world. Proves the recorder graph + flow without the
+// real ASR endpoint. Usage: node scripts/shoot-voice.cjs
+const { chromium } = require("playwright");
+const http = require("http");
+const fs = require("fs");
+const path = require("path");
+
+const ROOT = process.cwd();
+const PORT = 8804;
+const CT = {
+  ".html": "text/html", ".js": "application/javascript", ".mjs": "application/javascript",
+  ".css": "text/css", ".woff2": "font/woff2", ".webp": "image/webp", ".png": "image/png",
+  ".json": "application/json", ".svg": "image/svg+xml",
+};
+
+const server = http.createServer((req, res) => {
+  let p = decodeURIComponent(req.url.split("?")[0]);
+  if (p === "/") p = "/frontend/index.html";
+  if (p.startsWith("/shots/")) p = "/.shots/" + p.slice(7);
+  const file = path.join(ROOT, p);
+  fs.readFile(file, (err, data) => {
+    if (err) { res.writeHead(404); res.end("nf"); return; }
+    res.writeHead(200, { "content-type": CT[path.extname(file)] || "application/octet-stream" });
+    res.end(data);
+  });
+});
+
+(async () => {
+  await new Promise((r) => server.listen(PORT, r));
+  const browser = await chromium.launch({
+    args: [
+      "--use-gl=angle", "--use-angle=swiftshader", "--ignore-gpu-blocklist", "--enable-webgl",
+      "--use-fake-device-for-media-stream", "--use-fake-ui-for-media-stream",
+    ],
+  });
+  const ctx = await browser.newContext({ viewport: { width: 1680, height: 820 }, permissions: ["microphone"] });
+  const page = await ctx.newPage();
+  const errs = [];
+  page.on("console", (m) => { if (m.type() === "error") errs.push(m.text().slice(0, 200)); });
+  page.on("pageerror", (e) => errs.push("PAGEERROR: " + e.message.slice(0, 200)));
+
+  await page.goto(`http://localhost:${PORT}/frontend/index.html`, { waitUntil: "networkidle", timeout: 25000 }).catch((e) => errs.push("goto:" + e.message));
+  await page.waitForTimeout(1000);
+
+  // Stub the network bits: transcribe returns a known line; panorama reveals a real world.
+  await page.evaluate(() => {
+    const LL = window.LL;
+    const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+    window.__transcribeCalled = false;
+    window.__wavBytes = 0;
+    LL.api.transcribe = async (b64) => {
+      window.__transcribeCalled = true;
+      window.__wavBytes = (b64 || "").length;
+      return { text: "An old lighthouse keeper counted the waves at the edge of the world.", error: null };
+    };
+    LL.api.streamPanorama = async (text, lang, onEvent) => {
+      window.__panoText = text;
+      onEvent({ stage: "warming" });
+      await sleep(40);
+      onEvent({ stage: "painted", panorama: "/shots/real-pano.webp" });
+      await sleep(60);
+      onEvent({ stage: "depth", depth: "/shots/real-depth.png" });
+      onEvent({ stage: "done", session: "voice" });
+      return { session: "voice", done: true, cancelled: false };
+    };
+    return { supported: LL.recorder && LL.recorder.supported };
+  });
+
+  // Click Narrate (start recording) -> wait (capture fake audio) -> click again (stop).
+  const sup = await page.evaluate(() => window.LL.recorder && window.LL.recorder.supported);
+  await page.click("#btn-narrate").catch(() => {});
+  await page.waitForTimeout(1500);
+  const recState = await page.evaluate(() => window.LL.recorder.state);
+  await page.click("#btn-narrate").catch(() => {});
+  // Wait for transcribe -> recital -> world reveal.
+  await page.waitForFunction(() => window.LL.controller && window.LL.controller._worldShown, { timeout: 8000 }).catch(() => {});
+  await page.evaluate(() => window.dispatchEvent(new Event("resize")));
+  await page.waitForTimeout(1500);
+  await page.screenshot({ path: path.join(ROOT, ".shots", "voice-world.png") });
+
+  const st = await page.evaluate(() => ({
+    supported: window.LL.recorder.supported,
+    transcribeCalled: window.__transcribeCalled,
+    wavLen: window.__wavBytes,
+    panoText: window.__panoText,
+    worldShown: window.LL.controller._worldShown,
+    activeScreen: [...document.querySelectorAll(".screen.is-active")].map((s) => s.id),
+  }));
+  console.log("SUPPORTED:", sup, "REC_STATE_WHILE_RECORDING:", recState);
+  console.log("RESULT:", JSON.stringify(st));
+  console.log("JS_ERRORS:", JSON.stringify(errs.slice(0, 10)));
+  await browser.close();
+  server.close();
+})().catch((e) => { console.error("HARNESS ERROR:", e.message); process.exit(1); });