chore: deploy private lightloom build
Browse files- app.py +10 -3
- frontend/js/stage-corridor.js +104 -144
- scripts/_debug_asr.py +44 -0
- src/lightloom/audio_in/asr.py +40 -50
- src/lightloom/core/config.py +7 -3
app.py
CHANGED
|
@@ -144,7 +144,7 @@ def health() -> dict[str, Any]:
|
|
| 144 |
"privacy_mode": os.getenv("LIGHTLOOM_PRIVACY_MODE", "1") == "1",
|
| 145 |
"cpu_count": os.cpu_count(),
|
| 146 |
"director_backend": CONFIG.director_backend,
|
| 147 |
-
"build": "
|
| 148 |
}
|
| 149 |
|
| 150 |
|
|
@@ -482,12 +482,19 @@ def _corridor_world(text: str, session: str):
|
|
| 482 |
tokenizer, model = load_director_transformers()
|
| 483 |
klein = load_klein_pipeline()
|
| 484 |
depth_pipe = load_depth_pipeline()
|
| 485 |
-
state
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 486 |
|
| 487 |
for i, phrase in enumerate(phrases):
|
| 488 |
yield {"stage": "directing", "index": i, "text": phrase}
|
| 489 |
try:
|
| 490 |
-
shot, _m = generate_shot_transformers(tokenizer, model, phrase,
|
| 491 |
style = _corridor_style(shot)
|
| 492 |
scene_prompt = f"{shot.image_prompt_en}, {shot.lighting}. {style}, {_CORRIDOR_QUALITY}"
|
| 493 |
yield {"stage": "painting", "index": i, "scene": scene_prompt[:150]}
|
|
|
|
| 144 |
"privacy_mode": os.getenv("LIGHTLOOM_PRIVACY_MODE", "1") == "1",
|
| 145 |
"cpu_count": os.cpu_count(),
|
| 146 |
"director_backend": CONFIG.director_backend,
|
| 147 |
+
"build": "v2-parallax-whisper-1",
|
| 148 |
}
|
| 149 |
|
| 150 |
|
|
|
|
| 482 |
tokenizer, model = load_director_transformers()
|
| 483 |
klein = load_klein_pipeline()
|
| 484 |
depth_pipe = load_depth_pipeline()
|
| 485 |
+
# NEUTRAL state per scene (not the lighthouse-keeper seed in initial_state) so each
|
| 486 |
+
# scene is driven by what the narrator actually said — no "lighthouses everywhere".
|
| 487 |
+
from lightloom.director.state import SceneState
|
| 488 |
+
|
| 489 |
+
neutral = SceneState(
|
| 490 |
+
characters=[], setting="", lighting="", mood="",
|
| 491 |
+
palette=("#202428", "#7d8794", "#e6ddc8"),
|
| 492 |
+
).model_dump()
|
| 493 |
|
| 494 |
for i, phrase in enumerate(phrases):
|
| 495 |
yield {"stage": "directing", "index": i, "text": phrase}
|
| 496 |
try:
|
| 497 |
+
shot, _m = generate_shot_transformers(tokenizer, model, phrase, dict(neutral))
|
| 498 |
style = _corridor_style(shot)
|
| 499 |
scene_prompt = f"{shot.image_prompt_en}, {shot.lighting}. {style}, {_CORRIDOR_QUALITY}"
|
| 500 |
yield {"stage": "painting", "index": i, "scene": scene_prompt[:150]}
|
frontend/js/stage-corridor.js
CHANGED
|
@@ -1,19 +1,20 @@
|
|
| 1 |
/* Lightloom · frontend/js/stage-corridor.js
|
| 2 |
* ============================================================================
|
| 3 |
-
* THE
|
| 4 |
*
|
| 5 |
-
* Each phrase you narrate becomes
|
| 6 |
-
*
|
| 7 |
-
* the
|
| 8 |
-
*
|
| 9 |
-
*
|
| 10 |
-
*
|
|
|
|
|
|
|
| 11 |
*
|
| 12 |
* window.LL.corridor = {
|
| 13 |
-
* init(canvas), addScene({imageUrl, depthUrl}), reset(), dispose(),
|
| 14 |
-
* setReducedMotion(flag), get ready, get count
|
| 15 |
* }
|
| 16 |
-
* addScene() resolves once the panel's textures are loaded and it has been queued.
|
| 17 |
* ==========================================================================*/
|
| 18 |
"use strict";
|
| 19 |
import * as THREE from "three";
|
|
@@ -22,84 +23,69 @@ window.LL = window.LL || {};
|
|
| 22 |
|
| 23 |
const Corridor = (() => {
|
| 24 |
let renderer = null, scene = null, camera = null, canvas = null;
|
| 25 |
-
let raf = 0, t0 = 0,
|
| 26 |
-
const
|
| 27 |
-
let
|
| 28 |
-
let frontZ = 0; // z of the most recently added panel (grows negative)
|
| 29 |
-
let camZ = 0, targetZ = 0; // camera travel along -Z
|
| 30 |
const loader = new THREE.TextureLoader();
|
| 31 |
loader.crossOrigin = "anonymous";
|
| 32 |
|
| 33 |
-
// Geometry constants (world units). Tuned for a roomy, immersive corridor.
|
| 34 |
-
const START_Z = -56; // first panel sits here, ahead of the camera at z=0
|
| 35 |
-
const SPACING = 58; // distance between consecutive scenes
|
| 36 |
-
const APPROACH = 52; // camera settles this far in front of the newest panel
|
| 37 |
-
// Forward-facing 16:9 scene panels (NOT equirectangular), generously oversized so
|
| 38 |
-
// they always OVERFILL the view (no black borders), with a very gentle concave
|
| 39 |
-
// curve so the world wraps the periphery without distorting the frame.
|
| 40 |
-
const PANEL_W = 234, PANEL_H = 134, CURVE_R = 480;
|
| 41 |
-
|
| 42 |
const VERT = `
|
| 43 |
varying vec2 vUv;
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
float d = texture2D(uDepth, uv).r; // white = near
|
| 50 |
-
// push the surface toward the camera (+Z local) by the depth -> real relief
|
| 51 |
-
vec3 p = position + vec3(0.0, 0.0, 1.0) * (uRelief * d);
|
| 52 |
-
vec4 mv = modelViewMatrix * vec4(p, 1.0);
|
| 53 |
-
vFog = -mv.z; // view-space distance for fog
|
| 54 |
-
gl_Position = projectionMatrix * mv;
|
| 55 |
-
}`;
|
| 56 |
const FRAG = `
|
|
|
|
| 57 |
varying vec2 vUv;
|
| 58 |
-
|
| 59 |
-
uniform sampler2D
|
|
|
|
|
|
|
|
|
|
| 60 |
uniform float uOpacity;
|
| 61 |
-
uniform
|
| 62 |
-
uniform vec3 uFogColor;
|
| 63 |
-
uniform float uFogNear;
|
| 64 |
-
uniform float uFogFar;
|
| 65 |
void main() {
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
c = max(c, vec3(0.0));
|
| 72 |
vec3 lo = c * 12.92;
|
| 73 |
vec3 hi = 1.055 * pow(c, vec3(1.0 / 2.4)) - 0.055;
|
| 74 |
c = mix(lo, hi, step(vec3(0.0031308), c));
|
| 75 |
-
//
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
gl_FragColor = vec4(c, uOpacity * vig);
|
| 80 |
}`;
|
| 81 |
|
| 82 |
-
const
|
| 83 |
|
| 84 |
-
function
|
| 85 |
-
|
| 86 |
-
const
|
| 87 |
-
|
| 88 |
-
const x = pos.getX(i);
|
| 89 |
-
const theta = x / CURVE_R;
|
| 90 |
-
pos.setX(i, Math.sin(theta) * CURVE_R);
|
| 91 |
-
pos.setZ(i, (Math.cos(theta) - 1.0) * CURVE_R); // edges curl toward camera
|
| 92 |
-
}
|
| 93 |
-
pos.needsUpdate = true;
|
| 94 |
-
geo.computeVertexNormals();
|
| 95 |
-
return geo;
|
| 96 |
}
|
| 97 |
-
let GEO = null; // shared geometry (cloned per panel is unnecessary; reuse)
|
| 98 |
|
| 99 |
function loadTex(url) {
|
| 100 |
return new Promise((resolve) => {
|
| 101 |
if (!url) return resolve(null);
|
| 102 |
-
loader.load(url, (t) => {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
});
|
| 104 |
}
|
| 105 |
|
|
@@ -110,14 +96,10 @@ const Corridor = (() => {
|
|
| 110 |
renderer = new THREE.WebGLRenderer({ canvas, antialias: true, alpha: false });
|
| 111 |
renderer.setPixelRatio(Math.min(devicePixelRatio || 1, 2));
|
| 112 |
scene = new THREE.Scene();
|
| 113 |
-
|
| 114 |
-
camera = new THREE.PerspectiveCamera(72, 1, 0.1, 600);
|
| 115 |
-
camera.position.set(0, 0, 0);
|
| 116 |
-
GEO = curvedPanelGeo();
|
| 117 |
resize();
|
| 118 |
window.addEventListener("resize", resize);
|
| 119 |
-
ok = true;
|
| 120 |
-
t0 = performance.now();
|
| 121 |
loop();
|
| 122 |
} catch (e) { ok = false; }
|
| 123 |
}
|
|
@@ -127,7 +109,8 @@ const Corridor = (() => {
|
|
| 127 |
const w = canvas.clientWidth || canvas.parentElement?.clientWidth || 1280;
|
| 128 |
const h = canvas.clientHeight || canvas.parentElement?.clientHeight || 720;
|
| 129 |
renderer.setSize(w, h, false);
|
| 130 |
-
|
|
|
|
| 131 |
}
|
| 132 |
|
| 133 |
function reducedNow() {
|
|
@@ -135,103 +118,80 @@ const Corridor = (() => {
|
|
| 135 |
(window.matchMedia && window.matchMedia("(prefers-reduced-motion: reduce)").matches);
|
| 136 |
}
|
| 137 |
|
| 138 |
-
let onFocus = null; // (meta, index) => void — fires when a scene flies into focus
|
| 139 |
-
let focusKey = -1;
|
| 140 |
-
|
| 141 |
async function addScene({ imageUrl, depthUrl, meta } = {}) {
|
| 142 |
if (!ok) return null;
|
| 143 |
const [tex, depthTex] = await Promise.all([loadTex(imageUrl), loadTex(depthUrl)]);
|
| 144 |
if (!tex) return null;
|
|
|
|
| 145 |
const mat = new THREE.ShaderMaterial({
|
| 146 |
uniforms: {
|
| 147 |
-
|
| 148 |
uDepth: { value: depthTex || tex },
|
| 149 |
-
|
|
|
|
|
|
|
| 150 |
uOpacity: { value: 0.0 },
|
| 151 |
-
|
| 152 |
-
uFogColor: { value: FOG_COLOR.clone() },
|
| 153 |
-
uFogNear: { value: 60.0 },
|
| 154 |
-
uFogFar: { value: 230.0 },
|
| 155 |
},
|
| 156 |
-
vertexShader: VERT, fragmentShader: FRAG, transparent: true,
|
| 157 |
});
|
| 158 |
-
const mesh = new THREE.Mesh(
|
| 159 |
-
|
| 160 |
-
mesh.position.z = frontZ;
|
| 161 |
-
mesh.renderOrder = -frontZ; // draw far panels first (painter's order for blending)
|
| 162 |
scene.add(mesh);
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
return { id:
|
| 166 |
}
|
| 167 |
|
| 168 |
function reset() {
|
| 169 |
-
for (const
|
| 170 |
-
|
| 171 |
-
frontZ = 0; camZ = 0; targetZ = 0; focusKey = -1;
|
| 172 |
-
if (camera) camera.position.set(0, 0, 0);
|
| 173 |
}
|
| 174 |
|
| 175 |
function loop() {
|
| 176 |
raf = requestAnimationFrame(loop);
|
| 177 |
if (!renderer || !scene || !camera) return;
|
| 178 |
const now = performance.now();
|
| 179 |
-
const dt = Math.min(0.05, (now - (lastT || now)) / 1000);
|
| 180 |
-
lastT = now;
|
| 181 |
const t = (now - t0) / 1000;
|
| 182 |
const rm = reducedNow();
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
const fadeIn = smoothstep(160.0, 92.0, ahead); // far haze -> visible
|
| 202 |
-
const fadeOut = smoothstep(30.0, 50.0, ahead); // fully dissolved well before arrival
|
| 203 |
-
let op = Math.min(fadeIn, fadeOut);
|
| 204 |
-
// gentle birth fade so a freshly streamed scene doesn't pop
|
| 205 |
-
const age = (now - p.born) / 1000;
|
| 206 |
-
op *= Math.min(1, age / 0.7);
|
| 207 |
-
p.mesh.material.uniforms.uOpacity.value = op;
|
| 208 |
-
if (ahead < 28.0) { // dissolved: recycle before the camera can reach it
|
| 209 |
-
scene.remove(p.mesh); p.mesh.material.dispose(); panels.splice(i, 1);
|
| 210 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
}
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
for (const p of panels) {
|
| 216 |
-
const op = p.mesh.material.uniforms.uOpacity.value;
|
| 217 |
-
if (op > bestOp) { bestOp = op; best = p; }
|
| 218 |
-
}
|
| 219 |
-
if (best && best.id !== focusKey) {
|
| 220 |
-
focusKey = best.id;
|
| 221 |
-
if (onFocus && best.meta) { try { onFocus(best.meta, best.id); } catch (_) {} }
|
| 222 |
}
|
| 223 |
renderer.render(scene, camera);
|
| 224 |
}
|
| 225 |
|
| 226 |
-
function
|
| 227 |
-
const t = Math.max(0, Math.min(1, (x - edge0) / (edge1 - edge0)));
|
| 228 |
-
return t * t * (3 - 2 * t);
|
| 229 |
-
}
|
| 230 |
-
|
| 231 |
-
function setReducedMotion(flag) {
|
| 232 |
-
reduced = !!flag;
|
| 233 |
-
for (const p of panels) p.mesh.material.uniforms.uRelief.value = reducedNow() ? 3.0 : 12.0;
|
| 234 |
-
}
|
| 235 |
|
| 236 |
function dispose() {
|
| 237 |
cancelAnimationFrame(raf);
|
|
@@ -245,7 +205,7 @@ const Corridor = (() => {
|
|
| 245 |
init, addScene, reset, dispose, setReducedMotion,
|
| 246 |
set onFocus(fn) { onFocus = typeof fn === "function" ? fn : null; },
|
| 247 |
get ready() { return ok; },
|
| 248 |
-
get count() { return
|
| 249 |
};
|
| 250 |
})();
|
| 251 |
|
|
|
|
| 1 |
/* Lightloom · frontend/js/stage-corridor.js
|
| 2 |
* ============================================================================
|
| 3 |
+
* THE LIVING WORLD — a cinematic depth-parallax film of your story.
|
| 4 |
*
|
| 5 |
+
* Each phrase you narrate becomes one scene. Instead of displacing geometry by the
|
| 6 |
+
* depth map (which TEARS into jagged sawtooth edges at depth discontinuities), we
|
| 7 |
+
* keep the scene as a full-frame image and shift the *texture sample* per pixel by
|
| 8 |
+
* its depth — the "3D photo" / parallax technique (LeiaPix / Depthy / Facebook 3D
|
| 9 |
+
* photos). Near things slide more than far things as a slow cinematic camera path
|
| 10 |
+
* drifts and pushes in, so the frame breathes in real 3D — at the FULL quality of
|
| 11 |
+
* the painted image, with NO geometry tearing. Scenes cross-fade with a forward
|
| 12 |
+
* push, so the story keeps travelling ahead of you.
|
| 13 |
*
|
| 14 |
* window.LL.corridor = {
|
| 15 |
+
* init(canvas), addScene({imageUrl, depthUrl, meta}), reset(), dispose(),
|
| 16 |
+
* setReducedMotion(flag), set onFocus, get ready, get count
|
| 17 |
* }
|
|
|
|
| 18 |
* ==========================================================================*/
|
| 19 |
"use strict";
|
| 20 |
import * as THREE from "three";
|
|
|
|
| 23 |
|
| 24 |
const Corridor = (() => {
|
| 25 |
let renderer = null, scene = null, camera = null, canvas = null;
|
| 26 |
+
let raf = 0, t0 = 0, ok = false, reduced = false, aspect = 16 / 9;
|
| 27 |
+
const layers = []; // { mesh, mat, born, id, meta } newest last
|
| 28 |
+
let seq = 0, focusKey = -1, onFocus = null;
|
|
|
|
|
|
|
| 29 |
const loader = new THREE.TextureLoader();
|
| 30 |
loader.crossOrigin = "anonymous";
|
| 31 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
const VERT = `
|
| 33 |
varying vec2 vUv;
|
| 34 |
+
void main() { vUv = uv; gl_Position = vec4(position.xy, 0.0, 1.0); }`;
|
| 35 |
+
|
| 36 |
+
// Per-pixel depth parallax with a 2-step refinement (kills the "swimming" the
|
| 37 |
+
// naive single-tap shows). uParallax is the animated camera offset; near pixels
|
| 38 |
+
// (depth→1) shift most. uZoom is a slow Ken-Burns push for forward travel.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
const FRAG = `
|
| 40 |
+
precision highp float;
|
| 41 |
varying vec2 vUv;
|
| 42 |
+
uniform sampler2D uImage;
|
| 43 |
+
uniform sampler2D uDepth;
|
| 44 |
+
uniform vec2 uParallax;
|
| 45 |
+
uniform float uZoom;
|
| 46 |
+
uniform float uFocus;
|
| 47 |
uniform float uOpacity;
|
| 48 |
+
uniform vec2 uCover; // aspect-fill scale (cover the canvas, no bars)
|
|
|
|
|
|
|
|
|
|
| 49 |
void main() {
|
| 50 |
+
// center, aspect-cover, ken-burns zoom
|
| 51 |
+
vec2 uv = (vUv - 0.5) * uCover / uZoom + 0.5;
|
| 52 |
+
// depth-corrected parallax: sample depth, offset, refine once
|
| 53 |
+
float d = texture2D(uDepth, uv).r;
|
| 54 |
+
vec2 p = uParallax * (d - uFocus);
|
| 55 |
+
d = texture2D(uDepth, uv + p).r;
|
| 56 |
+
p = uParallax * (d - uFocus);
|
| 57 |
+
vec2 suv = uv + p;
|
| 58 |
+
vec3 c = texture2D(uImage, clamp(suv, 0.001, 0.999)).rgb;
|
| 59 |
+
// linear->sRGB (raw ShaderMaterial is not auto-encoded by three.js)
|
| 60 |
c = max(c, vec3(0.0));
|
| 61 |
vec3 lo = c * 12.92;
|
| 62 |
vec3 hi = 1.055 * pow(c, vec3(1.0 / 2.4)) - 0.055;
|
| 63 |
c = mix(lo, hi, step(vec3(0.0031308), c));
|
| 64 |
+
// a whisper of cinematic vignette
|
| 65 |
+
vec2 q = vUv - 0.5;
|
| 66 |
+
c *= 1.0 - dot(q, q) * 0.32;
|
| 67 |
+
gl_FragColor = vec4(c, uOpacity);
|
|
|
|
| 68 |
}`;
|
| 69 |
|
| 70 |
+
const QUAD = new THREE.PlaneGeometry(2, 2);
|
| 71 |
|
| 72 |
+
function texCover(texAspect) {
|
| 73 |
+
// aspect-fill: scale the SMALLER axis up so the image covers the canvas
|
| 74 |
+
const r = texAspect / aspect;
|
| 75 |
+
return r > 1 ? new THREE.Vector2(r, 1) : new THREE.Vector2(1, 1 / r);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
}
|
|
|
|
| 77 |
|
| 78 |
function loadTex(url) {
|
| 79 |
return new Promise((resolve) => {
|
| 80 |
if (!url) return resolve(null);
|
| 81 |
+
loader.load(url, (t) => {
|
| 82 |
+
t.colorSpace = THREE.SRGBColorSpace;
|
| 83 |
+
t.minFilter = THREE.LinearFilter; t.magFilter = THREE.LinearFilter;
|
| 84 |
+
t.generateMipmaps = false;
|
| 85 |
+
const a = renderer.capabilities.getMaxAnisotropy ? renderer.capabilities.getMaxAnisotropy() : 1;
|
| 86 |
+
t.anisotropy = Math.min(8, a || 1);
|
| 87 |
+
resolve(t);
|
| 88 |
+
}, undefined, () => resolve(null));
|
| 89 |
});
|
| 90 |
}
|
| 91 |
|
|
|
|
| 96 |
renderer = new THREE.WebGLRenderer({ canvas, antialias: true, alpha: false });
|
| 97 |
renderer.setPixelRatio(Math.min(devicePixelRatio || 1, 2));
|
| 98 |
scene = new THREE.Scene();
|
| 99 |
+
camera = new THREE.Camera(); // clip-space quads; no transforms needed
|
|
|
|
|
|
|
|
|
|
| 100 |
resize();
|
| 101 |
window.addEventListener("resize", resize);
|
| 102 |
+
ok = true; t0 = performance.now();
|
|
|
|
| 103 |
loop();
|
| 104 |
} catch (e) { ok = false; }
|
| 105 |
}
|
|
|
|
| 109 |
const w = canvas.clientWidth || canvas.parentElement?.clientWidth || 1280;
|
| 110 |
const h = canvas.clientHeight || canvas.parentElement?.clientHeight || 720;
|
| 111 |
renderer.setSize(w, h, false);
|
| 112 |
+
aspect = w / h;
|
| 113 |
+
for (const L of layers) L.mat.uniforms.uCover.value.copy(texCover(L.texAspect));
|
| 114 |
}
|
| 115 |
|
| 116 |
function reducedNow() {
|
|
|
|
| 118 |
(window.matchMedia && window.matchMedia("(prefers-reduced-motion: reduce)").matches);
|
| 119 |
}
|
| 120 |
|
|
|
|
|
|
|
|
|
|
| 121 |
async function addScene({ imageUrl, depthUrl, meta } = {}) {
|
| 122 |
if (!ok) return null;
|
| 123 |
const [tex, depthTex] = await Promise.all([loadTex(imageUrl), loadTex(depthUrl)]);
|
| 124 |
if (!tex) return null;
|
| 125 |
+
const texAspect = (tex.image && tex.image.width / tex.image.height) || 16 / 9;
|
| 126 |
const mat = new THREE.ShaderMaterial({
|
| 127 |
uniforms: {
|
| 128 |
+
uImage: { value: tex },
|
| 129 |
uDepth: { value: depthTex || tex },
|
| 130 |
+
uParallax: { value: new THREE.Vector2(0, 0) },
|
| 131 |
+
uZoom: { value: 1.0 },
|
| 132 |
+
uFocus: { value: 0.5 },
|
| 133 |
uOpacity: { value: 0.0 },
|
| 134 |
+
uCover: { value: texCover(texAspect) },
|
|
|
|
|
|
|
|
|
|
| 135 |
},
|
| 136 |
+
vertexShader: VERT, fragmentShader: FRAG, transparent: true, depthTest: false, depthWrite: false,
|
| 137 |
});
|
| 138 |
+
const mesh = new THREE.Mesh(QUAD, mat);
|
| 139 |
+
mesh.renderOrder = ++seq;
|
|
|
|
|
|
|
| 140 |
scene.add(mesh);
|
| 141 |
+
const L = { mesh, mat, born: performance.now(), id: seq, meta: meta || null, texAspect, phase: seq * 1.7 };
|
| 142 |
+
layers.push(L);
|
| 143 |
+
return { id: seq };
|
| 144 |
}
|
| 145 |
|
| 146 |
function reset() {
|
| 147 |
+
for (const L of layers) { scene.remove(L.mesh); L.mat.dispose(); L.mat.uniforms.uImage.value?.dispose?.(); }
|
| 148 |
+
layers.length = 0; focusKey = -1;
|
|
|
|
|
|
|
| 149 |
}
|
| 150 |
|
| 151 |
function loop() {
|
| 152 |
raf = requestAnimationFrame(loop);
|
| 153 |
if (!renderer || !scene || !camera) return;
|
| 154 |
const now = performance.now();
|
|
|
|
|
|
|
| 155 |
const t = (now - t0) / 1000;
|
| 156 |
const rm = reducedNow();
|
| 157 |
+
const FADE = rm ? 320 : 1300; // crossfade ms
|
| 158 |
+
const amp = rm ? 0.0 : 0.018; // parallax amplitude (UV units) — subtle, no tearing
|
| 159 |
+
|
| 160 |
+
let focus = null;
|
| 161 |
+
for (let i = layers.length - 1; i >= 0; i--) {
|
| 162 |
+
const L = layers[i];
|
| 163 |
+
const age = (now - L.born) / 1000;
|
| 164 |
+
const isNewest = i === layers.length - 1;
|
| 165 |
+
// crossfade: newest ramps in; everyone older ramps out
|
| 166 |
+
const k = Math.min(1, (now - L.born) / FADE);
|
| 167 |
+
const ease = k * k * (3 - 2 * k);
|
| 168 |
+
let op;
|
| 169 |
+
if (isNewest) op = ease;
|
| 170 |
+
else {
|
| 171 |
+
// older layers: fade out once a newer one exists (started fading at its sibling's birth)
|
| 172 |
+
const newer = layers[i + 1];
|
| 173 |
+
const ko = Math.min(1, (now - newer.born) / FADE);
|
| 174 |
+
op = 1 - ko * ko * (3 - 2 * ko);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
}
|
| 176 |
+
L.mat.uniforms.uOpacity.value = op;
|
| 177 |
+
// cinematic motion: a slow drifting parallax orbit + a gentle forward push (zoom-in)
|
| 178 |
+
const ph = L.phase;
|
| 179 |
+
L.mat.uniforms.uParallax.value.set(
|
| 180 |
+
amp * Math.sin(t * 0.16 + ph),
|
| 181 |
+
amp * 0.6 * Math.sin(t * 0.123 + ph * 1.3),
|
| 182 |
+
);
|
| 183 |
+
L.mat.uniforms.uZoom.value = 1.0 + (rm ? 0.0 : Math.min(0.16, age * 0.014)); // dolly-in feel
|
| 184 |
+
if (!isNewest && op <= 0.002) { scene.remove(L.mesh); L.mat.dispose(); layers.splice(i, 1); continue; }
|
| 185 |
+
if (op > 0.5 && (!focus || L.id > focus.id)) focus = L;
|
| 186 |
}
|
| 187 |
+
if (focus && focus.id !== focusKey) {
|
| 188 |
+
focusKey = focus.id;
|
| 189 |
+
if (onFocus && focus.meta) { try { onFocus(focus.meta, focus.id); } catch (_) {} }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 190 |
}
|
| 191 |
renderer.render(scene, camera);
|
| 192 |
}
|
| 193 |
|
| 194 |
+
function setReducedMotion(flag) { reduced = !!flag; }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 195 |
|
| 196 |
function dispose() {
|
| 197 |
cancelAnimationFrame(raf);
|
|
|
|
| 205 |
init, addScene, reset, dispose, setReducedMotion,
|
| 206 |
set onFocus(fn) { onFocus = typeof fn === "function" ? fn : null; },
|
| 207 |
get ready() { return ok; },
|
| 208 |
+
get count() { return layers.length; },
|
| 209 |
};
|
| 210 |
})();
|
| 211 |
|
scripts/_debug_asr.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Local definitive debug of Cohere Transcribe: does it load all weights, and does
|
| 2 |
+
it transcribe the demo clip correctly on CPU? Prints missing/unexpected keys."""
|
| 3 |
+
import os, sys
|
| 4 |
+
sys.path.insert(0, "src")
|
| 5 |
+
os.environ.setdefault("HF_TOKEN", "")
|
| 6 |
+
import torch
|
| 7 |
+
from huggingface_hub import hf_hub_download
|
| 8 |
+
from transformers import AutoProcessor, AutoModelForSpeechSeq2Seq
|
| 9 |
+
from lightloom.audio_in.asr import _patch_keys_to_ignore_coercion
|
| 10 |
+
|
| 11 |
+
REPO = "CohereLabs/cohere-transcribe-03-2026"
|
| 12 |
+
TOK = os.environ["HF_TOKEN"]
|
| 13 |
+
_patch_keys_to_ignore_coercion()
|
| 14 |
+
print("loading processor…", flush=True)
|
| 15 |
+
proc = AutoProcessor.from_pretrained(REPO, trust_remote_code=True, token=TOK)
|
| 16 |
+
print("loading model (fp32, cpu)…", flush=True)
|
| 17 |
+
model, info = AutoModelForSpeechSeq2Seq.from_pretrained(
|
| 18 |
+
REPO, trust_remote_code=True, token=TOK, output_loading_info=True
|
| 19 |
+
)
|
| 20 |
+
mk = list(info.get("missing_keys", []))
|
| 21 |
+
uk = list(info.get("unexpected_keys", []))
|
| 22 |
+
print(f"MISSING_KEYS: {len(mk)} (first 10: {mk[:10]})", flush=True)
|
| 23 |
+
print(f"UNEXPECTED_KEYS: {len(uk)} (first 10: {uk[:10]})", flush=True)
|
| 24 |
+
model.eval()
|
| 25 |
+
|
| 26 |
+
wav = hf_hub_download(REPO, "demo/voxpopuli_test_en_demo.wav", token=TOK)
|
| 27 |
+
try:
|
| 28 |
+
from transformers.audio_utils import load_audio
|
| 29 |
+
audio = load_audio(wav, sampling_rate=16000)
|
| 30 |
+
print("loaded audio via transformers.audio_utils.load_audio", flush=True)
|
| 31 |
+
except Exception as e:
|
| 32 |
+
from lightloom.audio_in.asr import read_wav
|
| 33 |
+
audio, _sr = read_wav(wav)
|
| 34 |
+
print("loaded audio via read_wav:", repr(e)[:80], flush=True)
|
| 35 |
+
|
| 36 |
+
inputs = proc(audio, sampling_rate=16000, return_tensors="pt", language="en")
|
| 37 |
+
print("processor output keys:", list(inputs.keys()), flush=True)
|
| 38 |
+
aci = inputs.get("audio_chunk_index")
|
| 39 |
+
with torch.inference_mode():
|
| 40 |
+
out = model.generate(**inputs, max_new_tokens=120)
|
| 41 |
+
txt = proc.decode(out, skip_special_tokens=True, audio_chunk_index=aci, language="en")
|
| 42 |
+
if isinstance(txt, list):
|
| 43 |
+
txt = txt[0] if txt else ""
|
| 44 |
+
sys.stdout.buffer.write(("LOCAL_TRANSCRIPT: " + txt[:300] + "\n").encode("utf-8", "replace"))
|
src/lightloom/audio_in/asr.py
CHANGED
|
@@ -1,9 +1,15 @@
|
|
| 1 |
-
"""
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
"""
|
| 8 |
|
| 9 |
from __future__ import annotations
|
|
@@ -12,8 +18,7 @@ from typing import Any
|
|
| 12 |
|
| 13 |
from lightloom.core.config import MODEL_REFS
|
| 14 |
|
| 15 |
-
#
|
| 16 |
-
# code on both processor() and decode(). Unknown codes fall back to English.
|
| 17 |
SUPPORTED_LANGS = {"en", "es", "fr", "pt", "de", "it", "nl", "ja", "ko", "zh", "ar", "hi", "ru", "tr"}
|
| 18 |
|
| 19 |
|
|
@@ -47,45 +52,33 @@ def read_wav(path: str) -> tuple[Any, int]:
|
|
| 47 |
return audio, sample_rate
|
| 48 |
|
| 49 |
|
| 50 |
-
def
|
| 51 |
-
"""
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
attribute to a set just before that step. Idempotent and guarded by ``isinstance``,
|
| 55 |
-
so it is a no-op for every model that already uses ``None``/``set`` (Director, FLUX)."""
|
| 56 |
-
import transformers.modeling_utils as _mu
|
| 57 |
-
|
| 58 |
-
base = _mu.PreTrainedModel
|
| 59 |
-
if getattr(base, "_ll_keys_patch", False):
|
| 60 |
-
return
|
| 61 |
-
original = base._adjust_missing_and_unexpected_keys
|
| 62 |
-
|
| 63 |
-
def _patched(self, *args, **kwargs):
|
| 64 |
-
value = getattr(self, "_keys_to_ignore_on_load_unexpected", None)
|
| 65 |
-
if isinstance(value, list):
|
| 66 |
-
self._keys_to_ignore_on_load_unexpected = set(value)
|
| 67 |
-
return original(self, *args, **kwargs)
|
| 68 |
|
| 69 |
-
|
| 70 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
|
| 72 |
|
| 73 |
def load_asr() -> tuple[Any, Any]:
|
| 74 |
-
"""Load
|
| 75 |
Returns ``(processor, model)``."""
|
| 76 |
import torch
|
| 77 |
-
from transformers import
|
| 78 |
|
| 79 |
-
_patch_keys_to_ignore_coercion()
|
| 80 |
ref = MODEL_REFS["asr"]
|
| 81 |
-
processor =
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
model = AutoModelForSpeechSeq2Seq.from_pretrained(
|
| 86 |
-
ref.repo_id, revision=ref.revision, trust_remote_code=True
|
| 87 |
-
)
|
| 88 |
-
model.to("cuda")
|
| 89 |
model.eval()
|
| 90 |
return processor, model
|
| 91 |
|
|
@@ -98,18 +91,15 @@ def transcribe(
|
|
| 98 |
language: str = "en",
|
| 99 |
max_new_tokens: int = 256,
|
| 100 |
) -> str:
|
| 101 |
-
"""Transcribe a
|
| 102 |
import torch
|
| 103 |
|
| 104 |
language = normalize_lang(language)
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
with torch.inference_mode():
|
| 109 |
-
|
| 110 |
-
text = processor.
|
| 111 |
-
|
| 112 |
-
)
|
| 113 |
-
if isinstance(text, list):
|
| 114 |
-
text = text[0] if text else ""
|
| 115 |
-
return text.strip()
|
|
|
|
| 1 |
+
"""Speech-to-text via Whisper (docs/02 §1).
|
| 2 |
+
|
| 3 |
+
We switched the runtime ASR from CohereLabs/cohere-transcribe-03-2026 to
|
| 4 |
+
openai/whisper-large-v3-turbo: the Cohere model has a custom architecture whose
|
| 5 |
+
remote code is incompatible with the transformers 5.11 that MiniCPM/FLUX require
|
| 6 |
+
(it loads with 0 missing weights but emits random multilingual tokens, both on the
|
| 7 |
+
Space and locally). Whisper-turbo is SOTA, fast, and natively supported. Cohere
|
| 8 |
+
stays in the pipeline via Tiny Aya (non-English narration -> English).
|
| 9 |
+
|
| 10 |
+
Parameter contribution: whisper-large-v3-turbo contributes 0.809B runtime
|
| 11 |
+
parameters (registered in the ledger). Loading/inference run under `@spaces.GPU`;
|
| 12 |
+
`normalize_lang`/`read_wav` are pure CPU and unit-tested.
|
| 13 |
"""
|
| 14 |
|
| 15 |
from __future__ import annotations
|
|
|
|
| 18 |
|
| 19 |
from lightloom.core.config import MODEL_REFS
|
| 20 |
|
| 21 |
+
# Whisper supports ~99 languages; we expose the ones the UI/Director use.
|
|
|
|
| 22 |
SUPPORTED_LANGS = {"en", "es", "fr", "pt", "de", "it", "nl", "ja", "ko", "zh", "ar", "hi", "ru", "tr"}
|
| 23 |
|
| 24 |
|
|
|
|
| 52 |
return audio, sample_rate
|
| 53 |
|
| 54 |
|
| 55 |
+
def _resample_to_16k(audio: Any, sr: int) -> Any:
|
| 56 |
+
"""Whisper expects 16 kHz mono. Linear-resample if the clip arrives at another
|
| 57 |
+
rate (the browser already sends 16 kHz, so this is usually a no-op)."""
|
| 58 |
+
import numpy as np
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
|
| 60 |
+
if sr == 16000:
|
| 61 |
+
return audio
|
| 62 |
+
ratio = 16000 / float(sr)
|
| 63 |
+
n = int(round(len(audio) * ratio))
|
| 64 |
+
if n <= 1:
|
| 65 |
+
return audio
|
| 66 |
+
xp = np.linspace(0.0, 1.0, num=len(audio), endpoint=False)
|
| 67 |
+
x = np.linspace(0.0, 1.0, num=n, endpoint=False)
|
| 68 |
+
return np.interp(x, xp, audio).astype("float32")
|
| 69 |
|
| 70 |
|
| 71 |
def load_asr() -> tuple[Any, Any]:
|
| 72 |
+
"""Load Whisper-turbo on CUDA (fp16). GPU only — call inside `@spaces.GPU`.
|
| 73 |
Returns ``(processor, model)``."""
|
| 74 |
import torch
|
| 75 |
+
from transformers import WhisperForConditionalGeneration, WhisperProcessor
|
| 76 |
|
|
|
|
| 77 |
ref = MODEL_REFS["asr"]
|
| 78 |
+
processor = WhisperProcessor.from_pretrained(ref.repo_id, revision=ref.revision)
|
| 79 |
+
model = WhisperForConditionalGeneration.from_pretrained(
|
| 80 |
+
ref.repo_id, revision=ref.revision, torch_dtype=torch.float16
|
| 81 |
+
).to("cuda")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
model.eval()
|
| 83 |
return processor, model
|
| 84 |
|
|
|
|
| 91 |
language: str = "en",
|
| 92 |
max_new_tokens: int = 256,
|
| 93 |
) -> str:
|
| 94 |
+
"""Transcribe a waveform (numpy array) in the given language."""
|
| 95 |
import torch
|
| 96 |
|
| 97 |
language = normalize_lang(language)
|
| 98 |
+
audio = _resample_to_16k(audio, sampling_rate)
|
| 99 |
+
inputs = processor(audio, sampling_rate=16000, return_tensors="pt")
|
| 100 |
+
feats = inputs.input_features.to(model.device, dtype=model.dtype)
|
| 101 |
with torch.inference_mode():
|
| 102 |
+
ids = model.generate(feats, language=language, task="transcribe", max_new_tokens=max_new_tokens)
|
| 103 |
+
text = processor.batch_decode(ids, skip_special_tokens=True)
|
| 104 |
+
out = text[0] if isinstance(text, list) and text else (text if isinstance(text, str) else "")
|
| 105 |
+
return out.strip()
|
|
|
|
|
|
|
|
|
src/lightloom/core/config.py
CHANGED
|
@@ -88,10 +88,14 @@ CONFIG = LOCAL_CONFIG if LIGHTLOOM_PROFILE == "local" else SPACE_CONFIG
|
|
| 88 |
MODEL_REFS: dict[str, ModelRef] = {
|
| 89 |
"asr": ModelRef(
|
| 90 |
role="ASR",
|
| 91 |
-
repo_id="
|
| 92 |
-
revision=
|
| 93 |
license="apache-2.0",
|
| 94 |
-
params=
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
),
|
| 96 |
"translator": ModelRef(
|
| 97 |
role="Translator (Tiny Aya)",
|
|
|
|
| 88 |
MODEL_REFS: dict[str, ModelRef] = {
|
| 89 |
"asr": ModelRef(
|
| 90 |
role="ASR",
|
| 91 |
+
repo_id="openai/whisper-large-v3-turbo",
|
| 92 |
+
revision=None,
|
| 93 |
license="apache-2.0",
|
| 94 |
+
params=809_000_000,
|
| 95 |
+
notes="Speech->text. (CohereLabs/cohere-transcribe-03-2026 has a custom "
|
| 96 |
+
"architecture incompatible with the transformers 5.11 that MiniCPM/FLUX "
|
| 97 |
+
"require — it loads but emits garbage; Whisper-turbo is the reliable choice. "
|
| 98 |
+
"Cohere stays in the pipeline via Tiny Aya for non-English narration.)",
|
| 99 |
),
|
| 100 |
"translator": ModelRef(
|
| 101 |
role="Translator (Tiny Aya)",
|