Efradeca commited on
Commit
d65f3e9
·
verified ·
1 Parent(s): 6569836

chore: deploy private lightloom build

Browse files
app.py CHANGED
@@ -144,7 +144,7 @@ def health() -> dict[str, Any]:
144
  "privacy_mode": os.getenv("LIGHTLOOM_PRIVACY_MODE", "1") == "1",
145
  "cpu_count": os.cpu_count(),
146
  "director_backend": CONFIG.director_backend,
147
- "build": "corridor-5",
148
  }
149
 
150
 
@@ -482,12 +482,19 @@ def _corridor_world(text: str, session: str):
482
  tokenizer, model = load_director_transformers()
483
  klein = load_klein_pipeline()
484
  depth_pipe = load_depth_pipeline()
485
- state = initial_state().model_dump()
 
 
 
 
 
 
 
486
 
487
  for i, phrase in enumerate(phrases):
488
  yield {"stage": "directing", "index": i, "text": phrase}
489
  try:
490
- shot, _m = generate_shot_transformers(tokenizer, model, phrase, state)
491
  style = _corridor_style(shot)
492
  scene_prompt = f"{shot.image_prompt_en}, {shot.lighting}. {style}, {_CORRIDOR_QUALITY}"
493
  yield {"stage": "painting", "index": i, "scene": scene_prompt[:150]}
 
144
  "privacy_mode": os.getenv("LIGHTLOOM_PRIVACY_MODE", "1") == "1",
145
  "cpu_count": os.cpu_count(),
146
  "director_backend": CONFIG.director_backend,
147
+ "build": "v2-parallax-whisper-1",
148
  }
149
 
150
 
 
482
  tokenizer, model = load_director_transformers()
483
  klein = load_klein_pipeline()
484
  depth_pipe = load_depth_pipeline()
485
+ # NEUTRAL state per scene (not the lighthouse-keeper seed in initial_state) so each
486
+ # scene is driven by what the narrator actually said — no "lighthouses everywhere".
487
+ from lightloom.director.state import SceneState
488
+
489
+ neutral = SceneState(
490
+ characters=[], setting="", lighting="", mood="",
491
+ palette=("#202428", "#7d8794", "#e6ddc8"),
492
+ ).model_dump()
493
 
494
  for i, phrase in enumerate(phrases):
495
  yield {"stage": "directing", "index": i, "text": phrase}
496
  try:
497
+ shot, _m = generate_shot_transformers(tokenizer, model, phrase, dict(neutral))
498
  style = _corridor_style(shot)
499
  scene_prompt = f"{shot.image_prompt_en}, {shot.lighting}. {style}, {_CORRIDOR_QUALITY}"
500
  yield {"stage": "painting", "index": i, "scene": scene_prompt[:150]}
frontend/js/stage-corridor.js CHANGED
@@ -1,19 +1,20 @@
1
  /* Lightloom · frontend/js/stage-corridor.js
2
  * ============================================================================
3
- * THE INFINITE 3D CORRIDOR — "speak, and your world unrolls ahead of you."
4
  *
5
- * Each phrase you narrate becomes a depth-displaced, curved scene panel placed
6
- * further down a forward-flying corridor. The camera continuously travels toward
7
- * the newest scene, passing THROUGH the older ones (which fade into atmosphere),
8
- * so the world literally grows ahead of you as you keep speaking. Every panel has
9
- * real relief from its depth map (near things loom as you approach) — Voice
10
- * Scroll's real-time unrolling + WonderWorld's explorable, connected 3D scenes.
 
 
11
  *
12
  * window.LL.corridor = {
13
- * init(canvas), addScene({imageUrl, depthUrl}), reset(), dispose(),
14
- * setReducedMotion(flag), get ready, get count
15
  * }
16
- * addScene() resolves once the panel's textures are loaded and it has been queued.
17
  * ==========================================================================*/
18
  "use strict";
19
  import * as THREE from "three";
@@ -22,84 +23,69 @@ window.LL = window.LL || {};
22
 
23
  const Corridor = (() => {
24
  let renderer = null, scene = null, camera = null, canvas = null;
25
- let raf = 0, t0 = 0, lastT = 0, ok = false, reduced = false;
26
- const panels = []; // { mesh, z, born, id, meta } in arrival order
27
- let sceneSeq = 0; // stable id per scene (panels are recycled, so indices shift)
28
- let frontZ = 0; // z of the most recently added panel (grows negative)
29
- let camZ = 0, targetZ = 0; // camera travel along -Z
30
  const loader = new THREE.TextureLoader();
31
  loader.crossOrigin = "anonymous";
32
 
33
- // Geometry constants (world units). Tuned for a roomy, immersive corridor.
34
- const START_Z = -56; // first panel sits here, ahead of the camera at z=0
35
- const SPACING = 58; // distance between consecutive scenes
36
- const APPROACH = 52; // camera settles this far in front of the newest panel
37
- // Forward-facing 16:9 scene panels (NOT equirectangular), generously oversized so
38
- // they always OVERFILL the view (no black borders), with a very gentle concave
39
- // curve so the world wraps the periphery without distorting the frame.
40
- const PANEL_W = 234, PANEL_H = 134, CURVE_R = 480;
41
-
42
  const VERT = `
43
  varying vec2 vUv;
44
- varying float vFog;
45
- uniform sampler2D uDepth;
46
- uniform float uRelief;
47
- void main() {
48
- vUv = uv;
49
- float d = texture2D(uDepth, uv).r; // white = near
50
- // push the surface toward the camera (+Z local) by the depth -> real relief
51
- vec3 p = position + vec3(0.0, 0.0, 1.0) * (uRelief * d);
52
- vec4 mv = modelViewMatrix * vec4(p, 1.0);
53
- vFog = -mv.z; // view-space distance for fog
54
- gl_Position = projectionMatrix * mv;
55
- }`;
56
  const FRAG = `
 
57
  varying vec2 vUv;
58
- varying float vFog;
59
- uniform sampler2D uTex;
 
 
 
60
  uniform float uOpacity;
61
- uniform float uExposure;
62
- uniform vec3 uFogColor;
63
- uniform float uFogNear;
64
- uniform float uFogFar;
65
  void main() {
66
- vec3 c = texture2D(uTex, vUv).rgb * uExposure;
67
- // atmospheric depth: blend distant panels into the corridor haze
68
- float fog = clamp((vFog - uFogNear) / (uFogFar - uFogNear), 0.0, 1.0);
69
- c = mix(c, uFogColor, fog * 0.9);
70
- // linear -> sRGB (raw ShaderMaterial is not auto-encoded by three.js)
 
 
 
 
 
71
  c = max(c, vec3(0.0));
72
  vec3 lo = c * 12.92;
73
  vec3 hi = 1.055 * pow(c, vec3(1.0 / 2.4)) - 0.055;
74
  c = mix(lo, hi, step(vec3(0.0031308), c));
75
- // soft edge vignette on alpha: melt the panel borders into atmosphere, which
76
- // also hides depth-displacement smear at the frame edges (a dream-frame look).
77
- float vig = smoothstep(0.0, 0.13, vUv.x) * smoothstep(1.0, 0.87, vUv.x)
78
- * smoothstep(0.0, 0.12, vUv.y) * smoothstep(1.0, 0.88, vUv.y);
79
- gl_FragColor = vec4(c, uOpacity * vig);
80
  }`;
81
 
82
- const FOG_COLOR = new THREE.Color(0x0b0d12);
83
 
84
- function curvedPanelGeo() {
85
- const geo = new THREE.PlaneGeometry(PANEL_W, PANEL_H, 96, 56);
86
- const pos = geo.attributes.position;
87
- for (let i = 0; i < pos.count; i++) {
88
- const x = pos.getX(i);
89
- const theta = x / CURVE_R;
90
- pos.setX(i, Math.sin(theta) * CURVE_R);
91
- pos.setZ(i, (Math.cos(theta) - 1.0) * CURVE_R); // edges curl toward camera
92
- }
93
- pos.needsUpdate = true;
94
- geo.computeVertexNormals();
95
- return geo;
96
  }
97
- let GEO = null; // shared geometry (cloned per panel is unnecessary; reuse)
98
 
99
  function loadTex(url) {
100
  return new Promise((resolve) => {
101
  if (!url) return resolve(null);
102
- loader.load(url, (t) => { t.colorSpace = THREE.SRGBColorSpace; resolve(t); }, undefined, () => resolve(null));
 
 
 
 
 
 
 
103
  });
104
  }
105
 
@@ -110,14 +96,10 @@ const Corridor = (() => {
110
  renderer = new THREE.WebGLRenderer({ canvas, antialias: true, alpha: false });
111
  renderer.setPixelRatio(Math.min(devicePixelRatio || 1, 2));
112
  scene = new THREE.Scene();
113
- scene.background = FOG_COLOR.clone();
114
- camera = new THREE.PerspectiveCamera(72, 1, 0.1, 600);
115
- camera.position.set(0, 0, 0);
116
- GEO = curvedPanelGeo();
117
  resize();
118
  window.addEventListener("resize", resize);
119
- ok = true;
120
- t0 = performance.now();
121
  loop();
122
  } catch (e) { ok = false; }
123
  }
@@ -127,7 +109,8 @@ const Corridor = (() => {
127
  const w = canvas.clientWidth || canvas.parentElement?.clientWidth || 1280;
128
  const h = canvas.clientHeight || canvas.parentElement?.clientHeight || 720;
129
  renderer.setSize(w, h, false);
130
- if (camera) { camera.aspect = w / h; camera.updateProjectionMatrix(); }
 
131
  }
132
 
133
  function reducedNow() {
@@ -135,103 +118,80 @@ const Corridor = (() => {
135
  (window.matchMedia && window.matchMedia("(prefers-reduced-motion: reduce)").matches);
136
  }
137
 
138
- let onFocus = null; // (meta, index) => void — fires when a scene flies into focus
139
- let focusKey = -1;
140
-
141
  async function addScene({ imageUrl, depthUrl, meta } = {}) {
142
  if (!ok) return null;
143
  const [tex, depthTex] = await Promise.all([loadTex(imageUrl), loadTex(depthUrl)]);
144
  if (!tex) return null;
 
145
  const mat = new THREE.ShaderMaterial({
146
  uniforms: {
147
- uTex: { value: tex },
148
  uDepth: { value: depthTex || tex },
149
- uRelief: { value: reducedNow() ? 3.0 : 12.0 },
 
 
150
  uOpacity: { value: 0.0 },
151
- uExposure: { value: 1.12 },
152
- uFogColor: { value: FOG_COLOR.clone() },
153
- uFogNear: { value: 60.0 },
154
- uFogFar: { value: 230.0 },
155
  },
156
- vertexShader: VERT, fragmentShader: FRAG, transparent: true, depthWrite: false, depthTest: false,
157
  });
158
- const mesh = new THREE.Mesh(GEO, mat);
159
- frontZ = panels.length ? frontZ - SPACING : START_Z;
160
- mesh.position.z = frontZ;
161
- mesh.renderOrder = -frontZ; // draw far panels first (painter's order for blending)
162
  scene.add(mesh);
163
- panels.push({ mesh, z: frontZ, born: performance.now(), id: ++sceneSeq, meta: meta || null });
164
- targetZ = frontZ + APPROACH; // travel to just in front of the newest scene
165
- return { id: sceneSeq };
166
  }
167
 
168
  function reset() {
169
- for (const p of panels) { scene.remove(p.mesh); p.mesh.material.dispose(); }
170
- panels.length = 0;
171
- frontZ = 0; camZ = 0; targetZ = 0; focusKey = -1;
172
- if (camera) camera.position.set(0, 0, 0);
173
  }
174
 
175
  function loop() {
176
  raf = requestAnimationFrame(loop);
177
  if (!renderer || !scene || !camera) return;
178
  const now = performance.now();
179
- const dt = Math.min(0.05, (now - (lastT || now)) / 1000);
180
- lastT = now;
181
  const t = (now - t0) / 1000;
182
  const rm = reducedNow();
183
-
184
- // Continuous forward travel toward the newest scene; always drifting, never a
185
- // dead stop, so even a single scene feels alive (the camera eases in, parallax).
186
- const baseGlide = rm ? 0 : 2.2 * dt; // gentle constant push deeper
187
- camZ += (targetZ - camZ) * (rm ? 0.06 : 0.022);
188
- if (panels.length && camZ > targetZ + 0.2) camZ -= baseGlide;
189
- camera.position.z = camZ;
190
- // subtle handheld life
191
- camera.position.x = rm ? 0 : Math.sin(t * 0.13) * 1.4;
192
- camera.position.y = rm ? 0 : Math.sin(t * 0.10) * 0.8;
193
- camera.rotation.set(rm ? 0 : Math.sin(t * 0.08) * 0.015, rm ? 0 : Math.sin(t * 0.06) * 0.02, 0, "YXZ");
194
-
195
- // Per-panel opacity envelope: emerge from the far haze, hold full through the
196
- // viewing band, then DISSOLVE before the camera arrives — so we never fly into
197
- // a panel's warped close-up; each scene melts into atmosphere as the next rises.
198
- for (let i = panels.length - 1; i >= 0; i--) {
199
- const p = panels[i];
200
- const ahead = camera.position.z - p.z; // >0 while the panel is in front
201
- const fadeIn = smoothstep(160.0, 92.0, ahead); // far haze -> visible
202
- const fadeOut = smoothstep(30.0, 50.0, ahead); // fully dissolved well before arrival
203
- let op = Math.min(fadeIn, fadeOut);
204
- // gentle birth fade so a freshly streamed scene doesn't pop
205
- const age = (now - p.born) / 1000;
206
- op *= Math.min(1, age / 0.7);
207
- p.mesh.material.uniforms.uOpacity.value = op;
208
- if (ahead < 28.0) { // dissolved: recycle before the camera can reach it
209
- scene.remove(p.mesh); p.mesh.material.dispose(); panels.splice(i, 1);
210
  }
 
 
 
 
 
 
 
 
 
 
211
  }
212
- // Focus = the most-visible scene; notify when it changes so the caption tracks
213
- // the scene the camera is actually looking at (not the one just streamed in).
214
- let best = null, bestOp = 0.5;
215
- for (const p of panels) {
216
- const op = p.mesh.material.uniforms.uOpacity.value;
217
- if (op > bestOp) { bestOp = op; best = p; }
218
- }
219
- if (best && best.id !== focusKey) {
220
- focusKey = best.id;
221
- if (onFocus && best.meta) { try { onFocus(best.meta, best.id); } catch (_) {} }
222
  }
223
  renderer.render(scene, camera);
224
  }
225
 
226
- function smoothstep(edge0, edge1, x) {
227
- const t = Math.max(0, Math.min(1, (x - edge0) / (edge1 - edge0)));
228
- return t * t * (3 - 2 * t);
229
- }
230
-
231
- function setReducedMotion(flag) {
232
- reduced = !!flag;
233
- for (const p of panels) p.mesh.material.uniforms.uRelief.value = reducedNow() ? 3.0 : 12.0;
234
- }
235
 
236
  function dispose() {
237
  cancelAnimationFrame(raf);
@@ -245,7 +205,7 @@ const Corridor = (() => {
245
  init, addScene, reset, dispose, setReducedMotion,
246
  set onFocus(fn) { onFocus = typeof fn === "function" ? fn : null; },
247
  get ready() { return ok; },
248
- get count() { return panels.length; },
249
  };
250
  })();
251
 
 
1
  /* Lightloom · frontend/js/stage-corridor.js
2
  * ============================================================================
3
+ * THE LIVING WORLD — a cinematic depth-parallax film of your story.
4
  *
5
+ * Each phrase you narrate becomes one scene. Instead of displacing geometry by the
6
+ * depth map (which TEARS into jagged sawtooth edges at depth discontinuities), we
7
+ * keep the scene as a full-frame image and shift the *texture sample* per pixel by
8
+ * its depth — the "3D photo" / parallax technique (LeiaPix / Depthy / Facebook 3D
9
+ * photos). Near things slide more than far things as a slow cinematic camera path
10
+ * drifts and pushes in, so the frame breathes in real 3D — at the FULL quality of
11
+ * the painted image, with NO geometry tearing. Scenes cross-fade with a forward
12
+ * push, so the story keeps travelling ahead of you.
13
  *
14
  * window.LL.corridor = {
15
+ * init(canvas), addScene({imageUrl, depthUrl, meta}), reset(), dispose(),
16
+ * setReducedMotion(flag), set onFocus, get ready, get count
17
  * }
 
18
  * ==========================================================================*/
19
  "use strict";
20
  import * as THREE from "three";
 
23
 
24
  const Corridor = (() => {
25
  let renderer = null, scene = null, camera = null, canvas = null;
26
+ let raf = 0, t0 = 0, ok = false, reduced = false, aspect = 16 / 9;
27
+ const layers = []; // { mesh, mat, born, id, meta } newest last
28
+ let seq = 0, focusKey = -1, onFocus = null;
 
 
29
  const loader = new THREE.TextureLoader();
30
  loader.crossOrigin = "anonymous";
31
 
 
 
 
 
 
 
 
 
 
32
  const VERT = `
33
  varying vec2 vUv;
34
+ void main() { vUv = uv; gl_Position = vec4(position.xy, 0.0, 1.0); }`;
35
+
36
+ // Per-pixel depth parallax with a 2-step refinement (kills the "swimming" the
37
+ // naive single-tap shows). uParallax is the animated camera offset; near pixels
38
+ // (depth→1) shift most. uZoom is a slow Ken-Burns push for forward travel.
 
 
 
 
 
 
 
39
  const FRAG = `
40
+ precision highp float;
41
  varying vec2 vUv;
42
+ uniform sampler2D uImage;
43
+ uniform sampler2D uDepth;
44
+ uniform vec2 uParallax;
45
+ uniform float uZoom;
46
+ uniform float uFocus;
47
  uniform float uOpacity;
48
+ uniform vec2 uCover; // aspect-fill scale (cover the canvas, no bars)
 
 
 
49
  void main() {
50
+ // center, aspect-cover, ken-burns zoom
51
+ vec2 uv = (vUv - 0.5) * uCover / uZoom + 0.5;
52
+ // depth-corrected parallax: sample depth, offset, refine once
53
+ float d = texture2D(uDepth, uv).r;
54
+ vec2 p = uParallax * (d - uFocus);
55
+ d = texture2D(uDepth, uv + p).r;
56
+ p = uParallax * (d - uFocus);
57
+ vec2 suv = uv + p;
58
+ vec3 c = texture2D(uImage, clamp(suv, 0.001, 0.999)).rgb;
59
+ // linear->sRGB (raw ShaderMaterial is not auto-encoded by three.js)
60
  c = max(c, vec3(0.0));
61
  vec3 lo = c * 12.92;
62
  vec3 hi = 1.055 * pow(c, vec3(1.0 / 2.4)) - 0.055;
63
  c = mix(lo, hi, step(vec3(0.0031308), c));
64
+ // a whisper of cinematic vignette
65
+ vec2 q = vUv - 0.5;
66
+ c *= 1.0 - dot(q, q) * 0.32;
67
+ gl_FragColor = vec4(c, uOpacity);
 
68
  }`;
69
 
70
+ const QUAD = new THREE.PlaneGeometry(2, 2);
71
 
72
+ function texCover(texAspect) {
73
+ // aspect-fill: scale the SMALLER axis up so the image covers the canvas
74
+ const r = texAspect / aspect;
75
+ return r > 1 ? new THREE.Vector2(r, 1) : new THREE.Vector2(1, 1 / r);
 
 
 
 
 
 
 
 
76
  }
 
77
 
78
  function loadTex(url) {
79
  return new Promise((resolve) => {
80
  if (!url) return resolve(null);
81
+ loader.load(url, (t) => {
82
+ t.colorSpace = THREE.SRGBColorSpace;
83
+ t.minFilter = THREE.LinearFilter; t.magFilter = THREE.LinearFilter;
84
+ t.generateMipmaps = false;
85
+ const a = renderer.capabilities.getMaxAnisotropy ? renderer.capabilities.getMaxAnisotropy() : 1;
86
+ t.anisotropy = Math.min(8, a || 1);
87
+ resolve(t);
88
+ }, undefined, () => resolve(null));
89
  });
90
  }
91
 
 
96
  renderer = new THREE.WebGLRenderer({ canvas, antialias: true, alpha: false });
97
  renderer.setPixelRatio(Math.min(devicePixelRatio || 1, 2));
98
  scene = new THREE.Scene();
99
+ camera = new THREE.Camera(); // clip-space quads; no transforms needed
 
 
 
100
  resize();
101
  window.addEventListener("resize", resize);
102
+ ok = true; t0 = performance.now();
 
103
  loop();
104
  } catch (e) { ok = false; }
105
  }
 
109
  const w = canvas.clientWidth || canvas.parentElement?.clientWidth || 1280;
110
  const h = canvas.clientHeight || canvas.parentElement?.clientHeight || 720;
111
  renderer.setSize(w, h, false);
112
+ aspect = w / h;
113
+ for (const L of layers) L.mat.uniforms.uCover.value.copy(texCover(L.texAspect));
114
  }
115
 
116
  function reducedNow() {
 
118
  (window.matchMedia && window.matchMedia("(prefers-reduced-motion: reduce)").matches);
119
  }
120
 
 
 
 
121
  async function addScene({ imageUrl, depthUrl, meta } = {}) {
122
  if (!ok) return null;
123
  const [tex, depthTex] = await Promise.all([loadTex(imageUrl), loadTex(depthUrl)]);
124
  if (!tex) return null;
125
+ const texAspect = (tex.image && tex.image.width / tex.image.height) || 16 / 9;
126
  const mat = new THREE.ShaderMaterial({
127
  uniforms: {
128
+ uImage: { value: tex },
129
  uDepth: { value: depthTex || tex },
130
+ uParallax: { value: new THREE.Vector2(0, 0) },
131
+ uZoom: { value: 1.0 },
132
+ uFocus: { value: 0.5 },
133
  uOpacity: { value: 0.0 },
134
+ uCover: { value: texCover(texAspect) },
 
 
 
135
  },
136
+ vertexShader: VERT, fragmentShader: FRAG, transparent: true, depthTest: false, depthWrite: false,
137
  });
138
+ const mesh = new THREE.Mesh(QUAD, mat);
139
+ mesh.renderOrder = ++seq;
 
 
140
  scene.add(mesh);
141
+ const L = { mesh, mat, born: performance.now(), id: seq, meta: meta || null, texAspect, phase: seq * 1.7 };
142
+ layers.push(L);
143
+ return { id: seq };
144
  }
145
 
146
  function reset() {
147
+ for (const L of layers) { scene.remove(L.mesh); L.mat.dispose(); L.mat.uniforms.uImage.value?.dispose?.(); }
148
+ layers.length = 0; focusKey = -1;
 
 
149
  }
150
 
151
  function loop() {
152
  raf = requestAnimationFrame(loop);
153
  if (!renderer || !scene || !camera) return;
154
  const now = performance.now();
 
 
155
  const t = (now - t0) / 1000;
156
  const rm = reducedNow();
157
+ const FADE = rm ? 320 : 1300; // crossfade ms
158
+ const amp = rm ? 0.0 : 0.018; // parallax amplitude (UV units) — subtle, no tearing
159
+
160
+ let focus = null;
161
+ for (let i = layers.length - 1; i >= 0; i--) {
162
+ const L = layers[i];
163
+ const age = (now - L.born) / 1000;
164
+ const isNewest = i === layers.length - 1;
165
+ // crossfade: newest ramps in; everyone older ramps out
166
+ const k = Math.min(1, (now - L.born) / FADE);
167
+ const ease = k * k * (3 - 2 * k);
168
+ let op;
169
+ if (isNewest) op = ease;
170
+ else {
171
+ // older layers: fade out once a newer one exists (started fading at its sibling's birth)
172
+ const newer = layers[i + 1];
173
+ const ko = Math.min(1, (now - newer.born) / FADE);
174
+ op = 1 - ko * ko * (3 - 2 * ko);
 
 
 
 
 
 
 
 
 
175
  }
176
+ L.mat.uniforms.uOpacity.value = op;
177
+ // cinematic motion: a slow drifting parallax orbit + a gentle forward push (zoom-in)
178
+ const ph = L.phase;
179
+ L.mat.uniforms.uParallax.value.set(
180
+ amp * Math.sin(t * 0.16 + ph),
181
+ amp * 0.6 * Math.sin(t * 0.123 + ph * 1.3),
182
+ );
183
+ L.mat.uniforms.uZoom.value = 1.0 + (rm ? 0.0 : Math.min(0.16, age * 0.014)); // dolly-in feel
184
+ if (!isNewest && op <= 0.002) { scene.remove(L.mesh); L.mat.dispose(); layers.splice(i, 1); continue; }
185
+ if (op > 0.5 && (!focus || L.id > focus.id)) focus = L;
186
  }
187
+ if (focus && focus.id !== focusKey) {
188
+ focusKey = focus.id;
189
+ if (onFocus && focus.meta) { try { onFocus(focus.meta, focus.id); } catch (_) {} }
 
 
 
 
 
 
 
190
  }
191
  renderer.render(scene, camera);
192
  }
193
 
194
+ function setReducedMotion(flag) { reduced = !!flag; }
 
 
 
 
 
 
 
 
195
 
196
  function dispose() {
197
  cancelAnimationFrame(raf);
 
205
  init, addScene, reset, dispose, setReducedMotion,
206
  set onFocus(fn) { onFocus = typeof fn === "function" ? fn : null; },
207
  get ready() { return ok; },
208
+ get count() { return layers.length; },
209
  };
210
  })();
211
 
scripts/_debug_asr.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Local definitive debug of Cohere Transcribe: does it load all weights, and does
2
+ it transcribe the demo clip correctly on CPU? Prints missing/unexpected keys."""
3
+ import os, sys
4
+ sys.path.insert(0, "src")
5
+ os.environ.setdefault("HF_TOKEN", "")
6
+ import torch
7
+ from huggingface_hub import hf_hub_download
8
+ from transformers import AutoProcessor, AutoModelForSpeechSeq2Seq
9
+ from lightloom.audio_in.asr import _patch_keys_to_ignore_coercion
10
+
11
+ REPO = "CohereLabs/cohere-transcribe-03-2026"
12
+ TOK = os.environ["HF_TOKEN"]
13
+ _patch_keys_to_ignore_coercion()
14
+ print("loading processor…", flush=True)
15
+ proc = AutoProcessor.from_pretrained(REPO, trust_remote_code=True, token=TOK)
16
+ print("loading model (fp32, cpu)…", flush=True)
17
+ model, info = AutoModelForSpeechSeq2Seq.from_pretrained(
18
+ REPO, trust_remote_code=True, token=TOK, output_loading_info=True
19
+ )
20
+ mk = list(info.get("missing_keys", []))
21
+ uk = list(info.get("unexpected_keys", []))
22
+ print(f"MISSING_KEYS: {len(mk)} (first 10: {mk[:10]})", flush=True)
23
+ print(f"UNEXPECTED_KEYS: {len(uk)} (first 10: {uk[:10]})", flush=True)
24
+ model.eval()
25
+
26
+ wav = hf_hub_download(REPO, "demo/voxpopuli_test_en_demo.wav", token=TOK)
27
+ try:
28
+ from transformers.audio_utils import load_audio
29
+ audio = load_audio(wav, sampling_rate=16000)
30
+ print("loaded audio via transformers.audio_utils.load_audio", flush=True)
31
+ except Exception as e:
32
+ from lightloom.audio_in.asr import read_wav
33
+ audio, _sr = read_wav(wav)
34
+ print("loaded audio via read_wav:", repr(e)[:80], flush=True)
35
+
36
+ inputs = proc(audio, sampling_rate=16000, return_tensors="pt", language="en")
37
+ print("processor output keys:", list(inputs.keys()), flush=True)
38
+ aci = inputs.get("audio_chunk_index")
39
+ with torch.inference_mode():
40
+ out = model.generate(**inputs, max_new_tokens=120)
41
+ txt = proc.decode(out, skip_special_tokens=True, audio_chunk_index=aci, language="en")
42
+ if isinstance(txt, list):
43
+ txt = txt[0] if txt else ""
44
+ sys.stdout.buffer.write(("LOCAL_TRANSCRIPT: " + txt[:300] + "\n").encode("utf-8", "replace"))
src/lightloom/audio_in/asr.py CHANGED
@@ -1,9 +1,15 @@
1
- """Cohere Transcribe — speech-to-text (docs/02 §1).
2
-
3
- Parameter contribution: cohere-transcribe-03-2026 contributes 2.0B runtime
4
- parameters (registered in the ledger). Model loading/inference run under
5
- `@spaces.GPU`. The model does NOT auto-detect language: the ISO code is required
6
- and comes from the UI selector (docs/04 §2). Gated repo — needs accepted terms.
 
 
 
 
 
 
7
  """
8
 
9
  from __future__ import annotations
@@ -12,8 +18,7 @@ from typing import Any
12
 
13
  from lightloom.core.config import MODEL_REFS
14
 
15
- # ISO codes the Director/UI use; the model supports 14 languages and expects the
16
- # code on both processor() and decode(). Unknown codes fall back to English.
17
  SUPPORTED_LANGS = {"en", "es", "fr", "pt", "de", "it", "nl", "ja", "ko", "zh", "ar", "hi", "ru", "tr"}
18
 
19
 
@@ -47,45 +52,33 @@ def read_wav(path: str) -> tuple[Any, int]:
47
  return audio, sample_rate
48
 
49
 
50
- def _patch_keys_to_ignore_coercion() -> None:
51
- """Cohere Transcribe's remote model class sets ``_keys_to_ignore_on_load_unexpected``
52
- as a *list*; transformers 5.11's ``_adjust_missing_and_unexpected_keys`` does
53
- ``(... or set()) | additional`` which raises ``TypeError: list | set``. Coerce the
54
- attribute to a set just before that step. Idempotent and guarded by ``isinstance``,
55
- so it is a no-op for every model that already uses ``None``/``set`` (Director, FLUX)."""
56
- import transformers.modeling_utils as _mu
57
-
58
- base = _mu.PreTrainedModel
59
- if getattr(base, "_ll_keys_patch", False):
60
- return
61
- original = base._adjust_missing_and_unexpected_keys
62
-
63
- def _patched(self, *args, **kwargs):
64
- value = getattr(self, "_keys_to_ignore_on_load_unexpected", None)
65
- if isinstance(value, list):
66
- self._keys_to_ignore_on_load_unexpected = set(value)
67
- return original(self, *args, **kwargs)
68
 
69
- base._adjust_missing_and_unexpected_keys = _patched
70
- base._ll_keys_patch = True
 
 
 
 
 
 
 
71
 
72
 
73
  def load_asr() -> tuple[Any, Any]:
74
- """Load Cohere Transcribe on CUDA. GPU only — call inside `@spaces.GPU`.
75
  Returns ``(processor, model)``."""
76
  import torch
77
- from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
78
 
79
- _patch_keys_to_ignore_coercion()
80
  ref = MODEL_REFS["asr"]
81
- processor = AutoProcessor.from_pretrained(ref.repo_id, revision=ref.revision, trust_remote_code=True)
82
- # Load in the model's native (fp32) precision, exactly as Cohere's official
83
- # example does. bf16 garbles the audio encoder -> random multilingual tokens.
84
- # auto_map routes AutoModelForSpeechSeq2Seq -> CohereAsrForConditionalGeneration.
85
- model = AutoModelForSpeechSeq2Seq.from_pretrained(
86
- ref.repo_id, revision=ref.revision, trust_remote_code=True
87
- )
88
- model.to("cuda")
89
  model.eval()
90
  return processor, model
91
 
@@ -98,18 +91,15 @@ def transcribe(
98
  language: str = "en",
99
  max_new_tokens: int = 256,
100
  ) -> str:
101
- """Transcribe a 16 kHz waveform (numpy array) in the given language."""
102
  import torch
103
 
104
  language = normalize_lang(language)
105
- inputs = processor(audio, sampling_rate=sampling_rate, return_tensors="pt", language=language)
106
- audio_chunk_index = inputs.get("audio_chunk_index")
107
- inputs = inputs.to(model.device, dtype=model.dtype) # fp32 model -> no-op cast
108
  with torch.inference_mode():
109
- outputs = model.generate(**inputs, max_new_tokens=max_new_tokens)
110
- text = processor.decode(
111
- outputs, skip_special_tokens=True, audio_chunk_index=audio_chunk_index, language=language
112
- )
113
- if isinstance(text, list):
114
- text = text[0] if text else ""
115
- return text.strip()
 
1
+ """Speech-to-text via Whisper (docs/02 §1).
2
+
3
+ We switched the runtime ASR from CohereLabs/cohere-transcribe-03-2026 to
4
+ openai/whisper-large-v3-turbo: the Cohere model has a custom architecture whose
5
+ remote code is incompatible with the transformers 5.11 that MiniCPM/FLUX require
6
+ (it loads with 0 missing weights but emits random multilingual tokens, both on the
7
+ Space and locally). Whisper-turbo is SOTA, fast, and natively supported. Cohere
8
+ stays in the pipeline via Tiny Aya (non-English narration -> English).
9
+
10
+ Parameter contribution: whisper-large-v3-turbo contributes 0.809B runtime
11
+ parameters (registered in the ledger). Loading/inference run under `@spaces.GPU`;
12
+ `normalize_lang`/`read_wav` are pure CPU and unit-tested.
13
  """
14
 
15
  from __future__ import annotations
 
18
 
19
  from lightloom.core.config import MODEL_REFS
20
 
21
+ # Whisper supports ~99 languages; we expose the ones the UI/Director use.
 
22
  SUPPORTED_LANGS = {"en", "es", "fr", "pt", "de", "it", "nl", "ja", "ko", "zh", "ar", "hi", "ru", "tr"}
23
 
24
 
 
52
  return audio, sample_rate
53
 
54
 
55
+ def _resample_to_16k(audio: Any, sr: int) -> Any:
56
+ """Whisper expects 16 kHz mono. Linear-resample if the clip arrives at another
57
+ rate (the browser already sends 16 kHz, so this is usually a no-op)."""
58
+ import numpy as np
 
 
 
 
 
 
 
 
 
 
 
 
 
 
59
 
60
+ if sr == 16000:
61
+ return audio
62
+ ratio = 16000 / float(sr)
63
+ n = int(round(len(audio) * ratio))
64
+ if n <= 1:
65
+ return audio
66
+ xp = np.linspace(0.0, 1.0, num=len(audio), endpoint=False)
67
+ x = np.linspace(0.0, 1.0, num=n, endpoint=False)
68
+ return np.interp(x, xp, audio).astype("float32")
69
 
70
 
71
  def load_asr() -> tuple[Any, Any]:
72
+ """Load Whisper-turbo on CUDA (fp16). GPU only — call inside `@spaces.GPU`.
73
  Returns ``(processor, model)``."""
74
  import torch
75
+ from transformers import WhisperForConditionalGeneration, WhisperProcessor
76
 
 
77
  ref = MODEL_REFS["asr"]
78
+ processor = WhisperProcessor.from_pretrained(ref.repo_id, revision=ref.revision)
79
+ model = WhisperForConditionalGeneration.from_pretrained(
80
+ ref.repo_id, revision=ref.revision, torch_dtype=torch.float16
81
+ ).to("cuda")
 
 
 
 
82
  model.eval()
83
  return processor, model
84
 
 
91
  language: str = "en",
92
  max_new_tokens: int = 256,
93
  ) -> str:
94
+ """Transcribe a waveform (numpy array) in the given language."""
95
  import torch
96
 
97
  language = normalize_lang(language)
98
+ audio = _resample_to_16k(audio, sampling_rate)
99
+ inputs = processor(audio, sampling_rate=16000, return_tensors="pt")
100
+ feats = inputs.input_features.to(model.device, dtype=model.dtype)
101
  with torch.inference_mode():
102
+ ids = model.generate(feats, language=language, task="transcribe", max_new_tokens=max_new_tokens)
103
+ text = processor.batch_decode(ids, skip_special_tokens=True)
104
+ out = text[0] if isinstance(text, list) and text else (text if isinstance(text, str) else "")
105
+ return out.strip()
 
 
 
src/lightloom/core/config.py CHANGED
@@ -88,10 +88,14 @@ CONFIG = LOCAL_CONFIG if LIGHTLOOM_PROFILE == "local" else SPACE_CONFIG
88
  MODEL_REFS: dict[str, ModelRef] = {
89
  "asr": ModelRef(
90
  role="ASR",
91
- repo_id="CohereLabs/cohere-transcribe-03-2026",
92
- revision="b1eacc2686a3d08ceaae5f24a88b1d519620bc09",
93
  license="apache-2.0",
94
- params=2_000_000_000,
 
 
 
 
95
  ),
96
  "translator": ModelRef(
97
  role="Translator (Tiny Aya)",
 
88
  MODEL_REFS: dict[str, ModelRef] = {
89
  "asr": ModelRef(
90
  role="ASR",
91
+ repo_id="openai/whisper-large-v3-turbo",
92
+ revision=None,
93
  license="apache-2.0",
94
+ params=809_000_000,
95
+ notes="Speech->text. (CohereLabs/cohere-transcribe-03-2026 has a custom "
96
+ "architecture incompatible with the transformers 5.11 that MiniCPM/FLUX "
97
+ "require — it loads but emits garbage; Whisper-turbo is the reliable choice. "
98
+ "Cohere stays in the pipeline via Tiny Aya for non-English narration.)",
99
  ),
100
  "translator": ModelRef(
101
  role="Translator (Tiny Aya)",