chore: deploy private lightloom build
Browse files- app.py +50 -12
- docs/RESEARCH_ROADMAP.md +94 -0
- src/lightloom/paint/scroll.py +17 -1
app.py
CHANGED
|
@@ -145,7 +145,7 @@ def health() -> dict[str, Any]:
|
|
| 145 |
"privacy_mode": os.getenv("LIGHTLOOM_PRIVACY_MODE", "1") == "1",
|
| 146 |
"cpu_count": os.cpu_count(),
|
| 147 |
"director_backend": CONFIG.director_backend,
|
| 148 |
-
"build": "live-narration-
|
| 149 |
}
|
| 150 |
|
| 151 |
|
|
@@ -793,11 +793,9 @@ def _scroll_world(text: str, session: str):
|
|
| 793 |
yield {"stage": "painting", "index": i, "scene": subject[:120]}
|
| 794 |
for k in range(strips_per_phrase):
|
| 795 |
# first strip of a new scene morphs from the previous one (smooth panorama)
|
| 796 |
-
|
| 797 |
-
|
| 798 |
-
else
|
| 799 |
-
strip_prompt = subject
|
| 800 |
-
sec = outpaint_right(pipe, ctx, strip_prompt + style, seg_w=seg, height=h, overlap=overlap, steps=4, seed=200 + g * 3)
|
| 801 |
ctx = sec
|
| 802 |
sec.save(out_dir / f"section-{g}.webp", "WEBP", quality=88)
|
| 803 |
yield {
|
|
@@ -1005,12 +1003,12 @@ def _scroll_live_phrase(session: str, text: str, audio_b64: str, lang: str):
|
|
| 1005 |
try:
|
| 1006 |
for k in range(strips_per_phrase):
|
| 1007 |
# the FIRST strip of a NEW scene MORPHS from the previous subject into this one
|
| 1008 |
-
# (klein paints the dissolve)
|
| 1009 |
-
|
| 1010 |
-
|
| 1011 |
-
else
|
| 1012 |
-
|
| 1013 |
-
sec = outpaint_right(pipe, ctx, strip_prompt + style, seg_w=seg, height=h, overlap=overlap, steps=4, seed=200 + g * 3)
|
| 1014 |
ctx = sec
|
| 1015 |
sec.save(out_dir / f"section-{g}.webp", "WEBP", quality=88)
|
| 1016 |
yield {
|
|
@@ -1048,6 +1046,46 @@ def scroll_live_api(session: str = "", text: str = "", audio_b64: str = "", lang
|
|
| 1048 |
yield {"stage": stage, "error": str(exc)[:200], "session": sess}
|
| 1049 |
|
| 1050 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1051 |
@app.post("/internal/bench/director-probe")
|
| 1052 |
def director_probe_endpoint(backend: str = "transformers", threads: int = 8) -> dict[str, Any]:
|
| 1053 |
if os.getenv("LIGHTLOOM_ENABLE_INTERNAL_BENCH", "0") != "1":
|
|
|
|
| 145 |
"privacy_mode": os.getenv("LIGHTLOOM_PRIVACY_MODE", "1") == "1",
|
| 146 |
"cpu_count": os.cpu_count(),
|
| 147 |
"director_backend": CONFIG.director_backend,
|
| 148 |
+
"build": "live-narration-8",
|
| 149 |
}
|
| 150 |
|
| 151 |
|
|
|
|
| 793 |
yield {"stage": "painting", "index": i, "scene": subject[:120]}
|
| 794 |
for k in range(strips_per_phrase):
|
| 795 |
# first strip of a new scene morphs from the previous one (smooth panorama)
|
| 796 |
+
is_change = k == 0 and prev_subject and prev_subject[:48].lower() != subject[:48].lower()
|
| 797 |
+
strip_prompt = _scroll_transition_prompt(prev_subject, subject) if is_change else subject
|
| 798 |
+
sec = outpaint_right(pipe, ctx, strip_prompt + style, seg_w=seg, height=h, overlap=overlap, steps=4, seed=200 + g * 3, morph_band=(96 if is_change else 0))
|
|
|
|
|
|
|
| 799 |
ctx = sec
|
| 800 |
sec.save(out_dir / f"section-{g}.webp", "WEBP", quality=88)
|
| 801 |
yield {
|
|
|
|
| 1003 |
try:
|
| 1004 |
for k in range(strips_per_phrase):
|
| 1005 |
# the FIRST strip of a NEW scene MORPHS from the previous subject into this one
|
| 1006 |
+
# (klein paints the dissolve) + a morph BAND that lets the new scene emerge out
|
| 1007 |
+
# of the carried pixels, so the panorama flows instead of cutting hard.
|
| 1008 |
+
is_change = k == 0 and prev_subject and prev_subject[:48].lower() != subject[:48].lower()
|
| 1009 |
+
strip_prompt = _scroll_transition_prompt(prev_subject, subject) if is_change else subject
|
| 1010 |
+
mb = 96 if is_change else 0
|
| 1011 |
+
sec = outpaint_right(pipe, ctx, strip_prompt + style, seg_w=seg, height=h, overlap=overlap, steps=4, seed=200 + g * 3, morph_band=mb)
|
| 1012 |
ctx = sec
|
| 1013 |
sec.save(out_dir / f"section-{g}.webp", "WEBP", quality=88)
|
| 1014 |
yield {
|
|
|
|
| 1046 |
yield {"stage": stage, "error": str(exc)[:200], "session": sess}
|
| 1047 |
|
| 1048 |
|
| 1049 |
+
@spaces.GPU(duration=120)
|
| 1050 |
+
def _introspect_klein_on_gpu() -> dict[str, Any]:
|
| 1051 |
+
"""Report the klein pipeline's call signature + LoRA/embeds capabilities, so we can
|
| 1052 |
+
build embedding-SLERP (prompt_embeds) and the style-LoRA fuse correctly (CLAUDE rule 4:
|
| 1053 |
+
read the real API instead of guessing)."""
|
| 1054 |
+
import inspect
|
| 1055 |
+
|
| 1056 |
+
from lightloom.paint.scroll import load_fast_pipeline
|
| 1057 |
+
|
| 1058 |
+
info: dict[str, Any] = {}
|
| 1059 |
+
try:
|
| 1060 |
+
pipe = load_fast_pipeline()
|
| 1061 |
+
sig = inspect.signature(pipe.__call__)
|
| 1062 |
+
params = list(sig.parameters.keys())
|
| 1063 |
+
info.update(
|
| 1064 |
+
ok=True,
|
| 1065 |
+
pipe_class=type(pipe).__name__,
|
| 1066 |
+
call_params=params,
|
| 1067 |
+
accepts_prompt_embeds="prompt_embeds" in params,
|
| 1068 |
+
accepts_mask_image="mask_image" in params,
|
| 1069 |
+
has_encode_prompt=hasattr(pipe, "encode_prompt"),
|
| 1070 |
+
has_load_lora_weights=hasattr(pipe, "load_lora_weights"),
|
| 1071 |
+
has_fuse_lora=hasattr(pipe, "fuse_lora"),
|
| 1072 |
+
components=list(getattr(pipe, "components", {}).keys()),
|
| 1073 |
+
text_encoder=type(getattr(pipe, "text_encoder", None)).__name__,
|
| 1074 |
+
)
|
| 1075 |
+
except Exception as exc: # noqa: BLE001 - diagnostic must report, not crash.
|
| 1076 |
+
import traceback
|
| 1077 |
+
|
| 1078 |
+
info.update(ok=False, error=f"{type(exc).__name__}: {exc!r}", trace=traceback.format_exc()[-800:])
|
| 1079 |
+
return info
|
| 1080 |
+
|
| 1081 |
+
|
| 1082 |
+
@app.post("/internal/introspect-klein")
|
| 1083 |
+
def introspect_klein_endpoint() -> dict[str, Any]:
|
| 1084 |
+
if os.getenv("LIGHTLOOM_ENABLE_INTERNAL_BENCH", "0") != "1":
|
| 1085 |
+
return {"ok": False, "error": "internal endpoint disabled"}
|
| 1086 |
+
return {"ok": True, "data": _introspect_klein_on_gpu()}
|
| 1087 |
+
|
| 1088 |
+
|
| 1089 |
@app.post("/internal/bench/director-probe")
|
| 1090 |
def director_probe_endpoint(backend: str = "transformers", threads: int = 8) -> dict[str, Any]:
|
| 1091 |
if os.getenv("LIGHTLOOM_ENABLE_INTERNAL_BENCH", "0") != "1":
|
docs/RESEARCH_ROADMAP.md
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Lightloom — Research Roadmap (faster + smoother + the innovation)
|
| 2 |
+
|
| 3 |
+
Source: deep June-2026 research (5 agents): FLUX/klein LoRA (style + continuation), pipeline
|
| 4 |
+
speed/architecture, Director fine-tune. Synthesized + cross-checked against the codebase.
|
| 5 |
+
|
| 6 |
+
**Bottom line:** speed is the *Director per-phrase + serial stages*, not klein/ASR. Smoothness is
|
| 7 |
+
a *hard-mask + hard-cut* problem, not a model problem. The highest-leverage trained win is one
|
| 8 |
+
cheap **style LoRA**. Ship the quick wins first; train one LoRA.
|
| 9 |
+
|
| 10 |
+
Measured latency (live-narration-6, on-box): first paint ~42s (cold model load), per-strip ~0.8s
|
| 11 |
+
(klein 0.7 + depth), per-phrase ~2.4s paint, phrase cadence ~4s (mostly speech duration).
|
| 12 |
+
|
| 13 |
+
---
|
| 14 |
+
|
| 15 |
+
## (1) Quick wins — NO training, bf16-safe, ~0ms GPU. Ordered by ROI.
|
| 16 |
+
|
| 17 |
+
- **QW-1 — Grammar-constrain the Director (`xgrammar`).** Replace the `retries=4` + `repetition_penalty`
|
| 18 |
+
hack with schema-constrained decoding → 100% valid JSON by construction, lower p50, lower
|
| 19 |
+
`max_new_tokens`. **RISK:** the codebase asserts "xgrammar incompatible with transformers 5.x"
|
| 20 |
+
(stale, from an older pair) — must verify a `cp311/cu12x` prebuilt wheel installs + constrains on
|
| 21 |
+
the **sm120 Space** before deleting the `_coerce_shot` salvage path. Keep retry+parser as fallback.
|
| 22 |
+
- **QW-2 — Director-once / World-Bible.** ✅ SHIPPED (simple form: Director runs only on phrase 1).
|
| 23 |
+
Upgrade: one MiniCPM call at session open → locked palette (3-hex), cast, setting, lighting, style
|
| 24 |
+
adjectives, **fixed base seed**; per-strip prompt = world-bible prefix + rolling transcript, NO
|
| 25 |
+
per-strip LLM. Fixed seed + locked palette also softens cuts for free (StyleAligned).
|
| 26 |
+
- **QW-3 — Soft/feathered continuation MASK (the seam fix).** `scroll.py` mask is a hard step (0 over
|
| 27 |
+
the 256px overlap, 255 over the new region). Replace with a horizontal **ramp**: 0 for the outer
|
| 28 |
+
~128px (deep context), 0→255 across an inner ~128px band, 255 in the new strip (numpy linspace, or
|
| 29 |
+
`pipe.mask_processor.blur(mask, blur_factor≈24-40)`). **CAVEAT (rule R4/CLAUDE §4):** verify
|
| 30 |
+
`Flux2KleinInpaintPipeline` does NOT internally binarize `mask_image` (diffusers #10690) — read
|
| 31 |
+
`site-packages/diffusers/pipelines/flux2/` first.
|
| 32 |
+
- **QW-4 — Embedding SLERP across scene cuts (the morph).** On a Director cut, spherically interpolate
|
| 33 |
+
`prompt_embeds` old→new across the next 3-5 strips (SLERP, never lerp — lerp corrupts T5 norms).
|
| 34 |
+
Pure CPU, 0ms GPU. Verify the klein inpaint signature accepts `prompt_embeds`.
|
| 35 |
+
- **QW-5 — Streaming ASR (LocalAgreement-2) + two-lane decoupling.** Rolling 16kHz buffer → Whisper
|
| 36 |
+
every 0.5-1s; commit a word when two overlapping decodes agree (`ufal/whisper_streaming` /
|
| 37 |
+
SimulStreaming, MIT). Run ASR on `concurrency_id="asr"`, paint/depth on `"gpu"`. Note: 2nd GPU lane
|
| 38 |
+
burns quota; the long-lived `@spaces.GPU` generator must self-relaunch before the ~120s cap.
|
| 39 |
+
- **QW-6 — Keep the 360-ERP LoRA OUT of the live scroll path** (needs klein-base + green ERP control,
|
| 40 |
+
for 360 *wrap* not open extension; `scroll.py` is already correct).
|
| 41 |
+
|
| 42 |
+
**Expected after QW-1→QW-5:** speak→first-reaction ~1.5-3s, a new strip every ~0.3-0.6s independent
|
| 43 |
+
of speech, scene cuts that morph instead of snap.
|
| 44 |
+
|
| 45 |
+
---
|
| 46 |
+
|
| 47 |
+
## (2) The ONE LoRA to train — painterly STYLE LoRA (highest ROI, cheap)
|
| 48 |
+
|
| 49 |
+
Train a **rank-16 style LoRA on FLUX.2-klein-base-4B** (`painter_base_train` in config), **fuse onto
|
| 50 |
+
the distilled klein-4B at warmup** (BFL: train base, load on distilled, adapters cross-compatible).
|
| 51 |
+
Every strip gets one invariant painterly "skin" → scene CHANGES (forest→interior→neon) read as one
|
| 52 |
+
hand-painted world. After `fuse_lora()` = **0ms/strip**. (Infra partly wired: `prompt.py` LORA_TRIGGER,
|
| 53 |
+
`config.py` painter_base_train.)
|
| 54 |
+
|
| 55 |
+
- **Data (~30-60 min, highest leverage):** render 24-30 imgs @1024 with the *current* distilled klein
|
| 56 |
+
(no LoRA) using ONE style suffix (e.g. "soft impasto oil painting, luminous diffuse light, muted
|
| 57 |
+
teal-amber palette, visible brushwork") across diverse subjects (forest, interior, neon city, water,
|
| 58 |
+
faces, sky). Hand-cull to ~24 (vary CONTENT, fix STYLE). Apache-2.0-clean, no scraping. Caption
|
| 59 |
+
content-only, prepend trigger `lghtlm style`, never write style words.
|
| 60 |
+
- **Method:** LoRA rank 16, alpha 16, target fused `attn.to_qkv_mlp_proj` + `attn.to_out` across
|
| 61 |
+
`single_transformer_blocks`; LR 1e-4, AdamW-8bit, batch 1 + ga 4, bf16, grad-checkpoint + cache
|
| 62 |
+
latents, 512→1024px, ~1500 steps, checkpoint every 250, **pick by eye** (~step 750-1500). Tool:
|
| 63 |
+
`ostris/ai-toolkit` or diffusers `train_dreambooth_lora_flux2_klein.py`.
|
| 64 |
+
- **Compute:** Modal **L40S (~$0.91/hr)** sweet spot; kick off via `training/modal_lora/launch.py`,
|
| 65 |
+
**poll — never block** (CLAUDE §3). **~45-75 min, ~$1-3 total** (Modal ~$30/mo free credit ≈ $0).
|
| 66 |
+
- **Integrate:** `load_*_pipeline()` → `pipe.load_lora_weights('<repo>'); pipe.fuse_lora()` after
|
| 67 |
+
`.to('cuda')`; add `peft` to requirements (full rebuild — batch with xgrammar). Flip `lora_loaded`.
|
| 68 |
+
**Verify (rule R4):** read `site-packages/diffusers/pipelines/flux2/` that the KV pipeline inherits
|
| 69 |
+
the LoRA mixin and `fuse_lora()` doesn't break the K/V reference path.
|
| 70 |
+
|
| 71 |
+
---
|
| 72 |
+
|
| 73 |
+
## (3) Backlog (effort / payoff)
|
| 74 |
+
|
| 75 |
+
| # | Item | Effort | Payoff |
|
| 76 |
+
|---|------|--------|--------|
|
| 77 |
+
| B1 | Continuation LoRA (L→R outpaint, sliding-window crops of wide pans) | High (1-2d + $40-120) | Med-High — only if QW-3/4 leave seams |
|
| 78 |
+
| B2 | Director QLoRA (800-1500 narration→Shot pairs, anti-lighthouse) | Med ($5-15) | Med — shrink few-shots, kill bias at root |
|
| 79 |
+
| B3 | KV prefix cache for the static system+few-shot block | Med | Med — pairs with QW-2 |
|
| 80 |
+
| B4 | Latent-carry / img2img feedback in the transition band only | Med-High | Med — most organic morph |
|
| 81 |
+
| B5 | AoTI behind try/except + eager bf16 fallback | Low | Low-Med — NEVER on critical path; unverified sm120 |
|
| 82 |
+
| B6 | Z-Image-Turbo (Apache-2) as painter Plan B | Med | Low — slower/strip; switch for quality not speed |
|
| 83 |
+
| B7 | Qwen3-0.6B Director A/B | Low-Med | Low-Med — only if MiniCPM5 misses latency |
|
| 84 |
+
|
| 85 |
+
## (4) Biggest risk
|
| 86 |
+
**xgrammar on sm120 must be verified on the Space, not locally** — QW-1 rides on it, and sm120 has a
|
| 87 |
+
history of breaking low-precision/compiled kernels. Smoke-test a 1-beat constrained decode inside
|
| 88 |
+
`@spaces.GPU` before deleting any salvage code; keep the retry loop as fallback.
|
| 89 |
+
|
| 90 |
+
## The innovation thesis
|
| 91 |
+
Real-time **voice → continuously-morphing generative panorama** (browser streaming ASR + Director-once
|
| 92 |
+
world-bible + fast latent-carry painter + a custom **style/continuation LoRA** for organic morphing) is
|
| 93 |
+
something nobody has shipped — David Rokeby's *Voice Scroll* realized with 2026 diffusion. The LoRA is a
|
| 94 |
+
shareable artifact (Sharing-is-Caring) and the streaming-voice-panorama is a publishable technique.
|
src/lightloom/paint/scroll.py
CHANGED
|
@@ -54,12 +54,21 @@ def outpaint_right(
|
|
| 54 |
overlap: int = 256,
|
| 55 |
steps: int = 8,
|
| 56 |
seed: int = 1,
|
|
|
|
| 57 |
) -> Any:
|
| 58 |
"""Return the next ``seg_w``-wide panorama section that continues ``context``.
|
| 59 |
|
| 60 |
The canvas is ``[overlap context | seg_w to-generate]``; the mask keeps the
|
| 61 |
overlap and generates the rest, so the model paints a continuation of the
|
| 62 |
-
carried-over edge. We return only the freshly generated right part.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
import torch
|
| 64 |
from PIL import Image
|
| 65 |
|
|
@@ -72,6 +81,13 @@ def outpaint_right(
|
|
| 72 |
ctx = ctx.resize((overlap, height))
|
| 73 |
canvas.paste(ctx, (0, 0))
|
| 74 |
mask.paste(Image.new("L", (overlap, height), 0), (0, 0)) # …but KEEP the overlap as context
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
generator = torch.Generator(device="cuda").manual_seed(seed)
|
| 76 |
out = pipe(
|
| 77 |
prompt=SCROLL_TRIGGER + prompt,
|
|
|
|
| 54 |
overlap: int = 256,
|
| 55 |
steps: int = 8,
|
| 56 |
seed: int = 1,
|
| 57 |
+
morph_band: int = 0,
|
| 58 |
) -> Any:
|
| 59 |
"""Return the next ``seg_w``-wide panorama section that continues ``context``.
|
| 60 |
|
| 61 |
The canvas is ``[overlap context | seg_w to-generate]``; the mask keeps the
|
| 62 |
overlap and generates the rest, so the model paints a continuation of the
|
| 63 |
+
carried-over edge. We return only the freshly generated right part.
|
| 64 |
+
|
| 65 |
+
``morph_band`` > 0 (use on a SCENE CHANGE): extend the carried scene a few px into
|
| 66 |
+
the start of the new section and RAMP the mask 0→255 across it, so the new scene
|
| 67 |
+
EMERGES organically out of the old one instead of cutting hard. This band lands in
|
| 68 |
+
the RETURNED region (``[overlap, width]``), so it actually reaches the panorama.
|
| 69 |
+
(R4: if the diffusers inpaint pipeline binarizes ``mask_image`` this degrades
|
| 70 |
+
gracefully to the current hard boundary — verify before relying on it.)"""
|
| 71 |
+
import numpy as np
|
| 72 |
import torch
|
| 73 |
from PIL import Image
|
| 74 |
|
|
|
|
| 81 |
ctx = ctx.resize((overlap, height))
|
| 82 |
canvas.paste(ctx, (0, 0))
|
| 83 |
mask.paste(Image.new("L", (overlap, height), 0), (0, 0)) # …but KEEP the overlap as context
|
| 84 |
+
band = min(int(morph_band), seg_w)
|
| 85 |
+
if band > 0:
|
| 86 |
+
# seed the band with a stretch of the carried edge, then ramp keep→generate
|
| 87 |
+
edge = ctx.crop((overlap - 8, 0, overlap, height)).resize((band, height))
|
| 88 |
+
canvas.paste(edge, (overlap, 0))
|
| 89 |
+
ramp = np.clip(np.linspace(0.0, 255.0, band), 0, 255).astype(np.uint8)
|
| 90 |
+
mask.paste(Image.fromarray(np.tile(ramp, (height, 1)), "L"), (overlap, 0))
|
| 91 |
generator = torch.Generator(device="cuda").manual_seed(seed)
|
| 92 |
out = pipe(
|
| 93 |
prompt=SCROLL_TRIGGER + prompt,
|