Efradeca commited on
Commit
c8b0a48
·
verified ·
1 Parent(s): 7a80517

chore: deploy private lightloom build

Browse files
Files changed (3) hide show
  1. app.py +50 -12
  2. docs/RESEARCH_ROADMAP.md +94 -0
  3. src/lightloom/paint/scroll.py +17 -1
app.py CHANGED
@@ -145,7 +145,7 @@ def health() -> dict[str, Any]:
145
  "privacy_mode": os.getenv("LIGHTLOOM_PRIVACY_MODE", "1") == "1",
146
  "cpu_count": os.cpu_count(),
147
  "director_backend": CONFIG.director_backend,
148
- "build": "live-narration-7",
149
  }
150
 
151
 
@@ -793,11 +793,9 @@ def _scroll_world(text: str, session: str):
793
  yield {"stage": "painting", "index": i, "scene": subject[:120]}
794
  for k in range(strips_per_phrase):
795
  # first strip of a new scene morphs from the previous one (smooth panorama)
796
- if k == 0 and prev_subject and prev_subject[:48].lower() != subject[:48].lower():
797
- strip_prompt = _scroll_transition_prompt(prev_subject, subject)
798
- else:
799
- strip_prompt = subject
800
- sec = outpaint_right(pipe, ctx, strip_prompt + style, seg_w=seg, height=h, overlap=overlap, steps=4, seed=200 + g * 3)
801
  ctx = sec
802
  sec.save(out_dir / f"section-{g}.webp", "WEBP", quality=88)
803
  yield {
@@ -1005,12 +1003,12 @@ def _scroll_live_phrase(session: str, text: str, audio_b64: str, lang: str):
1005
  try:
1006
  for k in range(strips_per_phrase):
1007
  # the FIRST strip of a NEW scene MORPHS from the previous subject into this one
1008
- # (klein paints the dissolve), so the panorama flows instead of cutting hard.
1009
- if k == 0 and prev_subject and prev_subject[:48].lower() != subject[:48].lower():
1010
- strip_prompt = _scroll_transition_prompt(prev_subject, subject)
1011
- else:
1012
- strip_prompt = subject
1013
- sec = outpaint_right(pipe, ctx, strip_prompt + style, seg_w=seg, height=h, overlap=overlap, steps=4, seed=200 + g * 3)
1014
  ctx = sec
1015
  sec.save(out_dir / f"section-{g}.webp", "WEBP", quality=88)
1016
  yield {
@@ -1048,6 +1046,46 @@ def scroll_live_api(session: str = "", text: str = "", audio_b64: str = "", lang
1048
  yield {"stage": stage, "error": str(exc)[:200], "session": sess}
1049
 
1050
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1051
  @app.post("/internal/bench/director-probe")
1052
  def director_probe_endpoint(backend: str = "transformers", threads: int = 8) -> dict[str, Any]:
1053
  if os.getenv("LIGHTLOOM_ENABLE_INTERNAL_BENCH", "0") != "1":
 
145
  "privacy_mode": os.getenv("LIGHTLOOM_PRIVACY_MODE", "1") == "1",
146
  "cpu_count": os.cpu_count(),
147
  "director_backend": CONFIG.director_backend,
148
+ "build": "live-narration-8",
149
  }
150
 
151
 
 
793
  yield {"stage": "painting", "index": i, "scene": subject[:120]}
794
  for k in range(strips_per_phrase):
795
  # first strip of a new scene morphs from the previous one (smooth panorama)
796
+ is_change = k == 0 and prev_subject and prev_subject[:48].lower() != subject[:48].lower()
797
+ strip_prompt = _scroll_transition_prompt(prev_subject, subject) if is_change else subject
798
+ sec = outpaint_right(pipe, ctx, strip_prompt + style, seg_w=seg, height=h, overlap=overlap, steps=4, seed=200 + g * 3, morph_band=(96 if is_change else 0))
 
 
799
  ctx = sec
800
  sec.save(out_dir / f"section-{g}.webp", "WEBP", quality=88)
801
  yield {
 
1003
  try:
1004
  for k in range(strips_per_phrase):
1005
  # the FIRST strip of a NEW scene MORPHS from the previous subject into this one
1006
+ # (klein paints the dissolve) + a morph BAND that lets the new scene emerge out
1007
+ # of the carried pixels, so the panorama flows instead of cutting hard.
1008
+ is_change = k == 0 and prev_subject and prev_subject[:48].lower() != subject[:48].lower()
1009
+ strip_prompt = _scroll_transition_prompt(prev_subject, subject) if is_change else subject
1010
+ mb = 96 if is_change else 0
1011
+ sec = outpaint_right(pipe, ctx, strip_prompt + style, seg_w=seg, height=h, overlap=overlap, steps=4, seed=200 + g * 3, morph_band=mb)
1012
  ctx = sec
1013
  sec.save(out_dir / f"section-{g}.webp", "WEBP", quality=88)
1014
  yield {
 
1046
  yield {"stage": stage, "error": str(exc)[:200], "session": sess}
1047
 
1048
 
1049
+ @spaces.GPU(duration=120)
1050
+ def _introspect_klein_on_gpu() -> dict[str, Any]:
1051
+ """Report the klein pipeline's call signature + LoRA/embeds capabilities, so we can
1052
+ build embedding-SLERP (prompt_embeds) and the style-LoRA fuse correctly (CLAUDE rule 4:
1053
+ read the real API instead of guessing)."""
1054
+ import inspect
1055
+
1056
+ from lightloom.paint.scroll import load_fast_pipeline
1057
+
1058
+ info: dict[str, Any] = {}
1059
+ try:
1060
+ pipe = load_fast_pipeline()
1061
+ sig = inspect.signature(pipe.__call__)
1062
+ params = list(sig.parameters.keys())
1063
+ info.update(
1064
+ ok=True,
1065
+ pipe_class=type(pipe).__name__,
1066
+ call_params=params,
1067
+ accepts_prompt_embeds="prompt_embeds" in params,
1068
+ accepts_mask_image="mask_image" in params,
1069
+ has_encode_prompt=hasattr(pipe, "encode_prompt"),
1070
+ has_load_lora_weights=hasattr(pipe, "load_lora_weights"),
1071
+ has_fuse_lora=hasattr(pipe, "fuse_lora"),
1072
+ components=list(getattr(pipe, "components", {}).keys()),
1073
+ text_encoder=type(getattr(pipe, "text_encoder", None)).__name__,
1074
+ )
1075
+ except Exception as exc: # noqa: BLE001 - diagnostic must report, not crash.
1076
+ import traceback
1077
+
1078
+ info.update(ok=False, error=f"{type(exc).__name__}: {exc!r}", trace=traceback.format_exc()[-800:])
1079
+ return info
1080
+
1081
+
1082
+ @app.post("/internal/introspect-klein")
1083
+ def introspect_klein_endpoint() -> dict[str, Any]:
1084
+ if os.getenv("LIGHTLOOM_ENABLE_INTERNAL_BENCH", "0") != "1":
1085
+ return {"ok": False, "error": "internal endpoint disabled"}
1086
+ return {"ok": True, "data": _introspect_klein_on_gpu()}
1087
+
1088
+
1089
  @app.post("/internal/bench/director-probe")
1090
  def director_probe_endpoint(backend: str = "transformers", threads: int = 8) -> dict[str, Any]:
1091
  if os.getenv("LIGHTLOOM_ENABLE_INTERNAL_BENCH", "0") != "1":
docs/RESEARCH_ROADMAP.md ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Lightloom — Research Roadmap (faster + smoother + the innovation)
2
+
3
+ Source: deep June-2026 research (5 agents): FLUX/klein LoRA (style + continuation), pipeline
4
+ speed/architecture, Director fine-tune. Synthesized + cross-checked against the codebase.
5
+
6
+ **Bottom line:** speed is the *Director per-phrase + serial stages*, not klein/ASR. Smoothness is
7
+ a *hard-mask + hard-cut* problem, not a model problem. The highest-leverage trained win is one
8
+ cheap **style LoRA**. Ship the quick wins first; train one LoRA.
9
+
10
+ Measured latency (live-narration-6, on-box): first paint ~42s (cold model load), per-strip ~0.8s
11
+ (klein 0.7 + depth), per-phrase ~2.4s paint, phrase cadence ~4s (mostly speech duration).
12
+
13
+ ---
14
+
15
+ ## (1) Quick wins — NO training, bf16-safe, ~0ms GPU. Ordered by ROI.
16
+
17
+ - **QW-1 — Grammar-constrain the Director (`xgrammar`).** Replace the `retries=4` + `repetition_penalty`
18
+ hack with schema-constrained decoding → 100% valid JSON by construction, lower p50, lower
19
+ `max_new_tokens`. **RISK:** the codebase asserts "xgrammar incompatible with transformers 5.x"
20
+ (stale, from an older pair) — must verify a `cp311/cu12x` prebuilt wheel installs + constrains on
21
+ the **sm120 Space** before deleting the `_coerce_shot` salvage path. Keep retry+parser as fallback.
22
+ - **QW-2 — Director-once / World-Bible.** ✅ SHIPPED (simple form: Director runs only on phrase 1).
23
+ Upgrade: one MiniCPM call at session open → locked palette (3-hex), cast, setting, lighting, style
24
+ adjectives, **fixed base seed**; per-strip prompt = world-bible prefix + rolling transcript, NO
25
+ per-strip LLM. Fixed seed + locked palette also softens cuts for free (StyleAligned).
26
+ - **QW-3 — Soft/feathered continuation MASK (the seam fix).** `scroll.py` mask is a hard step (0 over
27
+ the 256px overlap, 255 over the new region). Replace with a horizontal **ramp**: 0 for the outer
28
+ ~128px (deep context), 0→255 across an inner ~128px band, 255 in the new strip (numpy linspace, or
29
+ `pipe.mask_processor.blur(mask, blur_factor≈24-40)`). **CAVEAT (rule R4/CLAUDE §4):** verify
30
+ `Flux2KleinInpaintPipeline` does NOT internally binarize `mask_image` (diffusers #10690) — read
31
+ `site-packages/diffusers/pipelines/flux2/` first.
32
+ - **QW-4 — Embedding SLERP across scene cuts (the morph).** On a Director cut, spherically interpolate
33
+ `prompt_embeds` old→new across the next 3-5 strips (SLERP, never lerp — lerp corrupts T5 norms).
34
+ Pure CPU, 0ms GPU. Verify the klein inpaint signature accepts `prompt_embeds`.
35
+ - **QW-5 — Streaming ASR (LocalAgreement-2) + two-lane decoupling.** Rolling 16kHz buffer → Whisper
36
+ every 0.5-1s; commit a word when two overlapping decodes agree (`ufal/whisper_streaming` /
37
+ SimulStreaming, MIT). Run ASR on `concurrency_id="asr"`, paint/depth on `"gpu"`. Note: 2nd GPU lane
38
+ burns quota; the long-lived `@spaces.GPU` generator must self-relaunch before the ~120s cap.
39
+ - **QW-6 — Keep the 360-ERP LoRA OUT of the live scroll path** (needs klein-base + green ERP control,
40
+ for 360 *wrap* not open extension; `scroll.py` is already correct).
41
+
42
+ **Expected after QW-1→QW-5:** speak→first-reaction ~1.5-3s, a new strip every ~0.3-0.6s independent
43
+ of speech, scene cuts that morph instead of snap.
44
+
45
+ ---
46
+
47
+ ## (2) The ONE LoRA to train — painterly STYLE LoRA (highest ROI, cheap)
48
+
49
+ Train a **rank-16 style LoRA on FLUX.2-klein-base-4B** (`painter_base_train` in config), **fuse onto
50
+ the distilled klein-4B at warmup** (BFL: train base, load on distilled, adapters cross-compatible).
51
+ Every strip gets one invariant painterly "skin" → scene CHANGES (forest→interior→neon) read as one
52
+ hand-painted world. After `fuse_lora()` = **0ms/strip**. (Infra partly wired: `prompt.py` LORA_TRIGGER,
53
+ `config.py` painter_base_train.)
54
+
55
+ - **Data (~30-60 min, highest leverage):** render 24-30 imgs @1024 with the *current* distilled klein
56
+ (no LoRA) using ONE style suffix (e.g. "soft impasto oil painting, luminous diffuse light, muted
57
+ teal-amber palette, visible brushwork") across diverse subjects (forest, interior, neon city, water,
58
+ faces, sky). Hand-cull to ~24 (vary CONTENT, fix STYLE). Apache-2.0-clean, no scraping. Caption
59
+ content-only, prepend trigger `lghtlm style`, never write style words.
60
+ - **Method:** LoRA rank 16, alpha 16, target fused `attn.to_qkv_mlp_proj` + `attn.to_out` across
61
+ `single_transformer_blocks`; LR 1e-4, AdamW-8bit, batch 1 + ga 4, bf16, grad-checkpoint + cache
62
+ latents, 512→1024px, ~1500 steps, checkpoint every 250, **pick by eye** (~step 750-1500). Tool:
63
+ `ostris/ai-toolkit` or diffusers `train_dreambooth_lora_flux2_klein.py`.
64
+ - **Compute:** Modal **L40S (~$0.91/hr)** sweet spot; kick off via `training/modal_lora/launch.py`,
65
+ **poll — never block** (CLAUDE §3). **~45-75 min, ~$1-3 total** (Modal ~$30/mo free credit ≈ $0).
66
+ - **Integrate:** `load_*_pipeline()` → `pipe.load_lora_weights('<repo>'); pipe.fuse_lora()` after
67
+ `.to('cuda')`; add `peft` to requirements (full rebuild — batch with xgrammar). Flip `lora_loaded`.
68
+ **Verify (rule R4):** read `site-packages/diffusers/pipelines/flux2/` that the KV pipeline inherits
69
+ the LoRA mixin and `fuse_lora()` doesn't break the K/V reference path.
70
+
71
+ ---
72
+
73
+ ## (3) Backlog (effort / payoff)
74
+
75
+ | # | Item | Effort | Payoff |
76
+ |---|------|--------|--------|
77
+ | B1 | Continuation LoRA (L→R outpaint, sliding-window crops of wide pans) | High (1-2d + $40-120) | Med-High — only if QW-3/4 leave seams |
78
+ | B2 | Director QLoRA (800-1500 narration→Shot pairs, anti-lighthouse) | Med ($5-15) | Med — shrink few-shots, kill bias at root |
79
+ | B3 | KV prefix cache for the static system+few-shot block | Med | Med — pairs with QW-2 |
80
+ | B4 | Latent-carry / img2img feedback in the transition band only | Med-High | Med — most organic morph |
81
+ | B5 | AoTI behind try/except + eager bf16 fallback | Low | Low-Med — NEVER on critical path; unverified sm120 |
82
+ | B6 | Z-Image-Turbo (Apache-2) as painter Plan B | Med | Low — slower/strip; switch for quality not speed |
83
+ | B7 | Qwen3-0.6B Director A/B | Low-Med | Low-Med — only if MiniCPM5 misses latency |
84
+
85
+ ## (4) Biggest risk
86
+ **xgrammar on sm120 must be verified on the Space, not locally** — QW-1 rides on it, and sm120 has a
87
+ history of breaking low-precision/compiled kernels. Smoke-test a 1-beat constrained decode inside
88
+ `@spaces.GPU` before deleting any salvage code; keep the retry loop as fallback.
89
+
90
+ ## The innovation thesis
91
+ Real-time **voice → continuously-morphing generative panorama** (browser streaming ASR + Director-once
92
+ world-bible + fast latent-carry painter + a custom **style/continuation LoRA** for organic morphing) is
93
+ something nobody has shipped — David Rokeby's *Voice Scroll* realized with 2026 diffusion. The LoRA is a
94
+ shareable artifact (Sharing-is-Caring) and the streaming-voice-panorama is a publishable technique.
src/lightloom/paint/scroll.py CHANGED
@@ -54,12 +54,21 @@ def outpaint_right(
54
  overlap: int = 256,
55
  steps: int = 8,
56
  seed: int = 1,
 
57
  ) -> Any:
58
  """Return the next ``seg_w``-wide panorama section that continues ``context``.
59
 
60
  The canvas is ``[overlap context | seg_w to-generate]``; the mask keeps the
61
  overlap and generates the rest, so the model paints a continuation of the
62
- carried-over edge. We return only the freshly generated right part."""
 
 
 
 
 
 
 
 
63
  import torch
64
  from PIL import Image
65
 
@@ -72,6 +81,13 @@ def outpaint_right(
72
  ctx = ctx.resize((overlap, height))
73
  canvas.paste(ctx, (0, 0))
74
  mask.paste(Image.new("L", (overlap, height), 0), (0, 0)) # …but KEEP the overlap as context
 
 
 
 
 
 
 
75
  generator = torch.Generator(device="cuda").manual_seed(seed)
76
  out = pipe(
77
  prompt=SCROLL_TRIGGER + prompt,
 
54
  overlap: int = 256,
55
  steps: int = 8,
56
  seed: int = 1,
57
+ morph_band: int = 0,
58
  ) -> Any:
59
  """Return the next ``seg_w``-wide panorama section that continues ``context``.
60
 
61
  The canvas is ``[overlap context | seg_w to-generate]``; the mask keeps the
62
  overlap and generates the rest, so the model paints a continuation of the
63
+ carried-over edge. We return only the freshly generated right part.
64
+
65
+ ``morph_band`` > 0 (use on a SCENE CHANGE): extend the carried scene a few px into
66
+ the start of the new section and RAMP the mask 0→255 across it, so the new scene
67
+ EMERGES organically out of the old one instead of cutting hard. This band lands in
68
+ the RETURNED region (``[overlap, width]``), so it actually reaches the panorama.
69
+ (R4: if the diffusers inpaint pipeline binarizes ``mask_image`` this degrades
70
+ gracefully to the current hard boundary — verify before relying on it.)"""
71
+ import numpy as np
72
  import torch
73
  from PIL import Image
74
 
 
81
  ctx = ctx.resize((overlap, height))
82
  canvas.paste(ctx, (0, 0))
83
  mask.paste(Image.new("L", (overlap, height), 0), (0, 0)) # …but KEEP the overlap as context
84
+ band = min(int(morph_band), seg_w)
85
+ if band > 0:
86
+ # seed the band with a stretch of the carried edge, then ramp keep→generate
87
+ edge = ctx.crop((overlap - 8, 0, overlap, height)).resize((band, height))
88
+ canvas.paste(edge, (overlap, 0))
89
+ ramp = np.clip(np.linspace(0.0, 255.0, band), 0, 255).astype(np.uint8)
90
+ mask.paste(Image.fromarray(np.tile(ramp, (height, 1)), "L"), (overlap, 0))
91
  generator = torch.Generator(device="cuda").manual_seed(seed)
92
  out = pipe(
93
  prompt=SCROLL_TRIGGER + prompt,