JonathanColetti commited on
Commit
c6ab681
·
verified ·
1 Parent(s): 5395a71

rework the layout: two columns, no examples table, hardware notice

Browse files
Files changed (1) hide show
  1. app.py +73 -67
app.py CHANGED
@@ -213,9 +213,7 @@ PROMPTS = {f"{p['idx']:>2} · {p['text']}": p["idx"] for p in INFO["prompts"]}
213
  WORLDS = {f"world {w['idx']} · {w['prompt'][:70]}": w["idx"] for w in INFO["worlds"]}
214
  WORLD_LABELS = list(WORLDS)
215
  PROMPT_LABELS = list(PROMPTS)
216
- # by bank index, so the examples below can name the prompt they mean
217
- P = {i: lbl for lbl, i in PROMPTS.items()}
218
- W = {i: lbl for lbl, i in WORLDS.items()}
219
  print(f"[boot] ready: step {INFO['step']}, {len(PROMPTS)} prompts, "
220
  f"{len(WORLDS)} worlds", flush=True)
221
 
@@ -348,86 +346,94 @@ def run(world: str, steer_to: str, steer_text: str, steer_at: float,
348
  CSS = """
349
  #col-container { max-width: 1180px; margin: 0 auto; }
350
  .dark .gradio-container { color: var(--body-text-color); }
 
 
 
 
 
 
 
 
 
 
351
  """
352
 
353
  INTRO = f"""# LiveWan
354
 
355
  **Streaming text-to-video you can steer while it runs.** A 1.3B student distilled
356
  from Wan2.1-T2V-14B that generates video continuously instead of as a fixed clip:
357
- 750 ms of 640x368 at a time, extended block by block.
 
358
 
359
- Pick a world to open on, and optionally a prompt to swap to partway through. The
360
- swap keeps the K/V cache, so the scene *continues* rather than cutting. Checkpoint
361
- step {INFO['step']}.
362
 
363
  [Code]({GITHUB}) · [Weights](https://huggingface.co/{LIVEWAN_REPO})
364
  """
365
 
366
- NOTES = f"""
367
- - The live preview updates as each block lands, which is not the playback rate.
368
- The **mp4 below it is the real 16 fps clip** watch that one.
369
- - **This Space runs below real time: ~1.0 s of compute per 0.75 s of video (0.74x).**
370
- Generation is fine (386 ms/block); decoding is the cost, 631 ms against 343 ms
371
- locally, because `torch.compile` cannot run in a ZeroGPU worker and the VAE
372
- decoder is the thing that is normally compiled. On a local GPU with the decoder
373
- compiled the same checkpoint sustains about 2.7x real time.
374
- - Steer from the 96-prompt bank or from your own text. The bank is the conditioning
375
- every published number in the repo refers to; free text is encoded here by
376
- umt5-xxl and is **not** numerically comparable to it, because umt5 embeddings
377
- differ slightly by hardware the same string on the training box is a slightly
378
- different tensor. It looks fine; it just is not the same conditioning.
379
- - **How far you steer matters.** Swapping to a nearby scene (waterfall to mountain
380
- stream) continues cleanly, because the K/V cache the stream is conditioned on
381
- still describes something like the new prompt. Swapping somewhere very different
382
- (waterfall to an aurora at night) visibly falls apart over the following seconds.
383
- Both are worth trying the examples below include one of each.
384
- - One stream at a time: the engine holds a single K/V cache, so requests queue.
385
- - A stream cannot exceed 1024 latent frames (~4.3 min) that is where `WanModel`'s
386
- RoPE tables end. It stops itself and says so.
 
 
 
 
 
 
 
 
 
387
  """
388
 
389
  with gr.Blocks(title="LiveWan") as demo:
390
  with gr.Column(elem_id="col-container"):
391
  gr.Markdown(INTRO)
392
-
393
- with gr.Row():
394
- world = gr.Dropdown(WORLD_LABELS, value=WORLD_LABELS[2],
395
- label="Open on world", scale=1)
396
- steer_to = gr.Dropdown([NO_STEER] + PROMPT_LABELS, value=NO_STEER,
397
- label="Steer to (prompt bank)", scale=1)
398
- steer_text = gr.Textbox(
399
- label="…or steer to your own text",
400
- placeholder="A lighthouse in a storm",
401
- info="Encoded here with umt5-xxl. Overrides the bank selection above.")
402
- with gr.Row():
403
- steer_at = gr.Slider(1, 25, value=6, step=0.5,
404
- label="Steer at (seconds in)", scale=1)
405
- seconds = gr.Slider(5, 30, value=15, step=1,
406
- label="Generate (seconds of video)", scale=1)
407
- seed = gr.Number(value=0, precision=0, label="Seed", scale=0)
408
- run_btn = gr.Button("Stream", variant="primary")
409
-
410
- preview = gr.Image(label="Live preview, as each block lands", height=368)
411
- stats = gr.Markdown()
412
- video = gr.Video(label="The clip, at 16 fps", autoplay=True)
413
-
414
- gr.Examples(
415
- examples=[
416
- # left alone, the waterfall holds for the full 20 s
417
- [W[60], NO_STEER, "", 6, 20, 0],
418
- # a near steer: waterfall -> mountain stream. The scene carries on
419
- [W[60], P[57], "", 6, 18, 0],
420
- # a far steer: waterfall -> aurora at night. This is where it breaks
421
- [W[60], P[68], "", 6, 18, 0],
422
- # the same idea in free text, which is what the box is for
423
- [W[60], NO_STEER, "A lighthouse in a storm", 6, 18, 0],
424
- # a person, steered to another person
425
- [W[0], P[2], "", 7, 18, 0],
426
- ],
427
- inputs=[world, steer_to, steer_text, steer_at, seconds, seed],
428
- label="Try one",
429
- )
430
- gr.Markdown(NOTES)
431
 
432
  run_btn.click(run, inputs=[world, steer_to, steer_text, steer_at, seconds, seed],
433
  outputs=[preview, stats, video], concurrency_limit=1)
 
213
  WORLDS = {f"world {w['idx']} · {w['prompt'][:70]}": w["idx"] for w in INFO["worlds"]}
214
  WORLD_LABELS = list(WORLDS)
215
  PROMPT_LABELS = list(PROMPTS)
216
+ W = {i: lbl for lbl, i in WORLDS.items()} # by world id, for the default
 
 
217
  print(f"[boot] ready: step {INFO['step']}, {len(PROMPTS)} prompts, "
218
  f"{len(WORLDS)} worlds", flush=True)
219
 
 
346
  CSS = """
347
  #col-container { max-width: 1180px; margin: 0 auto; }
348
  .dark .gradio-container { color: var(--body-text-color); }
349
+ #hw-note {
350
+ border: 1px solid var(--border-color-primary);
351
+ border-left: 4px solid var(--color-accent);
352
+ border-radius: 6px;
353
+ padding: 10px 14px;
354
+ margin-bottom: 4px;
355
+ font-size: 0.92em;
356
+ line-height: 1.5;
357
+ }
358
+ #hw-note p { margin: 0; }
359
  """
360
 
361
  INTRO = f"""# LiveWan
362
 
363
  **Streaming text-to-video you can steer while it runs.** A 1.3B student distilled
364
  from Wan2.1-T2V-14B that generates video continuously instead of as a fixed clip:
365
+ 750 ms of 640x368 at a time, extended block by block. Checkpoint step
366
+ {INFO['step']}.
367
 
368
+ Open on a world, then optionally swap the prompt partway through. The swap keeps the
369
+ K/V cache, so the scene *continues* rather than cutting.
 
370
 
371
  [Code]({GITHUB}) · [Weights](https://huggingface.co/{LIVEWAN_REPO})
372
  """
373
 
374
+ HW_NOTE = """
375
+ **This Space is on shared ZeroGPU hardware, which is not what the project's numbers
376
+ were measured on.** It streams at roughly **0.7x real time** here; on a dedicated GPU
377
+ the same checkpoint sustains about **2.7x**. The clip you get back is correct, it
378
+ just takes longer to produce. Generation is not the bottleneck (about 470 ms per
379
+ block) decoding is, at about 640 ms against 343 ms, because `torch.compile` cannot
380
+ run inside a ZeroGPU worker and the VAE decoder is what normally gets compiled.
381
+ """
382
+
383
+ NOTES = """
384
+ **How far you steer matters.** Swapping to a nearby scene (waterfall to mountain
385
+ stream) continues cleanly, because the K/V cache the stream is conditioned on still
386
+ describes something close to the new prompt. Swapping somewhere very different
387
+ (waterfall to an aurora at night) visibly falls apart over the following seconds.
388
+ Both are worth trying.
389
+
390
+ **Bank prompts against free text.** The 96-prompt bank is the conditioning every
391
+ published number in the repo refers to. Free text is encoded here by umt5-xxl and is
392
+ not numerically comparable to it, because umt5 embeddings differ slightly by
393
+ hardware, so the same string on the training box is a slightly different tensor. It
394
+ looks fine, it just is not the same conditioning.
395
+
396
+ **Steering is scheduled, not clicked.** A GPU worker is forked per request and cannot
397
+ be reached from outside while it runs, so the swap is given up front. The swap itself
398
+ is the live one: it replaces the cross-attention conditioning and leaves the cache
399
+ alone.
400
+
401
+ **Limits.** One stream at a time, since the engine holds a single K/V cache, so
402
+ requests queue. A stream cannot pass 1024 latent frames (about 4.3 minutes), which is
403
+ where `WanModel`'s RoPE tables end; it stops itself and says so.
404
  """
405
 
406
  with gr.Blocks(title="LiveWan") as demo:
407
  with gr.Column(elem_id="col-container"):
408
  gr.Markdown(INTRO)
409
+ gr.Markdown(HW_NOTE, elem_id="hw-note")
410
+
411
+ with gr.Row(equal_height=False):
412
+ with gr.Column(scale=2):
413
+ world = gr.Dropdown(WORLD_LABELS, value=W[60],
414
+ label="Open on world")
415
+ steer_to = gr.Dropdown([NO_STEER] + PROMPT_LABELS, value=NO_STEER,
416
+ label="Steer to", info="From the prompt bank")
417
+ steer_text = gr.Textbox(
418
+ label="Or steer to your own text", lines=2,
419
+ placeholder="A lighthouse in a storm",
420
+ info="Encoded with umt5-xxl. Overrides the selection above.")
421
+ steer_at = gr.Slider(1, 25, value=6, step=0.5,
422
+ label="Swap at (seconds in)")
423
+ seconds = gr.Slider(5, 30, value=15, step=1,
424
+ label="Generate (seconds of video)")
425
+ seed = gr.Number(value=0, precision=0, label="Seed")
426
+ run_btn = gr.Button("Stream", variant="primary", size="lg")
427
+
428
+ with gr.Column(scale=3):
429
+ video = gr.Video(label="The clip, at 16 fps", autoplay=True,
430
+ height=368)
431
+ preview = gr.Image(label="Live preview, as each block lands",
432
+ height=200)
433
+ stats = gr.Markdown()
434
+
435
+ with gr.Accordion("How to read this", open=False):
436
+ gr.Markdown(NOTES)
 
 
 
 
 
 
 
 
 
 
 
437
 
438
  run_btn.click(run, inputs=[world, steer_to, steer_text, steer_at, seconds, seed],
439
  outputs=[preview, stats, video], concurrency_limit=1)