multimodalart HF Staff commited on
Commit
f0ad196
·
verified ·
1 Parent(s): 1fe46a3

Gradio 6 theme placement, tight duration calibration, length-scaled camera speed

Browse files
Files changed (2) hide show
  1. README.md +11 -0
  2. app.py +24 -11
README.md CHANGED
@@ -46,6 +46,17 @@ python -m miniworld.sample --dataset re10k \
46
  ray-encoding features (`freq=15`, unnormalized translations), so no
47
  ground-truth poses or reference video are needed.
48
  * Streaming causal VAE decode-on-commit, matching the reference pipeline.
 
 
 
 
 
 
 
 
 
 
 
49
 
50
  The action-conditioned DROID checkpoint is intentionally not exposed: its
51
  conditioning requires per-dataset `q01/q99` action normalization statistics that
 
46
  ray-encoding features (`freq=15`, unnormalized translations), so no
47
  ground-truth poses or reference video are needed.
48
  * Streaming causal VAE decode-on-commit, matching the reference pipeline.
49
+ * The **Camera speed** slider is `--trajectory_magnitude` expressed in
50
+ "magnitude at `total_len=64`" units. The trajectory builder spreads the whole
51
+ path evenly over the rollout, so the authors advise scaling magnitude linearly
52
+ with length to keep the apparent speed constant (3.0 @ 64 → 4.5 @ 96); the
53
+ Space does that for you, and at 64 latent frames it is the reference value
54
+ verbatim.
55
+
56
+ Measured on this Space's ZeroGPU slice: 46 s at 20 latent frames (77 output
57
+ frames), 72 s at 32 (125 frames), 157 s at 64 (253 frames). The
58
+ `@spaces.GPU(duration=...)` estimate replays the sampler's asynchronous step
59
+ schedule to cost each rollout, so short clips reserve proportionally less quota.
60
 
61
  The action-conditioned DROID checkpoint is intentionally not exposed: its
62
  conditioning requires per-dataset `q01/q99` action normalization statistics that
app.py CHANGED
@@ -275,10 +275,13 @@ def _rollout_tflops(total_len: int, steps: int) -> float:
275
  return tflops
276
 
277
 
278
- # Calibrated against measured wall-clock on ZeroGPU (see README).
279
- _TFLOPS_PER_SEC = 45.0
280
- _VAE_SEC_PER_LATENT_FRAME = 0.15
281
- _FIXED_OVERHEAD_SEC = 20.0
 
 
 
282
 
283
 
284
  def _duration(*args, **kwargs) -> int:
@@ -326,13 +329,20 @@ def simulate(
326
  denoiser.cfg_scale = float(cfg_scale)
327
  denoiser.steps = int(num_sampling_steps)
328
 
 
 
 
 
 
 
 
329
  videos = init_frame.unsqueeze(0).unsqueeze(0).to(device) # (1, 1, H, W, C)
330
  poses = (
331
  build_custom_trajectory(
332
  trajectory,
333
  num_frames=4 * (total_len - 1) + 1,
334
  focal_norm=float(focal_norm),
335
- magnitude=float(magnitude),
336
  )
337
  .unsqueeze(0)
338
  .to(device)
@@ -388,7 +398,8 @@ def simulate(
388
  path,
389
  seed,
390
  f"**{n} frames** @ {SAVE_FPS} fps ({n / SAVE_FPS:.1f}s) · "
391
- f"{total_len} latent frames · `{trajectory}` @ {magnitude:g} · "
 
392
  f"seed `{seed}` · {elapsed:.1f}s of GPU time",
393
  )
394
 
@@ -398,7 +409,7 @@ def simulate(
398
  # --------------------------------------------------------------------------- #
399
  CSS = "#col-container { max-width: 1060px; margin: 0 auto; }"
400
 
401
- with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo:
402
  with gr.Column(elem_id="col-container"):
403
  gr.Markdown(
404
  f"""
@@ -432,13 +443,15 @@ with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo:
432
  value="orbit_right",
433
  )
434
  magnitude = gr.Slider(
435
- label="Motion strength",
436
  minimum=0.5,
437
  maximum=8.0,
438
  step=0.5,
439
  value=3.0,
440
- info="3.0 gives clear, stable motion over a 32-frame rollout. "
441
- "Scale up for longer rollouts; lower it if late frames smear.",
 
 
442
  )
443
  total_len = gr.Slider(
444
  label="Rollout length (latent frames)",
@@ -519,4 +532,4 @@ with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo:
519
  outputs=[result, seed, info],
520
  )
521
 
522
- demo.queue().launch(mcp_server=True)
 
275
  return tflops
276
 
277
 
278
+ # Calibrated on this Space's ZeroGPU H200 slice: measured 46.0s / 72.3s / 157.2s
279
+ # at total_len 20 / 32 / 64 against 2395 / 3599 / 7703 modelled TFLOP, i.e. a
280
+ # very clean 47.8 TFLOP/s (streaming VAE decode overlaps the denoiser, so it
281
+ # needs no separate term).
282
+ _TFLOPS_PER_SEC = 47.8
283
+ _VAE_SEC_PER_LATENT_FRAME = 0.0
284
+ _FIXED_OVERHEAD_SEC = 2.0
285
 
286
 
287
  def _duration(*args, **kwargs) -> int:
 
329
  denoiser.cfg_scale = float(cfg_scale)
330
  denoiser.steps = int(num_sampling_steps)
331
 
332
+ # `build_custom_trajectory` spreads the whole path evenly over the rollout,
333
+ # so a fixed magnitude means *faster* per-frame motion in a shorter clip.
334
+ # The authors' guidance is to scale it linearly with length to keep the
335
+ # apparent speed constant (3.0 @ total_len 64 -> 4.5 @ 96), so the slider is
336
+ # exposed as a speed in "magnitude at 64 latent frames" units.
337
+ magnitude_eff = float(magnitude) * total_len / 64.0
338
+
339
  videos = init_frame.unsqueeze(0).unsqueeze(0).to(device) # (1, 1, H, W, C)
340
  poses = (
341
  build_custom_trajectory(
342
  trajectory,
343
  num_frames=4 * (total_len - 1) + 1,
344
  focal_norm=float(focal_norm),
345
+ magnitude=magnitude_eff,
346
  )
347
  .unsqueeze(0)
348
  .to(device)
 
398
  path,
399
  seed,
400
  f"**{n} frames** @ {SAVE_FPS} fps ({n / SAVE_FPS:.1f}s) · "
401
+ f"{total_len} latent frames · `{trajectory}` · speed {magnitude:g} "
402
+ f"(magnitude {magnitude_eff:.2f}) · "
403
  f"seed `{seed}` · {elapsed:.1f}s of GPU time",
404
  )
405
 
 
409
  # --------------------------------------------------------------------------- #
410
  CSS = "#col-container { max-width: 1060px; margin: 0 auto; }"
411
 
412
+ with gr.Blocks() as demo:
413
  with gr.Column(elem_id="col-container"):
414
  gr.Markdown(
415
  f"""
 
443
  value="orbit_right",
444
  )
445
  magnitude = gr.Slider(
446
+ label="Camera speed",
447
  minimum=0.5,
448
  maximum=8.0,
449
  step=0.5,
450
  value=3.0,
451
+ info="3.0 is the paper's default: clear, stable motion. "
452
+ "1.0 is nearly static, 8.0 breaks down late. Scaled "
453
+ "internally with rollout length so the apparent speed "
454
+ "stays constant.",
455
  )
456
  total_len = gr.Slider(
457
  label="Rollout length (latent frames)",
 
532
  outputs=[result, seed, info],
533
  )
534
 
535
+ demo.queue().launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)