Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/mpeg-diffusion: direct link, hf CLI and curl.
- Browser
- Download file 17.9 kB
-
https://huggingface.co/spaces/hugging-apps/mpeg-diffusion/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/mpeg-diffusion/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/mpeg-diffusion/resolve/main/app.py
17.9 kB
| """MPEG-Diffusion β mixed-resolution text-to-video generation on MiniMax-H3. | |
| This Space implements the MPEG-Diffusion inference recipe end to end: a low-resolution track | |
| denoises the whole frame, a linear region head self-selects where detail matters, and a | |
| high-resolution track refines only that region through dual-resolution attention with | |
| ridge-fitted up-maps, before the two tracks are merged in pixel space. | |
| The reference implementation is the paper's port for MiniMax-H3 | |
| (github.com/scrya-com/mpeg-diffusion, Apache-2.0), vendored here as ``lot/`` plus a | |
| subset of its musubi-tuner (``third_party/musubi_tuner/``). The weights are the published | |
| research artifacts (johndpope/MiniMax-H3-MPEG-Diffusion): the fitted KV up-maps and the | |
| v3 region head, applied to the *untrained* ComfyUI int8 ConvRot base DiT β exactly the | |
| "quality preset" the port's README documents (`--preset quality`: steps 20, switch 16, | |
| hr_steps 6, switch_upscale="crop", final="pixel", feather 4.0, alpha 0.5, block 25). | |
| The 62 GiB Qwen3-VL text encoder does not fit beside the transformer on a single ZeroGPU | |
| worker, so conditioning is delegated to ``multimodalart/qwen3vl-conditioner`` over the | |
| gradio API β the same split every MiniMax-H3 Space in this org uses. Its ``/encode`` | |
| returns the layer-50 hidden state ``prompt_embeds (1, L, 5120) bfloat16``, which is | |
| precisely the ``text`` input ``two_track_sample`` consumes. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| # expandable_segments avoids the fragmentation the 21 GB int8 streaming load would otherwise | |
| # hit; must be set before torch is imported. Same for the muted triton autotune cache dir. | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| # The int8 Triton kernels autotune; keep the cache on the ephemeral disk, not a read-only layer. | |
| os.environ.setdefault("TRITON_CACHE_DIR", "/tmp/triton-cache") | |
| import spaces # noqa: E402 β FIRST, before any CUDA-touching import | |
| import sys | |
| import tempfile | |
| import time | |
| import traceback | |
| from functools import cache | |
| from pathlib import Path | |
| import gradio as gr | |
| import torch | |
| SPACE_ROOT = Path(__file__).resolve().parent | |
| # The weights the Space downloads at runtime. The paths are exported *before* any ``lot`` | |
| # import because ``lot_paths`` reads them at import time (``procrustes_h3`` reads | |
| # ``LOT_H3_CHECKPOINT`` the moment it is imported). | |
| BASE_REPO = "Comfy-Org/MiniMax-H3" | |
| DIT_FILE = "diffusion_models/minimax_h3_fl2va_pruned_int8_convrot.safetensors" | |
| VAE_FILE = "vae/minimax_h3_video_vae_fp16.safetensors" | |
| TT_REPO = "johndpope/MiniMax-H3-MPEG-Diffusion" | |
| MAPS_FILE = "kv_maps/up_maps_tt.safetensors" | |
| REGION_FILE = "region_head_v3/region_head_tt.safetensors" | |
| CONDITIONER_SPACE = os.environ.get("H3_CONDITIONER", "multimodalart/qwen3vl-conditioner") | |
| # The quality preset from the port's README, pinned as defaults. 1344x768 is the canvas the | |
| # maps and the head were fitted/validated on. 22 pixel frames (latent_t 7, 0.92 s at 24 fps) | |
| # is the render length the port itself measures and ships: peak 26.8 GB with every block | |
| # resident. Longer clips blow up the HR dual-resolution attention β the author's own note | |
| # says 73 pixel frames OOM'd a 32 GB 5090 without streaming, and 56 frames OOM'd this | |
| # Space's 48 GB worker on the first live run. | |
| CANVAS_W, CANVAS_H = 1344, 768 | |
| NUM_FRAMES = 22 | |
| # The conditioner enforces a 2 s floor on num_frames, but its prompt_embeds are text-only | |
| # states that do NOT depend on clip length (its _encode never sees num_frames β the value | |
| # only gates validation and the optional prompt rewrite). So the /encode request passes 56 | |
| # (2.33 s, the shortest grid value over the floor) purely to clear that check, and the | |
| # sampler renders the 22-frame reference clip from those same states. | |
| CONDITIONER_FRAMES = 56 | |
| FPS = 24 | |
| DEFAULT_STEPS, DEFAULT_SWITCH, DEFAULT_HR_STEPS = 20, 16, 6 | |
| DEFAULT_FEATHER, DEFAULT_ALPHA = 4.0, 0.5 | |
| OUTPUT_DIR = tempfile.mkdtemp(prefix="mpeg-diffusion-out-") | |
| GPU_SIZE = "large" | |
| DIT: torch.nn.Module | None = None | |
| UPSCALE = None # lot.two_track_h3.VaeUpscaler | |
| MAPS = None # lot.mixed_res.UpMaps | |
| HEAD = None # lot.mixed_res.RegionHead | |
| HEAD_BLOCK = 25 | |
| LOAD_ERROR: str | None = None | |
| LOADED_IN: float | None = None | |
| # ββ Model loading (module scope, eager) βββββββββββββββββββββββββββββββββββββ | |
| def load_models() -> str | None: | |
| """Load DiT + VAE + up-maps + region head at startup, `.to("cuda")` so the ZeroGPU | |
| startup pack streams them disk->VRAM on a cold worker (26 GB total).""" | |
| global DIT, UPSCALE, MAPS, HEAD, HEAD_BLOCK, LOAD_ERROR, LOADED_IN | |
| if DIT is not None or LOAD_ERROR is not None: | |
| return LOAD_ERROR | |
| started = time.time() | |
| try: | |
| from huggingface_hub import hf_hub_download | |
| os.environ["LOT_H3_CHECKPOINT"] = hf_hub_download(BASE_REPO, DIT_FILE) | |
| os.environ["LOT_H3_VAE"] = hf_hub_download(BASE_REPO, VAE_FILE) | |
| maps_path = hf_hub_download(TT_REPO, MAPS_FILE) | |
| region_path = hf_hub_download(TT_REPO, REGION_FILE) | |
| sys.path.insert(0, str(SPACE_ROOT / "lot")) | |
| sys.path.insert(0, str(SPACE_ROOT / "third_party")) | |
| from two_track_h3 import load_maps, load_region, load_upscaler # lot/ is on sys.path | |
| from h3_musubi import load_dit | |
| print(f"[load] int8 ConvRot DiT from {BASE_REPO}/{DIT_FILE} ...", flush=True) | |
| DIT = load_dit(os.environ["LOT_H3_CHECKPOINT"], device="cuda", swap=0) | |
| print(f"[load] DiT resident ({time.time() - started:.0f}s), loading VAE ...", flush=True) | |
| UPSCALE = load_upscaler() | |
| print(f"[load] VAE resident, loading fitted up-maps + region head ...", flush=True) | |
| MAPS = load_maps(Path(maps_path)) | |
| saved = load_region(Path(region_path)) | |
| from mixed_res import RegionHead | |
| HEAD = RegionHead(saved["hidden"]).cuda().eval() | |
| HEAD.load_state_dict(saved["state"]) | |
| HEAD_BLOCK = int(saved["block"]) | |
| LOADED_IN = time.time() - started | |
| print(f"[load] ready in {LOADED_IN:.0f}s (region head on block {HEAD_BLOCK})", flush=True) | |
| except Exception as error: | |
| traceback.print_exc() | |
| LOAD_ERROR = f"**Loading failed** after {time.time() - started:.0f}s: `{type(error).__name__}: {error}`" | |
| return LOAD_ERROR | |
| load_models() | |
| # ββ Remote conditioning ββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def conditioner(): | |
| from gradio_client import Client | |
| return Client(CONDITIONER_SPACE) | |
| def conditioner_client(ip_token): | |
| if not ip_token: | |
| return conditioner() | |
| from gradio_client import Client | |
| return Client(CONDITIONER_SPACE, headers={"x-ip-token": ip_token}) | |
| def encode_remote(prompt, ip_token=None): | |
| """Call the conditioner Space for the layer-50 Qwen3-VL states ``two_track_sample`` consumes.""" | |
| from safetensors import safe_open | |
| def call(): | |
| return conditioner_client(ip_token).predict( | |
| prompt=prompt, | |
| image_path=None, | |
| last_image_path=None, | |
| canvas=f"{CANVAS_W}x{CANVAS_H} Β· 16:9 full", | |
| num_frames=CONDITIONER_FRAMES, | |
| rewrite_prompt=False, | |
| api_name="/encode", | |
| ) | |
| try: | |
| path, plan = call() | |
| except Exception as first: | |
| print(f"[cond] retrying with a fresh client after: {first}", flush=True) | |
| conditioner.cache_clear() | |
| path, plan = call() | |
| with safe_open(path, framework="pt") as handle: | |
| return handle.get_tensor("prompt_embeds"), plan | |
| def _caller_ip_token() -> str | None: | |
| from gradio.context import LocalContext | |
| request = LocalContext.request.get() | |
| return request.headers.get("x-ip-token") if request is not None else None | |
| # ββ GPU inference βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # measured 33s end to end (sampler 32s); 60 covers a cold | |
| # worker attach + the sliders' max settings (steps 30) | |
| # render() is grad-free end to end; without this the VAE decode of the merge | |
| # builds a full autograd graph (~24 GB of retained activations) and OOMs the worker | |
| def _generate(prompt_embeds, seed, steps, switch, hr_steps, feather, alpha, use_head, box): | |
| """Run the four-stage two-track sampler and compose to pixels, all on GPU time. | |
| Mirrors ``two_track_h3.render()`` with ``--preset quality``: 20 steps of which the | |
| first 16 run at half resolution, the region head picks M from block-25 states at the | |
| switch, the HR track takes 6 linear-sigma steps on M only (``switch_upscale="crop"`` | |
| round-trips just the bounding box through the VAE), and the merge happens in pixel | |
| space (``final="pixel"``) with a feather of 4 cells. | |
| ``two_track_sample`` has three return shapes: when the region comes back empty (or | |
| switch >= steps) it returns only the plain LR latent, so the pixel merge must be | |
| guarded exactly the way ``render()`` guards it. | |
| """ | |
| import torch | |
| from two_track_h3 import pixel_compose, save_clip, two_track_sample | |
| from h3_musubi import latent_frames | |
| latent_t = latent_frames(NUM_FRAMES) | |
| head = HEAD if use_head else None | |
| box = tuple(float(v) for v in box) if (box and not use_head) else None | |
| embeds = prompt_embeds.to("cuda", torch.bfloat16) | |
| latent, info = two_track_sample( | |
| DIT, embeds, UPSCALE, | |
| height=CANVAS_H, width=CANVAS_W, steps=int(steps), switch=int(switch), seed=int(seed), | |
| maps=MAPS, head=head, box=box, block=HEAD_BLOCK, | |
| alpha=float(alpha), feather=float(feather), hr_steps=int(hr_steps), | |
| latent_t=latent_t, switch_upscale="crop", final="pixel", | |
| ) | |
| # Stage boundary, as in the vendored code: the sampler's dual-resolution activations are | |
| # freed on return but the caching allocator still holds their segments β hand them back | |
| # so the merge starts from a clean pool. (The full-clip decode OOM here on the first two | |
| # live runs was the missing no_grad on this function: the VAE decode was retaining a | |
| # ~24 GB autograd graph that render()'s @torch.no_grad() never builds.) | |
| torch.cuda.empty_cache() | |
| if "lr" in info: # render()'s guard: the merge inputs exist only on the two-track path | |
| pixels = pixel_compose(UPSCALE.vae, latent, info["lr"], info["weight"]) | |
| else: | |
| pixels = UPSCALE.vae.decode(latent.to("cuda", UPSCALE.dtype)).to("cpu", torch.float16) | |
| out_path = os.path.join(OUTPUT_DIR, f"mpeg-diffusion-{int(time.time() * 1000)}.mp4") | |
| save_clip(pixels, Path(out_path), fps=FPS) | |
| share = float(info["mask"].float().mean()) if info.get("mask") is not None else 0.0 | |
| return out_path, share | |
| def generate( | |
| prompt, | |
| use_head=True, | |
| box_x0=0.25, | |
| box_y0=0.25, | |
| box_x1=0.75, | |
| box_y1=0.75, | |
| seed=0, | |
| steps=DEFAULT_STEPS, | |
| switch=DEFAULT_SWITCH, | |
| hr_steps=DEFAULT_HR_STEPS, | |
| feather=DEFAULT_FEATHER, | |
| alpha=DEFAULT_ALPHA, | |
| progress=gr.Progress(track_tqdm=True), | |
| ): | |
| """One request: prompt -> mixed-resolution video. Conditioning runs on the conditioner | |
| Space (outside GPU booking, charged to the caller's quota via x-ip-token); the sampler | |
| runs on this Space's ZeroGPU worker.""" | |
| if LOAD_ERROR: | |
| raise gr.Error(LOAD_ERROR.replace("**", "").replace("`", "")) | |
| if DIT is None: | |
| raise gr.Error("The model is still loading β watch the Space logs and retry shortly.") | |
| if not prompt or not prompt.strip(): | |
| raise gr.Error("A prompt is required.") | |
| if int(switch) >= int(steps): | |
| raise gr.Error("The region switch step must come before the last step.") | |
| box = (float(box_x0), float(box_y0), float(box_x1), float(box_y1)) | |
| if box[0] >= box[2] or box[1] >= box[3]: | |
| raise gr.Error("The manual region box needs x1 > x0 and y1 > y0.") | |
| progress(0.0, desc="Encoding the prompt ...") | |
| prompt_embeds, plan = encode_remote(prompt, ip_token=_caller_ip_token()) | |
| progress(0.1, desc=f"Two-track sampling {CANVAS_W}x{CANVAS_H} x {NUM_FRAMES} frames ...") | |
| started = time.time() | |
| out_path, share = _generate( | |
| prompt_embeds, int(seed), int(steps), int(switch), int(hr_steps), | |
| float(feather), float(alpha), bool(use_head), box, | |
| ) | |
| seconds = time.time() - started | |
| region = "region head" if use_head else f"manual box {tuple(round(v, 2) for v in box)}" | |
| report = ( | |
| f"{CANVAS_W}x{CANVAS_H} Β· {NUM_FRAMES} frames ({NUM_FRAMES / FPS:.2f}s) Β· " | |
| f"{int(steps)} steps, switch at {int(switch)}, {int(hr_steps)} HR steps Β· " | |
| f"HR region: {region} covering {share:.0%} of tokens Β· " | |
| f"conditioner {plan.get('seconds', '?')}s Β· sampler {seconds:.0f}s Β· seed {int(seed)}" | |
| ) | |
| print(f"[gen] {report}", flush=True) | |
| return out_path, report | |
| # ββ UI ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| INTRO = """# MPEG-Diffusion: Mixed-Resolution Video Generation | |
| <div align="center"> | |
| <a href="https://huggingface.co/papers/2611.12069" target="_blank" rel="noopener"><strong>[ paper ]</strong></a> | |
| <a href="https://github.com/scrya-com/mpeg-diffusion" target="_blank" rel="noopener"><strong>[ code ]</strong></a> | |
| <a href="https://huggingface.co/johndpope/MiniMax-H3-MPEG-Diffusion" target="_blank" rel="noopener"><strong>[ weights ]</strong></a> | |
| </div> | |
| **MPEG-Diffusion** (Menn et al.) generates video the way video codecs think about it: a | |
| **low-resolution track** denoises the whole frame cheaply, a tiny linear head **self-selects | |
| the region that deserves detail**, and a **high-resolution track** refines only that region | |
| through dual-resolution attention. The tracks merge in pixel space β full-res detail where it | |
| matters, ~2x the speed of rendering everything at high resolution. | |
| This Space runs the paper's MiniMax-H3 port with the published research weights: the fitted | |
| KV up-maps and the v3 region head on the int8 ConvRot base DiT, at the port's *quality preset* | |
| (20 steps, region chosen at step 16, 6 HR steps, pixel-space merge). By default the model | |
| picks its own detail region; force one with a manual box in *Advanced*.""" | |
| CSS = """ | |
| .main.fillable { max-width: 1100px !important; } | |
| """ | |
| EXAMPLE_PROMPTS = [ | |
| "A portrait of a young woman holding a red hibiscus flower against a rustic background", | |
| "A joyful woman with colorful flowers in her afro, exuding happiness and vibrant style", | |
| "A bearded man in a bathrobe uses a smartphone under studio lighting with a soft focus", | |
| "A fashionable man in checkered pants and cowboy hat poses outdoors on concrete blocks", | |
| "A serene portrait of a woman with curly hair surrounded by autumn foliage and berries", | |
| ] | |
| with gr.Blocks(title="MPEG-Diffusion") as demo: | |
| gr.Markdown(INTRO) | |
| with gr.Row(equal_height=True): | |
| with gr.Column(): | |
| prompt = gr.Textbox( | |
| label="Prompt", | |
| lines=3, | |
| value=EXAMPLE_PROMPTS[0], | |
| placeholder="Describe the scene β a clear subject helps the region head pick well ...", | |
| ) | |
| use_head = gr.Checkbox( | |
| label="Let the model choose the detail region (recommended)", | |
| value=True, | |
| ) | |
| run = gr.Button("Generate", variant="primary", size="lg") | |
| with gr.Accordion("Advanced", open=False): | |
| seed = gr.Number(label="Seed", value=0, precision=0) | |
| with gr.Row(): | |
| steps = gr.Slider(label="Steps", minimum=10, maximum=30, step=1, value=DEFAULT_STEPS) | |
| switch = gr.Slider(label="Region switch step", minimum=8, maximum=24, step=1, value=DEFAULT_SWITCH) | |
| with gr.Row(): | |
| hr_steps = gr.Slider(label="HR steps", minimum=2, maximum=10, step=1, value=DEFAULT_HR_STEPS) | |
| feather = gr.Slider(label="Feather (cells)", minimum=0, maximum=8, step=1, value=DEFAULT_FEATHER) | |
| with gr.Row(): | |
| alpha = gr.Slider(label="Region threshold Ξ±", minimum=0.1, maximum=0.9, step=0.05, value=DEFAULT_ALPHA) | |
| with gr.Accordion("Manual region (used when the checkbox is off)", open=False): | |
| with gr.Row(): | |
| box_x0 = gr.Slider(label="Left x0", minimum=0.0, maximum=1.0, step=0.01, value=0.25) | |
| box_y0 = gr.Slider(label="Top y0", minimum=0.0, maximum=1.0, step=0.01, value=0.25) | |
| with gr.Row(): | |
| box_x1 = gr.Slider(label="Right x1", minimum=0.0, maximum=1.0, step=0.01, value=0.75) | |
| box_y1 = gr.Slider(label="Bottom y1", minimum=0.0, maximum=1.0, step=0.01, value=0.75) | |
| with gr.Column(): | |
| result = gr.Video(label="Mixed-resolution video", autoplay=True) | |
| report = gr.Markdown() | |
| gr.Examples( | |
| examples=[[p] for p in EXAMPLE_PROMPTS], | |
| inputs=[prompt], | |
| fn=generate, | |
| outputs=[result, report], | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| inputs = [prompt, use_head, box_x0, box_y0, box_x1, box_y1, seed, steps, switch, hr_steps, feather, alpha] | |
| run.click(generate, inputs, [result, report], api_name="generate") | |
| prompt.submit(generate, inputs, [result, report], api_name="generate_submit") | |
| if __name__ == "__main__": | |
| demo.launch(show_error=True, theme=gr.themes.Citrus(), css=CSS, allowed_paths=[OUTPUT_DIR]) | |