invoker-render / app.py
multimodalart's picture
multimodalart HF Staff
Size GPU duration to measured runtime
2cf1517 verified
Raw History Blame Contribute Delete
13 kB
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 (must precede torch)
import gc # noqa: E402
import tempfile # noqa: E402
import time # noqa: E402
import gradio as gr # noqa: E402
import numpy as np # noqa: E402
import torch # noqa: E402
from huggingface_hub import hf_hub_download # noqa: E402
from PIL import Image, ImageOps # noqa: E402
import media # noqa: E402
from h3vae import H3VideoCodec, build_vae # noqa: E402
from invoker_dit import InvokerDiT, InvokerSession, load_invoker_dit # noqa: E402
from qwen_encoder import encode_condition, load_qwen # noqa: E402
MODEL_REPO = "inclusionAI/invoker-render-1.0"
H3_REPO = "MiniMaxAI/MiniMax-H3"
TOKEN = os.environ.get("HF_TOKEN")
FRAME_CHOICES = media.valid_frame_counts(124) # 39, 56, 73, 90, 107, 124
DEFAULT_FRAMES = 73
t0 = time.time()
# Qwen3-VL condition encoder: FP8 decoder linears, kept in host RAM and visiting the GPU only for
# the single encode at the start of a request (it cannot share the card with the DiT's KV cache).
QWEN, PROCESSOR = load_qwen(H3_REPO, "Ref2VA", token=TOKEN)
QWEN_MASTERS = {}
for _mname, _mod in QWEN.named_modules():
for _pname, _p in list(_mod._parameters.items()):
if _p is not None:
QWEN_MASTERS[(_mname, "p", _pname)] = _p.data
for _bname, _b in list(_mod._buffers.items()):
if _b is not None:
QWEN_MASTERS[(_mname, "b", _bname)] = _b
print(f"[load] qwen ready after {time.time() - t0:.0f}s", flush=True)
# Invoker-Render causal DiT (FP32 checkpoint -> FP8 block linears + precomputed AdaLN tables).
DIT_MODEL, ADALN_TABLES = load_invoker_dit(MODEL_REPO, token=TOKEN)
DIT_MODEL.to("cuda")
ADALN_TABLES = ADALN_TABLES.to("cuda")
DIT = InvokerDiT(DIT_MODEL, ADALN_TABLES)
print(f"[load] dit ready after {time.time() - t0:.0f}s", flush=True)
# Native MiniMax-H3 video VAE (BF16, posterior mean), from the Ref2VA partition.
VAE = build_vae(hf_hub_download(H3_REPO, "Ref2VA/video_vae/source/model.safetensors", token=TOKEN)).to("cuda")
CODEC = H3VideoCodec(VAE)
# Optional helper: real footage -> relative-depth proxy (Depth Anything V2, as in the paper's proxy pipeline).
from transformers import AutoImageProcessor, AutoModelForDepthEstimation # noqa: E402
DEPTH_PROCESSOR = AutoImageProcessor.from_pretrained(media.DEPTH_MODEL)
DEPTH_MODEL = AutoModelForDepthEstimation.from_pretrained(media.DEPTH_MODEL, torch_dtype=torch.float16).to("cuda").eval()
print(f"[load] all models ready after {time.time() - t0:.0f}s", flush=True)
def _qwen_to(device: str) -> None:
for (mname, kind, name), master in QWEN_MASTERS.items():
mod = QWEN.get_submodule(mname) if mname else QWEN
value = master if device == "cpu" else master.to(device, non_blocking=True)
if kind == "p":
mod._parameters[name].data = value
else:
mod._buffers[name] = value
def _latents_for(num_frames: int) -> int:
return 5 * ((num_frames - 5) // 17 + 1) - 3
def _duration(source_frames, *a, **k):
latents = _latents_for(len(source_frames))
# Measured on the live Space: ~56 s (12 latents), ~78 s (22), ~111 s (37) warm, plus up to ~25 s when the
# Qwen weights make their first host->GPU trip in a cold worker.
return int(55 + 2.3 * latents)
PROXY_KIND = "Structural proxy (use as-is)"
REAL_KIND = "Real footage → depth proxy"
@spaces.GPU(duration=_duration, size="xlarge")
def _render(source_frames, first_image, prompt, seed, input_kind):
stats = {}
started = time.time()
if input_kind == REAL_KIND:
proxy_small = media.depth_proxy(source_frames, DEPTH_MODEL, DEPTH_PROCESSOR, device="cuda")
else:
proxy_small = media.frames_to_canvas(source_frames, media.PROXY_CANVAS, Image.NEAREST)
stats["proxy"] = time.time() - started
# 1. Qwen3-VL static condition: `<Picture 1>: <first image> <prompt>`, layer-50 hidden state.
t = time.time()
_qwen_to("cuda")
try:
prompt_embeds, token_tags = encode_condition(QWEN, PROCESSOR, prompt, first_image, device="cuda")
finally:
_qwen_to("cpu")
gc.collect()
torch.cuda.empty_cache()
stats["qwen"] = time.time() - t
# 2. VAE: first image (single-frame path) and proxy (17-frame blocks, nearest resize to 1344x768).
t = time.time()
proxy_full = np.stack([np.asarray(Image.fromarray(f).resize(media.CANVAS, Image.NEAREST)) for f in proxy_small])
first_latent = CODEC.image(np.asarray(first_image))
proxy_latents = CODEC.video(proxy_full)
stats["encode"] = time.time() - t
# 3. Causal four-step streaming generation.
t = time.time()
generator = torch.Generator("cpu").manual_seed(int(seed))
session = InvokerSession(
DIT,
prompt_embeds=prompt_embeds,
token_tags=token_tags,
first_frame=first_latent.to("cuda"),
first_frame_noise=torch.randn(first_latent.shape, generator=generator).to("cuda"),
)
latents = session.generate(proxy_latents, generator=generator)
del session
gc.collect()
torch.cuda.empty_cache()
stats["dit"] = time.time() - t
# 4. Native H3 decode of the whole latent stream.
t = time.time()
video = CODEC.decode_video(latents, num_frames=len(source_frames))
stats["decode"] = time.time() - t
stats["peak_gb"] = torch.cuda.max_memory_allocated() / 2**30
return video, proxy_small, stats
def render(
proxy_video: str,
first_image: str | None,
prompt: str,
seed: int = 42,
num_frames: int = DEFAULT_FRAMES,
input_kind: str = PROXY_KIND,
progress=gr.Progress(track_tqdm=True),
):
"""Render a photorealistic video that follows a structural proxy video.
Args:
proxy_video: Structural proxy video (semantic map, depth, gray-box, wireframe, skeleton, ...),
or real footage when `input_kind` asks for an automatic depth proxy. Resampled to 24 FPS.
first_image: Appearance anchor - the opening RGB frame of the rendered video (cropped to 16:9,
1344x768). Optional only for real footage, where its first frame is used.
prompt: Description of the scene (environment, subjects, lighting, events, camera motion).
seed: Noise seed.
num_frames: Output length in frames at 24 FPS; must be 17*N+5 (39, 56, 73, ...).
input_kind: "Structural proxy (use as-is)" or "Real footage → depth proxy".
Returns:
The rendered MP4, the 448x256 proxy that conditioned it, and a timing summary.
"""
if not proxy_video:
raise gr.Error("Upload a proxy video (or real footage with the depth-proxy option).")
if not prompt or not prompt.strip():
raise gr.Error("Invoker-Render always takes a text description.")
num_frames = int(num_frames)
if num_frames not in FRAME_CHOICES:
raise gr.Error(f"Number of frames must be one of {FRAME_CHOICES}.")
frames = media.read_video_24fps(proxy_video, num_frames)
if len(frames) < 39:
raise gr.Error(f"The video is too short: {len(frames)} frames at 24 FPS, the model needs at least 39 (~1.6 s).")
usable = max(n for n in FRAME_CHOICES if n <= len(frames))
frames = frames[:usable]
if first_image:
image = ImageOps.exif_transpose(Image.open(first_image)).convert("RGB")
elif input_kind == REAL_KIND:
image = Image.fromarray(frames[0])
else:
raise gr.Error("Provide a first image (the appearance anchor) for a structural proxy.")
image = media.cover_crop(image, media.CANVAS)
video, proxy_small, stats = _render(frames, image, prompt.strip(), int(seed), input_kind)
out_dir = tempfile.mkdtemp()
out = media.write_mp4(video, os.path.join(out_dir, "invoker_render.mp4"))
proxy_out = media.write_mp4(proxy_small, os.path.join(out_dir, "proxy.mp4"))
info = (
f"{len(video)} frames ({len(video) / media.FPS:.2f} s) at {video.shape[2]}x{video.shape[1]} · "
f"{_latents_for(usable)} latents · seed {int(seed)} \n"
f"proxy {stats['proxy']:.1f}s · Qwen {stats['qwen']:.1f}s · VAE encode {stats['encode']:.1f}s · "
f"DiT {stats['dit']:.1f}s · decode {stats['decode']:.1f}s · peak {stats['peak_gb']:.1f} GiB"
)
print(f"[render] {info}", flush=True)
return out, proxy_out, info
EX = "examples"
EXAMPLES = [
[
f"{EX}/city_traffic_night_depth_proxy.mp4",
f"{EX}/city_traffic_night_first_frame.jpg",
"A wide multi-lane city road at night. Cars and a white bus drive from left to right with glowing "
"headlights and red taillights on dark wet asphalt; a long green neon storefront, street lamps and "
"colorful signs glow on the far side. Static wide shot, cool low-light night exposure.",
],
[
f"{EX}/cycling_forest_depth_proxy.mp4",
f"{EX}/cycling_forest_first_frame.jpg",
"A cyclist in a black jersey, white helmet and sunglasses rides a black road bike along a paved road "
"through a forest of tall pine trees. Muted overcast daylight; the camera tracks alongside the rider "
"as he pedals steadily.",
],
[
f"{EX}/running_jog_depth_proxy.mp4",
f"{EX}/running_jog_first_frame.jpg",
"A bearded man in a grey tank top jogs toward the camera on a red running track beside green trees "
"and an elevated highway overpass. Warm hazy daylight, shallow depth of field, the camera tracks "
"backward to keep him centered.",
],
[
f"{EX}/train_arriving_depth_proxy.mp4",
f"{EX}/train_arriving_first_frame.jpg",
"A white-and-red commuter train glides from left to right past a station sign across a busy rail "
"yard with overhead wires, cranes and city buildings behind. Bright sunny sky with scattered "
"clouds, stationary camera.",
],
]
INTRO = """
# Invoker-Render 1.0
<div style="display:flex;gap:12px;flex-wrap:wrap">
<a href="https://huggingface.co/inclusionAI/invoker-render-1.0">Model</a> ·
<a href="https://github.com/inclusionAI/CodeWorldModel">Code</a> ·
<a href="https://www.inclusion-ai.org/CodeWorldModel/">Project page</a> ·
<a href="https://arxiv.org/abs/2611.22925">Paper</a>
</div>
**Code World Model 2.0's renderer.** Give it a *structural proxy* video (depth, semantic map, gray-box, wireframe,
skeleton…) that fixes the camera path, layout and motion, an *appearance anchor* (the opening frame) and a
description; the 33B causal DiT renders the video in five-latent chunks with four steps each.
No proxy at hand? Upload real footage and pick **Real footage → depth proxy** under *Advanced*.
"""
CSS = """
#col-container { max-width: 1200px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks(title="Invoker-Render 1.0") as demo:
with gr.Column(elem_id="col-container"):
gr.Markdown(INTRO)
with gr.Row():
with gr.Column():
with gr.Row():
proxy_in = gr.Video(label="Proxy video (24 FPS)", height=260)
image_in = gr.Image(label="First image (appearance anchor)", type="filepath", height=260)
prompt_in = gr.Textbox(label="Description", lines=3,
placeholder="Environment, subjects, lighting, events and camera motion…")
run = gr.Button("Render", variant="primary")
with gr.Accordion("Advanced", open=False):
kind_in = gr.Radio([PROXY_KIND, REAL_KIND], value=PROXY_KIND, label="Input video is")
frames_in = gr.Dropdown(FRAME_CHOICES, value=DEFAULT_FRAMES, label="Frames (24 FPS, 17·N+5)")
seed_in = gr.Number(label="Seed", value=42, precision=0)
with gr.Column():
video_out = gr.Video(label="Rendered video", autoplay=True, height=360)
proxy_out = gr.Video(label="Proxy used (448x256)", height=200)
info_out = gr.Markdown()
gr.Examples(
examples=EXAMPLES,
inputs=[proxy_in, image_in, prompt_in],
outputs=[video_out, proxy_out, info_out],
fn=render,
cache_examples=True,
cache_mode="lazy",
)
gr.Markdown(
"Example clips: CC0 footage from "
"[linoyts/repo-to-space-example-videos](https://huggingface.co/datasets/linoyts/repo-to-space-example-videos), "
"converted to depth proxies with Depth Anything V2. Weights: Apache-2.0 / CC BY-NC 4.0 (non-commercial); "
"VAE and Qwen3-VL encoder from [MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3) "
"(MiniMax H3 Community License)."
)
run.click(render, [proxy_in, image_in, prompt_in, seed_in, frames_in, kind_in],
[video_out, proxy_out, info_out], api_name="render")
if __name__ == "__main__":
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)