import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # noqa: E402 (must precede torch) import gc # noqa: E402 import tempfile # noqa: E402 import time # noqa: E402 import gradio as gr # noqa: E402 import numpy as np # noqa: E402 import torch # noqa: E402 from huggingface_hub import hf_hub_download # noqa: E402 from PIL import Image, ImageOps # noqa: E402 import media # noqa: E402 from h3vae import H3VideoCodec, build_vae # noqa: E402 from invoker_dit import InvokerDiT, InvokerSession, load_invoker_dit # noqa: E402 from qwen_encoder import encode_condition, load_qwen # noqa: E402 MODEL_REPO = "inclusionAI/invoker-render-1.0" H3_REPO = "MiniMaxAI/MiniMax-H3" TOKEN = os.environ.get("HF_TOKEN") FRAME_CHOICES = media.valid_frame_counts(124) # 39, 56, 73, 90, 107, 124 DEFAULT_FRAMES = 73 t0 = time.time() # Qwen3-VL condition encoder: FP8 decoder linears, kept in host RAM and visiting the GPU only for # the single encode at the start of a request (it cannot share the card with the DiT's KV cache). QWEN, PROCESSOR = load_qwen(H3_REPO, "Ref2VA", token=TOKEN) QWEN_MASTERS = {} for _mname, _mod in QWEN.named_modules(): for _pname, _p in list(_mod._parameters.items()): if _p is not None: QWEN_MASTERS[(_mname, "p", _pname)] = _p.data for _bname, _b in list(_mod._buffers.items()): if _b is not None: QWEN_MASTERS[(_mname, "b", _bname)] = _b print(f"[load] qwen ready after {time.time() - t0:.0f}s", flush=True) # Invoker-Render causal DiT (FP32 checkpoint -> FP8 block linears + precomputed AdaLN tables). DIT_MODEL, ADALN_TABLES = load_invoker_dit(MODEL_REPO, token=TOKEN) DIT_MODEL.to("cuda") ADALN_TABLES = ADALN_TABLES.to("cuda") DIT = InvokerDiT(DIT_MODEL, ADALN_TABLES) print(f"[load] dit ready after {time.time() - t0:.0f}s", flush=True) # Native MiniMax-H3 video VAE (BF16, posterior mean), from the Ref2VA partition. VAE = build_vae(hf_hub_download(H3_REPO, "Ref2VA/video_vae/source/model.safetensors", token=TOKEN)).to("cuda") CODEC = H3VideoCodec(VAE) # Optional helper: real footage -> relative-depth proxy (Depth Anything V2, as in the paper's proxy pipeline). from transformers import AutoImageProcessor, AutoModelForDepthEstimation # noqa: E402 DEPTH_PROCESSOR = AutoImageProcessor.from_pretrained(media.DEPTH_MODEL) DEPTH_MODEL = AutoModelForDepthEstimation.from_pretrained(media.DEPTH_MODEL, torch_dtype=torch.float16).to("cuda").eval() print(f"[load] all models ready after {time.time() - t0:.0f}s", flush=True) def _qwen_to(device: str) -> None: for (mname, kind, name), master in QWEN_MASTERS.items(): mod = QWEN.get_submodule(mname) if mname else QWEN value = master if device == "cpu" else master.to(device, non_blocking=True) if kind == "p": mod._parameters[name].data = value else: mod._buffers[name] = value def _latents_for(num_frames: int) -> int: return 5 * ((num_frames - 5) // 17 + 1) - 3 def _duration(source_frames, *a, **k): latents = _latents_for(len(source_frames)) # Measured on the live Space: ~56 s (12 latents), ~78 s (22), ~111 s (37) warm, plus up to ~25 s when the # Qwen weights make their first host->GPU trip in a cold worker. return int(55 + 2.3 * latents) PROXY_KIND = "Structural proxy (use as-is)" REAL_KIND = "Real footage → depth proxy" @spaces.GPU(duration=_duration, size="xlarge") def _render(source_frames, first_image, prompt, seed, input_kind): stats = {} started = time.time() if input_kind == REAL_KIND: proxy_small = media.depth_proxy(source_frames, DEPTH_MODEL, DEPTH_PROCESSOR, device="cuda") else: proxy_small = media.frames_to_canvas(source_frames, media.PROXY_CANVAS, Image.NEAREST) stats["proxy"] = time.time() - started # 1. Qwen3-VL static condition: `: `, layer-50 hidden state. t = time.time() _qwen_to("cuda") try: prompt_embeds, token_tags = encode_condition(QWEN, PROCESSOR, prompt, first_image, device="cuda") finally: _qwen_to("cpu") gc.collect() torch.cuda.empty_cache() stats["qwen"] = time.time() - t # 2. VAE: first image (single-frame path) and proxy (17-frame blocks, nearest resize to 1344x768). t = time.time() proxy_full = np.stack([np.asarray(Image.fromarray(f).resize(media.CANVAS, Image.NEAREST)) for f in proxy_small]) first_latent = CODEC.image(np.asarray(first_image)) proxy_latents = CODEC.video(proxy_full) stats["encode"] = time.time() - t # 3. Causal four-step streaming generation. t = time.time() generator = torch.Generator("cpu").manual_seed(int(seed)) session = InvokerSession( DIT, prompt_embeds=prompt_embeds, token_tags=token_tags, first_frame=first_latent.to("cuda"), first_frame_noise=torch.randn(first_latent.shape, generator=generator).to("cuda"), ) latents = session.generate(proxy_latents, generator=generator) del session gc.collect() torch.cuda.empty_cache() stats["dit"] = time.time() - t # 4. Native H3 decode of the whole latent stream. t = time.time() video = CODEC.decode_video(latents, num_frames=len(source_frames)) stats["decode"] = time.time() - t stats["peak_gb"] = torch.cuda.max_memory_allocated() / 2**30 return video, proxy_small, stats def render( proxy_video: str, first_image: str | None, prompt: str, seed: int = 42, num_frames: int = DEFAULT_FRAMES, input_kind: str = PROXY_KIND, progress=gr.Progress(track_tqdm=True), ): """Render a photorealistic video that follows a structural proxy video. Args: proxy_video: Structural proxy video (semantic map, depth, gray-box, wireframe, skeleton, ...), or real footage when `input_kind` asks for an automatic depth proxy. Resampled to 24 FPS. first_image: Appearance anchor - the opening RGB frame of the rendered video (cropped to 16:9, 1344x768). Optional only for real footage, where its first frame is used. prompt: Description of the scene (environment, subjects, lighting, events, camera motion). seed: Noise seed. num_frames: Output length in frames at 24 FPS; must be 17*N+5 (39, 56, 73, ...). input_kind: "Structural proxy (use as-is)" or "Real footage → depth proxy". Returns: The rendered MP4, the 448x256 proxy that conditioned it, and a timing summary. """ if not proxy_video: raise gr.Error("Upload a proxy video (or real footage with the depth-proxy option).") if not prompt or not prompt.strip(): raise gr.Error("Invoker-Render always takes a text description.") num_frames = int(num_frames) if num_frames not in FRAME_CHOICES: raise gr.Error(f"Number of frames must be one of {FRAME_CHOICES}.") frames = media.read_video_24fps(proxy_video, num_frames) if len(frames) < 39: raise gr.Error(f"The video is too short: {len(frames)} frames at 24 FPS, the model needs at least 39 (~1.6 s).") usable = max(n for n in FRAME_CHOICES if n <= len(frames)) frames = frames[:usable] if first_image: image = ImageOps.exif_transpose(Image.open(first_image)).convert("RGB") elif input_kind == REAL_KIND: image = Image.fromarray(frames[0]) else: raise gr.Error("Provide a first image (the appearance anchor) for a structural proxy.") image = media.cover_crop(image, media.CANVAS) video, proxy_small, stats = _render(frames, image, prompt.strip(), int(seed), input_kind) out_dir = tempfile.mkdtemp() out = media.write_mp4(video, os.path.join(out_dir, "invoker_render.mp4")) proxy_out = media.write_mp4(proxy_small, os.path.join(out_dir, "proxy.mp4")) info = ( f"{len(video)} frames ({len(video) / media.FPS:.2f} s) at {video.shape[2]}x{video.shape[1]} · " f"{_latents_for(usable)} latents · seed {int(seed)} \n" f"proxy {stats['proxy']:.1f}s · Qwen {stats['qwen']:.1f}s · VAE encode {stats['encode']:.1f}s · " f"DiT {stats['dit']:.1f}s · decode {stats['decode']:.1f}s · peak {stats['peak_gb']:.1f} GiB" ) print(f"[render] {info}", flush=True) return out, proxy_out, info EX = "examples" EXAMPLES = [ [ f"{EX}/city_traffic_night_depth_proxy.mp4", f"{EX}/city_traffic_night_first_frame.jpg", "A wide multi-lane city road at night. Cars and a white bus drive from left to right with glowing " "headlights and red taillights on dark wet asphalt; a long green neon storefront, street lamps and " "colorful signs glow on the far side. Static wide shot, cool low-light night exposure.", ], [ f"{EX}/cycling_forest_depth_proxy.mp4", f"{EX}/cycling_forest_first_frame.jpg", "A cyclist in a black jersey, white helmet and sunglasses rides a black road bike along a paved road " "through a forest of tall pine trees. Muted overcast daylight; the camera tracks alongside the rider " "as he pedals steadily.", ], [ f"{EX}/running_jog_depth_proxy.mp4", f"{EX}/running_jog_first_frame.jpg", "A bearded man in a grey tank top jogs toward the camera on a red running track beside green trees " "and an elevated highway overpass. Warm hazy daylight, shallow depth of field, the camera tracks " "backward to keep him centered.", ], [ f"{EX}/train_arriving_depth_proxy.mp4", f"{EX}/train_arriving_first_frame.jpg", "A white-and-red commuter train glides from left to right past a station sign across a busy rail " "yard with overhead wires, cranes and city buildings behind. Bright sunny sky with scattered " "clouds, stationary camera.", ], ] INTRO = """ # Invoker-Render 1.0
Model · Code · Project page · Paper
**Code World Model 2.0's renderer.** Give it a *structural proxy* video (depth, semantic map, gray-box, wireframe, skeleton…) that fixes the camera path, layout and motion, an *appearance anchor* (the opening frame) and a description; the 33B causal DiT renders the video in five-latent chunks with four steps each. No proxy at hand? Upload real footage and pick **Real footage → depth proxy** under *Advanced*. """ CSS = """ #col-container { max-width: 1200px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks(title="Invoker-Render 1.0") as demo: with gr.Column(elem_id="col-container"): gr.Markdown(INTRO) with gr.Row(): with gr.Column(): with gr.Row(): proxy_in = gr.Video(label="Proxy video (24 FPS)", height=260) image_in = gr.Image(label="First image (appearance anchor)", type="filepath", height=260) prompt_in = gr.Textbox(label="Description", lines=3, placeholder="Environment, subjects, lighting, events and camera motion…") run = gr.Button("Render", variant="primary") with gr.Accordion("Advanced", open=False): kind_in = gr.Radio([PROXY_KIND, REAL_KIND], value=PROXY_KIND, label="Input video is") frames_in = gr.Dropdown(FRAME_CHOICES, value=DEFAULT_FRAMES, label="Frames (24 FPS, 17·N+5)") seed_in = gr.Number(label="Seed", value=42, precision=0) with gr.Column(): video_out = gr.Video(label="Rendered video", autoplay=True, height=360) proxy_out = gr.Video(label="Proxy used (448x256)", height=200) info_out = gr.Markdown() gr.Examples( examples=EXAMPLES, inputs=[proxy_in, image_in, prompt_in], outputs=[video_out, proxy_out, info_out], fn=render, cache_examples=True, cache_mode="lazy", ) gr.Markdown( "Example clips: CC0 footage from " "[linoyts/repo-to-space-example-videos](https://huggingface.co/datasets/linoyts/repo-to-space-example-videos), " "converted to depth proxies with Depth Anything V2. Weights: Apache-2.0 / CC BY-NC 4.0 (non-commercial); " "VAE and Qwen3-VL encoder from [MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3) " "(MiniMax H3 Community License)." ) run.click(render, [proxy_in, image_in, prompt_in, seed_in, frames_in, kind_in], [video_out, proxy_out, info_out], api_name="render") if __name__ == "__main__": demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)