"""AnyGameRenderer — re-render a crude gameplay blockout as a polished game clip, entity for entity. [`AlayaLab/AnyGameRenderer`](https://huggingface.co/AlayaLab/AnyGameRenderer) is a rank-128 LoRA for MiniMax-H3's `ref2va` partition. A request is one white-model video — pale-gray geometry with one flat, saturated proxy color per entity class — plus the paper's fixed six-section caption; the output is the same gameplay re-rendered in the prompted style, frame-aligned with the input, with a jointly generated soundtrack. This Space is the denoising half of a split deployment: the 37.5 GiB AdaLN-pruned `transformer_ref` partition and both autoencoders run here, the 62 GiB Qwen3-VL conditioner runs in [`multimodalart/qwen3vl-conditioner`](https://huggingface.co/spaces/multimodalart/qwen3vl-conditioner) and is called over the gradio API per request. `prompt_embeds` + `text_token_tags` is the whole wire format. """ from __future__ import annotations import os import tempfile import time import traceback # Before anything that could initialize CUDA: `import spaces` patches `torch.cuda` so the model load can happen at # startup rather than on GPU time. import spaces import gradio as gr import torch VERSION = "anygame-renderer/1" # The AdaLN-pruned `ref2va` partition in diffusers layout: 37.5 GiB against the released partition's 61.7 GiB. The # LoRA is AdaLN-free by construction — its musubi-tuner target pattern is # `blocks.\d+.(attn.(qkv_proj|out_proj)|mlp.(fc1|fc2))`, which never touches `adaln_proj` — so it applies verbatim to # the pruned weights. It was trained against Comfy-Org's `minimax_h3_ref2va_pruned_int8_convrot`, i.e. a quantization # of this same checkpoint family. MODEL_REPO = os.environ.get("H3_MODEL_REPO", "multimodalart/MiniMax-H3-Pruned") LORA_REPO = "AlayaLab/AnyGameRenderer" LORA_FILE = "anygamerenderer_lora.safetensors" ADAPTER = "anygame" CONDITIONER_SPACE = os.environ.get("H3_CONDITIONER", "multimodalart/qwen3vl-conditioner") # `lazy` moves the whole partition onto the card on the first GPU call and leaves it there. Startup placement is not # an option: `spaces`' startup `torch.pack()` writes every startup-resident CUDA tensor to a second copy on disk, and # the weights plus their pack do not fit the Space storage quota next to the VAEs. PLACEMENT = os.environ.get("H3_PLACEMENT", "lazy").lower() # cuDNN's fused attention, 10-20% faster than the SDPA default on this pool and needs nothing installed. # flash-attention 3 is sm90-only and this card is sm120 (the `zero-a10g` flavour name is legacy). ATTENTION = os.environ.get("H3_ATTENTION", "_native_cudnn").lower() GPU_SIZE = os.environ.get("H3_GPU_SIZE", "xlarge") MIN_GPU_DURATION = int(os.environ.get("H3_GPU_DURATION_MIN", "120")) MAX_GPU_DURATION = int(os.environ.get("H3_GPU_DURATION_MAX", "1500")) FPS, FRAMES_PER_CHUNK, LATENTS_PER_CHUNK = 24, 17, 5 # The LoRA's own training bucket: 1344x768 at 24 fps, 124 frames. Resolution is fixed to what it was trained on. CANVAS = "1344x768 · 16:9 full" HEIGHT, WIDTH = 768, 1344 # The bundled clips — 5.17 s each — snap down to 124 frames, the LoRA's training length. DEFAULT_STEPS = 20 # `infer.py`'s own default DEFAULT_SEED = 42 # `infer.py`'s own default MAX_DURATION = 15.0 # MiniMax-H3's ceiling MIN_UI_DURATION = 5.0 # the pipeline's floor; blockouts shorter than 5 s still render 5 s # Seconds of GPU one request needs, from the packed sequence it is about to denoise: linear in the rows for the # matmuls, quadratic for the attention. Fitted on the `t2va` half and checked against live `ref2va` requests by the # character-swap Space this deployment shares its layout with. A primitive clip *is* the reference video, and its # soundtrack is silent (a blockout has no audio stream), so the sequence is the text tokens plus the reference rows # plus the generated rows — smaller than the two-reference requests that fit was measured on, which leaves margin. STEP_LINEAR, STEP_QUADRATIC, SAFETY = 1.1745e-4, 3.8396e-9, 1.15 # The lazy `PIPE.to("cuda")` a cold worker pays inside its first GPU call; every request carries it, because nothing # here knows whether the worker it lands on is cold. The pruned `transformer_ref` plus the two float32 VAEs is # ~48 GiB resident, which the same deployment measured at 70 s. PLACEMENT_ALLOWANCE = int(os.environ.get("H3_PLACEMENT_ALLOWANCE", "70")) AUDIO_LATENTS_PER_SECOND, AUDIO_CHANNELS = 40, 2 CANVAS_MULTIPLE = 32 DECODE_BASE, DECODE_PER_DEFAULT_CANVAS, DEFAULT_CANVAS_PIXELS = 15, 25, 960 * 544 * 124 PIPE = None LOAD_ERROR: str | None = None LORA_STATUS: str = "" def snap_frames(seconds: float) -> int: """The frame count MiniMax-H3's video VAE can decode: the next `17 * n + 5` at 24 fps.""" frames = max(1, round(float(seconds) * FPS)) while frames % FRAMES_PER_CHUNK != LATENTS_PER_CHUNK: frames += 1 return frames def lower_duration_floor(seconds: float = MIN_UI_DURATION) -> None: """Let the pipeline generate below its 5 s floor is never needed here, but the property is cut off at the source: the bundled clips are 5.17 s and the UI never offers less.""" from diffusers.modular_pipelines.minimax_h3.modular_pipeline import MiniMaxH3ModularPipeline MiniMaxH3ModularPipeline.min_duration = property(lambda self: float(seconds)) def video_latent_frames(num_frames: int) -> int: """`17 * n + 5` frames become `5 * n + 2` video latents.""" return 5 * ((num_frames - LATENTS_PER_CHUNK) // FRAMES_PER_CHUNK) + 2 def target_rows(num_frames: int) -> int: """The generated rows of the packed sequence: video patched `(1, 2, 2)`, plus two audio rows per latent.""" video = video_latent_frames(num_frames) * (HEIGHT // CANVAS_MULTIPLE) * (WIDTH // CANVAS_MULTIPLE) return video + round(num_frames / FPS * AUDIO_LATENTS_PER_SECOND) * AUDIO_CHANNELS def reference_rows(primitive_path: str, num_frames: int) -> int: """The rows the primitive's reference block adds, from metadata alone — no decode. A primitive is put on the canvas its own aspect ratio resolves to, which at 1344x768 is the training canvas itself.""" import av with av.open(primitive_path) as container: stream = container.streams.video[0] source_height, source_width = stream.height, stream.width rate = float(stream.average_rate) if stream.average_rate else FPS duration = float(stream.duration * stream.time_base) if stream.duration is not None else 0.0 ratio = source_width / source_height short = 768 w, h = (short * ratio, short) if ratio >= 1.0 else (short, short / ratio) if w * h > 768 * 1344: scale = (768 * 1344 / (w * h)) ** 0.5 w, h = w * scale, h * scale canvas_width = max(CANVAS_MULTIPLE, round(w / CANVAS_MULTIPLE) * CANVAS_MULTIPLE) canvas_height = max(CANVAS_MULTIPLE, round(h / CANVAS_MULTIPLE) * CANVAS_MULTIPLE) frames = min(round(duration * rate), num_frames) snapped = max(1, (frames - LATENTS_PER_CHUNK) // FRAMES_PER_CHUNK) * FRAMES_PER_CHUNK + LATENTS_PER_CHUNK return video_latent_frames(snapped) * (canvas_height // CANVAS_MULTIPLE) * (canvas_width // CANVAS_MULTIPLE) def get_duration(prompt_embeds, text_token_tags, primitive, num_frames, steps, *_, **__): """Seconds of GPU to reserve for one request. Takes the arguments of the `@spaces.GPU` function it decorates and tolerates whatever else `spaces` injects.""" sequence = int(text_token_tags.shape[0]) + reference_rows(primitive, num_frames) + target_rows(num_frames) denoise = int(steps) * (STEP_LINEAR * sequence + STEP_QUADRATIC * sequence**2) * SAFETY encode = 5 + reference_rows(primitive, num_frames) * 1e-3 decode = DECODE_BASE + DECODE_PER_DEFAULT_CANVAS * (HEIGHT * WIDTH * num_frames) / DEFAULT_CANVAS_PIXELS total = PLACEMENT_ALLOWANCE + encode + denoise + decode + 10 duration = max(MIN_GPU_DURATION, min(MAX_GPU_DURATION, int(total))) print(f"[{VERSION}] S={sequence} -> reserving {duration}s ({denoise:.0f}s of denoise at {int(steps)} steps)", flush=True) return duration def load_models() -> str | None: """Load the denoising half at startup, but *not* onto the card. `MiniMaxH3Ref2VAGeneratorBlocks` declares `transformer_ref`, `vae`, `audio_vae`, the two schedulers and `video_processor`, so `load_components` fetches exactly those subfolders — `text_encoder/` and the `transformer/` partition are never touched. Both autoencoders carry `_keep_in_fp32_modules` and stay float32: a bfloat16 audio VAE decodes the soundtrack roughly 20 dB too quiet. """ global PIPE, LOAD_ERROR, LORA_STATUS if PIPE is not None or LOAD_ERROR is not None: return LOAD_ERROR started = time.time() try: from diffusers import ComponentsManager from h3_split_blocks import MiniMaxH3Ref2VAGeneratorBlocks lower_duration_floor() manager = ComponentsManager() blocks = MiniMaxH3Ref2VAGeneratorBlocks() print(f"[{VERSION}] loading {[c.name for c in blocks.expected_components]} from {MODEL_REPO} ...", flush=True) pipe = blocks.init_pipeline(MODEL_REPO, components_manager=manager, collection="h3") # The pruned DiT is served as remote code (`transformer_ref/modeling_minimax_h3_pruned.py`). `load_components` # forwards `trust_remote_code` only to components that live in the pipeline's own repo, so the two VAEs and # the schedulers — which `modular_model_index.json` still points at `MiniMaxAI/MiniMax-H3` — never see it. pipe.load_components(dtype=torch.bfloat16, trust_remote_code=True) # Both VAEs first, and explicitly. `set_attention_backend` also sets the registry's *global* backend, which # every processor that was not stamped fell through to, and the float32 audio VAE has no cuDNN kernel: # `RuntimeError: No available kernel. Aborting execution.` in its causal encoder attention. pipe.vae.set_attention_backend("native") pipe.audio_vae.set_attention_backend("native") pipe.transformer_ref.set_attention_backend(ATTENTION) # The LoRA is attached as a runtime PEFT adapter rather than folded into the base weights, so its strength # stays a per-request knob and 0.0 is the plain base `ref2va` model (the paper's own baseline). Before any # GPU placement, so the adapter's parameters travel with the transformer. LORA_STATUS = apply_lora_via_pipeline(pipe) print(f"[{VERSION}] {LORA_STATUS}", flush=True) PIPE = pipe print(f"[{VERSION}] ready in {time.time() - started:.0f}s", flush=True) except Exception as error: traceback.print_exc() LOAD_ERROR = ( f"**Loading `{MODEL_REPO}` + `{LORA_REPO}` failed** after {time.time() - started:.0f}s: " f"`{type(error).__name__}: {error}`" ) return LOAD_ERROR # ── LoRA loading (runtime PEFT adapter) ────────────────────────────────────── # # The adapter is a musubi-tuner kohya file: 200 flattened `lora_unet_blocks__` names (50 transformer # blocks x 4 modules) with `lora_down`/`lora_up` pairs and per-module `.alpha` scalars at rank 128. diffusers 0.41's # own `_convert_non_diffusers_minimax_h3_lora_to_diffusers` understands exactly this layout — it unflattens the # names, folds each alpha into the pair, splits the fused QKV into contiguous thirds, swaps the SwiGLU gate/value # halves and renames onto the diffusers module paths — so the conversion is delegated to `load_lora_weights` rather # than reimplemented here. Routing to `transformer_ref` (the partition `ref2va` denoises with) is what # `load_into_transformer_ref=True` is for. def apply_lora_via_pipeline(pipe) -> str: """Attach the AnyGameRenderer LoRA as a PEFT adapter so its strength stays a per-request knob. `MiniMaxH3LoraLoaderMixin.load_lora_weights` runs `_convert_non_diffusers_minimax_h3_lora_to_diffusers`'s musubi-tuner branch on the flattened `lora_unet_` names, folds each `.alpha` into the pair, splits the fused QKV into contiguous thirds, swaps the SwiGLU gate/value halves and renames onto the diffusers module paths — the same transforms `infer.py` applies through musubi-tuner's own merge. `load_into_transformer_ref=True` routes it to the `ref2va` partition, the one the LoRA was trained against. The pruned partition takes it verbatim: every module the adapter touches is identical between the pruned and the released partition, and the adapter has no AdaLN branch to reconcile (its target pattern excluded `adaln_proj` at training time). """ from huggingface_hub import hf_hub_download path = hf_hub_download(LORA_REPO, LORA_FILE) pipe.load_lora_weights(path, adapter_name=ADAPTER, load_into_transformer_ref=True) pipe.transformer_ref.set_adapters([ADAPTER], [1.0]) peft_modules = [name for name, module in pipe.transformer_ref.named_modules() if hasattr(module, "lora_A")] return ( f"AnyGameRenderer LoRA attached · rank 128, alpha 128 · 200 musubi targets -> " f"{len(peft_modules)} PEFT modules · default strength 1.0" ) def probe_duration(path: str) -> float | None: """Duration of a media file's video stream, in seconds.""" import av with av.open(path) as container: if not container.streams.video: return None stream = container.streams.video[0] if stream.duration is not None and stream.time_base is not None: return float(stream.duration * stream.time_base) return None if container.duration is None else container.duration / av.time_base def conditioner(): """The other half, over the gradio API. `gradio_client` attaches the caller's own ZeroGPU token per call, so the conditioner's booking is billed to whoever asked for the video.""" from gradio_client import Client return Client(CONDITIONER_SPACE) def build_reference(primitive_path: str): """The primitive as the request's one reference, decoded by diffusers' own reader (frames, fps, soundtrack).""" from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3VideoReference return MiniMaxH3VideoReference.from_file(primitive_path) def check(primitive_path: str | None, caption: str) -> None: """The request's own rules, before anything is uploaded or a card is allocated.""" if not caption or not caption.strip(): raise gr.Error("Add the caption — AnyGameRenderer always takes one. Start from the caption builder below.") if not primitive_path: raise gr.Error("Add the white-model clip to re-render.") seconds = probe_duration(primitive_path) if seconds is None: raise gr.Error("That file has no video stream.") if not 5 / FPS <= seconds <= MAX_DURATION: raise gr.Error(f"The clip is {seconds:.2f} s. Use one between 0.21 and {MAX_DURATION:g} seconds.") def encode_remote(caption: str, primitive_path: str, num_frames: int): """`/encode_ref2va` on the conditioner Space: a safetensors file holding `prompt_embeds` + `text_token_tags`, with the resolved `height` / `width` / `num_frames` in its metadata, plus the plan.""" from gradio_client import handle_file from safetensors import safe_open path, plan = conditioner().predict( prompt=caption, media=[handle_file(primitive_path)], kinds="video", canvas=CANVAS, num_frames=num_frames, rewrite_prompt=False, api_name="/encode_ref2va", ) with safe_open(path, framework="pt") as handle: return handle.get_tensor("prompt_embeds"), handle.get_tensor("text_token_tags"), handle.metadata(), plan @spaces.GPU(duration=get_duration, size=GPU_SIZE) def _generate(prompt_embeds, text_token_tags, primitive_path, num_frames, steps, seed, lora_scale): """The only thing on GPU time: the reference encoder, the packed-sequence denoise loop and the two decoders. The primitive crosses as a path and is decoded here; only the three generated outputs come back. A `@spaces.GPU` argument crosses a process boundary by pickling, and a 5 s 1344x768 clip is 370 MB of expanded frames once PyAV has opened it. """ if PLACEMENT == "lazy": PIPE.to("cuda") # Per request, inside the worker: the adapter's scale is a python attribute on its PEFT layers, so setting it on # this fork's copy leaves every concurrent request with its own strength. PIPE.transformer_ref.set_adapters([ADAPTER], [float(lora_scale)]) state = PIPE( prompt_embeds=prompt_embeds.to("cuda"), text_token_tags=text_token_tags, references=[build_reference(primitive_path)], height=HEIGHT, width=WIDTH, num_frames=int(num_frames), num_inference_steps=int(steps), generator=torch.Generator("cpu").manual_seed(int(seed)), ) return state.get("videos")[0], state.get("audio")[0].cpu(), state.get("sampling_rate") def generate( # The first two are the columns `gr.Examples` varies, and they lead the signature for that reason: an example # row is applied to `inputs` positionally. Every other parameter carries the same default as its UI component, # so clicking an example and pressing Render behave identically. primitive: str | None = None, caption: str = "", match_length: bool = True, duration: float = 5.17, steps: int = DEFAULT_STEPS, seed: int = DEFAULT_SEED, lora_scale: float = 1.0, progress=gr.Progress(track_tqdm=True), ): """Re-render a white-model gameplay clip as a polished game clip, entity for entity. Args: primitive: path to the blockout clip — pale-gray geometry, one flat saturated proxy color per entity class, 24 fps, 2-15 s. 1344x768 is the resolution the LoRA was trained on. caption: the AnyGameRenderer caption (six sections: subject_definitions, summary, retention_analysis, detailed_description, overall_soundscape, non_diegetic_music). Build it from the spec below. match_length: render for exactly as long as the clip runs, snapped to `17n+5` frames. duration: seconds to render when `match_length` is off, snapped to `17n+5` frames. steps: denoising steps. The paper's own inference default is 20. seed: RNG seed. Generation is deterministic for a given seed. lora_scale: AnyGameRenderer LoRA strength. 1.0 is the released model; 0 is the zero-shot MiniMax-H3 base. Returns: Path to the rendered MP4, video and its jointly generated soundtrack. """ if LOAD_ERROR: raise gr.Error(LOAD_ERROR) if PIPE is None: raise gr.Error("The denoiser is still loading; try again in a minute.") from diffusers.utils import encode_video check(primitive, caption) clip_seconds = probe_duration(primitive) requested = clip_seconds if (match_length and clip_seconds is not None) else float(duration) num_frames = snap_frames(requested) if num_frames / FPS > MAX_DURATION: num_frames = snap_frames(MAX_DURATION - 1 / FPS) progress(0.0, desc="Reading the caption and the blockout ...") conditioned = time.time() try: prompt_embeds, text_token_tags, metadata, plan = encode_remote(caption, primitive, num_frames) except gr.Error: raise except Exception as error: # gradio only puts the exception *type* on the wire, so the useful half of a conditioner-side failure is in # that Space's logs. traceback.print_exc() raise gr.Error( f"The conditioner ({CONDITIONER_SPACE}) failed with `{type(error).__name__}: {error}`. " "Its logs carry the full traceback." ) from error condition_seconds = time.time() - conditioned # The resolved count the conditioner aligned to is the one the denoising half pins. num_frames = int(metadata["num_frames"]) progress(0.1, desc=f"Rendering {num_frames / FPS:.1f} s at {WIDTH}x{HEIGHT} ...") started = time.time() frames, audio, sampling_rate = _generate( prompt_embeds, text_token_tags, primitive, num_frames, steps, seed, lora_scale ) generate_seconds = time.time() - started directory = os.path.join(tempfile.gettempdir(), "agr-outputs") os.makedirs(directory, exist_ok=True) path = os.path.join(directory, f"anygame-{int(time.time() * 1000)}.mp4") encode_video(frames, fps=FPS, output_path=path, audio=audio, audio_sample_rate=sampling_rate) print( f"[{VERSION}] `{WIDTH}x{HEIGHT}`, {num_frames} frames ({num_frames / FPS:.3f} s), {int(steps)} steps, " f"strength {float(lora_scale):g} · conditioner {condition_seconds:.0f}s " f"({plan['num_text_tokens']} tokens) · denoise + decode {generate_seconds:.0f}s " f"({generate_seconds / max(1, int(steps)):.1f} s/step) · seed {int(seed)}", flush=True, ) return path def build_caption(spec_json: str) -> str: """Build the six-section AnyGameRenderer caption from a spec, exactly as `make_caption.py` in the paper's repo fills the training layout. The spec's fields are documented in the accordion above.""" import json SHAPE_WORD = {"box": "box", "wire": "wireframe box", "cylinder": "cylinder", "sphere": "ellipsoid"} DEFAULT_STYLE = ( "a polished, photoreal AAA game visual style: physically based materials, crisp micro-detail, soft global " "illumination, filmic colour grading, subtle depth of field and motion blur" ) VID1 = ( "