Spaces:
Running on Zero
Running on Zero
Download app.py from AlayaLab/anygame-renderer: direct link, hf CLI and curl.
- Browser
- Download file 31.9 kB
-
https://huggingface.co/spaces/AlayaLab/anygame-renderer/resolve/main/app.py
- Command line
-
hf download hf://spaces/AlayaLab/anygame-renderer/app.py
-
curl -L -o app.py https://huggingface.co/spaces/AlayaLab/anygame-renderer/resolve/main/app.py
31.9 kB
| """AnyGameRenderer โ re-render a crude gameplay blockout as a polished game clip, entity for entity. | |
| [`AlayaLab/AnyGameRenderer`](https://huggingface.co/AlayaLab/AnyGameRenderer) is a rank-128 LoRA for MiniMax-H3's | |
| `ref2va` partition. A request is one white-model video โ pale-gray geometry with one flat, saturated proxy color per | |
| entity class โ plus the paper's fixed six-section caption; the output is the same gameplay re-rendered in the prompted | |
| style, frame-aligned with the input, with a jointly generated soundtrack. | |
| This Space is the denoising half of a split deployment: the 37.5 GiB AdaLN-pruned `transformer_ref` partition and both | |
| autoencoders run here, the 62 GiB Qwen3-VL conditioner runs in | |
| [`multimodalart/qwen3vl-conditioner`](https://huggingface.co/spaces/multimodalart/qwen3vl-conditioner) and is called | |
| over the gradio API per request. `prompt_embeds` + `text_token_tags` is the whole wire format. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| import tempfile | |
| import time | |
| import traceback | |
| # Before anything that could initialize CUDA: `import spaces` patches `torch.cuda` so the model load can happen at | |
| # startup rather than on GPU time. | |
| import spaces | |
| import gradio as gr | |
| import torch | |
| VERSION = "anygame-renderer/1" | |
| # The AdaLN-pruned `ref2va` partition in diffusers layout: 37.5 GiB against the released partition's 61.7 GiB. The | |
| # LoRA is AdaLN-free by construction โ its musubi-tuner target pattern is | |
| # `blocks.\d+.(attn.(qkv_proj|out_proj)|mlp.(fc1|fc2))`, which never touches `adaln_proj` โ so it applies verbatim to | |
| # the pruned weights. It was trained against Comfy-Org's `minimax_h3_ref2va_pruned_int8_convrot`, i.e. a quantization | |
| # of this same checkpoint family. | |
| MODEL_REPO = os.environ.get("H3_MODEL_REPO", "multimodalart/MiniMax-H3-Pruned") | |
| LORA_REPO = "AlayaLab/AnyGameRenderer" | |
| LORA_FILE = "anygamerenderer_lora.safetensors" | |
| ADAPTER = "anygame" | |
| CONDITIONER_SPACE = os.environ.get("H3_CONDITIONER", "multimodalart/qwen3vl-conditioner") | |
| # `lazy` moves the whole partition onto the card on the first GPU call and leaves it there. Startup placement is not | |
| # an option: `spaces`' startup `torch.pack()` writes every startup-resident CUDA tensor to a second copy on disk, and | |
| # the weights plus their pack do not fit the Space storage quota next to the VAEs. | |
| PLACEMENT = os.environ.get("H3_PLACEMENT", "lazy").lower() | |
| # cuDNN's fused attention, 10-20% faster than the SDPA default on this pool and needs nothing installed. | |
| # flash-attention 3 is sm90-only and this card is sm120 (the `zero-a10g` flavour name is legacy). | |
| ATTENTION = os.environ.get("H3_ATTENTION", "_native_cudnn").lower() | |
| GPU_SIZE = os.environ.get("H3_GPU_SIZE", "xlarge") | |
| MIN_GPU_DURATION = int(os.environ.get("H3_GPU_DURATION_MIN", "120")) | |
| MAX_GPU_DURATION = int(os.environ.get("H3_GPU_DURATION_MAX", "1500")) | |
| FPS, FRAMES_PER_CHUNK, LATENTS_PER_CHUNK = 24, 17, 5 | |
| # The LoRA's own training bucket: 1344x768 at 24 fps, 124 frames. Resolution is fixed to what it was trained on. | |
| CANVAS = "1344x768 ยท 16:9 full" | |
| HEIGHT, WIDTH = 768, 1344 | |
| # The bundled clips โ 5.17 s each โ snap down to 124 frames, the LoRA's training length. | |
| DEFAULT_STEPS = 20 # `infer.py`'s own default | |
| DEFAULT_SEED = 42 # `infer.py`'s own default | |
| MAX_DURATION = 15.0 # MiniMax-H3's ceiling | |
| MIN_UI_DURATION = 5.0 # the pipeline's floor; blockouts shorter than 5 s still render 5 s | |
| # Seconds of GPU one request needs, from the packed sequence it is about to denoise: linear in the rows for the | |
| # matmuls, quadratic for the attention. Fitted on the `t2va` half and checked against live `ref2va` requests by the | |
| # character-swap Space this deployment shares its layout with. A primitive clip *is* the reference video, and its | |
| # soundtrack is silent (a blockout has no audio stream), so the sequence is the text tokens plus the reference rows | |
| # plus the generated rows โ smaller than the two-reference requests that fit was measured on, which leaves margin. | |
| STEP_LINEAR, STEP_QUADRATIC, SAFETY = 1.1745e-4, 3.8396e-9, 1.15 | |
| # The lazy `PIPE.to("cuda")` a cold worker pays inside its first GPU call; every request carries it, because nothing | |
| # here knows whether the worker it lands on is cold. The pruned `transformer_ref` plus the two float32 VAEs is | |
| # ~48 GiB resident, which the same deployment measured at 70 s. | |
| PLACEMENT_ALLOWANCE = int(os.environ.get("H3_PLACEMENT_ALLOWANCE", "70")) | |
| AUDIO_LATENTS_PER_SECOND, AUDIO_CHANNELS = 40, 2 | |
| CANVAS_MULTIPLE = 32 | |
| DECODE_BASE, DECODE_PER_DEFAULT_CANVAS, DEFAULT_CANVAS_PIXELS = 15, 25, 960 * 544 * 124 | |
| PIPE = None | |
| LOAD_ERROR: str | None = None | |
| LORA_STATUS: str = "" | |
| def snap_frames(seconds: float) -> int: | |
| """The frame count MiniMax-H3's video VAE can decode: the next `17 * n + 5` at 24 fps.""" | |
| frames = max(1, round(float(seconds) * FPS)) | |
| while frames % FRAMES_PER_CHUNK != LATENTS_PER_CHUNK: | |
| frames += 1 | |
| return frames | |
| def lower_duration_floor(seconds: float = MIN_UI_DURATION) -> None: | |
| """Let the pipeline generate below its 5 s floor is never needed here, but the property is cut off at the source: | |
| the bundled clips are 5.17 s and the UI never offers less.""" | |
| from diffusers.modular_pipelines.minimax_h3.modular_pipeline import MiniMaxH3ModularPipeline | |
| MiniMaxH3ModularPipeline.min_duration = property(lambda self: float(seconds)) | |
| def video_latent_frames(num_frames: int) -> int: | |
| """`17 * n + 5` frames become `5 * n + 2` video latents.""" | |
| return 5 * ((num_frames - LATENTS_PER_CHUNK) // FRAMES_PER_CHUNK) + 2 | |
| def target_rows(num_frames: int) -> int: | |
| """The generated rows of the packed sequence: video patched `(1, 2, 2)`, plus two audio rows per latent.""" | |
| video = video_latent_frames(num_frames) * (HEIGHT // CANVAS_MULTIPLE) * (WIDTH // CANVAS_MULTIPLE) | |
| return video + round(num_frames / FPS * AUDIO_LATENTS_PER_SECOND) * AUDIO_CHANNELS | |
| def reference_rows(primitive_path: str, num_frames: int) -> int: | |
| """The rows the primitive's reference block adds, from metadata alone โ no decode. A primitive is put on the | |
| canvas its own aspect ratio resolves to, which at 1344x768 is the training canvas itself.""" | |
| import av | |
| with av.open(primitive_path) as container: | |
| stream = container.streams.video[0] | |
| source_height, source_width = stream.height, stream.width | |
| rate = float(stream.average_rate) if stream.average_rate else FPS | |
| duration = float(stream.duration * stream.time_base) if stream.duration is not None else 0.0 | |
| ratio = source_width / source_height | |
| short = 768 | |
| w, h = (short * ratio, short) if ratio >= 1.0 else (short, short / ratio) | |
| if w * h > 768 * 1344: | |
| scale = (768 * 1344 / (w * h)) ** 0.5 | |
| w, h = w * scale, h * scale | |
| canvas_width = max(CANVAS_MULTIPLE, round(w / CANVAS_MULTIPLE) * CANVAS_MULTIPLE) | |
| canvas_height = max(CANVAS_MULTIPLE, round(h / CANVAS_MULTIPLE) * CANVAS_MULTIPLE) | |
| frames = min(round(duration * rate), num_frames) | |
| snapped = max(1, (frames - LATENTS_PER_CHUNK) // FRAMES_PER_CHUNK) * FRAMES_PER_CHUNK + LATENTS_PER_CHUNK | |
| return video_latent_frames(snapped) * (canvas_height // CANVAS_MULTIPLE) * (canvas_width // CANVAS_MULTIPLE) | |
| def get_duration(prompt_embeds, text_token_tags, primitive, num_frames, steps, *_, **__): | |
| """Seconds of GPU to reserve for one request. Takes the arguments of the `@spaces.GPU` function it decorates and | |
| tolerates whatever else `spaces` injects.""" | |
| sequence = int(text_token_tags.shape[0]) + reference_rows(primitive, num_frames) + target_rows(num_frames) | |
| denoise = int(steps) * (STEP_LINEAR * sequence + STEP_QUADRATIC * sequence**2) * SAFETY | |
| encode = 5 + reference_rows(primitive, num_frames) * 1e-3 | |
| decode = DECODE_BASE + DECODE_PER_DEFAULT_CANVAS * (HEIGHT * WIDTH * num_frames) / DEFAULT_CANVAS_PIXELS | |
| total = PLACEMENT_ALLOWANCE + encode + denoise + decode + 10 | |
| duration = max(MIN_GPU_DURATION, min(MAX_GPU_DURATION, int(total))) | |
| print(f"[{VERSION}] S={sequence} -> reserving {duration}s ({denoise:.0f}s of denoise at {int(steps)} steps)", flush=True) | |
| return duration | |
| def load_models() -> str | None: | |
| """Load the denoising half at startup, but *not* onto the card. | |
| `MiniMaxH3Ref2VAGeneratorBlocks` declares `transformer_ref`, `vae`, `audio_vae`, the two schedulers and | |
| `video_processor`, so `load_components` fetches exactly those subfolders โ `text_encoder/` and the `transformer/` | |
| partition are never touched. Both autoencoders carry `_keep_in_fp32_modules` and stay float32: a bfloat16 audio | |
| VAE decodes the soundtrack roughly 20 dB too quiet. | |
| """ | |
| global PIPE, LOAD_ERROR, LORA_STATUS | |
| if PIPE is not None or LOAD_ERROR is not None: | |
| return LOAD_ERROR | |
| started = time.time() | |
| try: | |
| from diffusers import ComponentsManager | |
| from h3_split_blocks import MiniMaxH3Ref2VAGeneratorBlocks | |
| lower_duration_floor() | |
| manager = ComponentsManager() | |
| blocks = MiniMaxH3Ref2VAGeneratorBlocks() | |
| print(f"[{VERSION}] loading {[c.name for c in blocks.expected_components]} from {MODEL_REPO} ...", flush=True) | |
| pipe = blocks.init_pipeline(MODEL_REPO, components_manager=manager, collection="h3") | |
| # The pruned DiT is served as remote code (`transformer_ref/modeling_minimax_h3_pruned.py`). `load_components` | |
| # forwards `trust_remote_code` only to components that live in the pipeline's own repo, so the two VAEs and | |
| # the schedulers โ which `modular_model_index.json` still points at `MiniMaxAI/MiniMax-H3` โ never see it. | |
| pipe.load_components(dtype=torch.bfloat16, trust_remote_code=True) | |
| # Both VAEs first, and explicitly. `set_attention_backend` also sets the registry's *global* backend, which | |
| # every processor that was not stamped fell through to, and the float32 audio VAE has no cuDNN kernel: | |
| # `RuntimeError: No available kernel. Aborting execution.` in its causal encoder attention. | |
| pipe.vae.set_attention_backend("native") | |
| pipe.audio_vae.set_attention_backend("native") | |
| pipe.transformer_ref.set_attention_backend(ATTENTION) | |
| # The LoRA is attached as a runtime PEFT adapter rather than folded into the base weights, so its strength | |
| # stays a per-request knob and 0.0 is the plain base `ref2va` model (the paper's own baseline). Before any | |
| # GPU placement, so the adapter's parameters travel with the transformer. | |
| LORA_STATUS = apply_lora_via_pipeline(pipe) | |
| print(f"[{VERSION}] {LORA_STATUS}", flush=True) | |
| PIPE = pipe | |
| print(f"[{VERSION}] ready in {time.time() - started:.0f}s", flush=True) | |
| except Exception as error: | |
| traceback.print_exc() | |
| LOAD_ERROR = ( | |
| f"**Loading `{MODEL_REPO}` + `{LORA_REPO}` failed** after {time.time() - started:.0f}s: " | |
| f"`{type(error).__name__}: {error}`" | |
| ) | |
| return LOAD_ERROR | |
| # โโ LoRA loading (runtime PEFT adapter) โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ | |
| # | |
| # The adapter is a musubi-tuner kohya file: 200 flattened `lora_unet_blocks_<N>_<module>` names (50 transformer | |
| # blocks x 4 modules) with `lora_down`/`lora_up` pairs and per-module `.alpha` scalars at rank 128. diffusers 0.41's | |
| # own `_convert_non_diffusers_minimax_h3_lora_to_diffusers` understands exactly this layout โ it unflattens the | |
| # names, folds each alpha into the pair, splits the fused QKV into contiguous thirds, swaps the SwiGLU gate/value | |
| # halves and renames onto the diffusers module paths โ so the conversion is delegated to `load_lora_weights` rather | |
| # than reimplemented here. Routing to `transformer_ref` (the partition `ref2va` denoises with) is what | |
| # `load_into_transformer_ref=True` is for. | |
| def apply_lora_via_pipeline(pipe) -> str: | |
| """Attach the AnyGameRenderer LoRA as a PEFT adapter so its strength stays a per-request knob. | |
| `MiniMaxH3LoraLoaderMixin.load_lora_weights` runs `_convert_non_diffusers_minimax_h3_lora_to_diffusers`'s | |
| musubi-tuner branch on the flattened `lora_unet_` names, folds each `.alpha` into the pair, splits the fused | |
| QKV into contiguous thirds, swaps the SwiGLU gate/value halves and renames onto the diffusers module paths โ | |
| the same transforms `infer.py` applies through musubi-tuner's own merge. `load_into_transformer_ref=True` | |
| routes it to the `ref2va` partition, the one the LoRA was trained against. The pruned partition takes it | |
| verbatim: every module the adapter touches is identical between the pruned and the released partition, and the | |
| adapter has no AdaLN branch to reconcile (its target pattern excluded `adaln_proj` at training time). | |
| """ | |
| from huggingface_hub import hf_hub_download | |
| path = hf_hub_download(LORA_REPO, LORA_FILE) | |
| pipe.load_lora_weights(path, adapter_name=ADAPTER, load_into_transformer_ref=True) | |
| pipe.transformer_ref.set_adapters([ADAPTER], [1.0]) | |
| peft_modules = [name for name, module in pipe.transformer_ref.named_modules() if hasattr(module, "lora_A")] | |
| return ( | |
| f"AnyGameRenderer LoRA attached ยท rank 128, alpha 128 ยท 200 musubi targets -> " | |
| f"{len(peft_modules)} PEFT modules ยท default strength 1.0" | |
| ) | |
| def probe_duration(path: str) -> float | None: | |
| """Duration of a media file's video stream, in seconds.""" | |
| import av | |
| with av.open(path) as container: | |
| if not container.streams.video: | |
| return None | |
| stream = container.streams.video[0] | |
| if stream.duration is not None and stream.time_base is not None: | |
| return float(stream.duration * stream.time_base) | |
| return None if container.duration is None else container.duration / av.time_base | |
| def conditioner(): | |
| """The other half, over the gradio API. `gradio_client` attaches the caller's own ZeroGPU token per call, so the | |
| conditioner's booking is billed to whoever asked for the video.""" | |
| from gradio_client import Client | |
| return Client(CONDITIONER_SPACE) | |
| def build_reference(primitive_path: str): | |
| """The primitive as the request's one reference, decoded by diffusers' own reader (frames, fps, soundtrack).""" | |
| from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3VideoReference | |
| return MiniMaxH3VideoReference.from_file(primitive_path) | |
| def check(primitive_path: str | None, caption: str) -> None: | |
| """The request's own rules, before anything is uploaded or a card is allocated.""" | |
| if not caption or not caption.strip(): | |
| raise gr.Error("Add the caption โ AnyGameRenderer always takes one. Start from the caption builder below.") | |
| if not primitive_path: | |
| raise gr.Error("Add the white-model clip to re-render.") | |
| seconds = probe_duration(primitive_path) | |
| if seconds is None: | |
| raise gr.Error("That file has no video stream.") | |
| if not 5 / FPS <= seconds <= MAX_DURATION: | |
| raise gr.Error(f"The clip is {seconds:.2f} s. Use one between 0.21 and {MAX_DURATION:g} seconds.") | |
| def encode_remote(caption: str, primitive_path: str, num_frames: int): | |
| """`/encode_ref2va` on the conditioner Space: a safetensors file holding `prompt_embeds` + `text_token_tags`, | |
| with the resolved `height` / `width` / `num_frames` in its metadata, plus the plan.""" | |
| from gradio_client import handle_file | |
| from safetensors import safe_open | |
| path, plan = conditioner().predict( | |
| prompt=caption, | |
| media=[handle_file(primitive_path)], | |
| kinds="video", | |
| canvas=CANVAS, | |
| num_frames=num_frames, | |
| rewrite_prompt=False, | |
| api_name="/encode_ref2va", | |
| ) | |
| with safe_open(path, framework="pt") as handle: | |
| return handle.get_tensor("prompt_embeds"), handle.get_tensor("text_token_tags"), handle.metadata(), plan | |
| def _generate(prompt_embeds, text_token_tags, primitive_path, num_frames, steps, seed, lora_scale): | |
| """The only thing on GPU time: the reference encoder, the packed-sequence denoise loop and the two decoders. | |
| The primitive crosses as a path and is decoded here; only the three generated outputs come back. A | |
| `@spaces.GPU` argument crosses a process boundary by pickling, and a 5 s 1344x768 clip is 370 MB of expanded | |
| frames once PyAV has opened it. | |
| """ | |
| if PLACEMENT == "lazy": | |
| PIPE.to("cuda") | |
| # Per request, inside the worker: the adapter's scale is a python attribute on its PEFT layers, so setting it on | |
| # this fork's copy leaves every concurrent request with its own strength. | |
| PIPE.transformer_ref.set_adapters([ADAPTER], [float(lora_scale)]) | |
| state = PIPE( | |
| prompt_embeds=prompt_embeds.to("cuda"), | |
| text_token_tags=text_token_tags, | |
| references=[build_reference(primitive_path)], | |
| height=HEIGHT, | |
| width=WIDTH, | |
| num_frames=int(num_frames), | |
| num_inference_steps=int(steps), | |
| generator=torch.Generator("cpu").manual_seed(int(seed)), | |
| ) | |
| return state.get("videos")[0], state.get("audio")[0].cpu(), state.get("sampling_rate") | |
| def generate( | |
| # The first two are the columns `gr.Examples` varies, and they lead the signature for that reason: an example | |
| # row is applied to `inputs` positionally. Every other parameter carries the same default as its UI component, | |
| # so clicking an example and pressing Render behave identically. | |
| primitive: str | None = None, | |
| caption: str = "", | |
| match_length: bool = True, | |
| duration: float = 5.17, | |
| steps: int = DEFAULT_STEPS, | |
| seed: int = DEFAULT_SEED, | |
| lora_scale: float = 1.0, | |
| progress=gr.Progress(track_tqdm=True), | |
| ): | |
| """Re-render a white-model gameplay clip as a polished game clip, entity for entity. | |
| Args: | |
| primitive: path to the blockout clip โ pale-gray geometry, one flat saturated proxy color per entity class, | |
| 24 fps, 2-15 s. 1344x768 is the resolution the LoRA was trained on. | |
| caption: the AnyGameRenderer caption (six sections: subject_definitions, summary, retention_analysis, | |
| detailed_description, overall_soundscape, non_diegetic_music). Build it from the spec below. | |
| match_length: render for exactly as long as the clip runs, snapped to `17n+5` frames. | |
| duration: seconds to render when `match_length` is off, snapped to `17n+5` frames. | |
| steps: denoising steps. The paper's own inference default is 20. | |
| seed: RNG seed. Generation is deterministic for a given seed. | |
| lora_scale: AnyGameRenderer LoRA strength. 1.0 is the released model; 0 is the zero-shot MiniMax-H3 base. | |
| Returns: | |
| Path to the rendered MP4, video and its jointly generated soundtrack. | |
| """ | |
| if LOAD_ERROR: | |
| raise gr.Error(LOAD_ERROR) | |
| if PIPE is None: | |
| raise gr.Error("The denoiser is still loading; try again in a minute.") | |
| from diffusers.utils import encode_video | |
| check(primitive, caption) | |
| clip_seconds = probe_duration(primitive) | |
| requested = clip_seconds if (match_length and clip_seconds is not None) else float(duration) | |
| num_frames = snap_frames(requested) | |
| if num_frames / FPS > MAX_DURATION: | |
| num_frames = snap_frames(MAX_DURATION - 1 / FPS) | |
| progress(0.0, desc="Reading the caption and the blockout ...") | |
| conditioned = time.time() | |
| try: | |
| prompt_embeds, text_token_tags, metadata, plan = encode_remote(caption, primitive, num_frames) | |
| except gr.Error: | |
| raise | |
| except Exception as error: | |
| # gradio only puts the exception *type* on the wire, so the useful half of a conditioner-side failure is in | |
| # that Space's logs. | |
| traceback.print_exc() | |
| raise gr.Error( | |
| f"The conditioner ({CONDITIONER_SPACE}) failed with `{type(error).__name__}: {error}`. " | |
| "Its logs carry the full traceback." | |
| ) from error | |
| condition_seconds = time.time() - conditioned | |
| # The resolved count the conditioner aligned to is the one the denoising half pins. | |
| num_frames = int(metadata["num_frames"]) | |
| progress(0.1, desc=f"Rendering {num_frames / FPS:.1f} s at {WIDTH}x{HEIGHT} ...") | |
| started = time.time() | |
| frames, audio, sampling_rate = _generate( | |
| prompt_embeds, text_token_tags, primitive, num_frames, steps, seed, lora_scale | |
| ) | |
| generate_seconds = time.time() - started | |
| directory = os.path.join(tempfile.gettempdir(), "agr-outputs") | |
| os.makedirs(directory, exist_ok=True) | |
| path = os.path.join(directory, f"anygame-{int(time.time() * 1000)}.mp4") | |
| encode_video(frames, fps=FPS, output_path=path, audio=audio, audio_sample_rate=sampling_rate) | |
| print( | |
| f"[{VERSION}] `{WIDTH}x{HEIGHT}`, {num_frames} frames ({num_frames / FPS:.3f} s), {int(steps)} steps, " | |
| f"strength {float(lora_scale):g} ยท conditioner {condition_seconds:.0f}s " | |
| f"({plan['num_text_tokens']} tokens) ยท denoise + decode {generate_seconds:.0f}s " | |
| f"({generate_seconds / max(1, int(steps)):.1f} s/step) ยท seed {int(seed)}", | |
| flush=True, | |
| ) | |
| return path | |
| def build_caption(spec_json: str) -> str: | |
| """Build the six-section AnyGameRenderer caption from a spec, exactly as `make_caption.py` in the paper's repo | |
| fills the training layout. The spec's fields are documented in the accordion above.""" | |
| import json | |
| SHAPE_WORD = {"box": "box", "wire": "wireframe box", "cylinder": "cylinder", "sphere": "ellipsoid"} | |
| DEFAULT_STYLE = ( | |
| "a polished, photoreal AAA game visual style: physically based materials, crisp micro-detail, soft global " | |
| "illumination, filmic colour grading, subtle depth of field and motion blur" | |
| ) | |
| VID1 = ( | |
| "<Video 1> is the source video for the target video edit. It is an untextured blockout capture of this exact " | |
| "take: pale grey primitive geometry for the whole environment and plain coloured {S}s, one per character or " | |
| "vehicle, sliding along their exact paths, rendered with the exact camera motion of the take." | |
| ) | |
| RET_V = ( | |
| "<Video 1> (source video editing): partially_preserved - its camera trajectory, scene layout, object " | |
| "placement and temporal structure are preserved exactly; every grey blockout surface is re-rendered in place " | |
| "as the corresponding real-world surface with the same silhouette, scale and position, and every coloured {S} " | |
| "reads as its finished subject in every frame, at the same position, size and heading, appearing exactly when " | |
| "its {S} appears and vanishing exactly when its {S} vanishes; areas empty in <Video 1> stay empty, and no " | |
| "character, vehicle or creature is added that has no {S} in <Video 1>." | |
| ) | |
| OCCL = ( | |
| " Every grey blockout volume of <Video 1>, including any that passes close to or in front of the camera, is " | |
| "rendered as solid opaque geometry of the setting at exactly the same place and size, occluding whatever lies " | |
| "behind it exactly as it does in <Video 1>; nothing is seen through blockout geometry, and no surface that is " | |
| "hidden in <Video 1> becomes visible." | |
| ) | |
| CONSISTENCY = " Consistency with <Video 1> takes priority over everything else: geometry, camera and characters follow <Video 1> frame by frame." | |
| try: | |
| spec = json.loads(spec_json) | |
| except json.JSONDecodeError as error: | |
| raise gr.Error(f"The spec is not valid JSON: {error}") | |
| try: | |
| S = SHAPE_WORD[spec["shape"]] | |
| setting = spec["setting"] | |
| subjects = spec["subjects"] | |
| sound = spec["sound"] | |
| except (KeyError, TypeError) as error: | |
| raise gr.Error(f"The spec is missing a required field ({error}); see the template in the accordion.") | |
| style = spec.get("style") or DEFAULT_STYLE | |
| occl = OCCL if spec.get("occlusion", True) else "" | |
| subs, rets, shots = [], [], [] | |
| for n, s in enumerate(subjects, 1): | |
| who, col, verb = f"<Subject {n}>", s["color"], s.get("motion", "moving as recorded") | |
| if s.get("many"): | |
| subs.append( | |
| f"{who} is {s['desc']}; every {col} {S} in <Video 1> is one such subject, and the whole {col} " | |
| "group is a group of identical ones." | |
| ) | |
| rets.append( | |
| f"{who}: partially_preserved - the position, apparent size, count, heading and recorded movement of " | |
| f"every {col} {S} from <Video 1> are preserved exactly, while each flat {col} {S} is replaced by one " | |
| f"subject {verb}." | |
| ) | |
| shots.append( | |
| f"Each {col} {S} becomes one {who}, exactly along its recorded path and timing, no more and no fewer " | |
| "than the {S}s present in each frame." | |
| ) | |
| else: | |
| subs.append(f"{who} is {s['desc']}; represented in <Video 1> by the {col} {S}.") | |
| rets.append( | |
| f"{who}: partially_preserved - its position, apparent size, heading and recorded movement from " | |
| f"<Video 1> are preserved exactly, while the flat {col} {S} is replaced by the subject {verb}." | |
| ) | |
| shots.append(f"The {col} {S} becomes {who}, exactly along its recorded path and timing.") | |
| caption = ( | |
| "subject_definitions:\n" | |
| + "\n".join(subs) | |
| + "\n" | |
| + VID1.format(S=S) | |
| + f"\n\nsummary:\n[video editing] The target video is an edited version of <Video 1>. The flat blockout " | |
| "visuals are replaced with a polished, photoreal AAA game render. The target setting is " | |
| f"{setting}.\n\nretention_analysis:\n" | |
| + "\n".join(rets) | |
| + "\n" | |
| + RET_V.format(S=S) | |
| + occl | |
| + f"\n\ndetailed_description:\nThe target video is in {style}.{CONSISTENCY}{occl}\n" | |
| "[Shot 1] The shot follows <Video 1> as one continuous view; the camera repeats the recorded motion exactly. " | |
| "Every grey blockout block of <Video 1> becomes the matching structure of the setting in the same place, with " | |
| "the same silhouette and scale, and the grey ground becomes the setting's ground. " | |
| + " ".join(shots) | |
| + " Nothing moves that does not move in <Video 1>." | |
| + f"\n\noverall_soundscape:\n{sound}\n\nnon_diegetic_music:\nN/A" | |
| ) | |
| return caption | |
| load_models() | |
| INTRO = """# AnyGameRenderer ยท blockout gameplay โ rendered game clip | |
| <div align="center"> | |
| <a href="https://huggingface.co/AlayaLab/AnyGameRenderer" target="_blank" rel="noopener"><strong>[ LoRA ]</strong></a> | |
| <a href="https://github.com/AlayaLab/AnyGameRenderer" target="_blank" rel="noopener"><strong>[ code ]</strong></a> | |
| <a href="https://alaya-lab.github.io/AnyGameRenderer/" target="_blank" rel="noopener"><strong>[ project page ]</strong></a> | |
| <a href="https://arxiv.org/pdf/2610.31466" target="_blank" rel="noopener"><strong>[ paper ]</strong></a> | |
| </div> | |
| AnyGameRenderer turns the "programmer art" a game agent produces โ pale-gray blockouts with one flat, saturated proxy | |
| color per entity class โ into the same gameplay re-rendered as a polished game clip, frame-aligned with the input, | |
| group counts and appearance timing intact. One checkpoint covers shooters, kart racers, city driving, survivors-likes | |
| and more; the soundtrack is generated with the frames. | |
| Hand it the **blockout clip** (`<Video 1>`) and the **caption** the paper trains on โ six fixed sections built from a | |
| short spec by the caption builder below. | |
| """ | |
| SPEC_TEMPLATE = """{ | |
| "shape": "box", | |
| "setting": "a First World War no man's land: churned mud and craters, sandbag revetments, barbed wire, shattered trees, under a heavy overcast sky", | |
| "subjects": [ | |
| {"color": "blue", "desc": "the player's own weapon seen in first person: a long wooden bolt-action field rifle", "many": false, "motion": "locked to the viewpoint, swaying gently with each step"}, | |
| {"color": "blue", "desc": "an infantryman crossing no man's land in a gray-brown wool uniform and steel helmet", "many": true, "motion": "moving as recorded"} | |
| ], | |
| "sound": "rifle cracks and distant artillery, boots in mud, wind over the craters", | |
| "style": null, | |
| "occlusion": true | |
| }""" | |
| SPEC_HELP = """#### Spec fields | |
| | field | what it is | | |
| |---|---| | |
| | `shape` | proxy shape in the white model: `box` \\| `wire` \\| `cylinder` \\| `sphere` | | |
| | `setting` | one sentence describing the target environment (no trailing period) | | |
| | `subjects` | one entry per proxy color: `color`, `desc`, `many` (every proxy of that color = one instance), `motion` | | |
| | `sound` | the soundscape sentence | | |
| | `style` | optional; default is photoreal AAA. Keep it to lighting / weather / grade, never geometry | | |
| | `occlusion` | optional, default `true`: adds the clause that blockout volumes stay opaque | | |
| Proxy colors: red, orange, yellow, yellow-green, green, cyan, blue, purple, magenta.""" | |
| CSS = """ | |
| .main.fillable { max-width: 1250px !important; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| with gr.Blocks(title="AnyGameRenderer") as demo: | |
| gr.Markdown(INTRO) | |
| with gr.Row(): | |
| with gr.Column(): | |
| primitive_in = gr.Video(label="Blockout clip ยท <Video 1>", height=260) | |
| caption = gr.Textbox( | |
| label="Caption (six sections, from the builder below)", | |
| lines=10, | |
| placeholder="Paste the caption here, or build one from a spec ...", | |
| ) | |
| run = gr.Button("Render the clip", variant="primary", size="lg") | |
| with gr.Accordion("Caption builder (spec โ caption)", open=False): | |
| gr.Markdown(SPEC_HELP) | |
| spec_in = gr.Textbox(label="Spec JSON", lines=14, value=SPEC_TEMPLATE) | |
| build_btn = gr.Button("Build the caption") | |
| build_btn.click(build_caption, spec_in, caption, api_name="build_caption") | |
| with gr.Accordion("Advanced options", open=False): | |
| lora_scale = gr.Slider( | |
| label="AnyGameRenderer LoRA strength (0 = zero-shot MiniMax-H3 base)", | |
| minimum=0.0, | |
| maximum=1.5, | |
| step=0.05, | |
| value=1.0, | |
| ) | |
| match_length = gr.Checkbox(label="Match the blockout's length", value=True) | |
| duration = gr.Slider( | |
| label="Duration (s) โ used when the clip is shorter or match-length is off", | |
| minimum=MIN_UI_DURATION, | |
| maximum=14.0, | |
| step=0.5, | |
| value=5.17, | |
| ) | |
| steps = gr.Slider(label="Steps", minimum=10, maximum=40, step=1, value=DEFAULT_STEPS) | |
| seed = gr.Number(label="Seed", value=DEFAULT_SEED, precision=0) | |
| with gr.Column(): | |
| result = gr.Video(label="Rendered clip + soundtrack") | |
| request = [primitive_in, caption, match_length, duration, steps, seed, lora_scale] | |
| gr.Examples( | |
| # The paper's own bundled examples: the crude recording and the caption its repo ships, one per genre. | |
| examples=[ | |
| ["examples/trenchfire_primitive.mp4", open("examples/trenchfire_caption.txt").read().strip()], | |
| ["examples/eggheist_primitive.mp4", open("examples/eggheist_caption.txt").read().strip()], | |
| ["examples/cityroam_primitive.mp4", open("examples/cityroam_caption.txt").read().strip()], | |
| ], | |
| inputs=[primitive_in, caption], | |
| outputs=result, | |
| fn=generate, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| run.click(generate, request, result, api_name="render") | |
| if __name__ == "__main__": | |
| # Gradio 6 moved `theme` / `css` off the `Blocks` constructor and onto `launch`. | |
| demo.launch(show_error=True, theme=gr.themes.Citrus(), css=CSS, max_threads=1000) | |