File size: 31,867 Bytes
d932406
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
"""AnyGameRenderer โ€” re-render a crude gameplay blockout as a polished game clip, entity for entity.

[`AlayaLab/AnyGameRenderer`](https://huggingface.co/AlayaLab/AnyGameRenderer) is a rank-128 LoRA for MiniMax-H3's
`ref2va` partition. A request is one white-model video โ€” pale-gray geometry with one flat, saturated proxy color per
entity class โ€” plus the paper's fixed six-section caption; the output is the same gameplay re-rendered in the prompted
style, frame-aligned with the input, with a jointly generated soundtrack.

This Space is the denoising half of a split deployment: the 37.5 GiB AdaLN-pruned `transformer_ref` partition and both
autoencoders run here, the 62 GiB Qwen3-VL conditioner runs in
[`multimodalart/qwen3vl-conditioner`](https://huggingface.co/spaces/multimodalart/qwen3vl-conditioner) and is called
over the gradio API per request. `prompt_embeds` + `text_token_tags` is the whole wire format.
"""

from __future__ import annotations

import os
import tempfile
import time
import traceback

# Before anything that could initialize CUDA: `import spaces` patches `torch.cuda` so the model load can happen at
# startup rather than on GPU time.
import spaces
import gradio as gr
import torch

VERSION = "anygame-renderer/1"

# The AdaLN-pruned `ref2va` partition in diffusers layout: 37.5 GiB against the released partition's 61.7 GiB. The
# LoRA is AdaLN-free by construction โ€” its musubi-tuner target pattern is
# `blocks.\d+.(attn.(qkv_proj|out_proj)|mlp.(fc1|fc2))`, which never touches `adaln_proj` โ€” so it applies verbatim to
# the pruned weights. It was trained against Comfy-Org's `minimax_h3_ref2va_pruned_int8_convrot`, i.e. a quantization
# of this same checkpoint family.
MODEL_REPO = os.environ.get("H3_MODEL_REPO", "multimodalart/MiniMax-H3-Pruned")
LORA_REPO = "AlayaLab/AnyGameRenderer"
LORA_FILE = "anygamerenderer_lora.safetensors"
ADAPTER = "anygame"

CONDITIONER_SPACE = os.environ.get("H3_CONDITIONER", "multimodalart/qwen3vl-conditioner")
# `lazy` moves the whole partition onto the card on the first GPU call and leaves it there. Startup placement is not
# an option: `spaces`' startup `torch.pack()` writes every startup-resident CUDA tensor to a second copy on disk, and
# the weights plus their pack do not fit the Space storage quota next to the VAEs.
PLACEMENT = os.environ.get("H3_PLACEMENT", "lazy").lower()
# cuDNN's fused attention, 10-20% faster than the SDPA default on this pool and needs nothing installed.
# flash-attention 3 is sm90-only and this card is sm120 (the `zero-a10g` flavour name is legacy).
ATTENTION = os.environ.get("H3_ATTENTION", "_native_cudnn").lower()
GPU_SIZE = os.environ.get("H3_GPU_SIZE", "xlarge")
MIN_GPU_DURATION = int(os.environ.get("H3_GPU_DURATION_MIN", "120"))
MAX_GPU_DURATION = int(os.environ.get("H3_GPU_DURATION_MAX", "1500"))

FPS, FRAMES_PER_CHUNK, LATENTS_PER_CHUNK = 24, 17, 5
# The LoRA's own training bucket: 1344x768 at 24 fps, 124 frames. Resolution is fixed to what it was trained on.
CANVAS = "1344x768 ยท 16:9 full"
HEIGHT, WIDTH = 768, 1344
# The bundled clips โ€” 5.17 s each โ€” snap down to 124 frames, the LoRA's training length.
DEFAULT_STEPS = 20       # `infer.py`'s own default
DEFAULT_SEED = 42        # `infer.py`'s own default
MAX_DURATION = 15.0      # MiniMax-H3's ceiling
MIN_UI_DURATION = 5.0    # the pipeline's floor; blockouts shorter than 5 s still render 5 s

# Seconds of GPU one request needs, from the packed sequence it is about to denoise: linear in the rows for the
# matmuls, quadratic for the attention. Fitted on the `t2va` half and checked against live `ref2va` requests by the
# character-swap Space this deployment shares its layout with. A primitive clip *is* the reference video, and its
# soundtrack is silent (a blockout has no audio stream), so the sequence is the text tokens plus the reference rows
# plus the generated rows โ€” smaller than the two-reference requests that fit was measured on, which leaves margin.
STEP_LINEAR, STEP_QUADRATIC, SAFETY = 1.1745e-4, 3.8396e-9, 1.15
# The lazy `PIPE.to("cuda")` a cold worker pays inside its first GPU call; every request carries it, because nothing
# here knows whether the worker it lands on is cold. The pruned `transformer_ref` plus the two float32 VAEs is
# ~48 GiB resident, which the same deployment measured at 70 s.
PLACEMENT_ALLOWANCE = int(os.environ.get("H3_PLACEMENT_ALLOWANCE", "70"))
AUDIO_LATENTS_PER_SECOND, AUDIO_CHANNELS = 40, 2
CANVAS_MULTIPLE = 32
DECODE_BASE, DECODE_PER_DEFAULT_CANVAS, DEFAULT_CANVAS_PIXELS = 15, 25, 960 * 544 * 124

PIPE = None
LOAD_ERROR: str | None = None
LORA_STATUS: str = ""


def snap_frames(seconds: float) -> int:
    """The frame count MiniMax-H3's video VAE can decode: the next `17 * n + 5` at 24 fps."""
    frames = max(1, round(float(seconds) * FPS))
    while frames % FRAMES_PER_CHUNK != LATENTS_PER_CHUNK:
        frames += 1
    return frames


def lower_duration_floor(seconds: float = MIN_UI_DURATION) -> None:
    """Let the pipeline generate below its 5 s floor is never needed here, but the property is cut off at the source:
    the bundled clips are 5.17 s and the UI never offers less."""
    from diffusers.modular_pipelines.minimax_h3.modular_pipeline import MiniMaxH3ModularPipeline

    MiniMaxH3ModularPipeline.min_duration = property(lambda self: float(seconds))


def video_latent_frames(num_frames: int) -> int:
    """`17 * n + 5` frames become `5 * n + 2` video latents."""
    return 5 * ((num_frames - LATENTS_PER_CHUNK) // FRAMES_PER_CHUNK) + 2


def target_rows(num_frames: int) -> int:
    """The generated rows of the packed sequence: video patched `(1, 2, 2)`, plus two audio rows per latent."""
    video = video_latent_frames(num_frames) * (HEIGHT // CANVAS_MULTIPLE) * (WIDTH // CANVAS_MULTIPLE)
    return video + round(num_frames / FPS * AUDIO_LATENTS_PER_SECOND) * AUDIO_CHANNELS


def reference_rows(primitive_path: str, num_frames: int) -> int:
    """The rows the primitive's reference block adds, from metadata alone โ€” no decode. A primitive is put on the
    canvas its own aspect ratio resolves to, which at 1344x768 is the training canvas itself."""
    import av

    with av.open(primitive_path) as container:
        stream = container.streams.video[0]
        source_height, source_width = stream.height, stream.width
        rate = float(stream.average_rate) if stream.average_rate else FPS
        duration = float(stream.duration * stream.time_base) if stream.duration is not None else 0.0

    ratio = source_width / source_height
    short = 768
    w, h = (short * ratio, short) if ratio >= 1.0 else (short, short / ratio)
    if w * h > 768 * 1344:
        scale = (768 * 1344 / (w * h)) ** 0.5
        w, h = w * scale, h * scale
    canvas_width = max(CANVAS_MULTIPLE, round(w / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)
    canvas_height = max(CANVAS_MULTIPLE, round(h / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)

    frames = min(round(duration * rate), num_frames)
    snapped = max(1, (frames - LATENTS_PER_CHUNK) // FRAMES_PER_CHUNK) * FRAMES_PER_CHUNK + LATENTS_PER_CHUNK
    return video_latent_frames(snapped) * (canvas_height // CANVAS_MULTIPLE) * (canvas_width // CANVAS_MULTIPLE)


def get_duration(prompt_embeds, text_token_tags, primitive, num_frames, steps, *_, **__):
    """Seconds of GPU to reserve for one request. Takes the arguments of the `@spaces.GPU` function it decorates and
    tolerates whatever else `spaces` injects."""
    sequence = int(text_token_tags.shape[0]) + reference_rows(primitive, num_frames) + target_rows(num_frames)
    denoise = int(steps) * (STEP_LINEAR * sequence + STEP_QUADRATIC * sequence**2) * SAFETY
    encode = 5 + reference_rows(primitive, num_frames) * 1e-3
    decode = DECODE_BASE + DECODE_PER_DEFAULT_CANVAS * (HEIGHT * WIDTH * num_frames) / DEFAULT_CANVAS_PIXELS
    total = PLACEMENT_ALLOWANCE + encode + denoise + decode + 10
    duration = max(MIN_GPU_DURATION, min(MAX_GPU_DURATION, int(total)))
    print(f"[{VERSION}] S={sequence} -> reserving {duration}s ({denoise:.0f}s of denoise at {int(steps)} steps)", flush=True)
    return duration


def load_models() -> str | None:
    """Load the denoising half at startup, but *not* onto the card.

    `MiniMaxH3Ref2VAGeneratorBlocks` declares `transformer_ref`, `vae`, `audio_vae`, the two schedulers and
    `video_processor`, so `load_components` fetches exactly those subfolders โ€” `text_encoder/` and the `transformer/`
    partition are never touched. Both autoencoders carry `_keep_in_fp32_modules` and stay float32: a bfloat16 audio
    VAE decodes the soundtrack roughly 20 dB too quiet.
    """
    global PIPE, LOAD_ERROR, LORA_STATUS

    if PIPE is not None or LOAD_ERROR is not None:
        return LOAD_ERROR

    started = time.time()
    try:
        from diffusers import ComponentsManager

        from h3_split_blocks import MiniMaxH3Ref2VAGeneratorBlocks

        lower_duration_floor()
        manager = ComponentsManager()
        blocks = MiniMaxH3Ref2VAGeneratorBlocks()
        print(f"[{VERSION}] loading {[c.name for c in blocks.expected_components]} from {MODEL_REPO} ...", flush=True)
        pipe = blocks.init_pipeline(MODEL_REPO, components_manager=manager, collection="h3")
        # The pruned DiT is served as remote code (`transformer_ref/modeling_minimax_h3_pruned.py`). `load_components`
        # forwards `trust_remote_code` only to components that live in the pipeline's own repo, so the two VAEs and
        # the schedulers โ€” which `modular_model_index.json` still points at `MiniMaxAI/MiniMax-H3` โ€” never see it.
        pipe.load_components(dtype=torch.bfloat16, trust_remote_code=True)

        # Both VAEs first, and explicitly. `set_attention_backend` also sets the registry's *global* backend, which
        # every processor that was not stamped fell through to, and the float32 audio VAE has no cuDNN kernel:
        # `RuntimeError: No available kernel. Aborting execution.` in its causal encoder attention.
        pipe.vae.set_attention_backend("native")
        pipe.audio_vae.set_attention_backend("native")
        pipe.transformer_ref.set_attention_backend(ATTENTION)

        # The LoRA is attached as a runtime PEFT adapter rather than folded into the base weights, so its strength
        # stays a per-request knob and 0.0 is the plain base `ref2va` model (the paper's own baseline). Before any
        # GPU placement, so the adapter's parameters travel with the transformer.
        LORA_STATUS = apply_lora_via_pipeline(pipe)
        print(f"[{VERSION}] {LORA_STATUS}", flush=True)

        PIPE = pipe
        print(f"[{VERSION}] ready in {time.time() - started:.0f}s", flush=True)
    except Exception as error:
        traceback.print_exc()
        LOAD_ERROR = (
            f"**Loading `{MODEL_REPO}` + `{LORA_REPO}` failed** after {time.time() - started:.0f}s: "
            f"`{type(error).__name__}: {error}`"
        )
    return LOAD_ERROR


# โ”€โ”€ LoRA loading (runtime PEFT adapter) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
#
# The adapter is a musubi-tuner kohya file: 200 flattened `lora_unet_blocks_<N>_<module>` names (50 transformer
# blocks x 4 modules) with `lora_down`/`lora_up` pairs and per-module `.alpha` scalars at rank 128. diffusers 0.41's
# own `_convert_non_diffusers_minimax_h3_lora_to_diffusers` understands exactly this layout โ€” it unflattens the
# names, folds each alpha into the pair, splits the fused QKV into contiguous thirds, swaps the SwiGLU gate/value
# halves and renames onto the diffusers module paths โ€” so the conversion is delegated to `load_lora_weights` rather
# than reimplemented here. Routing to `transformer_ref` (the partition `ref2va` denoises with) is what
# `load_into_transformer_ref=True` is for.


def apply_lora_via_pipeline(pipe) -> str:
    """Attach the AnyGameRenderer LoRA as a PEFT adapter so its strength stays a per-request knob.

    `MiniMaxH3LoraLoaderMixin.load_lora_weights` runs `_convert_non_diffusers_minimax_h3_lora_to_diffusers`'s
    musubi-tuner branch on the flattened `lora_unet_` names, folds each `.alpha` into the pair, splits the fused
    QKV into contiguous thirds, swaps the SwiGLU gate/value halves and renames onto the diffusers module paths โ€”
    the same transforms `infer.py` applies through musubi-tuner's own merge. `load_into_transformer_ref=True`
    routes it to the `ref2va` partition, the one the LoRA was trained against. The pruned partition takes it
    verbatim: every module the adapter touches is identical between the pruned and the released partition, and the
    adapter has no AdaLN branch to reconcile (its target pattern excluded `adaln_proj` at training time).
    """
    from huggingface_hub import hf_hub_download

    path = hf_hub_download(LORA_REPO, LORA_FILE)
    pipe.load_lora_weights(path, adapter_name=ADAPTER, load_into_transformer_ref=True)
    pipe.transformer_ref.set_adapters([ADAPTER], [1.0])
    peft_modules = [name for name, module in pipe.transformer_ref.named_modules() if hasattr(module, "lora_A")]
    return (
        f"AnyGameRenderer LoRA attached ยท rank 128, alpha 128 ยท 200 musubi targets -> "
        f"{len(peft_modules)} PEFT modules ยท default strength 1.0"
    )


def probe_duration(path: str) -> float | None:
    """Duration of a media file's video stream, in seconds."""
    import av

    with av.open(path) as container:
        if not container.streams.video:
            return None
        stream = container.streams.video[0]
        if stream.duration is not None and stream.time_base is not None:
            return float(stream.duration * stream.time_base)
        return None if container.duration is None else container.duration / av.time_base


def conditioner():
    """The other half, over the gradio API. `gradio_client` attaches the caller's own ZeroGPU token per call, so the
    conditioner's booking is billed to whoever asked for the video."""
    from gradio_client import Client

    return Client(CONDITIONER_SPACE)


def build_reference(primitive_path: str):
    """The primitive as the request's one reference, decoded by diffusers' own reader (frames, fps, soundtrack)."""
    from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3VideoReference

    return MiniMaxH3VideoReference.from_file(primitive_path)


def check(primitive_path: str | None, caption: str) -> None:
    """The request's own rules, before anything is uploaded or a card is allocated."""
    if not caption or not caption.strip():
        raise gr.Error("Add the caption โ€” AnyGameRenderer always takes one. Start from the caption builder below.")
    if not primitive_path:
        raise gr.Error("Add the white-model clip to re-render.")
    seconds = probe_duration(primitive_path)
    if seconds is None:
        raise gr.Error("That file has no video stream.")
    if not 5 / FPS <= seconds <= MAX_DURATION:
        raise gr.Error(f"The clip is {seconds:.2f} s. Use one between 0.21 and {MAX_DURATION:g} seconds.")


def encode_remote(caption: str, primitive_path: str, num_frames: int):
    """`/encode_ref2va` on the conditioner Space: a safetensors file holding `prompt_embeds` + `text_token_tags`,
    with the resolved `height` / `width` / `num_frames` in its metadata, plus the plan."""
    from gradio_client import handle_file
    from safetensors import safe_open

    path, plan = conditioner().predict(
        prompt=caption,
        media=[handle_file(primitive_path)],
        kinds="video",
        canvas=CANVAS,
        num_frames=num_frames,
        rewrite_prompt=False,
        api_name="/encode_ref2va",
    )
    with safe_open(path, framework="pt") as handle:
        return handle.get_tensor("prompt_embeds"), handle.get_tensor("text_token_tags"), handle.metadata(), plan


@spaces.GPU(duration=get_duration, size=GPU_SIZE)
def _generate(prompt_embeds, text_token_tags, primitive_path, num_frames, steps, seed, lora_scale):
    """The only thing on GPU time: the reference encoder, the packed-sequence denoise loop and the two decoders.

    The primitive crosses as a path and is decoded here; only the three generated outputs come back. A
    `@spaces.GPU` argument crosses a process boundary by pickling, and a 5 s 1344x768 clip is 370 MB of expanded
    frames once PyAV has opened it.
    """
    if PLACEMENT == "lazy":
        PIPE.to("cuda")

    # Per request, inside the worker: the adapter's scale is a python attribute on its PEFT layers, so setting it on
    # this fork's copy leaves every concurrent request with its own strength.
    PIPE.transformer_ref.set_adapters([ADAPTER], [float(lora_scale)])

    state = PIPE(
        prompt_embeds=prompt_embeds.to("cuda"),
        text_token_tags=text_token_tags,
        references=[build_reference(primitive_path)],
        height=HEIGHT,
        width=WIDTH,
        num_frames=int(num_frames),
        num_inference_steps=int(steps),
        generator=torch.Generator("cpu").manual_seed(int(seed)),
    )
    return state.get("videos")[0], state.get("audio")[0].cpu(), state.get("sampling_rate")


def generate(
    # The first two are the columns `gr.Examples` varies, and they lead the signature for that reason: an example
    # row is applied to `inputs` positionally. Every other parameter carries the same default as its UI component,
    # so clicking an example and pressing Render behave identically.
    primitive: str | None = None,
    caption: str = "",
    match_length: bool = True,
    duration: float = 5.17,
    steps: int = DEFAULT_STEPS,
    seed: int = DEFAULT_SEED,
    lora_scale: float = 1.0,
    progress=gr.Progress(track_tqdm=True),
):
    """Re-render a white-model gameplay clip as a polished game clip, entity for entity.

    Args:
        primitive: path to the blockout clip โ€” pale-gray geometry, one flat saturated proxy color per entity class,
            24 fps, 2-15 s. 1344x768 is the resolution the LoRA was trained on.
        caption: the AnyGameRenderer caption (six sections: subject_definitions, summary, retention_analysis,
            detailed_description, overall_soundscape, non_diegetic_music). Build it from the spec below.
        match_length: render for exactly as long as the clip runs, snapped to `17n+5` frames.
        duration: seconds to render when `match_length` is off, snapped to `17n+5` frames.
        steps: denoising steps. The paper's own inference default is 20.
        seed: RNG seed. Generation is deterministic for a given seed.
        lora_scale: AnyGameRenderer LoRA strength. 1.0 is the released model; 0 is the zero-shot MiniMax-H3 base.

    Returns:
        Path to the rendered MP4, video and its jointly generated soundtrack.
    """
    if LOAD_ERROR:
        raise gr.Error(LOAD_ERROR)
    if PIPE is None:
        raise gr.Error("The denoiser is still loading; try again in a minute.")

    from diffusers.utils import encode_video

    check(primitive, caption)

    clip_seconds = probe_duration(primitive)
    requested = clip_seconds if (match_length and clip_seconds is not None) else float(duration)
    num_frames = snap_frames(requested)
    if num_frames / FPS > MAX_DURATION:
        num_frames = snap_frames(MAX_DURATION - 1 / FPS)

    progress(0.0, desc="Reading the caption and the blockout ...")
    conditioned = time.time()
    try:
        prompt_embeds, text_token_tags, metadata, plan = encode_remote(caption, primitive, num_frames)
    except gr.Error:
        raise
    except Exception as error:
        # gradio only puts the exception *type* on the wire, so the useful half of a conditioner-side failure is in
        # that Space's logs.
        traceback.print_exc()
        raise gr.Error(
            f"The conditioner ({CONDITIONER_SPACE}) failed with `{type(error).__name__}: {error}`. "
            "Its logs carry the full traceback."
        ) from error
    condition_seconds = time.time() - conditioned
    # The resolved count the conditioner aligned to is the one the denoising half pins.
    num_frames = int(metadata["num_frames"])

    progress(0.1, desc=f"Rendering {num_frames / FPS:.1f} s at {WIDTH}x{HEIGHT} ...")
    started = time.time()
    frames, audio, sampling_rate = _generate(
        prompt_embeds, text_token_tags, primitive, num_frames, steps, seed, lora_scale
    )
    generate_seconds = time.time() - started

    directory = os.path.join(tempfile.gettempdir(), "agr-outputs")
    os.makedirs(directory, exist_ok=True)
    path = os.path.join(directory, f"anygame-{int(time.time() * 1000)}.mp4")
    encode_video(frames, fps=FPS, output_path=path, audio=audio, audio_sample_rate=sampling_rate)

    print(
        f"[{VERSION}] `{WIDTH}x{HEIGHT}`, {num_frames} frames ({num_frames / FPS:.3f} s), {int(steps)} steps, "
        f"strength {float(lora_scale):g} ยท conditioner {condition_seconds:.0f}s "
        f"({plan['num_text_tokens']} tokens) ยท denoise + decode {generate_seconds:.0f}s "
        f"({generate_seconds / max(1, int(steps)):.1f} s/step) ยท seed {int(seed)}",
        flush=True,
    )
    return path


def build_caption(spec_json: str) -> str:
    """Build the six-section AnyGameRenderer caption from a spec, exactly as `make_caption.py` in the paper's repo
    fills the training layout. The spec's fields are documented in the accordion above."""
    import json

    SHAPE_WORD = {"box": "box", "wire": "wireframe box", "cylinder": "cylinder", "sphere": "ellipsoid"}
    DEFAULT_STYLE = (
        "a polished, photoreal AAA game visual style: physically based materials, crisp micro-detail, soft global "
        "illumination, filmic colour grading, subtle depth of field and motion blur"
    )
    VID1 = (
        "<Video 1> is the source video for the target video edit. It is an untextured blockout capture of this exact "
        "take: pale grey primitive geometry for the whole environment and plain coloured {S}s, one per character or "
        "vehicle, sliding along their exact paths, rendered with the exact camera motion of the take."
    )
    RET_V = (
        "<Video 1> (source video editing): partially_preserved - its camera trajectory, scene layout, object "
        "placement and temporal structure are preserved exactly; every grey blockout surface is re-rendered in place "
        "as the corresponding real-world surface with the same silhouette, scale and position, and every coloured {S} "
        "reads as its finished subject in every frame, at the same position, size and heading, appearing exactly when "
        "its {S} appears and vanishing exactly when its {S} vanishes; areas empty in <Video 1> stay empty, and no "
        "character, vehicle or creature is added that has no {S} in <Video 1>."
    )
    OCCL = (
        " Every grey blockout volume of <Video 1>, including any that passes close to or in front of the camera, is "
        "rendered as solid opaque geometry of the setting at exactly the same place and size, occluding whatever lies "
        "behind it exactly as it does in <Video 1>; nothing is seen through blockout geometry, and no surface that is "
        "hidden in <Video 1> becomes visible."
    )
    CONSISTENCY = " Consistency with <Video 1> takes priority over everything else: geometry, camera and characters follow <Video 1> frame by frame."

    try:
        spec = json.loads(spec_json)
    except json.JSONDecodeError as error:
        raise gr.Error(f"The spec is not valid JSON: {error}")
    try:
        S = SHAPE_WORD[spec["shape"]]
        setting = spec["setting"]
        subjects = spec["subjects"]
        sound = spec["sound"]
    except (KeyError, TypeError) as error:
        raise gr.Error(f"The spec is missing a required field ({error}); see the template in the accordion.")

    style = spec.get("style") or DEFAULT_STYLE
    occl = OCCL if spec.get("occlusion", True) else ""
    subs, rets, shots = [], [], []
    for n, s in enumerate(subjects, 1):
        who, col, verb = f"<Subject {n}>", s["color"], s.get("motion", "moving as recorded")
        if s.get("many"):
            subs.append(
                f"{who} is {s['desc']}; every {col} {S} in <Video 1> is one such subject, and the whole {col} "
                "group is a group of identical ones."
            )
            rets.append(
                f"{who}: partially_preserved - the position, apparent size, count, heading and recorded movement of "
                f"every {col} {S} from <Video 1> are preserved exactly, while each flat {col} {S} is replaced by one "
                f"subject {verb}."
            )
            shots.append(
                f"Each {col} {S} becomes one {who}, exactly along its recorded path and timing, no more and no fewer "
                "than the {S}s present in each frame."
            )
        else:
            subs.append(f"{who} is {s['desc']}; represented in <Video 1> by the {col} {S}.")
            rets.append(
                f"{who}: partially_preserved - its position, apparent size, heading and recorded movement from "
                f"<Video 1> are preserved exactly, while the flat {col} {S} is replaced by the subject {verb}."
            )
            shots.append(f"The {col} {S} becomes {who}, exactly along its recorded path and timing.")

    caption = (
        "subject_definitions:\n"
        + "\n".join(subs)
        + "\n"
        + VID1.format(S=S)
        + f"\n\nsummary:\n[video editing] The target video is an edited version of <Video 1>. The flat blockout "
        "visuals are replaced with a polished, photoreal AAA game render. The target setting is "
        f"{setting}.\n\nretention_analysis:\n"
        + "\n".join(rets)
        + "\n"
        + RET_V.format(S=S)
        + occl
        + f"\n\ndetailed_description:\nThe target video is in {style}.{CONSISTENCY}{occl}\n"
        "[Shot 1] The shot follows <Video 1> as one continuous view; the camera repeats the recorded motion exactly. "
        "Every grey blockout block of <Video 1> becomes the matching structure of the setting in the same place, with "
        "the same silhouette and scale, and the grey ground becomes the setting's ground. "
        + " ".join(shots)
        + " Nothing moves that does not move in <Video 1>."
        + f"\n\noverall_soundscape:\n{sound}\n\nnon_diegetic_music:\nN/A"
    )
    return caption


load_models()

INTRO = """# AnyGameRenderer ยท blockout gameplay โ†’ rendered game clip

<div align="center">
  <a href="https://huggingface.co/AlayaLab/AnyGameRenderer" target="_blank" rel="noopener"><strong>[ LoRA ]</strong></a> &nbsp;
  <a href="https://github.com/AlayaLab/AnyGameRenderer" target="_blank" rel="noopener"><strong>[ code ]</strong></a> &nbsp;
  <a href="https://alaya-lab.github.io/AnyGameRenderer/" target="_blank" rel="noopener"><strong>[ project page ]</strong></a> &nbsp;
  <a href="https://arxiv.org/pdf/2610.31466" target="_blank" rel="noopener"><strong>[ paper ]</strong></a>
</div>

AnyGameRenderer turns the "programmer art" a game agent produces โ€” pale-gray blockouts with one flat, saturated proxy
color per entity class โ€” into the same gameplay re-rendered as a polished game clip, frame-aligned with the input,
group counts and appearance timing intact. One checkpoint covers shooters, kart racers, city driving, survivors-likes
and more; the soundtrack is generated with the frames.

Hand it the **blockout clip** (`<Video 1>`) and the **caption** the paper trains on โ€” six fixed sections built from a
short spec by the caption builder below.
"""

SPEC_TEMPLATE = """{
  "shape": "box",
  "setting": "a First World War no man's land: churned mud and craters, sandbag revetments, barbed wire, shattered trees, under a heavy overcast sky",
  "subjects": [
    {"color": "blue", "desc": "the player's own weapon seen in first person: a long wooden bolt-action field rifle", "many": false, "motion": "locked to the viewpoint, swaying gently with each step"},
    {"color": "blue", "desc": "an infantryman crossing no man's land in a gray-brown wool uniform and steel helmet", "many": true, "motion": "moving as recorded"}
  ],
  "sound": "rifle cracks and distant artillery, boots in mud, wind over the craters",
  "style": null,
  "occlusion": true
}"""

SPEC_HELP = """#### Spec fields

| field | what it is |
|---|---|
| `shape` | proxy shape in the white model: `box` \\| `wire` \\| `cylinder` \\| `sphere` |
| `setting` | one sentence describing the target environment (no trailing period) |
| `subjects` | one entry per proxy color: `color`, `desc`, `many` (every proxy of that color = one instance), `motion` |
| `sound` | the soundscape sentence |
| `style` | optional; default is photoreal AAA. Keep it to lighting / weather / grade, never geometry |
| `occlusion` | optional, default `true`: adds the clause that blockout volumes stay opaque |

Proxy colors: red, orange, yellow, yellow-green, green, cyan, blue, purple, magenta."""

CSS = """
.main.fillable { max-width: 1250px !important; }
.dark .gradio-container { color: var(--body-text-color); }
"""

with gr.Blocks(title="AnyGameRenderer") as demo:
    gr.Markdown(INTRO)

    with gr.Row():
        with gr.Column():
            primitive_in = gr.Video(label="Blockout clip ยท <Video 1>", height=260)
            caption = gr.Textbox(
                label="Caption (six sections, from the builder below)",
                lines=10,
                placeholder="Paste the caption here, or build one from a spec ...",
            )
            run = gr.Button("Render the clip", variant="primary", size="lg")
            with gr.Accordion("Caption builder (spec โ†’ caption)", open=False):
                gr.Markdown(SPEC_HELP)
                spec_in = gr.Textbox(label="Spec JSON", lines=14, value=SPEC_TEMPLATE)
                build_btn = gr.Button("Build the caption")
                build_btn.click(build_caption, spec_in, caption, api_name="build_caption")
            with gr.Accordion("Advanced options", open=False):
                lora_scale = gr.Slider(
                    label="AnyGameRenderer LoRA strength (0 = zero-shot MiniMax-H3 base)",
                    minimum=0.0,
                    maximum=1.5,
                    step=0.05,
                    value=1.0,
                )
                match_length = gr.Checkbox(label="Match the blockout's length", value=True)
                duration = gr.Slider(
                    label="Duration (s) โ€” used when the clip is shorter or match-length is off",
                    minimum=MIN_UI_DURATION,
                    maximum=14.0,
                    step=0.5,
                    value=5.17,
                )
                steps = gr.Slider(label="Steps", minimum=10, maximum=40, step=1, value=DEFAULT_STEPS)
                seed = gr.Number(label="Seed", value=DEFAULT_SEED, precision=0)

        with gr.Column():
            result = gr.Video(label="Rendered clip + soundtrack")

    request = [primitive_in, caption, match_length, duration, steps, seed, lora_scale]

    gr.Examples(
        # The paper's own bundled examples: the crude recording and the caption its repo ships, one per genre.
        examples=[
            ["examples/trenchfire_primitive.mp4", open("examples/trenchfire_caption.txt").read().strip()],
            ["examples/eggheist_primitive.mp4", open("examples/eggheist_caption.txt").read().strip()],
            ["examples/cityroam_primitive.mp4", open("examples/cityroam_caption.txt").read().strip()],
        ],
        inputs=[primitive_in, caption],
        outputs=result,
        fn=generate,
        cache_examples=True,
        cache_mode="lazy",
    )

    run.click(generate, request, result, api_name="render")


if __name__ == "__main__":
    # Gradio 6 moved `theme` / `css` off the `Blocks` constructor and onto `launch`.
    demo.launch(show_error=True, theme=gr.themes.Citrus(), css=CSS, max_threads=1000)