Upload MiniMax-H3 acceleration LoRAs and examples

Upload the MiniMax-H3 FL2VA and Ref2VA acceleration LoRAs, inference examples, documentation, assets, and generated comparison videos. Excludes __pycache__ and samples.

.gitattributes

@@ -33,3 +33,24 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+asset/ref2va_test_5_2.jpg filter=lfs diff=lfs merge=lfs -text
+asset/ref2va_test_5_3.jpg filter=lfs diff=lfs merge=lfs -text
+asset/ref2va_test_5_4.jpg filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_fl2va_acc_8step_v1_1.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_fl2va_acc_8step_v1_2.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_fl2va_acc_8step_v1_3.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_fl2va_baseline_1.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_fl2va_baseline_2.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_fl2va_baseline_3.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_fl2va_turbo_4step_v1.1_768p_1.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_fl2va_turbo_4step_v1.1_768p_2.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_fl2va_turbo_4step_v1.1_768p_3.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_ref2va_acc_8step_v1_1.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_ref2va_acc_8step_v1_2.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_ref2va_acc_8step_v1_3.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_ref2va_baseline_1.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_ref2va_baseline_2.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_ref2va_baseline_3.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_ref2va_turbo_4step_v0.1_768p_1.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_ref2va_turbo_4step_v0.1_768p_2.mp4 filter=lfs diff=lfs merge=lfs -text
+results/minimax_h3_ref2va_turbo_4step_v0.1_768p_3.mp4 filter=lfs diff=lfs merge=lfs -text

MiniMax-H3-FL2VA-Acc-8Step.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b29be7042d883970eb0c20774a9ba03d95669ed80a721bb4d21be8ea0d0a196
+size 1372450680

MiniMax-H3-Ref2VA-Acc-8Step.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:111c82e669f6e20e628228172edf39395f1a9fc3ad049793895e542c0f55b18c
+size 1372450680

README.md

@@ -1,3 +1,126 @@
 ---
 license: apache-2.0
+base_model:
+- alibaba-pai/MiniMax-H3-Acc-LoRAs
+- MiniMaxAI/MiniMax-H3
+library_name: videox_fun
+pipeline_tag: text-to-video
 ---
+# MiniMax-H3-Acc-LoRAs
+## Introduction
+We apply Parallel Decoding Distillation (PDD) <sup>[1](#ref1)</sup> to [MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3), enabling efficient video generation in only a few inference steps.
+
+For more details, please refer to our [GitHub repo](https://github.com/aigc-apps/VideoX-Fun).
+
+| Name | Base Model | Hugging Face | Description |
+|--|--|--|--|
+| MiniMax-H3-FL2VA-Acc-8Step.safetensors | [MiniMax-H3 (FL2VA)](https://huggingface.co/MiniMaxAI/MiniMax-H3/tree/main/FL2VA) | [🤗Link](https://huggingface.co/alibaba-pai/MiniMax-H3-Acc-LoRAs/resolve/main/MiniMax-H3-FL2VA-Acc-8Step.safetensors) | Official 8 Step Acc LoRA (`rank=64` and `network_alpha=64` in BF16) for MiniMax-H3 (FL2VA).|
+| MiniMax-H3-Ref2VA-Acc-8Step.safetensors | [MiniMax-H3 (Ref2VA)](https://huggingface.co/MiniMaxAI/MiniMax-H3/tree/main/Ref2VA) | [🤗Link](https://huggingface.co/alibaba-pai/MiniMax-H3-Acc-LoRAs/resolve/main/MiniMax-H3-Ref2VA-Acc-8Step.safetensors) | Official 8 Step Acc LoRA (`rank=64` and `network_alpha=64` in BF16) for MiniMax-H3 (Ref2VA).|
+
+## Demo
+
+### FL2VA (768p)
+
+<table border="0" style="width: 100%; text-align: center; margin-top: 20px;">
+    <thead>
+        <tr>
+            <th style="text-align: center;" width="33%">MiniMax-H3-FL2VA</th>
+            <th style="text-align: center;" width="33%"><a href="https://huggingface.co/lightx2v/Minimax-h3-Turbo/blob/main/minimax_h3_fl2v_turbo_4step_v1.1_768p_bf16.safetensors">Minimax-h3-Turbo <br> (fl2v_turbo_4step_v1.1_768p)</a></th>
+            <th style="text-align: center;" width="33%">MiniMax-H3-FL2VA-Acc-8Step </th>
+        </tr>
+    </thead>
+    <tr>
+        <td>
+            <video src="results/minimax_h3_fl2va_baseline_1.mp4" width="100%" controls autoplay loop></video>
+        </td>
+        <td>
+            <video src="results/minimax_h3_fl2va_turbo_4step_v1.1_768p_1.mp4" width="100%" controls autoplay loop></video>
+        </td>
+        <td>
+            <video src="results/minimax_h3_fl2va_acc_8step_v1_1.mp4" width="100%" controls autoplay loop></video>
+        </td>
+    </tr>
+    <tr>
+        <td>
+            <video src="results/minimax_h3_fl2va_baseline_2.mp4" width="100%" controls autoplay loop></video>
+        </td>
+        <td>
+            <video src="results/minimax_h3_fl2va_turbo_4step_v1.1_768p_2.mp4" width="100%" controls autoplay loop></video>
+        </td>
+        <td>
+            <video src="results/minimax_h3_fl2va_acc_8step_v1_2.mp4" width="100%" controls autoplay loop></video>
+        </td>
+    </tr>
+    <tr>
+        <td>
+            <video src="results/minimax_h3_fl2va_baseline_3.mp4" width="100%" controls autoplay loop></video>
+        </td>
+        <td>
+            <video src="results/minimax_h3_fl2va_turbo_4step_v1.1_768p_3.mp4" width="100%" controls autoplay loop></video>
+        </td>
+        <td>
+            <video src="results/minimax_h3_fl2va_acc_8step_v1_3.mp4" width="100%" controls autoplay loop></video>
+        </td>
+    </tr>
+</table>
+
+### Ref2VA
+
+<table border="0" style="width: 100%; text-align: center; margin-top: 20px;">
+    <thead>
+        <tr>
+            <th style="text-align: center;" width="33%">MiniMax-H3-Ref2VA</th>
+            <th style="text-align: center;" width="33%"><a href="https://huggingface.co/lightx2v/Minimax-h3-Turbo/blob/main/minimax_h3_ref2v_turbo_4step_v0.1_bf16.safetensors">Minimax-h3-Turbo <br> (ref2v_turbo_4step_v0.1)</a></th>
+            <th style="text-align: center;" width="33%">MiniMax-H3-Ref2VA-Acc-8Step</th>
+        </tr>
+    </thead>
+    <tr>
+        <td>
+            <video src="results/minimax_h3_ref2va_baseline_1.mp4" width="100%" controls autoplay loop></video>
+        </td>
+        <td>
+            <video src="results/minimax_h3_ref2va_turbo_4step_v0.1_768p_1.mp4" width="100%" controls autoplay loop></video>
+        </td>
+        <td>
+            <video src="results/minimax_h3_ref2va_acc_8step_v1_1.mp4" width="100%" controls autoplay loop></video>
+        </td>
+    </tr>
+    <tr>
+        <td>
+            <video src="results/minimax_h3_ref2va_baseline_2.mp4" width="100%" controls autoplay loop></video>
+        </td>
+        <td>
+            <video src="results/minimax_h3_ref2va_turbo_4step_v0.1_2.mp4" width="100%" controls autoplay loop></video>
+        </td>
+        <td>
+            <video src="results/minimax_h3_ref2va_acc_8step_v1_2.mp4" width="100%" controls autoplay loop></video>
+        </td>
+    </tr>
+    <tr>
+        <td>
+            <video src="results/minimax_h3_ref2va_baseline_3.mp4" width="100%" controls autoplay loop></video>
+        </td>
+        <td>
+            <video src="results/minimax_h3_ref2v_turbo_4step_v0.1_768p_3.mp4" width="100%" controls autoplay loop></video>
+        </td>
+        <td>
+            <video src="asset/minimax_h3_ref2va_acc_8step_v1_3.mp4" width="100%" controls autoplay loop></video>
+        </td>
+    </tr>
+</table>
+
+
+> [!NOTE]
+> The above test cases are from <a href="https://github.com/ModelTC/Minimax-H3-Turbo/tree/main/examples">Minimax-H3-Turbo</a>.
+> Videos are generated with a LoRA weight of 1.0 at both 4 and 8 NFE.
+
+## Quick Start
+Set `model_path` and `pdd_lora_path` to the MiniMax-H3 model and the matching acceleration LoRA checkpoint in [predict_t2v.py](https://github.com/aigc-apps/VideoX-Fun/blob/main/examples/MiniMax-H3-Acc-LoRAs/predict_t2v.py) for FL2VA or [predict_ref2v.py](https://github.com/aigc-apps/VideoX-Fun/blob/main/examples/MiniMax-H3-Acc-LoRAs/predict_ref2v.py) for Ref2VA, then run the corresponding script. Each example uses `apply_pdd_lora` to load the checkpoint and derive the required number of inference steps from its configuration.
+
+> [!NOTE]
+> These scripts use Diffusers' MiniMax-H3 `ModularPipeline` and require `diffusers >= 0.40.0`.
+
+## Reference
+<ol>
+  <li id="ref1">Neta Shaul, et al. "Parallel Decoding Distillation for Fast Image and Video Generation.". arXiv preprint arXiv:2607.26004 (2026).</li>
+</ol>

minimax_h3_pdd.py

@@ -0,0 +1,290 @@
+"""PDD LoRA adapter for MiniMax-H3 on Diffusers 0.40.
+
+Parallel Decoding Distillation (PDD, arXiv 2607.26004) is not a PEFT LoRA: the
+backbone gets low-rank updates, and the two final heads (`proj_out`,
+`audio_proj_out`) are repeated once per interval of a length-`N` grid. Each
+generation step fuses a block of those heads into one Euler velocity, so
+`NFE = N / L` transformer evaluations cover the whole trajectory.
+
+This module is inference-only. It mutates a loaded
+`MiniMaxH3Transformer3DModel` in place and arms the fused heads on every
+forward, so the stock Diffusers modular pipeline does not need a step callback.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Sequence
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+DEFAULT_PDD_CONFIG = {
+    "pdd_num_steps": 32,
+    "pdd_block_size": 4,
+    "lora_rank": 64,
+    "lora_alpha": 64.0,
+    "lora_targets": "to_q,to_k,to_v,to_out.0,ff.net.0.proj,ff.net.2,adaln_proj.linear",
+}
+
+
+def shifted_sigma(shift: float, sigma: torch.Tensor) -> torch.Tensor:
+    return shift * sigma / (1 + (shift - 1) * sigma)
+
+
+def pdd_time_grid(shift: float, num_steps: int) -> torch.Tensor:
+    """Ascending grid `0 = t_0 < ... < t_N = 1` of one MiniMax-H3 schedule."""
+    sigma = torch.linspace(1.0, 0.0, num_steps + 1, dtype=torch.float64)
+    return 1.0 - shifted_sigma(shift, sigma)
+
+
+def pdd_sampling_plan(step_sizes: torch.Tensor, start: int, block_size: int) -> torch.Tensor:
+    """Mean velocity of one block, which an Euler step over the block boundaries consumes."""
+    plan = torch.zeros(1, step_sizes.shape[0], dtype=step_sizes.dtype, device=step_sizes.device)
+    span = step_sizes[start : start + block_size].sum()
+    plan[0, start : start + block_size] = step_sizes[start : start + block_size] / span
+    return plan
+
+
+class MiniMaxH3ParallelHead(nn.Module):
+    """`N` per-interval output heads in place of one final linear layer."""
+
+    def __init__(self, source: nn.Linear, num_steps: int):
+        super().__init__()
+        self.num_steps = num_steps
+        self.in_features = source.in_features
+        self.out_features = source.out_features
+        self.weight = nn.Parameter(source.weight.detach()[None].repeat(num_steps, 1, 1).clone())
+        self.bias = (
+            None if source.bias is None else nn.Parameter(source.bias.detach()[None].repeat(num_steps, 1).clone())
+        )
+        self.plan = torch.zeros(1, num_steps)
+        self.plan[0, 0] = 1.0
+
+    def set_plan(self, plan: torch.Tensor) -> None:
+        if plan.ndim != 2 or plan.shape[1] != self.num_steps:
+            raise ValueError(f"A PDD plan must be `(num_directions, {self.num_steps})`, got {list(plan.shape)}.")
+        self.plan = plan
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        plan = self.plan.to(device=self.weight.device, dtype=self.weight.dtype)
+        weight = torch.einsum("pn,noi->poi", plan, self.weight).flatten(0, 1)
+        bias = None if self.bias is None else torch.einsum("pn,no->po", plan, self.bias).flatten()
+        return F.linear(hidden_states, weight, bias)
+
+
+class LoRALinear(nn.Module):
+    """Frozen `nn.Linear` plus `y += (alpha / rank) * B A x`."""
+
+    # Diffusers' AdaLN casts activations with `get_parameter_dtype(linear)`. That
+    # helper returns the first floating parameter, so skip the float32 adapters
+    # and expose the frozen backbone dtype.
+    _keep_in_fp32_modules = ["lora_down", "lora_up"]
+
+    def __init__(self, base: nn.Linear, rank: int, alpha: float):
+        super().__init__()
+        self.base = base
+        self.base.requires_grad_(False)
+        self.scaling = alpha / rank
+        self.lora_down = nn.Parameter(torch.empty(rank, base.in_features, dtype=torch.float32))
+        self.lora_up = nn.Parameter(torch.zeros(base.out_features, rank, dtype=torch.float32))
+        nn.init.kaiming_uniform_(self.lora_down, a=5**0.5)
+
+    @property
+    def weight(self) -> torch.Tensor:
+        return self.base.weight
+
+    @property
+    def bias(self):
+        return self.base.bias
+
+    @property
+    def in_features(self) -> int:
+        return self.base.in_features
+
+    @property
+    def out_features(self) -> int:
+        return self.base.out_features
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        out = self.base(hidden_states)
+        update = F.linear(
+            F.linear(hidden_states, self.lora_down.to(device=hidden_states.device, dtype=hidden_states.dtype)),
+            self.lora_up.to(device=hidden_states.device, dtype=hidden_states.dtype),
+        )
+        return out + self.scaling * update.to(out.dtype)
+
+
+def attach_parallel_decoder(transformer: nn.Module, num_steps: int) -> None:
+    transformer.proj_out = MiniMaxH3ParallelHead(transformer.proj_out, num_steps)
+    transformer.audio_proj_out = MiniMaxH3ParallelHead(transformer.audio_proj_out, num_steps)
+
+
+def add_lora(module: nn.Module, target_names: Sequence[str], rank: int, alpha: float) -> int:
+    targets = [
+        (name, child)
+        for name, child in module.named_modules()
+        if isinstance(child, nn.Linear) and any(name.endswith(suffix) for suffix in target_names)
+    ]
+    for name, child in targets:
+        parent_name, _, attribute = name.rpartition(".")
+        parent = module.get_submodule(parent_name) if parent_name else module
+        setattr(parent, attribute, LoRALinear(child, rank, alpha))
+    return len(targets)
+
+
+def set_parallel_plan(transformer: nn.Module, video_plan: torch.Tensor, audio_plan: torch.Tensor) -> None:
+    transformer.proj_out.set_plan(video_plan)
+    transformer.audio_proj_out.set_plan(audio_plan)
+
+
+def _load_pdd_config(config_path: Path | None) -> dict:
+    config = dict(DEFAULT_PDD_CONFIG)
+    if config_path is None or not config_path.is_file():
+        return config
+    with config_path.open(encoding="utf-8") as handle:
+        saved = json.load(handle)
+    for key in config:
+        if key in saved:
+            config[key] = saved[key]
+    if not isinstance(config["lora_targets"], str):
+        config["lora_targets"] = ",".join(config["lora_targets"])
+    return config
+
+
+def _load_state_dict(path: Path) -> dict[str, torch.Tensor]:
+    if path.suffix.lower() == ".safetensors":
+        from safetensors.torch import load_file
+
+        checkpoint = load_file(path, device="cpu")
+    else:
+        try:
+            checkpoint = torch.load(path, map_location="cpu", weights_only=True, mmap=True)
+        except TypeError:
+            checkpoint = torch.load(path, map_location="cpu", weights_only=True)
+    if isinstance(checkpoint, dict) and isinstance(checkpoint.get("state_dict"), dict):
+        checkpoint = checkpoint["state_dict"]
+    if not isinstance(checkpoint, dict):
+        raise TypeError(f"Expected a state-dict mapping in {path}, got {type(checkpoint).__name__}.")
+    return checkpoint
+
+
+def resolve_pdd_checkpoint(path: str) -> tuple[Path, Path | None]:
+    """Return `(weights_path, config_path)` from a local file/folder or a Hub repo id."""
+    local = Path(path).expanduser()
+    if local.is_file():
+        return local.resolve(), local.resolve().parent / "pdd_config.json"
+    if local.is_dir():
+        for name in ("pdd.pt", "pdd.safetensors"):
+            candidate = local / name
+            if candidate.is_file():
+                return candidate.resolve(), (local / "pdd_config.json").resolve()
+        raise FileNotFoundError(f"No pdd.pt or pdd.safetensors in {local}")
+    if "/" not in path:
+        raise FileNotFoundError(
+            f"PDD checkpoint not found: {path}. Pass a local pdd.pt / folder, or a Hugging Face repo id."
+        )
+
+    from huggingface_hub import hf_hub_download
+    from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
+
+    try:
+        weights = Path(hf_hub_download(path, "pdd.pt"))
+    except (EntryNotFoundError, RepositoryNotFoundError):
+        weights = Path(hf_hub_download(path, "pdd.safetensors"))
+    try:
+        config = Path(hf_hub_download(path, "pdd_config.json"))
+    except EntryNotFoundError:
+        config = None
+    return weights, config
+
+
+class _PDDStepArm:
+    """Arm the fused heads before each transformer forward, then advance one block."""
+
+    def __init__(
+        self,
+        transformer: nn.Module,
+        video_steps: torch.Tensor,
+        audio_steps: torch.Tensor,
+        block_size: int,
+        nfe: int,
+    ):
+        self.transformer = transformer
+        self.video_steps = video_steps
+        self.audio_steps = audio_steps
+        self.block_size = block_size
+        self.nfe = nfe
+        self.index = 0
+        self.arm(0)
+
+    def arm(self, step_index: int) -> None:
+        start = step_index * self.block_size
+        set_parallel_plan(
+            self.transformer,
+            pdd_sampling_plan(self.video_steps, start, self.block_size).float(),
+            pdd_sampling_plan(self.audio_steps, start, self.block_size).float(),
+        )
+
+    def __call__(self, _module, _args, output):
+        self.index += 1
+        if self.index < self.nfe:
+            self.arm(self.index)
+        else:
+            self.index = 0
+            self.arm(0)
+        return output
+
+
+def apply_pdd_lora(transformer: nn.Module, checkpoint: str, video_shift: float, audio_shift: float) -> int:
+    """Inject PDD LoRA + parallel heads, load weights, and arm them on every forward.
+
+    Args:
+        transformer: MiniMax-H3 `transformer` or `transformer_ref`.
+        checkpoint: Local `pdd.pt` / folder / Hugging Face repo id.
+        video_shift, audio_shift: Scheduler shifts (12.0 / 3.0 as released).
+
+    Returns:
+        `nfe`, the number of transformer evaluations. Diffusers' MiniMax-H3
+        scheduler counts the terminal sigma, so call the pipeline with
+        `num_inference_steps=nfe + 1`.
+    """
+    weights_path, config_path = resolve_pdd_checkpoint(checkpoint)
+    config = _load_pdd_config(config_path)
+    num_steps = int(config["pdd_num_steps"])
+    block_size = int(config["pdd_block_size"])
+    if block_size < 1 or num_steps % block_size != 0:
+        raise ValueError(f"pdd_num_steps={num_steps} must be divisible by pdd_block_size={block_size}.")
+    nfe = num_steps // block_size
+
+    add_lora(
+        transformer,
+        config["lora_targets"].split(","),
+        int(config["lora_rank"]),
+        float(config["lora_alpha"]),
+    )
+    attach_parallel_decoder(transformer, num_steps)
+
+    state_dict = _load_state_dict(weights_path)
+    incompatible = transformer.load_state_dict(state_dict, strict=False)
+    if incompatible.unexpected_keys:
+        preview = ", ".join(incompatible.unexpected_keys[:3])
+        raise RuntimeError(
+            f"{weights_path} holds keys the parallel decoder does not have, e.g. {preview}."
+        )
+    print(
+        f"Loaded PDD LoRA: {weights_path} ({len(state_dict)} tensors, "
+        f"grid={num_steps}, block={block_size}, nfe={nfe})",
+        flush=True,
+    )
+
+    video_steps = pdd_time_grid(float(video_shift), num_steps).diff()
+    audio_steps = pdd_time_grid(float(audio_shift), num_steps).diff()
+    controller = _PDDStepArm(transformer, video_steps, audio_steps, block_size, nfe)
+    transformer.register_forward_hook(controller)
+    transformer._pdd_step_arm = controller
+    transformer.requires_grad_(False)
+    transformer.eval()
+    return nfe

predict_ref2v.py

@@ -0,0 +1,51 @@
+import sys
+from pathlib import Path
+
+import torch
+from diffusers import ComponentsManager, ModularPipeline
+from diffusers.modular_pipelines.minimax_h3 import MiniMaxH3ImageReference
+from diffusers.utils.export_utils import encode_video
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from minimax_h3_pdd import apply_pdd_lora
+
+here = Path(__file__).resolve().parent
+model_path = "MiniMaxAI/MiniMax-H3"
+pdd_lora_path = str(here / "MiniMax-H3-Ref2VA-Acc-8Step.safetensors")
+reference_images = [here / "asset" / f"ref2va_test_5_{i}.jpg" for i in range(1, 5)]
+prompt = "subject_definitions:\n<Subject 1> is the East Asian woman sword fighter shown in <Picture 1>, preserving her facial identity, black hair tied in a high bun, focused expression, cream martial-arts jacket, and black-handled katana.\n<Subject 2> is the pale-skinned woman fighter shown in <Picture 2>, preserving her facial identity, short layered black hair with bangs, dark eye makeup, black lipstick, black leather jacket, and black clothing. The motorcycle is not present in the target video.\n<Subject 3> is the outdoor competition swimming pool shown in <Picture 3>, preserving the bright turquoise water, blue-and-white lane dividers, pale pool deck, and strong daylight.\n<Subject 4> is the underwater pool environment shown in <Picture 4>, preserving the cyan water, tiled pool floor, dark lane marking, moving caustic light, suspended bubbles, and refracted surface light.\n\nsummary:\n[reference generation] In one continuous 10-second action shot, <Subject 1> and <Subject 2> sprint into a violent exchange beside <Subject 3>. A rolling dodge leads into a left-fist versus right-hand-sword collision, a jumping counter, and a connected two-body tumble across the wet deck. Their momentum throws both women into the pool, where the confrontation finishes underwater inside <Subject 4>.\n\nretention_analysis:\n<Subject 1> (appears throughout): fully_preserved - retain the same face, high bun, cream jacket, body proportions, and single black-handled katana before and after entering the water.\n<Subject 2> (appears throughout): fully_preserved - retain the same face, short black hair, dark makeup, black lipstick, black leather jacket, black clothing, and body proportions before and after entering the water.\n<Subject 3> (appears before the underwater transition): fully_preserved - retain the competition-pool geometry, turquoise water, blue-and-white lane ropes, sunlit deck, and bright daylight.\n<Subject 4> (appears after the underwater transition): fully_preserved - retain the tiled floor, dark lane marking, cyan water, caustic illumination, bubbles, and refracted sunlight.\n\ndetailed_description:\nPhotorealistic live-action action cinema, 16:9 composition, hard bright daylight above water and luminous cyan light below water. Use one continuous aggressively tracking shot with no cuts: low lateral sprint tracking, one fast camera roll during the deck tumble, then a seamless plunge through the waterline. Keep both adult women clearly distinguishable by face, hairstyle, clothing color, and silhouette. <Subject 1> holds exactly one katana in her right hand throughout. <Subject 2> remains unarmed. Any fist-to-sword contact strikes the flat near the guard, never the sharpened edge. No blood, wounds, severed contact, duplicate people, duplicate weapons, motorcycle, readable text, logos, or costume changes.\n\n[Shot 1, 00:00.000-00:02.000] A low handheld camera races parallel to the wet edge of <Subject 3>. <Subject 1> in the cream jacket and <Subject 2> in black leather sprint toward each other from opposite ends of the pool deck. <Subject 1> carries the single katana low in her right hand, blade trailing safely behind. She plants one foot on a low starting block, leaps forward, and brings the sword into a fast diagonal arc. <Subject 2> drops beneath it into a forward shoulder roll, slides across the wet tiles, and rises inside the sword's range as water sprays from both fighters' shoes.\n\n[Shot 1, 00:02.000-00:04.200] <Subject 2> drives upward from the roll with one explosive left punch. <Subject 1> snaps the katana across in her right hand so the punch collides with the flat of the blade immediately beside the guard. The impact produces one sharp metallic crack, a compact spray of sparks, and a ring of displaced droplets. Their faces pass within centimeters. <Subject 2> hooks the sword wrist with her right hand and whips a low leg sweep; <Subject 1> jumps over it, twists in the air, and lands one step from the pool edge without releasing the sword.\n\n[Shot 1, 00:04.200-00:06.800] <Subject 2> immediately tackles <Subject 1> around the waist. <Subject 1> braces her left forearm across <Subject 2>'s shoulder and extends her right sword arm away from both bodies. Locked together, they crash onto the wet deck and complete two fast connected shoulder-over-shoulder rolls. The camera rolls with them at tile level as sky, lane ropes, faces, black leather, and cream fabric whip across frame. <Subject 1> hooks <Subject 2>'s ankle during the second rotation; their combined momentum carries both bodies over the pool lip.\n\n[Shot 1, 00:06.800-00:07.900] Both women rotate once in open air and strike the turquoise surface together in an enormous splash. The camera plunges between them without a cut, crossing the violently folding waterline as daylight becomes refracted cyan and a dense wall of bubbles engulfs the frame.\n\n[Shot 1, 00:07.900-00:10.000] Inside <Subject 4>, their clothes and hair surge in opposite currents while both identities remain clear. <Subject 2> throws one slowed left punch through the bubbles. <Subject 1>, still gripping the single katana in her right hand, blocks with the hilt and her left forearm while keeping the blade pointed safely away. The impact rolls both women apart above the dark lane marking. They kick backward, regain facing guard positions, and end suspended on opposite sides of a rising column of bubbles as caustic light flashes across their faces and the tiled floor.\n\noverall_soundscape:\nRapid wet footfalls, starting-block impact, sword whoosh, rolling bodies, one metallic fist-to-blade crack, sparks, tile scrapes, a heavy synchronized splash, dense bubbles, muffled underwater impact, and filtered pool ambience.\n\nnon_diegetic_music:\nFast taiko-like percussion accelerates through the sprint and deck rolls, cuts completely at the splash, then returns underwater as two muffled bass pulses ending on one suspended metallic note.\n"
+sample_size = [768, 1344]
+video_length = 243
+
+manager = ComponentsManager()
+manager.enable_auto_cpu_offload(device="cuda", memory_reserve_margin="12GB")
+pipeline = ModularPipeline.from_pretrained(model_path, workflow="ref2va", components_manager=manager)
+pipeline.load_components(dtype=torch.bfloat16, pretrained_model_name_or_path=model_path)
+
+nfe = apply_pdd_lora(
+    pipeline.transformer_ref, pdd_lora_path, pipeline.scheduler.shift, pipeline.audio_scheduler.shift
+)
+
+generator = torch.Generator().manual_seed(42)
+result = pipeline(
+    prompt=prompt,
+    references=[MiniMaxH3ImageReference.from_file(str(path)) for path in reference_images],
+    height=sample_size[0],
+    width=sample_size[1],
+    num_frames=video_length,
+    num_inference_steps=nfe + 1,
+    generator=generator,
+    output_type="np",
+    output=["videos", "audio", "sampling_rate"],
+)
+
+output_path = here / "samples" / "ref2v.mp4"
+output_path.parent.mkdir(parents=True, exist_ok=True)
+encode_video(
+    result["videos"][0],
+    fps=24,
+    output_path=str(output_path),
+    audio=result["audio"][0],
+    audio_sample_rate=int(result["sampling_rate"]),
+)
+print(f"Saved {output_path}", flush=True)

predict_t2v.py

@@ -0,0 +1,60 @@
+import sys
+from pathlib import Path
+
+import torch
+from diffusers import ComponentsManager, ModularPipeline
+from diffusers.utils.export_utils import encode_video
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from minimax_h3_pdd import apply_pdd_lora
+
+model_path = "MiniMaxAI/MiniMax-H3"
+pdd_lora_path = str(Path(__file__).resolve().parent / "MiniMax-H3-FL2VA-Acc-8Step.safetensors")
+prompt = (
+    "[Shot 1] Cinematic wide shot, low angle, camera pushing in fast. A knight in dented steel plate armour "
+    "charges across a muddy tournament field toward a shaggy brown warhorse-mounted rival, his tattered blue "
+    "surcoat streaming behind him. He raises a notched longsword in both hands and brings it down; the rival "
+    "parries with a round wooden shield that splinters, chips of wood spinning into the air. Clods of wet earth "
+    "fly up from their boots and hooves. Grey banners snap in a strong crosswind under an overcast sky, and a "
+    "crowd of blurred spectators presses against a wooden barricade in the background. The camera swings around "
+    "the clash as the knight shoulders into the shield and both men stagger sideways."
+    "Ringing steel-on-steel impacts with long metallic decay, the dry crack of splitting wood, heavy plate "
+    "rattling with every stride, boots sucking out of deep mud, a horse snorting and stamping, wind buffeting the "
+    "microphone and a roaring crowd swelling behind it."
+    "Full orchestral battle cue, fast ostinato strings under blaring low brass, timpani accenting each sword "
+    "impact."
+)
+sample_size = [704, 1280]
+video_length = 124
+
+manager = ComponentsManager()
+manager.enable_auto_cpu_offload(device="cuda", memory_reserve_margin="12GB")
+pipeline = ModularPipeline.from_pretrained(model_path, workflow="t2va", components_manager=manager)
+pipeline.load_components(dtype=torch.bfloat16, pretrained_model_name_or_path=model_path)
+
+nfe = apply_pdd_lora(
+    pipeline.transformer, pdd_lora_path, pipeline.scheduler.shift, pipeline.audio_scheduler.shift
+)
+
+generator = torch.Generator().manual_seed(42)
+result = pipeline(
+    prompt=prompt,
+    height=sample_size[0],
+    width=sample_size[1],
+    num_frames=video_length,
+    num_inference_steps=nfe + 1,
+    generator=generator,
+    output_type="np",
+    output=["videos", "audio", "sampling_rate"],
+)
+
+output_path = Path(__file__).resolve().parent / "samples" / "t2v.mp4"
+output_path.parent.mkdir(parents=True, exist_ok=True)
+encode_video(
+    result["videos"][0],
+    fps=24,
+    output_path=str(output_path),
+    audio=result["audio"][0],
+    audio_sample_rate=int(result["sampling_rate"]),
+)
+print(f"Saved {output_path}", flush=True)

results/minimax_h3_fl2va_acc_8step_v1_1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fcef5076fec4cfad72b9e1a2104744ace87a6c0c2f804088b45d9817841741e
+size 851760

results/minimax_h3_fl2va_acc_8step_v1_2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e15ed74a9883384f3a3662f959067de0793e8aefff20e6fd55885fb882529d23
+size 8798213

results/minimax_h3_fl2va_acc_8step_v1_3.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf02dd4516a9453bb1bc012fb912f1ef83f854d07151686dfbcd16757cab0219
+size 1961347

results/minimax_h3_fl2va_baseline_1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afcbd76e7e5b6107df3b5b0a77fbdc09253d2e20beca55403730d45c55032584
+size 835589

results/minimax_h3_fl2va_baseline_2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6673f964eda77ada13384af36cca1fb3fa058a45a788780ff9da154569cdfd44
+size 7719897

results/minimax_h3_fl2va_baseline_3.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cc312bdafbf84b3a9d9da6c0af8a5b5cd75888b3d695d33f6bb5016c811d253
+size 1814474

results/minimax_h3_fl2va_turbo_4step_v1.1_768p_1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b92514db1f7a59517a1ef165748e5d7fab6e317fc51e8212e1b95645e3912f18
+size 893312

results/minimax_h3_fl2va_turbo_4step_v1.1_768p_2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f21d92548605a8e05c2d8bee608df81c47a584ed483d3b7c9aea17f1ddaeaa0c
+size 9806891

results/minimax_h3_fl2va_turbo_4step_v1.1_768p_3.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:831a974e4f25cb01c60845f1b6dc5720966c12f88801d5aefe8a077d76f99fda
+size 2326695

results/minimax_h3_ref2va_acc_8step_v1_1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c38d56649aeb5420e6e595808d4f5f92aabdcc059fc52350ce09806fa7f5e1a6
+size 6085204

results/minimax_h3_ref2va_acc_8step_v1_2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49db1d2d853a189c5f04987439fe2f4ca7b8d99827cddf9e5cf93ce0a0cafa4b
+size 2006767

results/minimax_h3_ref2va_acc_8step_v1_3.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87955def3dbb1ea49726ae9c07fc8709dbb15f9026ecc38a15ee300771d944f6
+size 4151912

results/minimax_h3_ref2va_baseline_1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b06369445bba07ed87581b49e163ec47ea4219c4624f95ff59d0c8870d4fad09
+size 5930077

results/minimax_h3_ref2va_baseline_2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bd6e9b1b8e992d00d9c7754215ce47314be76a81d461b5d50a1ad250b808afc
+size 1659004

results/minimax_h3_ref2va_baseline_3.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:233d92ae3bedad301581c1a7df1c19fef6746967620902c9330950a52c1ae30c
+size 4891750

results/minimax_h3_ref2va_turbo_4step_v0.1_768p_1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa71f4bc139274893d2ffcf07a1c461bc04afd54f8959c94843a2c6418f7222f
+size 5273075

results/minimax_h3_ref2va_turbo_4step_v0.1_768p_2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67b13c561a0d34387c0fc46904435edfc618bce23923fcebaab8102806638cc7
+size 1012341

results/minimax_h3_ref2va_turbo_4step_v0.1_768p_3.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfce9c6dd70388c1eef57ae8c2455a492d2d2104a55c6ee31ad5188c0e3dcf9c
+size 3734857