File size: 8,316 Bytes
cad2439 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 | import gradio as gr
MODEL_INFO = {
"name": "MiniMax-H3-Fun-Controlnet-Union",
"author": "alibaba-pai",
"base_model": "MiniMaxAI/MiniMax-H3",
"description": (
"ControlNet-Union for MiniMax-H3 — a single checkpoint that conditions the "
"MiniMax-H3 video generator on Canny, Depth, HED, MLSD, or Pose control videos, "
"and also runs video inpainting. Trained with the VideoX-Fun pipeline."
),
}
RESULTS = [
("Canny", "A Tokyo street scene", "canny_tokyo_street"),
("Depth", "An astronaut floating in space", "depth_astronaut"),
("HED", "A T-Rex riding a BMX", "hed_trex_bmx"),
("MLSD", "A village with straight-line architecture", "mlsd_village"),
("Pose", "A dancer transformed into a flamenco dancer", "pose_dance"),
]
ASSET_URL = "https://huggingface.co/alibaba-pai/MiniMax-H3-Fun-Controlnet-Union/resolve/main"
RESULT_URL = "https://huggingface.co/alibaba-pai/MiniMax-H3-Fun-Controlnet-Union/resolve/main/results"
def result_section(control_type, caption, key):
return gr.HTML(
f"""
<div style="display:flex; gap:12px; flex-wrap:wrap; margin:8px 0;">
<div style="flex:1; min-width:240px;">
<p style="font-weight:600; margin:0 0 4px;">Control — {control_type}</p>
<video src="{ASSET_URL}/asset/{key}.mp4"
width="100%" controls muted loop
style="border-radius:8px; background:#111;"></video>
<p style="font-size:0.8em; color:#888; margin:4px 0 0;">{caption}</p>
</div>
<div style="flex:1; min-width:240px;">
<p style="font-weight:600; margin:0 0 4px;">Output — MiniMax-H3 + ControlNet-Union</p>
<video src="{RESULT_URL}/{key}.mp4"
width="100%" controls muted loop
style="border-radius:8px; background:#111;"></video>
</div>
</div>
"""
)
with gr.Blocks(
theme=gr.themes.Soft(),
title="MiniMax-H3-Fun-Controlnet-Union",
css="""
.model-header { text-align:center; padding:1.5rem 0 0.5rem; }
.model-header h1 { font-size:1.8rem; font-weight:700; }
.model-header .badge { display:inline-block; background:#6366f1; color:#fff;
padding:0.2rem 0.7rem; border-radius:999px; font-size:0.75rem; margin-left:0.5rem; }
video { box-shadow:0 4px 16px rgba(0,0,0,0.3); }
.section-title { font-size:1.1rem; font-weight:600; margin-bottom:0.5rem;
border-bottom:2px solid #e5e7eb; padding-bottom:0.3rem; }
.info-table td { padding:0.4rem 0.8rem; vertical-align:top; }
.info-table td:first-child { font-weight:600; white-space:nowrap; width:160px; }
"""
) as demo:
gr.HTML(
"""
<div class="model-header">
<h1>MiniMax-H3-Fun-Controlnet-Union
<span class="badge">alibaba-pai</span>
</h1>
<p style="color:#666; margin-top:0.3rem;">
ControlNet-Union for <a href="https://huggingface.co/MiniMaxAI/MiniMax-H3" target="_blank">MiniMax-H3</a>
· <a href="https://github.com/aigc-apps/VideoX-Fun" target="_blank">VideoX-Fun</a> pipeline
</p>
</div>
"""
)
gr.Markdown("""
**MiniMax-H3-Fun-Controlnet-Union** is a single ControlNet-Union checkpoint for the
[MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3) video diffusion transformer.
One model handles **Canny, Depth, HED, MLSD, and Pose** control conditions for
video-to-video generation — no per-condition checkpoint switching — and also supports
video inpainting.
| File | Description |
|------|-------------|
| `MiniMax-H3-Fun-Controlnet-Union.safetensors` | Control branch weights (~6.8 GB): `control_proj_in` + 5 `control_blocks`. Loaded on top of the base MiniMax-H3 transformer. |
""")
with gr.Row():
with gr.Column(scale=1):
gr.Markdown("### Model Features")
gr.Markdown("""
- **Union control** — one checkpoint for Canny, Depth, HED, MLSD, and Pose.
- **5 control injection points** — layers 0, 10, 20, 30, 40 of the 50-block transformer.
- **Guidance-distilled** — run with `guidance_scale = 1.0`; one forward pass per step.
- **Inpainting** — control input widened to `control_in_dim = 49` (latent + masked latent + mask channels).
- **`control_context_scale`** — scales every control skip before adding to the main branch:
`1.0` = strongest control, `0.0` = control branch off.
- **Frame snap** — frame count snaps to the largest `17*n + 5` the video VAE can decode
(duration capped at 15 s), canvas keeps the control video's aspect ratio.
""")
with gr.Column(scale=1):
gr.Markdown("### Inference Defaults")
gr.HTML(
"""
<table class="info-table">
<tr><td>num_inference_steps</td><td>40</td></tr>
<tr><td>guidance_scale</td><td>1.0 (guidance-distilled)</td></tr>
<tr><td>control_context_scale</td><td>1.00</td></tr>
<tr><td>seed</td><td>43</td></tr>
<tr><td>fps</td><td>24</td></tr>
</table>
"""
)
gr.Markdown("---")
gr.Markdown("### Results — All 5 Control Conditions")
gr.Markdown(
"All samples generated with `num_inference_steps=40`, `guidance_scale=1.0`, "
"`control_context_scale=1.00`, seed 43."
)
for control_type, caption, key in RESULTS:
result_section(control_type, caption, key)
gr.Markdown("---")
gr.Markdown("### How to Run Inference")
gr.Markdown(
"""
1. Clone the [VideoX-Fun](https://github.com/aigc-apps/VideoX-Fun) repository.
2. Download the base **MiniMax-H3** model and this ControlNet-Union checkpoint.
3. Place them under `models/Diffusion_Transformer/`:
```
models/
└── Diffusion_Transformer/
├── MiniMax-H3/ # base transformer (~62 GB)
└── MiniMax-H3-Fun-Controlnet-Union/
└── MiniMax-H3-Fun-Controlnet-Union.safetensors # control branch (~6.8 GB)
```
4. Edit the variables at the top of `examples/minimax_h3_fun/predict_v2v_control.py`:
```python
model_name = "models/Diffusion_Transformer/MiniMax-H3"
config_path = "config/minimax_h3/minimax_h3_control.yaml"
transformer_path = "models/Diffusion_Transformer/MiniMax-H3-Fun-Controlnet-Union/MiniMax-H3-Fun-Controlnet-Union.safetensors"
control_video = "your_control_video.mp4"
prompt = "your prompt"
```
5. Run: `python examples/minimax_h3_fun/predict_v2v_control.py`
**Important notes:**
- `config_path` must use the exact trained layout: `control_blocks_places: [0, 10, 20, 30, 40]`,
`control_in_dim: 49`, `control_apply_audio: false`.
- Keep `guidance_scale = 1.0` — higher values apply guidance twice and degrade output.
- The control checkpoint carries **only the control branch**; the base MiniMax-H3 weights
must be present at `model_name`.
- **Memory:** transformer (~62 GB) + Qwen3-VL text encoder (~62 GB) do **not** fit a single
80 GB GPU fully loaded. Use `model_group_offload` (fastest) or `model_cpu_offload_and_qfloat8`.
"""
)
gr.Markdown("---")
gr.Markdown("### Links")
gr.HTML(
"""
<p>
<a href="https://huggingface.co/alibaba-pai/MiniMax-H3-Fun-Controlnet-Union" target="_blank">
Model page on Hugging Face
</a>
·
<a href="https://huggingface.co/MiniMaxAI/MiniMax-H3" target="_blank">
MiniMax-H3 base model
</a>
·
<a href="https://github.com/aigc-apps/VideoX-Fun" target="_blank">
VideoX-Fun repository
</a>
</p>
"""
)
if __name__ == "__main__":
demo.launch(server_name="0.0.0.0", server_port=7860)
|