File size: 10,700 Bytes

9018dbd

---
library_name: diffusers
pipeline_tag: image-text-to-video
base_model:
- MiniMaxAI/MiniMax-H3
---

This tiny model is for debugging. It is randomly initialized with the config adapted from [MiniMaxAI/MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3).

File size:
- ~12MB text_encoder/model.safetensors
- ~0.5MB transformer/diffusion_pytorch_model.safetensors
- ~0.5MB transformer_ref/diffusion_pytorch_model.safetensors
- ~4MB vae/diffusion_pytorch_model.safetensors
- ~64MB audio_vae/diffusion_pytorch_model.safetensors

Requires `diffusers` from source (`main`) with MiniMax-H3 modular blocks, and a recent `transformers` that ships `Qwen3VLForConditionalGeneration`.

Notes:
- The conditioner must have `num_hidden_layers > 50` because MiniMax-H3 reads `hidden_states[50]`.
- Transformer RoPE needs `attention_head_dim >= 6 * rope_freq_dim`.
- Audio VAE decoder needs `decoder_dim >= 128` for the released 7-stage upsample stack.

| File path | Size |
|------|------|
| audio_vae/diffusion_pytorch_model.safetensors | 66.7MB |
| text_encoder/model.safetensors | 12.1MB |
| transformer/diffusion_pytorch_model.safetensors | 0.5MB |
| transformer_ref/diffusion_pytorch_model.safetensors | 0.5MB |
| vae/diffusion_pytorch_model.safetensors | 4.5MB |


### Example usage:

```python
import torch
from diffusers import ModularPipeline

model_id = "tiny-random/minimax-h3"
device = 'cuda' if torch.cuda.is_available() else 'cpu'
pipe = ModularPipeline.from_pretrained(model_id, workflow='t2va')
pipe.load_components(dtype=torch.bfloat16)
if device == 'cuda':
    pipe.to(device)

outputs = pipe(
    prompt='A red fox trotting through a snowy pine forest',
    num_frames=124,
    height=64,
    width=64,
    num_inference_steps=2,
    generator=torch.Generator(device=device).manual_seed(42),
    output=['videos', 'audio', 'sampling_rate'],
)
print(type(outputs['videos'][0]), getattr(outputs['videos'][0], 'shape', None))
print(type(outputs['audio'][0]), getattr(outputs['audio'][0], 'shape', None), outputs['sampling_rate'])
```

### Codes to create this repo:

```python
import json
from pathlib import Path

import torch
from diffusers import (
    AutoencoderKLMiniMaxH3,
    AutoencoderKLMiniMaxH3Audio,
    MiniMaxH3Blocks,
    MiniMaxH3Scheduler,
    MiniMaxH3Transformer3DModel,
)
from huggingface_hub import hf_hub_download
from transformers import AutoConfig, AutoProcessor, AutoTokenizer, Qwen3VLForConditionalGeneration

source_model_id = "MiniMaxAI/MiniMax-H3"
save_folder = "/tmp/tiny-random/minimax-h3"

def save_json(path, obj):
    Path(path).parent.mkdir(parents=True, exist_ok=True)
    with open(path, 'w', encoding='utf-8') as f:
        json.dump(obj, f, indent=2, ensure_ascii=False)

def init_weights(model):
    torch.manual_seed(42)
    model = model.cpu()
    with torch.no_grad():
        for name, p in sorted(model.named_parameters()):
            torch.nn.init.normal_(p, 0, 0.1)
            print(name, p.shape, p.dtype, p.device)

torch.set_default_dtype(torch.bfloat16)
text_dim = 32
Path(save_folder).mkdir(parents=True, exist_ok=True)

AutoTokenizer.from_pretrained(source_model_id, subfolder='tokenizer').save_pretrained(
    f'{save_folder}/tokenizer'
)
AutoProcessor.from_pretrained(source_model_id, subfolder='processor').save_pretrained(
    f'{save_folder}/processor'
)

with open(hf_hub_download(source_model_id, filename='text_encoder/config.json', repo_type='model'), 'r', encoding='utf-8') as f:
    config = json.load(f)
# MiniMax-H3 conditions on hidden_states[50], so keep >50 layers with a tiny width.
config['text_config'].update({
    'head_dim': 8,
    'hidden_size': text_dim,
    'intermediate_size': 64,
    'num_attention_heads': 4,
    'num_key_value_heads': 2,
    'num_hidden_layers': 51,
    'tie_word_embeddings': True,
})
config['text_config']['rope_scaling']['mrope_section'] = [2, 1, 1]
config['vision_config'].update({
    'depth': 4,
    'hidden_size': 64,
    'intermediate_size': 128,
    'num_heads': 4,
    'out_hidden_size': text_dim,
    'deepstack_visual_indexes': [1, 2, 3],
})
config['tie_word_embeddings'] = True
save_json(f'{save_folder}/text_encoder/config.json', config)
text_encoder = Qwen3VLForConditionalGeneration(
    AutoConfig.from_pretrained(f'{save_folder}/text_encoder')
).to(torch.bfloat16)
init_weights(text_encoder)
text_encoder.save_pretrained(f'{save_folder}/text_encoder')

# attention_head_dim must cover 2 * 3 * rope_freq_dim rotary channels.
transformer_kwargs = dict(
    num_attention_heads=2,
    attention_head_dim=32,
    hidden_size=64,
    num_layers=2,
    num_refiner_layers=1,
    ffn_dim=128,
    in_channels=8,
    audio_in_channels=8,
    patch_size=(1, 2, 2),
    text_dim=text_dim,
    freq_dim=64,
    time_embed_hidden_dim=64,
    time_embed_dim=32,
    rope_freq_dim=4,
)
for subfolder in ('transformer', 'transformer_ref'):
    transformer = MiniMaxH3Transformer3DModel(**transformer_kwargs)
    init_weights(transformer)
    transformer.save_pretrained(f'{save_folder}/{subfolder}')

with open(hf_hub_download(source_model_id, filename='vae/config.json', repo_type='model'), 'r', encoding='utf-8') as f:
    vae_config = json.load(f)
vae_config.update({
    'latent_channels': 8,
    'block_out_channels': [32, 32, 32, 64, 64, 64],
    'layers_per_block': 1,
    'spatial_downsample_factors': [2, 2, 2, 2, 1, 1],
    'temporal_downsample_factors': [1, 2, 2, 1, 1, 1],
    'norm_num_groups': 8,
    'decoder_num_layers': 2,
    'decoder_num_attention_heads': 2,
    'decoder_attention_head_dim': 16,
    'decoder_num_register_tokens': 2,
    'decoder_ffn_mult': 2,
    'latents_mean': [0.0] * 8,
    'latents_std': [1.0] * 8,
})
save_json(f'{save_folder}/vae/config.json', vae_config)
vae = AutoencoderKLMiniMaxH3.from_config(
    AutoencoderKLMiniMaxH3.load_config(f'{save_folder}/vae')
)
init_weights(vae)
vae.save_pretrained(f'{save_folder}/vae')

# Keep hop length 800 (=32000/40Hz). decoder_dim must stay >= 128 for 7 upsample stages.
with open(hf_hub_download(source_model_id, filename='audio_vae/config.json', repo_type='model'), 'r', encoding='utf-8') as f:
    audio_config = json.load(f)
audio_config.update({
    'encoder_dim': 32,
    'latent_dim': 128,
    'latent_channels': 8,
    'num_attention_heads': 4,
    'decoder_dim': 128,
    'latents_mean': [0.0] * 8,
    'latents_std': [1.0] * 8,
})
save_json(f'{save_folder}/audio_vae/config.json', audio_config)
audio_vae = AutoencoderKLMiniMaxH3Audio.from_config(
    AutoencoderKLMiniMaxH3Audio.load_config(f'{save_folder}/audio_vae')
)
init_weights(audio_vae)
audio_vae.save_pretrained(f'{save_folder}/audio_vae')

MiniMaxH3Scheduler.from_pretrained(source_model_id, subfolder='scheduler').save_pretrained(
    f'{save_folder}/scheduler'
)
MiniMaxH3Scheduler.from_pretrained(source_model_id, subfolder='audio_scheduler').save_pretrained(
    f'{save_folder}/audio_scheduler'
)

for index_name in ('model_index.json', 'modular_model_index.json'):
    index = json.load(open(hf_hub_download(source_model_id, filename=index_name, repo_type='model'), encoding='utf-8'))
    for value in index.values():
        if isinstance(value, list) and len(value) >= 3 and isinstance(value[2], dict):
            value[2]['pretrained_model_name_or_path'] = save_folder
    save_json(f'{save_folder}/{index_name}', index)

pipe = MiniMaxH3Blocks().init_pipeline(save_folder)
pipe.load_components(dtype=torch.bfloat16)
pipe.save_pretrained(save_folder, safe_serialization=True, overwrite_modular_index=True)
torch.set_default_dtype(torch.float32)
print(pipe)
```

### Printing the model:

```text
MiniMaxH3ModularPipeline {
  "_blocks_class_name": "MiniMaxH3Blocks",
  "_class_name": "MiniMaxH3ModularPipeline",
  "_diffusers_version": "0.40.0.dev0",
  "audio_scheduler": [
    "diffusers",
    "MiniMaxH3Scheduler",
    {
      "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3",
      "revision": null,
      "subfolder": "audio_scheduler",
      "type_hint": [
        "diffusers",
        "MiniMaxH3Scheduler"
      ],
      "variant": null
    }
  ],
  "audio_vae": [
    "diffusers",
    "AutoencoderKLMiniMaxH3Audio",
    {
      "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3",
      "revision": null,
      "subfolder": "audio_vae",
      "type_hint": [
        "diffusers",
        "AutoencoderKLMiniMaxH3Audio"
      ],
      "variant": null
    }
  ],
  "canvas_max_pixels": 1032192,
  "canvas_short_edge": 768,
  "processor": [
    "transformers",
    "Qwen3VLProcessor",
    {
      "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3",
      "revision": null,
      "subfolder": "processor",
      "type_hint": [
        "transformers",
        "Qwen3VLProcessor"
      ],
      "variant": null
    }
  ],
  "reference_image_short_edge": 2048,
  "scheduler": [
    "diffusers",
    "MiniMaxH3Scheduler",
    {
      "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3",
      "revision": null,
      "subfolder": "scheduler",
      "type_hint": [
        "diffusers",
        "MiniMaxH3Scheduler"
      ],
      "variant": null
    }
  ],
  "text_encoder": [
    "transformers",
    "Qwen3VLForConditionalGeneration",
    {
      "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3",
      "revision": null,
      "subfolder": "text_encoder",
      "type_hint": [
        "transformers",
        "Qwen3VLForConditionalGeneration"
      ],
      "variant": null
    }
  ],
  "tokenizer": [
    "transformers",
    "Qwen2Tokenizer",
    {
      "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3",
      "revision": null,
      "subfolder": "tokenizer",
      "type_hint": [
        "transformers",
        "Qwen2Tokenizer"
      ],
      "variant": null
    }
  ],
  "transformer": [
    "diffusers",
    "MiniMaxH3Transformer3DModel",
    {
      "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3",
      "revision": null,
      "subfolder": "transformer",
      "type_hint": [
        "diffusers",
        "MiniMaxH3Transformer3DModel"
      ],
      "variant": null
    }
  ],
  "transformer_ref": [
    "diffusers",
    "MiniMaxH3Transformer3DModel",
    {
      "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3",
      "revision": null,
      "subfolder": "transformer_ref",
      "type_hint": [
        "diffusers",
        "MiniMaxH3Transformer3DModel"
      ],
      "variant": null
    }
  ],
  "vae": [
    "diffusers",
    "AutoencoderKLMiniMaxH3",
    {
      "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3",
      "revision": null,
      "subfolder": "vae",
      "type_hint": [
        "diffusers",
        "AutoencoderKLMiniMaxH3"
      ],
      "variant": null
    }
  ]
}
```