| --- |
| library_name: diffusers |
| pipeline_tag: image-text-to-video |
| base_model: |
| - MiniMaxAI/MiniMax-H3 |
| --- |
| |
| This tiny model is for debugging. It is randomly initialized with the config adapted from [MiniMaxAI/MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3). |
|
|
| File size: |
| - ~12MB text_encoder/model.safetensors |
| - ~0.5MB transformer/diffusion_pytorch_model.safetensors |
| - ~0.5MB transformer_ref/diffusion_pytorch_model.safetensors |
| - ~4MB vae/diffusion_pytorch_model.safetensors |
| - ~64MB audio_vae/diffusion_pytorch_model.safetensors |
| |
| Requires `diffusers` from source (`main`) with MiniMax-H3 modular blocks, and a recent `transformers` that ships `Qwen3VLForConditionalGeneration`. |
| |
| Notes: |
| - The conditioner must have `num_hidden_layers > 50` because MiniMax-H3 reads `hidden_states[50]`. |
| - Transformer RoPE needs `attention_head_dim >= 6 * rope_freq_dim`. |
| - Audio VAE decoder needs `decoder_dim >= 128` for the released 7-stage upsample stack. |
|
|
| | File path | Size | |
| |------|------| |
| | audio_vae/diffusion_pytorch_model.safetensors | 66.7MB | |
| | text_encoder/model.safetensors | 12.1MB | |
| | transformer/diffusion_pytorch_model.safetensors | 0.5MB | |
| | transformer_ref/diffusion_pytorch_model.safetensors | 0.5MB | |
| | vae/diffusion_pytorch_model.safetensors | 4.5MB | |
| |
| |
| ### Example usage: |
| |
| ```python |
| import torch |
| from diffusers import ModularPipeline |
| |
| model_id = "tiny-random/minimax-h3" |
| device = 'cuda' if torch.cuda.is_available() else 'cpu' |
| pipe = ModularPipeline.from_pretrained(model_id, workflow='t2va') |
| pipe.load_components(dtype=torch.bfloat16) |
| if device == 'cuda': |
| pipe.to(device) |
| |
| outputs = pipe( |
| prompt='A red fox trotting through a snowy pine forest', |
| num_frames=124, |
| height=64, |
| width=64, |
| num_inference_steps=2, |
| generator=torch.Generator(device=device).manual_seed(42), |
| output=['videos', 'audio', 'sampling_rate'], |
| ) |
| print(type(outputs['videos'][0]), getattr(outputs['videos'][0], 'shape', None)) |
| print(type(outputs['audio'][0]), getattr(outputs['audio'][0], 'shape', None), outputs['sampling_rate']) |
| ``` |
| |
| ### Codes to create this repo: |
|
|
| ```python |
| import json |
| from pathlib import Path |
| |
| import torch |
| from diffusers import ( |
| AutoencoderKLMiniMaxH3, |
| AutoencoderKLMiniMaxH3Audio, |
| MiniMaxH3Blocks, |
| MiniMaxH3Scheduler, |
| MiniMaxH3Transformer3DModel, |
| ) |
| from huggingface_hub import hf_hub_download |
| from transformers import AutoConfig, AutoProcessor, AutoTokenizer, Qwen3VLForConditionalGeneration |
| |
| source_model_id = "MiniMaxAI/MiniMax-H3" |
| save_folder = "/tmp/tiny-random/minimax-h3" |
| |
| def save_json(path, obj): |
| Path(path).parent.mkdir(parents=True, exist_ok=True) |
| with open(path, 'w', encoding='utf-8') as f: |
| json.dump(obj, f, indent=2, ensure_ascii=False) |
| |
| def init_weights(model): |
| torch.manual_seed(42) |
| model = model.cpu() |
| with torch.no_grad(): |
| for name, p in sorted(model.named_parameters()): |
| torch.nn.init.normal_(p, 0, 0.1) |
| print(name, p.shape, p.dtype, p.device) |
| |
| torch.set_default_dtype(torch.bfloat16) |
| text_dim = 32 |
| Path(save_folder).mkdir(parents=True, exist_ok=True) |
| |
| AutoTokenizer.from_pretrained(source_model_id, subfolder='tokenizer').save_pretrained( |
| f'{save_folder}/tokenizer' |
| ) |
| AutoProcessor.from_pretrained(source_model_id, subfolder='processor').save_pretrained( |
| f'{save_folder}/processor' |
| ) |
| |
| with open(hf_hub_download(source_model_id, filename='text_encoder/config.json', repo_type='model'), 'r', encoding='utf-8') as f: |
| config = json.load(f) |
| # MiniMax-H3 conditions on hidden_states[50], so keep >50 layers with a tiny width. |
| config['text_config'].update({ |
| 'head_dim': 8, |
| 'hidden_size': text_dim, |
| 'intermediate_size': 64, |
| 'num_attention_heads': 4, |
| 'num_key_value_heads': 2, |
| 'num_hidden_layers': 51, |
| 'tie_word_embeddings': True, |
| }) |
| config['text_config']['rope_scaling']['mrope_section'] = [2, 1, 1] |
| config['vision_config'].update({ |
| 'depth': 4, |
| 'hidden_size': 64, |
| 'intermediate_size': 128, |
| 'num_heads': 4, |
| 'out_hidden_size': text_dim, |
| 'deepstack_visual_indexes': [1, 2, 3], |
| }) |
| config['tie_word_embeddings'] = True |
| save_json(f'{save_folder}/text_encoder/config.json', config) |
| text_encoder = Qwen3VLForConditionalGeneration( |
| AutoConfig.from_pretrained(f'{save_folder}/text_encoder') |
| ).to(torch.bfloat16) |
| init_weights(text_encoder) |
| text_encoder.save_pretrained(f'{save_folder}/text_encoder') |
| |
| # attention_head_dim must cover 2 * 3 * rope_freq_dim rotary channels. |
| transformer_kwargs = dict( |
| num_attention_heads=2, |
| attention_head_dim=32, |
| hidden_size=64, |
| num_layers=2, |
| num_refiner_layers=1, |
| ffn_dim=128, |
| in_channels=8, |
| audio_in_channels=8, |
| patch_size=(1, 2, 2), |
| text_dim=text_dim, |
| freq_dim=64, |
| time_embed_hidden_dim=64, |
| time_embed_dim=32, |
| rope_freq_dim=4, |
| ) |
| for subfolder in ('transformer', 'transformer_ref'): |
| transformer = MiniMaxH3Transformer3DModel(**transformer_kwargs) |
| init_weights(transformer) |
| transformer.save_pretrained(f'{save_folder}/{subfolder}') |
| |
| with open(hf_hub_download(source_model_id, filename='vae/config.json', repo_type='model'), 'r', encoding='utf-8') as f: |
| vae_config = json.load(f) |
| vae_config.update({ |
| 'latent_channels': 8, |
| 'block_out_channels': [32, 32, 32, 64, 64, 64], |
| 'layers_per_block': 1, |
| 'spatial_downsample_factors': [2, 2, 2, 2, 1, 1], |
| 'temporal_downsample_factors': [1, 2, 2, 1, 1, 1], |
| 'norm_num_groups': 8, |
| 'decoder_num_layers': 2, |
| 'decoder_num_attention_heads': 2, |
| 'decoder_attention_head_dim': 16, |
| 'decoder_num_register_tokens': 2, |
| 'decoder_ffn_mult': 2, |
| 'latents_mean': [0.0] * 8, |
| 'latents_std': [1.0] * 8, |
| }) |
| save_json(f'{save_folder}/vae/config.json', vae_config) |
| vae = AutoencoderKLMiniMaxH3.from_config( |
| AutoencoderKLMiniMaxH3.load_config(f'{save_folder}/vae') |
| ) |
| init_weights(vae) |
| vae.save_pretrained(f'{save_folder}/vae') |
| |
| # Keep hop length 800 (=32000/40Hz). decoder_dim must stay >= 128 for 7 upsample stages. |
| with open(hf_hub_download(source_model_id, filename='audio_vae/config.json', repo_type='model'), 'r', encoding='utf-8') as f: |
| audio_config = json.load(f) |
| audio_config.update({ |
| 'encoder_dim': 32, |
| 'latent_dim': 128, |
| 'latent_channels': 8, |
| 'num_attention_heads': 4, |
| 'decoder_dim': 128, |
| 'latents_mean': [0.0] * 8, |
| 'latents_std': [1.0] * 8, |
| }) |
| save_json(f'{save_folder}/audio_vae/config.json', audio_config) |
| audio_vae = AutoencoderKLMiniMaxH3Audio.from_config( |
| AutoencoderKLMiniMaxH3Audio.load_config(f'{save_folder}/audio_vae') |
| ) |
| init_weights(audio_vae) |
| audio_vae.save_pretrained(f'{save_folder}/audio_vae') |
| |
| MiniMaxH3Scheduler.from_pretrained(source_model_id, subfolder='scheduler').save_pretrained( |
| f'{save_folder}/scheduler' |
| ) |
| MiniMaxH3Scheduler.from_pretrained(source_model_id, subfolder='audio_scheduler').save_pretrained( |
| f'{save_folder}/audio_scheduler' |
| ) |
| |
| for index_name in ('model_index.json', 'modular_model_index.json'): |
| index = json.load(open(hf_hub_download(source_model_id, filename=index_name, repo_type='model'), encoding='utf-8')) |
| for value in index.values(): |
| if isinstance(value, list) and len(value) >= 3 and isinstance(value[2], dict): |
| value[2]['pretrained_model_name_or_path'] = save_folder |
| save_json(f'{save_folder}/{index_name}', index) |
| |
| pipe = MiniMaxH3Blocks().init_pipeline(save_folder) |
| pipe.load_components(dtype=torch.bfloat16) |
| pipe.save_pretrained(save_folder, safe_serialization=True, overwrite_modular_index=True) |
| torch.set_default_dtype(torch.float32) |
| print(pipe) |
| ``` |
|
|
| ### Printing the model: |
|
|
| ```text |
| MiniMaxH3ModularPipeline { |
| "_blocks_class_name": "MiniMaxH3Blocks", |
| "_class_name": "MiniMaxH3ModularPipeline", |
| "_diffusers_version": "0.40.0.dev0", |
| "audio_scheduler": [ |
| "diffusers", |
| "MiniMaxH3Scheduler", |
| { |
| "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3", |
| "revision": null, |
| "subfolder": "audio_scheduler", |
| "type_hint": [ |
| "diffusers", |
| "MiniMaxH3Scheduler" |
| ], |
| "variant": null |
| } |
| ], |
| "audio_vae": [ |
| "diffusers", |
| "AutoencoderKLMiniMaxH3Audio", |
| { |
| "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3", |
| "revision": null, |
| "subfolder": "audio_vae", |
| "type_hint": [ |
| "diffusers", |
| "AutoencoderKLMiniMaxH3Audio" |
| ], |
| "variant": null |
| } |
| ], |
| "canvas_max_pixels": 1032192, |
| "canvas_short_edge": 768, |
| "processor": [ |
| "transformers", |
| "Qwen3VLProcessor", |
| { |
| "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3", |
| "revision": null, |
| "subfolder": "processor", |
| "type_hint": [ |
| "transformers", |
| "Qwen3VLProcessor" |
| ], |
| "variant": null |
| } |
| ], |
| "reference_image_short_edge": 2048, |
| "scheduler": [ |
| "diffusers", |
| "MiniMaxH3Scheduler", |
| { |
| "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3", |
| "revision": null, |
| "subfolder": "scheduler", |
| "type_hint": [ |
| "diffusers", |
| "MiniMaxH3Scheduler" |
| ], |
| "variant": null |
| } |
| ], |
| "text_encoder": [ |
| "transformers", |
| "Qwen3VLForConditionalGeneration", |
| { |
| "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3", |
| "revision": null, |
| "subfolder": "text_encoder", |
| "type_hint": [ |
| "transformers", |
| "Qwen3VLForConditionalGeneration" |
| ], |
| "variant": null |
| } |
| ], |
| "tokenizer": [ |
| "transformers", |
| "Qwen2Tokenizer", |
| { |
| "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3", |
| "revision": null, |
| "subfolder": "tokenizer", |
| "type_hint": [ |
| "transformers", |
| "Qwen2Tokenizer" |
| ], |
| "variant": null |
| } |
| ], |
| "transformer": [ |
| "diffusers", |
| "MiniMaxH3Transformer3DModel", |
| { |
| "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3", |
| "revision": null, |
| "subfolder": "transformer", |
| "type_hint": [ |
| "diffusers", |
| "MiniMaxH3Transformer3DModel" |
| ], |
| "variant": null |
| } |
| ], |
| "transformer_ref": [ |
| "diffusers", |
| "MiniMaxH3Transformer3DModel", |
| { |
| "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3", |
| "revision": null, |
| "subfolder": "transformer_ref", |
| "type_hint": [ |
| "diffusers", |
| "MiniMaxH3Transformer3DModel" |
| ], |
| "variant": null |
| } |
| ], |
| "vae": [ |
| "diffusers", |
| "AutoencoderKLMiniMaxH3", |
| { |
| "pretrained_model_name_or_path": "./tmp/tiny-random/minimax-h3", |
| "revision": null, |
| "subfolder": "vae", |
| "type_hint": [ |
| "diffusers", |
| "AutoencoderKLMiniMaxH3" |
| ], |
| "variant": null |
| } |
| ] |
| } |
| ``` |
