| { |
| "transformer": { |
| "in_channels": 128, |
| "out_channels": 128, |
| "audio_in_channels": 8, |
| "audio_out_channels": 8, |
| "num_layers": 4, |
| "num_attention_heads": 2, |
| "attention_head_dim": 32, |
| "audio_num_attention_heads": 2, |
| "audio_attention_head_dim": 32, |
| "cross_attention_dim": 64, |
| "audio_cross_attention_dim": 64, |
| "caption_channels": 8, |
| "use_prompt_embeddings": false, |
| "cross_attn_mod": true, |
| "audio_cross_attn_mod": true, |
| "use_prompt_adaln_single": false, |
| "use_keyframes_abs_pos_embedding": true |
| }, |
| "connectors": { |
| "caption_channels": 8, |
| "text_proj_in_factor": 3, |
| "video_connector_num_attention_heads": 2, |
| "video_connector_attention_head_dim": 32, |
| "video_connector_num_layers": 2, |
| "video_connector_num_learnable_registers": 4, |
| "audio_connector_num_attention_heads": 2, |
| "audio_connector_attention_head_dim": 32, |
| "audio_connector_num_layers": 2, |
| "audio_connector_num_learnable_registers": 4, |
| "video_hidden_dim": 64, |
| "audio_hidden_dim": 64, |
| "per_modality_projections": true |
| }, |
| "vae": { |
| "in_channels": 3, |
| "out_channels": 3, |
| "latent_channels": 128, |
| "block_out_channels": [ |
| 16, |
| 32 |
| ], |
| "decoder_block_out_channels": [ |
| 16 |
| ], |
| "layers_per_block": [ |
| 1, |
| 1, |
| 1 |
| ], |
| "decoder_layers_per_block": [ |
| 1, |
| 1 |
| ], |
| "spatio_temporal_scaling": [ |
| true, |
| true |
| ], |
| "decoder_spatio_temporal_scaling": [ |
| true |
| ], |
| "downsample_type": [ |
| "spatiotemporal", |
| "spatiotemporal" |
| ], |
| "upsample_type": [ |
| "spatiotemporal" |
| ], |
| "upsample_residual": [ |
| true |
| ], |
| "upsample_factor": [ |
| 2 |
| ], |
| "patch_size": 4, |
| "patch_size_t": 1 |
| }, |
| "audio_vae": { |
| "base_channels": 8, |
| "output_channels": 2, |
| "ch_mult": [ |
| 1, |
| 2 |
| ], |
| "num_res_blocks": 1, |
| "in_channels": 2, |
| "resolution": 64, |
| "latent_channels": 8, |
| "mel_bins": 4 |
| }, |
| "vocoder": { |
| "in_channels": 8, |
| "hidden_channels": 32, |
| "out_channels": 2, |
| "upsample_kernel_sizes": [ |
| 8, |
| 8 |
| ], |
| "upsample_factors": [ |
| 4, |
| 4 |
| ], |
| "resnet_kernel_sizes": [ |
| 3 |
| ], |
| "resnet_dilations": [ |
| [ |
| 1, |
| 1, |
| 1 |
| ] |
| ] |
| }, |
| "text_encoder": { |
| "vocab_size": 262144, |
| "hidden_size": 8, |
| "intermediate_size": 16, |
| "num_hidden_layers": 2, |
| "num_attention_heads": 1, |
| "num_key_value_heads": 1, |
| "head_dim": 32, |
| "max_position_embeddings": 4096, |
| "tie_word_embeddings": true |
| } |
| } |
|
|
