| { |
| "format_version": 1, |
| "config": { |
| "_class_name": "MiniMaxH3Transformer3DModel", |
| "_diffusers_version": "0.36.0.dev0", |
| "num_attention_heads": 56, |
| "attention_head_dim": 128, |
| "hidden_size": 5376, |
| "num_layers": 50, |
| "num_refiner_layers": 2, |
| "ffn_dim": 14336, |
| "in_channels": 24, |
| "audio_in_channels": 32, |
| "patch_size": [ |
| 1, |
| 2, |
| 2 |
| ], |
| "text_dim": 5120, |
| "freq_dim": 256, |
| "time_embed_hidden_dim": 5376, |
| "time_embed_dim": 2688, |
| "rope_freq_dim": 16, |
| "rope_theta": 10000.0, |
| "norm_eps": 1e-05, |
| "qk_norm_eps": 1e-05, |
| "final_norm_eps": 1e-05 |
| }, |
| "num_blocks": 50, |
| "num_refiner_blocks": 2, |
| "quantization": { |
| "mode": "affine", |
| "bits": 4, |
| "group_size": 64 |
| }, |
| "quantized_keys": { |
| "blocks.0.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.0.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.0.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.0.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.0.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.0.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.1.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.1.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.1.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.1.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.1.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.1.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.2.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.2.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.2.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.2.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.2.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.2.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.3.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.3.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.3.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.3.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.3.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.3.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.4.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.4.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.4.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.4.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.4.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.4.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.5.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.5.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.5.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.5.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.5.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.5.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.6.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.6.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.6.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.6.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.6.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.6.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.7.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.7.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.7.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.7.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.7.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.7.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.8.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.8.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.8.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.8.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.8.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.8.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.9.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.9.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.9.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.9.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.9.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.9.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.10.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.10.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.10.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.10.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.10.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.10.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.11.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.11.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.11.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.11.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.11.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.11.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.12.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.12.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.12.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.12.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.12.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.12.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.13.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.13.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.13.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.13.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.13.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.13.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.14.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.14.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.14.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.14.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.14.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.14.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.15.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.15.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.15.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.15.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.15.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.15.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.16.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.16.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.16.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.16.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.16.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.16.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.17.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.17.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.17.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.17.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.17.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.17.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.18.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.18.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.18.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.18.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.18.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.18.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.19.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.19.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.19.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.19.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.19.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.19.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.20.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.20.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.20.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.20.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.20.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.20.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.21.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.21.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.21.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.21.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.21.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.21.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.22.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.22.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.22.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.22.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.22.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.22.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.23.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.23.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.23.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.23.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.23.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.23.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.24.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.24.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.24.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.24.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.24.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.24.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.25.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.25.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.25.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.25.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.25.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.25.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.26.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.26.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.26.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.26.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.26.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.26.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.27.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.27.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.27.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.27.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.27.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.27.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.28.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.28.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.28.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.28.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.28.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.28.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.29.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.29.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.29.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.29.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.29.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.29.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.30.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.30.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.30.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.30.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.30.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.30.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.31.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.31.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.31.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.31.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.31.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.31.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.32.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.32.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.32.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.32.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.32.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.32.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.33.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.33.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.33.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.33.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.33.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.33.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.34.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.34.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.34.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.34.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.34.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.34.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.35.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.35.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.35.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.35.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.35.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.35.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.36.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.36.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.36.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.36.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.36.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.36.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.37.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.37.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.37.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.37.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.37.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.37.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.38.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.38.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.38.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.38.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.38.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.38.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.39.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.39.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.39.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.39.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.39.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.39.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.40.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.40.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.40.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.40.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.40.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.40.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.41.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.41.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.41.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.41.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.41.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.41.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.42.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.42.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.42.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.42.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.42.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.42.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.43.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.43.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.43.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.43.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.43.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.43.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.44.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.44.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.44.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.44.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.44.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.44.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.45.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.45.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.45.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.45.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.45.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.45.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.46.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.46.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.46.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.46.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.46.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.46.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.47.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.47.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.47.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.47.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.47.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.47.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.48.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.48.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.48.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.48.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.48.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.48.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.49.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.49.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.49.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.49.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.49.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "blocks.49.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "refiner.0.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "refiner.0.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "refiner.0.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "refiner.0.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "refiner.0.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "refiner.0.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "refiner.1.ff.net.2.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "refiner.1.attn.to_v.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "refiner.1.attn.to_out.0.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "refiner.1.attn.to_k.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "refiner.1.ff.net.0.proj.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| }, |
| "refiner.1.attn.to_q.weight": { |
| "dequantized_dtype": "bf16", |
| "has_biases": true |
| } |
| }, |
| "adaln_cache": { |
| "timesteps": [ |
| 0.0, |
| 0.027027010917663574, |
| 0.07692307233810425, |
| 0.10000002384185791, |
| 0.19999998807907104, |
| 0.25, |
| 0.5, |
| 1.0 |
| ], |
| "num_blocks": 50, |
| "tables_per_block": 6 |
| }, |
| "vsa": { |
| "capable": false, |
| "num_gate_matrices": 0, |
| "gate_key_suffix": "attn.to_gate_compress.weight", |
| "attention_activations": "bf16", |
| "note": "Gate projections are quantized with the checkpoint's affine INT8/INT6/INT4 weight-only grid. VSA attention Q/K/V remain unquantized activations." |
| } |
| } |
