| { |
| "schemaVersion": 1, |
| "modelFamily": "ming_3_tts_tiny_language", |
| "alignment": 16384, |
| "file": "weights.bin", |
| "quantizedLinears": { |
| "model.model.layers.1.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.1.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.1.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.10.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.10.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.10.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.11.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.11.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.11.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.12.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.12.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.12.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.13.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.13.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.13.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.14.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.14.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.14.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.15.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.15.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.15.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.16.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.16.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.16.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.17.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.17.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.17.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.18.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.18.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.18.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.19.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.19.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.19.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.2.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.2.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.2.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.20.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.20.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.20.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.21.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.21.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.21.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.22.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.22.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.22.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.23.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.23.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.23.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.3.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.3.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.3.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.4.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.4.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.4.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.5.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.5.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.5.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.6.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.6.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.6.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.7.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.7.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.7.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.8.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.8.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.8.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.9.mlp.experts.down_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.9.mlp.experts.gate_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| }, |
| "model.model.layers.9.mlp.experts.up_proj": { |
| "bits": 4, |
| "groupSize": 64 |
| } |
| }, |
| "tensors": { |
| "model.model.layers.1.mlp.experts.down_proj.weight": { |
| "offset": 0, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.1.mlp.experts.down_proj.scales": { |
| "offset": 50331648, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.1.mlp.experts.down_proj.biases": { |
| "offset": 56623104, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.1.mlp.experts.gate_proj.weight": { |
| "offset": 62914560, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.1.mlp.experts.gate_proj.scales": { |
| "offset": 113246208, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.1.mlp.experts.gate_proj.biases": { |
| "offset": 119537664, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.1.mlp.experts.up_proj.weight": { |
| "offset": 125829120, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.1.mlp.experts.up_proj.scales": { |
| "offset": 176160768, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.1.mlp.experts.up_proj.biases": { |
| "offset": 182452224, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.10.mlp.experts.down_proj.weight": { |
| "offset": 188743680, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.10.mlp.experts.down_proj.scales": { |
| "offset": 239075328, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.10.mlp.experts.down_proj.biases": { |
| "offset": 245366784, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.10.mlp.experts.gate_proj.weight": { |
| "offset": 251658240, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.10.mlp.experts.gate_proj.scales": { |
| "offset": 301989888, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.10.mlp.experts.gate_proj.biases": { |
| "offset": 308281344, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.10.mlp.experts.up_proj.weight": { |
| "offset": 314572800, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.10.mlp.experts.up_proj.scales": { |
| "offset": 364904448, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.10.mlp.experts.up_proj.biases": { |
| "offset": 371195904, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.11.mlp.experts.down_proj.weight": { |
| "offset": 377487360, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.11.mlp.experts.down_proj.scales": { |
| "offset": 427819008, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.11.mlp.experts.down_proj.biases": { |
| "offset": 434110464, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.11.mlp.experts.gate_proj.weight": { |
| "offset": 440401920, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.11.mlp.experts.gate_proj.scales": { |
| "offset": 490733568, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.11.mlp.experts.gate_proj.biases": { |
| "offset": 497025024, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.11.mlp.experts.up_proj.weight": { |
| "offset": 503316480, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.11.mlp.experts.up_proj.scales": { |
| "offset": 553648128, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.11.mlp.experts.up_proj.biases": { |
| "offset": 559939584, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.12.mlp.experts.down_proj.weight": { |
| "offset": 566231040, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.12.mlp.experts.down_proj.scales": { |
| "offset": 616562688, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.12.mlp.experts.down_proj.biases": { |
| "offset": 622854144, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.12.mlp.experts.gate_proj.weight": { |
| "offset": 629145600, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.12.mlp.experts.gate_proj.scales": { |
| "offset": 679477248, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.12.mlp.experts.gate_proj.biases": { |
| "offset": 685768704, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.12.mlp.experts.up_proj.weight": { |
| "offset": 692060160, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.12.mlp.experts.up_proj.scales": { |
| "offset": 742391808, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.12.mlp.experts.up_proj.biases": { |
| "offset": 748683264, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.13.mlp.experts.down_proj.weight": { |
| "offset": 754974720, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.13.mlp.experts.down_proj.scales": { |
| "offset": 805306368, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.13.mlp.experts.down_proj.biases": { |
| "offset": 811597824, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.13.mlp.experts.gate_proj.weight": { |
| "offset": 817889280, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.13.mlp.experts.gate_proj.scales": { |
| "offset": 868220928, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.13.mlp.experts.gate_proj.biases": { |
| "offset": 874512384, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.13.mlp.experts.up_proj.weight": { |
| "offset": 880803840, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.13.mlp.experts.up_proj.scales": { |
| "offset": 931135488, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.13.mlp.experts.up_proj.biases": { |
| "offset": 937426944, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.14.mlp.experts.down_proj.weight": { |
| "offset": 943718400, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.14.mlp.experts.down_proj.scales": { |
| "offset": 994050048, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.14.mlp.experts.down_proj.biases": { |
| "offset": 1000341504, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.14.mlp.experts.gate_proj.weight": { |
| "offset": 1006632960, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.14.mlp.experts.gate_proj.scales": { |
| "offset": 1056964608, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.14.mlp.experts.gate_proj.biases": { |
| "offset": 1063256064, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.14.mlp.experts.up_proj.weight": { |
| "offset": 1069547520, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.14.mlp.experts.up_proj.scales": { |
| "offset": 1119879168, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.14.mlp.experts.up_proj.biases": { |
| "offset": 1126170624, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.15.mlp.experts.down_proj.weight": { |
| "offset": 1132462080, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.15.mlp.experts.down_proj.scales": { |
| "offset": 1182793728, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.15.mlp.experts.down_proj.biases": { |
| "offset": 1189085184, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.15.mlp.experts.gate_proj.weight": { |
| "offset": 1195376640, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.15.mlp.experts.gate_proj.scales": { |
| "offset": 1245708288, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.15.mlp.experts.gate_proj.biases": { |
| "offset": 1251999744, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.15.mlp.experts.up_proj.weight": { |
| "offset": 1258291200, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.15.mlp.experts.up_proj.scales": { |
| "offset": 1308622848, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.15.mlp.experts.up_proj.biases": { |
| "offset": 1314914304, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.16.mlp.experts.down_proj.weight": { |
| "offset": 1321205760, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.16.mlp.experts.down_proj.scales": { |
| "offset": 1371537408, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.16.mlp.experts.down_proj.biases": { |
| "offset": 1377828864, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.16.mlp.experts.gate_proj.weight": { |
| "offset": 1384120320, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.16.mlp.experts.gate_proj.scales": { |
| "offset": 1434451968, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.16.mlp.experts.gate_proj.biases": { |
| "offset": 1440743424, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.16.mlp.experts.up_proj.weight": { |
| "offset": 1447034880, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.16.mlp.experts.up_proj.scales": { |
| "offset": 1497366528, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.16.mlp.experts.up_proj.biases": { |
| "offset": 1503657984, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.17.mlp.experts.down_proj.weight": { |
| "offset": 1509949440, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.17.mlp.experts.down_proj.scales": { |
| "offset": 1560281088, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.17.mlp.experts.down_proj.biases": { |
| "offset": 1566572544, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.17.mlp.experts.gate_proj.weight": { |
| "offset": 1572864000, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.17.mlp.experts.gate_proj.scales": { |
| "offset": 1623195648, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.17.mlp.experts.gate_proj.biases": { |
| "offset": 1629487104, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.17.mlp.experts.up_proj.weight": { |
| "offset": 1635778560, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.17.mlp.experts.up_proj.scales": { |
| "offset": 1686110208, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.17.mlp.experts.up_proj.biases": { |
| "offset": 1692401664, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.18.mlp.experts.down_proj.weight": { |
| "offset": 1698693120, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.18.mlp.experts.down_proj.scales": { |
| "offset": 1749024768, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.18.mlp.experts.down_proj.biases": { |
| "offset": 1755316224, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.18.mlp.experts.gate_proj.weight": { |
| "offset": 1761607680, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.18.mlp.experts.gate_proj.scales": { |
| "offset": 1811939328, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.18.mlp.experts.gate_proj.biases": { |
| "offset": 1818230784, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.18.mlp.experts.up_proj.weight": { |
| "offset": 1824522240, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.18.mlp.experts.up_proj.scales": { |
| "offset": 1874853888, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.18.mlp.experts.up_proj.biases": { |
| "offset": 1881145344, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.19.mlp.experts.down_proj.weight": { |
| "offset": 1887436800, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.19.mlp.experts.down_proj.scales": { |
| "offset": 1937768448, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.19.mlp.experts.down_proj.biases": { |
| "offset": 1944059904, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.19.mlp.experts.gate_proj.weight": { |
| "offset": 1950351360, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.19.mlp.experts.gate_proj.scales": { |
| "offset": 2000683008, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.19.mlp.experts.gate_proj.biases": { |
| "offset": 2006974464, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.19.mlp.experts.up_proj.weight": { |
| "offset": 2013265920, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.19.mlp.experts.up_proj.scales": { |
| "offset": 2063597568, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.19.mlp.experts.up_proj.biases": { |
| "offset": 2069889024, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.2.mlp.experts.down_proj.weight": { |
| "offset": 2076180480, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.2.mlp.experts.down_proj.scales": { |
| "offset": 2126512128, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.2.mlp.experts.down_proj.biases": { |
| "offset": 2132803584, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.2.mlp.experts.gate_proj.weight": { |
| "offset": 2139095040, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.2.mlp.experts.gate_proj.scales": { |
| "offset": 2189426688, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.2.mlp.experts.gate_proj.biases": { |
| "offset": 2195718144, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.2.mlp.experts.up_proj.weight": { |
| "offset": 2202009600, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.2.mlp.experts.up_proj.scales": { |
| "offset": 2252341248, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.2.mlp.experts.up_proj.biases": { |
| "offset": 2258632704, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.20.mlp.experts.down_proj.weight": { |
| "offset": 2264924160, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.20.mlp.experts.down_proj.scales": { |
| "offset": 2315255808, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.20.mlp.experts.down_proj.biases": { |
| "offset": 2321547264, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.20.mlp.experts.gate_proj.weight": { |
| "offset": 2327838720, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.20.mlp.experts.gate_proj.scales": { |
| "offset": 2378170368, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.20.mlp.experts.gate_proj.biases": { |
| "offset": 2384461824, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.20.mlp.experts.up_proj.weight": { |
| "offset": 2390753280, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.20.mlp.experts.up_proj.scales": { |
| "offset": 2441084928, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.20.mlp.experts.up_proj.biases": { |
| "offset": 2447376384, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.21.mlp.experts.down_proj.weight": { |
| "offset": 2453667840, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.21.mlp.experts.down_proj.scales": { |
| "offset": 2503999488, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.21.mlp.experts.down_proj.biases": { |
| "offset": 2510290944, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.21.mlp.experts.gate_proj.weight": { |
| "offset": 2516582400, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.21.mlp.experts.gate_proj.scales": { |
| "offset": 2566914048, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.21.mlp.experts.gate_proj.biases": { |
| "offset": 2573205504, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.21.mlp.experts.up_proj.weight": { |
| "offset": 2579496960, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.21.mlp.experts.up_proj.scales": { |
| "offset": 2629828608, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.21.mlp.experts.up_proj.biases": { |
| "offset": 2636120064, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.22.mlp.experts.down_proj.weight": { |
| "offset": 2642411520, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.22.mlp.experts.down_proj.scales": { |
| "offset": 2692743168, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.22.mlp.experts.down_proj.biases": { |
| "offset": 2699034624, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.22.mlp.experts.gate_proj.weight": { |
| "offset": 2705326080, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.22.mlp.experts.gate_proj.scales": { |
| "offset": 2755657728, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.22.mlp.experts.gate_proj.biases": { |
| "offset": 2761949184, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.22.mlp.experts.up_proj.weight": { |
| "offset": 2768240640, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.22.mlp.experts.up_proj.scales": { |
| "offset": 2818572288, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.22.mlp.experts.up_proj.biases": { |
| "offset": 2824863744, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.23.mlp.experts.down_proj.weight": { |
| "offset": 2831155200, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.23.mlp.experts.down_proj.scales": { |
| "offset": 2881486848, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.23.mlp.experts.down_proj.biases": { |
| "offset": 2887778304, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.23.mlp.experts.gate_proj.weight": { |
| "offset": 2894069760, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.23.mlp.experts.gate_proj.scales": { |
| "offset": 2944401408, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.23.mlp.experts.gate_proj.biases": { |
| "offset": 2950692864, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.23.mlp.experts.up_proj.weight": { |
| "offset": 2956984320, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.23.mlp.experts.up_proj.scales": { |
| "offset": 3007315968, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.23.mlp.experts.up_proj.biases": { |
| "offset": 3013607424, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.3.mlp.experts.down_proj.weight": { |
| "offset": 3019898880, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.3.mlp.experts.down_proj.scales": { |
| "offset": 3070230528, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.3.mlp.experts.down_proj.biases": { |
| "offset": 3076521984, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.3.mlp.experts.gate_proj.weight": { |
| "offset": 3082813440, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.3.mlp.experts.gate_proj.scales": { |
| "offset": 3133145088, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.3.mlp.experts.gate_proj.biases": { |
| "offset": 3139436544, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.3.mlp.experts.up_proj.weight": { |
| "offset": 3145728000, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.3.mlp.experts.up_proj.scales": { |
| "offset": 3196059648, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.3.mlp.experts.up_proj.biases": { |
| "offset": 3202351104, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.4.mlp.experts.down_proj.weight": { |
| "offset": 3208642560, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.4.mlp.experts.down_proj.scales": { |
| "offset": 3258974208, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.4.mlp.experts.down_proj.biases": { |
| "offset": 3265265664, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.4.mlp.experts.gate_proj.weight": { |
| "offset": 3271557120, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.4.mlp.experts.gate_proj.scales": { |
| "offset": 3321888768, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.4.mlp.experts.gate_proj.biases": { |
| "offset": 3328180224, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.4.mlp.experts.up_proj.weight": { |
| "offset": 3334471680, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.4.mlp.experts.up_proj.scales": { |
| "offset": 3384803328, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.4.mlp.experts.up_proj.biases": { |
| "offset": 3391094784, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.5.mlp.experts.down_proj.weight": { |
| "offset": 3397386240, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.5.mlp.experts.down_proj.scales": { |
| "offset": 3447717888, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.5.mlp.experts.down_proj.biases": { |
| "offset": 3454009344, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.5.mlp.experts.gate_proj.weight": { |
| "offset": 3460300800, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.5.mlp.experts.gate_proj.scales": { |
| "offset": 3510632448, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.5.mlp.experts.gate_proj.biases": { |
| "offset": 3516923904, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.5.mlp.experts.up_proj.weight": { |
| "offset": 3523215360, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.5.mlp.experts.up_proj.scales": { |
| "offset": 3573547008, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.5.mlp.experts.up_proj.biases": { |
| "offset": 3579838464, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.6.mlp.experts.down_proj.weight": { |
| "offset": 3586129920, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.6.mlp.experts.down_proj.scales": { |
| "offset": 3636461568, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.6.mlp.experts.down_proj.biases": { |
| "offset": 3642753024, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.6.mlp.experts.gate_proj.weight": { |
| "offset": 3649044480, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.6.mlp.experts.gate_proj.scales": { |
| "offset": 3699376128, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.6.mlp.experts.gate_proj.biases": { |
| "offset": 3705667584, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.6.mlp.experts.up_proj.weight": { |
| "offset": 3711959040, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.6.mlp.experts.up_proj.scales": { |
| "offset": 3762290688, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.6.mlp.experts.up_proj.biases": { |
| "offset": 3768582144, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.7.mlp.experts.down_proj.weight": { |
| "offset": 3774873600, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.7.mlp.experts.down_proj.scales": { |
| "offset": 3825205248, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.7.mlp.experts.down_proj.biases": { |
| "offset": 3831496704, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.7.mlp.experts.gate_proj.weight": { |
| "offset": 3837788160, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.7.mlp.experts.gate_proj.scales": { |
| "offset": 3888119808, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.7.mlp.experts.gate_proj.biases": { |
| "offset": 3894411264, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.7.mlp.experts.up_proj.weight": { |
| "offset": 3900702720, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.7.mlp.experts.up_proj.scales": { |
| "offset": 3951034368, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.7.mlp.experts.up_proj.biases": { |
| "offset": 3957325824, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.8.mlp.experts.down_proj.weight": { |
| "offset": 3963617280, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.8.mlp.experts.down_proj.scales": { |
| "offset": 4013948928, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.8.mlp.experts.down_proj.biases": { |
| "offset": 4020240384, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.8.mlp.experts.gate_proj.weight": { |
| "offset": 4026531840, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.8.mlp.experts.gate_proj.scales": { |
| "offset": 4076863488, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.8.mlp.experts.gate_proj.biases": { |
| "offset": 4083154944, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.8.mlp.experts.up_proj.weight": { |
| "offset": 4089446400, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.8.mlp.experts.up_proj.scales": { |
| "offset": 4139778048, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.8.mlp.experts.up_proj.biases": { |
| "offset": 4146069504, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.9.mlp.experts.down_proj.weight": { |
| "offset": 4152360960, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 1536, |
| 64 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.9.mlp.experts.down_proj.scales": { |
| "offset": 4202692608, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.9.mlp.experts.down_proj.biases": { |
| "offset": 4208984064, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 1536, |
| 8 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.9.mlp.experts.gate_proj.weight": { |
| "offset": 4215275520, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.9.mlp.experts.gate_proj.scales": { |
| "offset": 4265607168, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.9.mlp.experts.gate_proj.biases": { |
| "offset": 4271898624, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.9.mlp.experts.up_proj.weight": { |
| "offset": 4278190080, |
| "bytes": 50331648, |
| "shape": [ |
| 128, |
| 512, |
| 192 |
| ], |
| "dtype": "uint32" |
| }, |
| "model.model.layers.9.mlp.experts.up_proj.scales": { |
| "offset": 4328521728, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.9.mlp.experts.up_proj.biases": { |
| "offset": 4334813184, |
| "bytes": 6291456, |
| "shape": [ |
| 128, |
| 512, |
| 24 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.0.attention.A_log": { |
| "offset": 4341104640, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.0.attention.b_proj.weight": { |
| "offset": 4341121024, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.attention.dt_bias": { |
| "offset": 4341170176, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.0.attention.f_proj.weight": { |
| "offset": 4341186560, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.attention.g_proj.weight": { |
| "offset": 4347478016, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.attention.k_conv1d.weight": { |
| "offset": 4353769472, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.attention.k_proj.weight": { |
| "offset": 4353785856, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.attention.o_norm.weight": { |
| "offset": 4360077312, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.attention.o_proj.weight": { |
| "offset": 4360093696, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.attention.q_conv1d.weight": { |
| "offset": 4366385152, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.attention.q_proj.weight": { |
| "offset": 4366401536, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.attention.v_conv1d.weight": { |
| "offset": 4372692992, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.attention.v_proj.weight": { |
| "offset": 4372709376, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.input_layernorm.weight": { |
| "offset": 4379000832, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.mlp.down_proj.weight": { |
| "offset": 4379017216, |
| "bytes": 14155776, |
| "shape": [ |
| 1536, |
| 4608 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.mlp.gate_proj.weight": { |
| "offset": 4393172992, |
| "bytes": 14155776, |
| "shape": [ |
| 4608, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.mlp.up_proj.weight": { |
| "offset": 4407328768, |
| "bytes": 14155776, |
| "shape": [ |
| 4608, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.0.post_attention_layernorm.weight": { |
| "offset": 4421484544, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.attention.A_log": { |
| "offset": 4421500928, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.1.attention.b_proj.weight": { |
| "offset": 4421517312, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.attention.dt_bias": { |
| "offset": 4421566464, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.1.attention.f_proj.weight": { |
| "offset": 4421582848, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.attention.g_proj.weight": { |
| "offset": 4427874304, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.attention.k_conv1d.weight": { |
| "offset": 4434165760, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.attention.k_proj.weight": { |
| "offset": 4434182144, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.attention.o_norm.weight": { |
| "offset": 4440473600, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.attention.o_proj.weight": { |
| "offset": 4440489984, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.attention.q_conv1d.weight": { |
| "offset": 4446781440, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.attention.q_proj.weight": { |
| "offset": 4446797824, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.attention.v_conv1d.weight": { |
| "offset": 4453089280, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.attention.v_proj.weight": { |
| "offset": 4453105664, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.input_layernorm.weight": { |
| "offset": 4459397120, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.mlp.gate.expert_bias": { |
| "offset": 4459413504, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.1.mlp.gate.weight": { |
| "offset": 4459429888, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.1.mlp.shared_experts.down_proj.weight": { |
| "offset": 4460216320, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4461789184, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.mlp.shared_experts.up_proj.weight": { |
| "offset": 4463362048, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.1.post_attention_layernorm.weight": { |
| "offset": 4464934912, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.attention.A_log": { |
| "offset": 4464951296, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.10.attention.b_proj.weight": { |
| "offset": 4464967680, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.attention.dt_bias": { |
| "offset": 4465016832, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.10.attention.f_proj.weight": { |
| "offset": 4465033216, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.attention.g_proj.weight": { |
| "offset": 4471324672, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.attention.k_conv1d.weight": { |
| "offset": 4477616128, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.attention.k_proj.weight": { |
| "offset": 4477632512, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.attention.o_norm.weight": { |
| "offset": 4483923968, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.attention.o_proj.weight": { |
| "offset": 4483940352, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.attention.q_conv1d.weight": { |
| "offset": 4490231808, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.attention.q_proj.weight": { |
| "offset": 4490248192, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.attention.v_conv1d.weight": { |
| "offset": 4496539648, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.attention.v_proj.weight": { |
| "offset": 4496556032, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.input_layernorm.weight": { |
| "offset": 4502847488, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.mlp.gate.expert_bias": { |
| "offset": 4502863872, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.10.mlp.gate.weight": { |
| "offset": 4502880256, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.10.mlp.shared_experts.down_proj.weight": { |
| "offset": 4503666688, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4505239552, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.mlp.shared_experts.up_proj.weight": { |
| "offset": 4506812416, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.10.post_attention_layernorm.weight": { |
| "offset": 4508385280, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.11.attention.dense.weight": { |
| "offset": 4508401664, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.11.attention.g_proj.weight": { |
| "offset": 4514693120, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.11.attention.kv_a_layernorm.weight": { |
| "offset": 4514742272, |
| "bytes": 1024, |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.11.attention.kv_a_proj_with_mqa.weight": { |
| "offset": 4514758656, |
| "bytes": 1769472, |
| "shape": [ |
| 576, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.11.attention.kv_b_proj.weight": { |
| "offset": 4516528128, |
| "bytes": 4194304, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.11.attention.q_a_layernorm.weight": { |
| "offset": 4520722432, |
| "bytes": 512, |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.11.attention.q_a_proj.weight": { |
| "offset": 4520738816, |
| "bytes": 786432, |
| "shape": [ |
| 256, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.11.attention.q_b_proj.weight": { |
| "offset": 4521525248, |
| "bytes": 1572864, |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.11.input_layernorm.weight": { |
| "offset": 4523098112, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.11.mlp.gate.expert_bias": { |
| "offset": 4523114496, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.11.mlp.gate.weight": { |
| "offset": 4523130880, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.11.mlp.shared_experts.down_proj.weight": { |
| "offset": 4523917312, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.11.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4525490176, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.11.mlp.shared_experts.up_proj.weight": { |
| "offset": 4527063040, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.11.post_attention_layernorm.weight": { |
| "offset": 4528635904, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.attention.A_log": { |
| "offset": 4528652288, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.12.attention.b_proj.weight": { |
| "offset": 4528668672, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.attention.dt_bias": { |
| "offset": 4528717824, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.12.attention.f_proj.weight": { |
| "offset": 4528734208, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.attention.g_proj.weight": { |
| "offset": 4535025664, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.attention.k_conv1d.weight": { |
| "offset": 4541317120, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.attention.k_proj.weight": { |
| "offset": 4541333504, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.attention.o_norm.weight": { |
| "offset": 4547624960, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.attention.o_proj.weight": { |
| "offset": 4547641344, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.attention.q_conv1d.weight": { |
| "offset": 4553932800, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.attention.q_proj.weight": { |
| "offset": 4553949184, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.attention.v_conv1d.weight": { |
| "offset": 4560240640, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.attention.v_proj.weight": { |
| "offset": 4560257024, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.input_layernorm.weight": { |
| "offset": 4566548480, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.mlp.gate.expert_bias": { |
| "offset": 4566564864, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.12.mlp.gate.weight": { |
| "offset": 4566581248, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.12.mlp.shared_experts.down_proj.weight": { |
| "offset": 4567367680, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4568940544, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.mlp.shared_experts.up_proj.weight": { |
| "offset": 4570513408, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.12.post_attention_layernorm.weight": { |
| "offset": 4572086272, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.attention.A_log": { |
| "offset": 4572102656, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.13.attention.b_proj.weight": { |
| "offset": 4572119040, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.attention.dt_bias": { |
| "offset": 4572168192, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.13.attention.f_proj.weight": { |
| "offset": 4572184576, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.attention.g_proj.weight": { |
| "offset": 4578476032, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.attention.k_conv1d.weight": { |
| "offset": 4584767488, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.attention.k_proj.weight": { |
| "offset": 4584783872, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.attention.o_norm.weight": { |
| "offset": 4591075328, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.attention.o_proj.weight": { |
| "offset": 4591091712, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.attention.q_conv1d.weight": { |
| "offset": 4597383168, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.attention.q_proj.weight": { |
| "offset": 4597399552, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.attention.v_conv1d.weight": { |
| "offset": 4603691008, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.attention.v_proj.weight": { |
| "offset": 4603707392, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.input_layernorm.weight": { |
| "offset": 4609998848, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.mlp.gate.expert_bias": { |
| "offset": 4610015232, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.13.mlp.gate.weight": { |
| "offset": 4610031616, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.13.mlp.shared_experts.down_proj.weight": { |
| "offset": 4610818048, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4612390912, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.mlp.shared_experts.up_proj.weight": { |
| "offset": 4613963776, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.13.post_attention_layernorm.weight": { |
| "offset": 4615536640, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.attention.A_log": { |
| "offset": 4615553024, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.14.attention.b_proj.weight": { |
| "offset": 4615569408, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.attention.dt_bias": { |
| "offset": 4615618560, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.14.attention.f_proj.weight": { |
| "offset": 4615634944, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.attention.g_proj.weight": { |
| "offset": 4621926400, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.attention.k_conv1d.weight": { |
| "offset": 4628217856, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.attention.k_proj.weight": { |
| "offset": 4628234240, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.attention.o_norm.weight": { |
| "offset": 4634525696, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.attention.o_proj.weight": { |
| "offset": 4634542080, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.attention.q_conv1d.weight": { |
| "offset": 4640833536, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.attention.q_proj.weight": { |
| "offset": 4640849920, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.attention.v_conv1d.weight": { |
| "offset": 4647141376, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.attention.v_proj.weight": { |
| "offset": 4647157760, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.input_layernorm.weight": { |
| "offset": 4653449216, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.mlp.gate.expert_bias": { |
| "offset": 4653465600, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.14.mlp.gate.weight": { |
| "offset": 4653481984, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.14.mlp.shared_experts.down_proj.weight": { |
| "offset": 4654268416, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4655841280, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.mlp.shared_experts.up_proj.weight": { |
| "offset": 4657414144, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.14.post_attention_layernorm.weight": { |
| "offset": 4658987008, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.15.attention.dense.weight": { |
| "offset": 4659003392, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.15.attention.g_proj.weight": { |
| "offset": 4665294848, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.15.attention.kv_a_layernorm.weight": { |
| "offset": 4665344000, |
| "bytes": 1024, |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.15.attention.kv_a_proj_with_mqa.weight": { |
| "offset": 4665360384, |
| "bytes": 1769472, |
| "shape": [ |
| 576, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.15.attention.kv_b_proj.weight": { |
| "offset": 4667129856, |
| "bytes": 4194304, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.15.attention.q_a_layernorm.weight": { |
| "offset": 4671324160, |
| "bytes": 512, |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.15.attention.q_a_proj.weight": { |
| "offset": 4671340544, |
| "bytes": 786432, |
| "shape": [ |
| 256, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.15.attention.q_b_proj.weight": { |
| "offset": 4672126976, |
| "bytes": 1572864, |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.15.input_layernorm.weight": { |
| "offset": 4673699840, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.15.mlp.gate.expert_bias": { |
| "offset": 4673716224, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.15.mlp.gate.weight": { |
| "offset": 4673732608, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.15.mlp.shared_experts.down_proj.weight": { |
| "offset": 4674519040, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.15.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4676091904, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.15.mlp.shared_experts.up_proj.weight": { |
| "offset": 4677664768, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.15.post_attention_layernorm.weight": { |
| "offset": 4679237632, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.attention.A_log": { |
| "offset": 4679254016, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.16.attention.b_proj.weight": { |
| "offset": 4679270400, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.attention.dt_bias": { |
| "offset": 4679319552, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.16.attention.f_proj.weight": { |
| "offset": 4679335936, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.attention.g_proj.weight": { |
| "offset": 4685627392, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.attention.k_conv1d.weight": { |
| "offset": 4691918848, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.attention.k_proj.weight": { |
| "offset": 4691935232, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.attention.o_norm.weight": { |
| "offset": 4698226688, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.attention.o_proj.weight": { |
| "offset": 4698243072, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.attention.q_conv1d.weight": { |
| "offset": 4704534528, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.attention.q_proj.weight": { |
| "offset": 4704550912, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.attention.v_conv1d.weight": { |
| "offset": 4710842368, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.attention.v_proj.weight": { |
| "offset": 4710858752, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.input_layernorm.weight": { |
| "offset": 4717150208, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.mlp.gate.expert_bias": { |
| "offset": 4717166592, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.16.mlp.gate.weight": { |
| "offset": 4717182976, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.16.mlp.shared_experts.down_proj.weight": { |
| "offset": 4717969408, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4719542272, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.mlp.shared_experts.up_proj.weight": { |
| "offset": 4721115136, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.16.post_attention_layernorm.weight": { |
| "offset": 4722688000, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.attention.A_log": { |
| "offset": 4722704384, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.17.attention.b_proj.weight": { |
| "offset": 4722720768, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.attention.dt_bias": { |
| "offset": 4722769920, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.17.attention.f_proj.weight": { |
| "offset": 4722786304, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.attention.g_proj.weight": { |
| "offset": 4729077760, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.attention.k_conv1d.weight": { |
| "offset": 4735369216, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.attention.k_proj.weight": { |
| "offset": 4735385600, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.attention.o_norm.weight": { |
| "offset": 4741677056, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.attention.o_proj.weight": { |
| "offset": 4741693440, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.attention.q_conv1d.weight": { |
| "offset": 4747984896, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.attention.q_proj.weight": { |
| "offset": 4748001280, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.attention.v_conv1d.weight": { |
| "offset": 4754292736, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.attention.v_proj.weight": { |
| "offset": 4754309120, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.input_layernorm.weight": { |
| "offset": 4760600576, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.mlp.gate.expert_bias": { |
| "offset": 4760616960, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.17.mlp.gate.weight": { |
| "offset": 4760633344, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.17.mlp.shared_experts.down_proj.weight": { |
| "offset": 4761419776, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4762992640, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.mlp.shared_experts.up_proj.weight": { |
| "offset": 4764565504, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.17.post_attention_layernorm.weight": { |
| "offset": 4766138368, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.attention.A_log": { |
| "offset": 4766154752, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.18.attention.b_proj.weight": { |
| "offset": 4766171136, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.attention.dt_bias": { |
| "offset": 4766220288, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.18.attention.f_proj.weight": { |
| "offset": 4766236672, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.attention.g_proj.weight": { |
| "offset": 4772528128, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.attention.k_conv1d.weight": { |
| "offset": 4778819584, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.attention.k_proj.weight": { |
| "offset": 4778835968, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.attention.o_norm.weight": { |
| "offset": 4785127424, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.attention.o_proj.weight": { |
| "offset": 4785143808, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.attention.q_conv1d.weight": { |
| "offset": 4791435264, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.attention.q_proj.weight": { |
| "offset": 4791451648, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.attention.v_conv1d.weight": { |
| "offset": 4797743104, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.attention.v_proj.weight": { |
| "offset": 4797759488, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.input_layernorm.weight": { |
| "offset": 4804050944, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.mlp.gate.expert_bias": { |
| "offset": 4804067328, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.18.mlp.gate.weight": { |
| "offset": 4804083712, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.18.mlp.shared_experts.down_proj.weight": { |
| "offset": 4804870144, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4806443008, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.mlp.shared_experts.up_proj.weight": { |
| "offset": 4808015872, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.18.post_attention_layernorm.weight": { |
| "offset": 4809588736, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.19.attention.dense.weight": { |
| "offset": 4809605120, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.19.attention.g_proj.weight": { |
| "offset": 4815896576, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.19.attention.kv_a_layernorm.weight": { |
| "offset": 4815945728, |
| "bytes": 1024, |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.19.attention.kv_a_proj_with_mqa.weight": { |
| "offset": 4815962112, |
| "bytes": 1769472, |
| "shape": [ |
| 576, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.19.attention.kv_b_proj.weight": { |
| "offset": 4817731584, |
| "bytes": 4194304, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.19.attention.q_a_layernorm.weight": { |
| "offset": 4821925888, |
| "bytes": 512, |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.19.attention.q_a_proj.weight": { |
| "offset": 4821942272, |
| "bytes": 786432, |
| "shape": [ |
| 256, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.19.attention.q_b_proj.weight": { |
| "offset": 4822728704, |
| "bytes": 1572864, |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.19.input_layernorm.weight": { |
| "offset": 4824301568, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.19.mlp.gate.expert_bias": { |
| "offset": 4824317952, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.19.mlp.gate.weight": { |
| "offset": 4824334336, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.19.mlp.shared_experts.down_proj.weight": { |
| "offset": 4825120768, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.19.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4826693632, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.19.mlp.shared_experts.up_proj.weight": { |
| "offset": 4828266496, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.19.post_attention_layernorm.weight": { |
| "offset": 4829839360, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.attention.A_log": { |
| "offset": 4829855744, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.2.attention.b_proj.weight": { |
| "offset": 4829872128, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.attention.dt_bias": { |
| "offset": 4829921280, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.2.attention.f_proj.weight": { |
| "offset": 4829937664, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.attention.g_proj.weight": { |
| "offset": 4836229120, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.attention.k_conv1d.weight": { |
| "offset": 4842520576, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.attention.k_proj.weight": { |
| "offset": 4842536960, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.attention.o_norm.weight": { |
| "offset": 4848828416, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.attention.o_proj.weight": { |
| "offset": 4848844800, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.attention.q_conv1d.weight": { |
| "offset": 4855136256, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.attention.q_proj.weight": { |
| "offset": 4855152640, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.attention.v_conv1d.weight": { |
| "offset": 4861444096, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.attention.v_proj.weight": { |
| "offset": 4861460480, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.input_layernorm.weight": { |
| "offset": 4867751936, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.mlp.gate.expert_bias": { |
| "offset": 4867768320, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.2.mlp.gate.weight": { |
| "offset": 4867784704, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.2.mlp.shared_experts.down_proj.weight": { |
| "offset": 4868571136, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4870144000, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.mlp.shared_experts.up_proj.weight": { |
| "offset": 4871716864, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.2.post_attention_layernorm.weight": { |
| "offset": 4873289728, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.attention.A_log": { |
| "offset": 4873306112, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.20.attention.b_proj.weight": { |
| "offset": 4873322496, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.attention.dt_bias": { |
| "offset": 4873371648, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.20.attention.f_proj.weight": { |
| "offset": 4873388032, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.attention.g_proj.weight": { |
| "offset": 4879679488, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.attention.k_conv1d.weight": { |
| "offset": 4885970944, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.attention.k_proj.weight": { |
| "offset": 4885987328, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.attention.o_norm.weight": { |
| "offset": 4892278784, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.attention.o_proj.weight": { |
| "offset": 4892295168, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.attention.q_conv1d.weight": { |
| "offset": 4898586624, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.attention.q_proj.weight": { |
| "offset": 4898603008, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.attention.v_conv1d.weight": { |
| "offset": 4904894464, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.attention.v_proj.weight": { |
| "offset": 4904910848, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.input_layernorm.weight": { |
| "offset": 4911202304, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.mlp.gate.expert_bias": { |
| "offset": 4911218688, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.20.mlp.gate.weight": { |
| "offset": 4911235072, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.20.mlp.shared_experts.down_proj.weight": { |
| "offset": 4912021504, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4913594368, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.mlp.shared_experts.up_proj.weight": { |
| "offset": 4915167232, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.20.post_attention_layernorm.weight": { |
| "offset": 4916740096, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.attention.A_log": { |
| "offset": 4916756480, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.21.attention.b_proj.weight": { |
| "offset": 4916772864, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.attention.dt_bias": { |
| "offset": 4916822016, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.21.attention.f_proj.weight": { |
| "offset": 4916838400, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.attention.g_proj.weight": { |
| "offset": 4923129856, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.attention.k_conv1d.weight": { |
| "offset": 4929421312, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.attention.k_proj.weight": { |
| "offset": 4929437696, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.attention.o_norm.weight": { |
| "offset": 4935729152, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.attention.o_proj.weight": { |
| "offset": 4935745536, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.attention.q_conv1d.weight": { |
| "offset": 4942036992, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.attention.q_proj.weight": { |
| "offset": 4942053376, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.attention.v_conv1d.weight": { |
| "offset": 4948344832, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.attention.v_proj.weight": { |
| "offset": 4948361216, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.input_layernorm.weight": { |
| "offset": 4954652672, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.mlp.gate.expert_bias": { |
| "offset": 4954669056, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.21.mlp.gate.weight": { |
| "offset": 4954685440, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.21.mlp.shared_experts.down_proj.weight": { |
| "offset": 4955471872, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.mlp.shared_experts.gate_proj.weight": { |
| "offset": 4957044736, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.mlp.shared_experts.up_proj.weight": { |
| "offset": 4958617600, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.21.post_attention_layernorm.weight": { |
| "offset": 4960190464, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.attention.A_log": { |
| "offset": 4960206848, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.22.attention.b_proj.weight": { |
| "offset": 4960223232, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.attention.dt_bias": { |
| "offset": 4960272384, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.22.attention.f_proj.weight": { |
| "offset": 4960288768, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.attention.g_proj.weight": { |
| "offset": 4966580224, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.attention.k_conv1d.weight": { |
| "offset": 4972871680, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.attention.k_proj.weight": { |
| "offset": 4972888064, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.attention.o_norm.weight": { |
| "offset": 4979179520, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.attention.o_proj.weight": { |
| "offset": 4979195904, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.attention.q_conv1d.weight": { |
| "offset": 4985487360, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.attention.q_proj.weight": { |
| "offset": 4985503744, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.attention.v_conv1d.weight": { |
| "offset": 4991795200, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.attention.v_proj.weight": { |
| "offset": 4991811584, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.input_layernorm.weight": { |
| "offset": 4998103040, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.mlp.gate.expert_bias": { |
| "offset": 4998119424, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.22.mlp.gate.weight": { |
| "offset": 4998135808, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.22.mlp.shared_experts.down_proj.weight": { |
| "offset": 4998922240, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.mlp.shared_experts.gate_proj.weight": { |
| "offset": 5000495104, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.mlp.shared_experts.up_proj.weight": { |
| "offset": 5002067968, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.22.post_attention_layernorm.weight": { |
| "offset": 5003640832, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.23.attention.dense.weight": { |
| "offset": 5003657216, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.23.attention.g_proj.weight": { |
| "offset": 5009948672, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.23.attention.kv_a_layernorm.weight": { |
| "offset": 5009997824, |
| "bytes": 1024, |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.23.attention.kv_a_proj_with_mqa.weight": { |
| "offset": 5010014208, |
| "bytes": 1769472, |
| "shape": [ |
| 576, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.23.attention.kv_b_proj.weight": { |
| "offset": 5011783680, |
| "bytes": 4194304, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.23.attention.q_a_layernorm.weight": { |
| "offset": 5015977984, |
| "bytes": 512, |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.23.attention.q_a_proj.weight": { |
| "offset": 5015994368, |
| "bytes": 786432, |
| "shape": [ |
| 256, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.23.attention.q_b_proj.weight": { |
| "offset": 5016780800, |
| "bytes": 1572864, |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.23.input_layernorm.weight": { |
| "offset": 5018353664, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.23.mlp.gate.expert_bias": { |
| "offset": 5018370048, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.23.mlp.gate.weight": { |
| "offset": 5018386432, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.23.mlp.shared_experts.down_proj.weight": { |
| "offset": 5019172864, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.23.mlp.shared_experts.gate_proj.weight": { |
| "offset": 5020745728, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.23.mlp.shared_experts.up_proj.weight": { |
| "offset": 5022318592, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.23.post_attention_layernorm.weight": { |
| "offset": 5023891456, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.3.attention.dense.weight": { |
| "offset": 5023907840, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.3.attention.g_proj.weight": { |
| "offset": 5030199296, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.3.attention.kv_a_layernorm.weight": { |
| "offset": 5030248448, |
| "bytes": 1024, |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.3.attention.kv_a_proj_with_mqa.weight": { |
| "offset": 5030264832, |
| "bytes": 1769472, |
| "shape": [ |
| 576, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.3.attention.kv_b_proj.weight": { |
| "offset": 5032034304, |
| "bytes": 4194304, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.3.attention.q_a_layernorm.weight": { |
| "offset": 5036228608, |
| "bytes": 512, |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.3.attention.q_a_proj.weight": { |
| "offset": 5036244992, |
| "bytes": 786432, |
| "shape": [ |
| 256, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.3.attention.q_b_proj.weight": { |
| "offset": 5037031424, |
| "bytes": 1572864, |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.3.input_layernorm.weight": { |
| "offset": 5038604288, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.3.mlp.gate.expert_bias": { |
| "offset": 5038620672, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.3.mlp.gate.weight": { |
| "offset": 5038637056, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.3.mlp.shared_experts.down_proj.weight": { |
| "offset": 5039423488, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.3.mlp.shared_experts.gate_proj.weight": { |
| "offset": 5040996352, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.3.mlp.shared_experts.up_proj.weight": { |
| "offset": 5042569216, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.3.post_attention_layernorm.weight": { |
| "offset": 5044142080, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.attention.A_log": { |
| "offset": 5044158464, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.4.attention.b_proj.weight": { |
| "offset": 5044174848, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.attention.dt_bias": { |
| "offset": 5044224000, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.4.attention.f_proj.weight": { |
| "offset": 5044240384, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.attention.g_proj.weight": { |
| "offset": 5050531840, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.attention.k_conv1d.weight": { |
| "offset": 5056823296, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.attention.k_proj.weight": { |
| "offset": 5056839680, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.attention.o_norm.weight": { |
| "offset": 5063131136, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.attention.o_proj.weight": { |
| "offset": 5063147520, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.attention.q_conv1d.weight": { |
| "offset": 5069438976, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.attention.q_proj.weight": { |
| "offset": 5069455360, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.attention.v_conv1d.weight": { |
| "offset": 5075746816, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.attention.v_proj.weight": { |
| "offset": 5075763200, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.input_layernorm.weight": { |
| "offset": 5082054656, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.mlp.gate.expert_bias": { |
| "offset": 5082071040, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.4.mlp.gate.weight": { |
| "offset": 5082087424, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.4.mlp.shared_experts.down_proj.weight": { |
| "offset": 5082873856, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.mlp.shared_experts.gate_proj.weight": { |
| "offset": 5084446720, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.mlp.shared_experts.up_proj.weight": { |
| "offset": 5086019584, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.4.post_attention_layernorm.weight": { |
| "offset": 5087592448, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.attention.A_log": { |
| "offset": 5087608832, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.5.attention.b_proj.weight": { |
| "offset": 5087625216, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.attention.dt_bias": { |
| "offset": 5087674368, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.5.attention.f_proj.weight": { |
| "offset": 5087690752, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.attention.g_proj.weight": { |
| "offset": 5093982208, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.attention.k_conv1d.weight": { |
| "offset": 5100273664, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.attention.k_proj.weight": { |
| "offset": 5100290048, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.attention.o_norm.weight": { |
| "offset": 5106581504, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.attention.o_proj.weight": { |
| "offset": 5106597888, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.attention.q_conv1d.weight": { |
| "offset": 5112889344, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.attention.q_proj.weight": { |
| "offset": 5112905728, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.attention.v_conv1d.weight": { |
| "offset": 5119197184, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.attention.v_proj.weight": { |
| "offset": 5119213568, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.input_layernorm.weight": { |
| "offset": 5125505024, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.mlp.gate.expert_bias": { |
| "offset": 5125521408, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.5.mlp.gate.weight": { |
| "offset": 5125537792, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.5.mlp.shared_experts.down_proj.weight": { |
| "offset": 5126324224, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.mlp.shared_experts.gate_proj.weight": { |
| "offset": 5127897088, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.mlp.shared_experts.up_proj.weight": { |
| "offset": 5129469952, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.5.post_attention_layernorm.weight": { |
| "offset": 5131042816, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.attention.A_log": { |
| "offset": 5131059200, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.6.attention.b_proj.weight": { |
| "offset": 5131075584, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.attention.dt_bias": { |
| "offset": 5131124736, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.6.attention.f_proj.weight": { |
| "offset": 5131141120, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.attention.g_proj.weight": { |
| "offset": 5137432576, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.attention.k_conv1d.weight": { |
| "offset": 5143724032, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.attention.k_proj.weight": { |
| "offset": 5143740416, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.attention.o_norm.weight": { |
| "offset": 5150031872, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.attention.o_proj.weight": { |
| "offset": 5150048256, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.attention.q_conv1d.weight": { |
| "offset": 5156339712, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.attention.q_proj.weight": { |
| "offset": 5156356096, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.attention.v_conv1d.weight": { |
| "offset": 5162647552, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.attention.v_proj.weight": { |
| "offset": 5162663936, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.input_layernorm.weight": { |
| "offset": 5168955392, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.mlp.gate.expert_bias": { |
| "offset": 5168971776, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.6.mlp.gate.weight": { |
| "offset": 5168988160, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.6.mlp.shared_experts.down_proj.weight": { |
| "offset": 5169774592, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.mlp.shared_experts.gate_proj.weight": { |
| "offset": 5171347456, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.mlp.shared_experts.up_proj.weight": { |
| "offset": 5172920320, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.6.post_attention_layernorm.weight": { |
| "offset": 5174493184, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.7.attention.dense.weight": { |
| "offset": 5174509568, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.7.attention.g_proj.weight": { |
| "offset": 5180801024, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.7.attention.kv_a_layernorm.weight": { |
| "offset": 5180850176, |
| "bytes": 1024, |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.7.attention.kv_a_proj_with_mqa.weight": { |
| "offset": 5180866560, |
| "bytes": 1769472, |
| "shape": [ |
| 576, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.7.attention.kv_b_proj.weight": { |
| "offset": 5182636032, |
| "bytes": 4194304, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.7.attention.q_a_layernorm.weight": { |
| "offset": 5186830336, |
| "bytes": 512, |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.7.attention.q_a_proj.weight": { |
| "offset": 5186846720, |
| "bytes": 786432, |
| "shape": [ |
| 256, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.7.attention.q_b_proj.weight": { |
| "offset": 5187633152, |
| "bytes": 1572864, |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.7.input_layernorm.weight": { |
| "offset": 5189206016, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.7.mlp.gate.expert_bias": { |
| "offset": 5189222400, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.7.mlp.gate.weight": { |
| "offset": 5189238784, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.7.mlp.shared_experts.down_proj.weight": { |
| "offset": 5190025216, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.7.mlp.shared_experts.gate_proj.weight": { |
| "offset": 5191598080, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.7.mlp.shared_experts.up_proj.weight": { |
| "offset": 5193170944, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.7.post_attention_layernorm.weight": { |
| "offset": 5194743808, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.attention.A_log": { |
| "offset": 5194760192, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.8.attention.b_proj.weight": { |
| "offset": 5194776576, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.attention.dt_bias": { |
| "offset": 5194825728, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.8.attention.f_proj.weight": { |
| "offset": 5194842112, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.attention.g_proj.weight": { |
| "offset": 5201133568, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.attention.k_conv1d.weight": { |
| "offset": 5207425024, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.attention.k_proj.weight": { |
| "offset": 5207441408, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.attention.o_norm.weight": { |
| "offset": 5213732864, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.attention.o_proj.weight": { |
| "offset": 5213749248, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.attention.q_conv1d.weight": { |
| "offset": 5220040704, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.attention.q_proj.weight": { |
| "offset": 5220057088, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.attention.v_conv1d.weight": { |
| "offset": 5226348544, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.attention.v_proj.weight": { |
| "offset": 5226364928, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.input_layernorm.weight": { |
| "offset": 5232656384, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.mlp.gate.expert_bias": { |
| "offset": 5232672768, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.8.mlp.gate.weight": { |
| "offset": 5232689152, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.8.mlp.shared_experts.down_proj.weight": { |
| "offset": 5233475584, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.mlp.shared_experts.gate_proj.weight": { |
| "offset": 5235048448, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.mlp.shared_experts.up_proj.weight": { |
| "offset": 5236621312, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.8.post_attention_layernorm.weight": { |
| "offset": 5238194176, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.attention.A_log": { |
| "offset": 5238210560, |
| "bytes": 64, |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.9.attention.b_proj.weight": { |
| "offset": 5238226944, |
| "bytes": 49152, |
| "shape": [ |
| 16, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.attention.dt_bias": { |
| "offset": 5238276096, |
| "bytes": 8192, |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.9.attention.f_proj.weight": { |
| "offset": 5238292480, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.attention.g_proj.weight": { |
| "offset": 5244583936, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.attention.k_conv1d.weight": { |
| "offset": 5250875392, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.attention.k_proj.weight": { |
| "offset": 5250891776, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.attention.o_norm.weight": { |
| "offset": 5257183232, |
| "bytes": 256, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.attention.o_proj.weight": { |
| "offset": 5257199616, |
| "bytes": 6291456, |
| "shape": [ |
| 1536, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.attention.q_conv1d.weight": { |
| "offset": 5263491072, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.attention.q_proj.weight": { |
| "offset": 5263507456, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.attention.v_conv1d.weight": { |
| "offset": 5269798912, |
| "bytes": 16384, |
| "shape": [ |
| 2048, |
| 1, |
| 4 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.attention.v_proj.weight": { |
| "offset": 5269815296, |
| "bytes": 6291456, |
| "shape": [ |
| 2048, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.input_layernorm.weight": { |
| "offset": 5276106752, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.mlp.gate.expert_bias": { |
| "offset": 5276123136, |
| "bytes": 512, |
| "shape": [ |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.9.mlp.gate.weight": { |
| "offset": 5276139520, |
| "bytes": 786432, |
| "shape": [ |
| 128, |
| 1536 |
| ], |
| "dtype": "float32" |
| }, |
| "model.model.layers.9.mlp.shared_experts.down_proj.weight": { |
| "offset": 5276925952, |
| "bytes": 1572864, |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.mlp.shared_experts.gate_proj.weight": { |
| "offset": 5278498816, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.mlp.shared_experts.up_proj.weight": { |
| "offset": 5280071680, |
| "bytes": 1572864, |
| "shape": [ |
| 512, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.layers.9.post_attention_layernorm.weight": { |
| "offset": 5281644544, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.norm.weight": { |
| "offset": 5281660928, |
| "bytes": 3072, |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.model.word_embeddings.weight": { |
| "offset": 5281677312, |
| "bytes": 482869248, |
| "shape": [ |
| 157184, |
| 1536 |
| ], |
| "dtype": "bfloat16" |
| } |
| } |
| } |
|
|
