| { |
| "metadata": { |
| "ParamSize": 325, |
| "ParamBytes": 2149644288.0, |
| "BitsPerParam": 4.500600961055312 |
| }, |
| "records": [ |
| { |
| "dataPath": "params_shard_0.bin", |
| "format": "raw-shard", |
| "nbytes": 49250304, |
| "records": [ |
| { |
| "name": "lm_head.q_weight", |
| "shape": [ |
| 32064, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 49250304, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "92ebb5de33ba5a5f23296b9f224b7d1e" |
| }, |
| { |
| "dataPath": "params_shard_1.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.21.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "291fa3d154d803e65ea132a1a148ba94" |
| }, |
| { |
| "dataPath": "params_shard_2.bin", |
| "format": "raw-shard", |
| "nbytes": 23470080, |
| "records": [ |
| { |
| "name": "lm_head.q_scale", |
| "shape": [ |
| 32064, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6156288, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.21.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 6156288 |
| }, |
| { |
| "name": "transformer.h.21.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 6162432 |
| }, |
| { |
| "name": "transformer.h.21.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 18745344 |
| }, |
| { |
| "name": "transformer.h.21.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 20318208 |
| }, |
| { |
| "name": "transformer.h.21.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 23463936 |
| } |
| ], |
| "md5sum": "b5685c2039a1f0e1cfadd9fc6d047847" |
| }, |
| { |
| "dataPath": "params_shard_3.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.22.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "6aa51bc82e13a0a09d2bd79ecf706f56" |
| }, |
| { |
| "dataPath": "params_shard_4.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.21.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.21.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.22.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.22.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.22.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.22.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.22.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "4366e38eacc6ea8eb94d141e44410591" |
| }, |
| { |
| "dataPath": "params_shard_5.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.22.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.22.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.22.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.22.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.23.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "5ee32b18d843413d3dfc9e3b4f46339f" |
| }, |
| { |
| "dataPath": "params_shard_6.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.23.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "038753ac7a61c5c8fc1f37e48a163184" |
| }, |
| { |
| "dataPath": "params_shard_7.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.23.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.23.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.23.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.23.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.23.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.23.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "0d855411c37b4b8a8c86503ab2253569" |
| }, |
| { |
| "dataPath": "params_shard_8.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.24.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "10ec010d75596ab4288c0101cdaeaed3" |
| }, |
| { |
| "dataPath": "params_shard_9.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.23.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.23.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.24.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.24.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.24.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.24.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.24.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "3cfae286d4c269cc4346ec4eb7dc4af8" |
| }, |
| { |
| "dataPath": "params_shard_10.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.24.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.24.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.24.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.24.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.25.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "1f50e9bfcacd87299a0def811b3e53f4" |
| }, |
| { |
| "dataPath": "params_shard_11.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.25.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "d0e4431ab49bbe7282f00262b88e11ac" |
| }, |
| { |
| "dataPath": "params_shard_12.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.25.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.25.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.25.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.25.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.25.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.25.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "330d2998d083b2bac1816eb588166dbd" |
| }, |
| { |
| "dataPath": "params_shard_13.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.26.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "244eba2a84e4a3bc56632585f826328b" |
| }, |
| { |
| "dataPath": "params_shard_14.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.25.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.25.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.26.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.26.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.26.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.26.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.26.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "d7a94d445c78370f01676b100b7c7877" |
| }, |
| { |
| "dataPath": "params_shard_15.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.26.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.26.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.26.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.26.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.27.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "97d8b4a962ad51e291b494f840236c0c" |
| }, |
| { |
| "dataPath": "params_shard_16.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.27.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "e77dfa662ddf8497df0f796e0dfb7797" |
| }, |
| { |
| "dataPath": "params_shard_17.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.27.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.27.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.27.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.27.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.27.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.27.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "64accd8b4e0174e5997cd0489699af2c" |
| }, |
| { |
| "dataPath": "params_shard_18.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.28.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "468d07c6c0d4decfadcb24eb2384f21f" |
| }, |
| { |
| "dataPath": "params_shard_19.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.27.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.27.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.28.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.28.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.28.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.28.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.28.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "feea61e1c79af19a012f2d07c1f85d3e" |
| }, |
| { |
| "dataPath": "params_shard_20.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.28.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.28.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.28.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.28.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.29.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "d4d16f50c4216ef4ec90789dacb04e39" |
| }, |
| { |
| "dataPath": "params_shard_21.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.29.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "e516301a253bb0a02a4cfabc742f095a" |
| }, |
| { |
| "dataPath": "params_shard_22.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.29.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.29.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.29.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.29.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.29.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.29.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "2c003186a0f02ad129bf443f527ed9ae" |
| }, |
| { |
| "dataPath": "params_shard_23.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.30.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "b7a3d71b718be0cd079fc4592ae36cee" |
| }, |
| { |
| "dataPath": "params_shard_24.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.29.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.29.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.30.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.30.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.30.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.30.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.30.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "1fbcc39db59bc343193575b272631c18" |
| }, |
| { |
| "dataPath": "params_shard_25.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.30.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.30.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.30.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.30.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.31.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "891310753baa264db60398ca8b111786" |
| }, |
| { |
| "dataPath": "params_shard_26.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.31.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "42aec8a789db093d5a850365beca1b9a" |
| }, |
| { |
| "dataPath": "params_shard_27.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.31.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.31.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.31.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.31.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.31.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.31.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "68c35c01ba2d353ab3d845e2efe74ba8" |
| }, |
| { |
| "dataPath": "params_shard_28.bin", |
| "format": "raw-shard", |
| "nbytes": 49250304, |
| "records": [ |
| { |
| "name": "transformer.embd.q_weight", |
| "shape": [ |
| 32064, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 49250304, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "e65cd6e13f78175ce2f5fcb3844136e4" |
| }, |
| { |
| "dataPath": "params_shard_29.bin", |
| "format": "raw-shard", |
| "nbytes": 22093824, |
| "records": [ |
| { |
| "name": "transformer.h.31.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.31.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.norm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.embd.q_scale", |
| "shape": [ |
| 32064, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6156288, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.0.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 22087680 |
| } |
| ], |
| "md5sum": "13906ed5091c3b4cabe7f7448ea53d40" |
| }, |
| { |
| "dataPath": "params_shard_30.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.0.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "01a5d0a50e64f55fba559fee50cf9fa9" |
| }, |
| { |
| "dataPath": "params_shard_31.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.0.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.0.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.0.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.0.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.0.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.0.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "9bf2169e2a6a905d653f259afc821a74" |
| }, |
| { |
| "dataPath": "params_shard_32.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.1.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "06b55aec3bd00e550e033e2c8d6ac0ac" |
| }, |
| { |
| "dataPath": "params_shard_33.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.0.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.0.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.1.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.1.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.1.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.1.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.1.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "286d25d70b6955d7f03d4206c4598888" |
| }, |
| { |
| "dataPath": "params_shard_34.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.1.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.1.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.1.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.1.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.10.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "6cf2b18f2ebae017af06f6f493dc1621" |
| }, |
| { |
| "dataPath": "params_shard_35.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.10.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "34d6d5bb2604bf2c325306582370cc74" |
| }, |
| { |
| "dataPath": "params_shard_36.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.10.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.10.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.10.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.10.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.10.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.10.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "d35cd5125d0ea0c1dca10aea7057a564" |
| }, |
| { |
| "dataPath": "params_shard_37.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.11.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "978d47b4e8ab28da90916bdf8b5fd352" |
| }, |
| { |
| "dataPath": "params_shard_38.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.10.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.10.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.11.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.11.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.11.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.11.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.11.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "c29753d0e13239b0f4587ac02fa35c4b" |
| }, |
| { |
| "dataPath": "params_shard_39.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.11.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.11.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.11.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.11.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.12.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "3e56173a0ee3a45828db96ba45c2b473" |
| }, |
| { |
| "dataPath": "params_shard_40.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.12.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "e79ed66deccd5e690ccfe389f77031a2" |
| }, |
| { |
| "dataPath": "params_shard_41.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.12.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.12.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.12.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.12.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.12.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.12.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "17cbd5c7bca284055f12ee3b4caac8e8" |
| }, |
| { |
| "dataPath": "params_shard_42.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.13.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "3908c105ab14b5ca5619d2a24327a377" |
| }, |
| { |
| "dataPath": "params_shard_43.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.12.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.12.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.13.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.13.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.13.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.13.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.13.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "9d92155e270502f1b41ae65a60b9c893" |
| }, |
| { |
| "dataPath": "params_shard_44.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.13.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.13.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.13.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.13.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.14.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "af72397827d4259544e64aa00d6a8690" |
| }, |
| { |
| "dataPath": "params_shard_45.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.14.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "2330e02f28e6f4061b18954cb306e561" |
| }, |
| { |
| "dataPath": "params_shard_46.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.14.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.14.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.14.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.14.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.14.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.14.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "303293a5da1a1a819dcc4b9101eabd3f" |
| }, |
| { |
| "dataPath": "params_shard_47.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.15.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "55cae218fc755e702bdda6f5a540fe35" |
| }, |
| { |
| "dataPath": "params_shard_48.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.14.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.14.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.15.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.15.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.15.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.15.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.15.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "d8db651a933e3d2bc57d046947a1ba6a" |
| }, |
| { |
| "dataPath": "params_shard_49.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.15.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.15.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.15.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.15.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.16.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "6698ece06dbded78bee76b95e360969b" |
| }, |
| { |
| "dataPath": "params_shard_50.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.16.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "88768cd8dbf6131dd218b737b0fe319e" |
| }, |
| { |
| "dataPath": "params_shard_51.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.16.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.16.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.16.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.16.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.16.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.16.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "a03e719a387c138929dc162258845aac" |
| }, |
| { |
| "dataPath": "params_shard_52.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.17.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "917019b1ae81f19b9ff17e7be8b1c46d" |
| }, |
| { |
| "dataPath": "params_shard_53.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.16.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.16.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.17.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.17.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.17.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.17.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.17.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "e20a4d29c4ea4fa8f15fc05d9c490ff5" |
| }, |
| { |
| "dataPath": "params_shard_54.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.17.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.17.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.17.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.17.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.18.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "a00eb580f4e97fbdbc40e7d24ef76477" |
| }, |
| { |
| "dataPath": "params_shard_55.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.18.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "f07b0ba7133454ef1ff215d1804b42ee" |
| }, |
| { |
| "dataPath": "params_shard_56.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.18.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.18.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.18.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.18.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.18.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.18.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "93c65c80bfb472abf7ded1c1f1cc8599" |
| }, |
| { |
| "dataPath": "params_shard_57.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.19.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "2d72f59f7c995f4bf601ad7987b9f241" |
| }, |
| { |
| "dataPath": "params_shard_58.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.18.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.18.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.19.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.19.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.19.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.19.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.19.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "73dfeaadd8f46ce891258ce762b7f3e2" |
| }, |
| { |
| "dataPath": "params_shard_59.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.19.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.19.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.19.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.19.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.2.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "74281733e066b662bf055e5fdceb30b5" |
| }, |
| { |
| "dataPath": "params_shard_60.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.2.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "98300e11ca6111c9098dcc9de6154007" |
| }, |
| { |
| "dataPath": "params_shard_61.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.2.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.2.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.2.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.2.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.2.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.2.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "45f46ac7a171ad5c90c22ee83f3024e7" |
| }, |
| { |
| "dataPath": "params_shard_62.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.20.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "9ba4f8920d7bf59afdc3c51ad021ed92" |
| }, |
| { |
| "dataPath": "params_shard_63.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.2.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.2.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.20.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.20.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.20.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.20.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.20.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "d61c778d345905231dac05085fbdfd22" |
| }, |
| { |
| "dataPath": "params_shard_64.bin", |
| "format": "raw-shard", |
| "nbytes": 26548224, |
| "records": [ |
| { |
| "name": "transformer.h.20.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.20.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.20.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.20.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.21.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 21233664 |
| }, |
| { |
| "name": "transformer.h.21.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 25952256 |
| }, |
| { |
| "name": "transformer.h.3.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 26542080 |
| } |
| ], |
| "md5sum": "58ccf26e06e6d12ac530ff375453b806" |
| }, |
| { |
| "dataPath": "params_shard_65.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.3.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "cd3b102866adb064501e901e4023606e" |
| }, |
| { |
| "dataPath": "params_shard_66.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.3.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.3.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.3.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.3.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.3.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.3.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "1a153dbb44b10061584585431991d5aa" |
| }, |
| { |
| "dataPath": "params_shard_67.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.4.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "99d2df12ac0f50534380eecf23a3c3de" |
| }, |
| { |
| "dataPath": "params_shard_68.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.3.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.3.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.4.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.4.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.4.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.4.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.4.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "c0ce59f090970b3f72d488ce39e4b410" |
| }, |
| { |
| "dataPath": "params_shard_69.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.4.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.4.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.4.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.4.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.5.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "e9599b78c0d00c3309711421df8e0152" |
| }, |
| { |
| "dataPath": "params_shard_70.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.5.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "f8f17a39b5b58070a21f5dae32d5b353" |
| }, |
| { |
| "dataPath": "params_shard_71.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.5.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.5.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.5.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.5.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.5.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.5.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "e986fefed19d2767754ac5341378efd7" |
| }, |
| { |
| "dataPath": "params_shard_72.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.6.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "7b010c64b1812476c23014b0f22c7c86" |
| }, |
| { |
| "dataPath": "params_shard_73.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.5.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.5.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.6.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.6.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.6.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.6.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.6.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "066a3d6a69924ee20d11e6d0d0334833" |
| }, |
| { |
| "dataPath": "params_shard_74.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.6.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.6.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.6.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.6.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.7.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "dc44fe882a6addc8d96d9d945b925638" |
| }, |
| { |
| "dataPath": "params_shard_75.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.7.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "060ba9a25db7e4dcfa972264cf1448a0" |
| }, |
| { |
| "dataPath": "params_shard_76.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.7.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.7.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.7.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.7.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.7.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.7.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "288ca75c3654091733ce30274718cde2" |
| }, |
| { |
| "dataPath": "params_shard_77.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.8.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "65b9831a3c07e0a5d2693ee65fc900d0" |
| }, |
| { |
| "dataPath": "params_shard_78.bin", |
| "format": "raw-shard", |
| "nbytes": 33239040, |
| "records": [ |
| { |
| "name": "transformer.h.7.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.7.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.8.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 15925248 |
| }, |
| { |
| "name": "transformer.h.8.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 15931392 |
| }, |
| { |
| "name": "transformer.h.8.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 28514304 |
| }, |
| { |
| "name": "transformer.h.8.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 30087168 |
| }, |
| { |
| "name": "transformer.h.8.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 33232896 |
| } |
| ], |
| "md5sum": "eeada502d5c94487aff9ca24190d896e" |
| }, |
| { |
| "dataPath": "params_shard_79.bin", |
| "format": "raw-shard", |
| "nbytes": 21239808, |
| "records": [ |
| { |
| "name": "transformer.h.8.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.8.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 4718592 |
| }, |
| { |
| "name": "transformer.h.8.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 5308416 |
| }, |
| { |
| "name": "transformer.h.8.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 19464192 |
| }, |
| { |
| "name": "transformer.h.9.ln.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 21233664 |
| } |
| ], |
| "md5sum": "41a85312cc0b92a045b0b0b6c9b224f7" |
| }, |
| { |
| "dataPath": "params_shard_80.bin", |
| "format": "raw-shard", |
| "nbytes": 25165824, |
| "records": [ |
| { |
| "name": "transformer.h.9.mlp.gate_up_proj.q_weight", |
| "shape": [ |
| 16384, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 25165824, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "6684c5aadf0f546b1bdec716959167e1" |
| }, |
| { |
| "dataPath": "params_shard_81.bin", |
| "format": "raw-shard", |
| "nbytes": 22616064, |
| "records": [ |
| { |
| "name": "transformer.h.9.mlp.down_proj.q_weight", |
| "shape": [ |
| 3072, |
| 1024 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 12582912, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.9.mlp.down_proj.q_scale", |
| "shape": [ |
| 3072, |
| 256 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1572864, |
| "byteOffset": 12582912 |
| }, |
| { |
| "name": "transformer.h.9.mlp.gate_up_proj.q_scale", |
| "shape": [ |
| 16384, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 3145728, |
| "byteOffset": 14155776 |
| }, |
| { |
| "name": "transformer.h.9.post_attention_layernorm.weight", |
| "shape": [ |
| 3072 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 6144, |
| "byteOffset": 17301504 |
| }, |
| { |
| "name": "transformer.h.9.mixer.out_proj.q_weight", |
| "shape": [ |
| 3072, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 4718592, |
| "byteOffset": 17307648 |
| }, |
| { |
| "name": "transformer.h.9.mixer.out_proj.q_scale", |
| "shape": [ |
| 3072, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 589824, |
| "byteOffset": 22026240 |
| } |
| ], |
| "md5sum": "85aceb764e32ad900e812880aead03cb" |
| }, |
| { |
| "dataPath": "params_shard_82.bin", |
| "format": "raw-shard", |
| "nbytes": 15925248, |
| "records": [ |
| { |
| "name": "transformer.h.9.mixer.qkv_proj.q_weight", |
| "shape": [ |
| 9216, |
| 384 |
| ], |
| "dtype": "uint32", |
| "format": "f32-to-bf16", |
| "nbytes": 14155776, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "transformer.h.9.mixer.qkv_proj.q_scale", |
| "shape": [ |
| 9216, |
| 96 |
| ], |
| "dtype": "float16", |
| "format": "f32-to-bf16", |
| "nbytes": 1769472, |
| "byteOffset": 14155776 |
| } |
| ], |
| "md5sum": "865b527be9a21333488551e85239379f" |
| } |
| ] |
| } |
