Hugging Face's logo

mawriyo
/
phi-mini4k

Model card Files
xet
Community
phi-mini4k / ndarray-cache.json
Keny Melgar
init
0ed4215
raw
history blame contribute delete
142 kB
 {
"metadata": {
"ParamSize": 325,
"ParamBytes": 2149644288.0,
"BitsPerParam": 4.500600961055312
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 49250304,
"records": [
{
"name": "lm_head.q_weight",
"shape": [
32064,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 49250304,
"byteOffset": 0
}
],
"md5sum": "c9e354046db2a906d8794caaaaee850d"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.21.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "4b283c7e90f84f5871b8383564b1ea72"
},
{
"dataPath": "params_shard_2.bin",
"format": "raw-shard",
"nbytes": 23470080,
"records": [
{
"name": "lm_head.q_scale",
"shape": [
32064,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6156288,
"byteOffset": 0
},
{
"name": "transformer.h.21.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 6156288
},
{
"name": "transformer.h.21.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 6162432
},
{
"name": "transformer.h.21.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 18745344
},
{
"name": "transformer.h.21.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 20318208
},
{
"name": "transformer.h.21.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23463936
}
],
"md5sum": "c8cabb53c75d1f4815c50cde31324df0"
},
{
"dataPath": "params_shard_3.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.22.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "d5f4b37d33c671a5a4b9dcbb37020281"
},
{
"dataPath": "params_shard_4.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.21.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.21.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.22.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.22.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.22.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.22.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.22.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "2604e8c55ed43faf77a4d21126eee0d6"
},
{
"dataPath": "params_shard_5.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.22.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.22.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.22.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.22.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.23.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "94ac0ff50fa28b04cdd0919dd30ea721"
},
{
"dataPath": "params_shard_6.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.23.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "d9f2b9e22b94f4f9d5cceb4eb7afa70f"
},
{
"dataPath": "params_shard_7.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.23.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.23.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.23.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.23.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.23.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.23.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "4dc3b6375d1a2ae962fe5b863b682d76"
},
{
"dataPath": "params_shard_8.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.24.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "f9a47ddd48acfdfb91d72d76b41ae57b"
},
{
"dataPath": "params_shard_9.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.23.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.23.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.24.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.24.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.24.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.24.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.24.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "3bb92bd753f0ea516c9abbd528c2c48a"
},
{
"dataPath": "params_shard_10.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.24.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.24.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.24.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.24.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.25.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "d6bffaa9f087578bf43667f8b3fbda0f"
},
{
"dataPath": "params_shard_11.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.25.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "aa9119b9950a5f945c86b5fc643c7060"
},
{
"dataPath": "params_shard_12.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.25.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.25.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.25.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.25.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.25.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.25.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "6629af1e1e27dbe16f1a6d5de38df159"
},
{
"dataPath": "params_shard_13.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.26.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "e1f99e218b8d83f487dceab5980183aa"
},
{
"dataPath": "params_shard_14.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.25.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.25.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.26.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.26.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.26.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.26.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.26.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "ed57c2bd7cee3d8a324b1bd99ce38e66"
},
{
"dataPath": "params_shard_15.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.26.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.26.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.26.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.26.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.27.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "62bdac2c473c7416578d1a54ea77b8c7"
},
{
"dataPath": "params_shard_16.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.27.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "64e63f78f23ad59e138a32293f5cc7f3"
},
{
"dataPath": "params_shard_17.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.27.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.27.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.27.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.27.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.27.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.27.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "e6064b29b1fa2ae29d0730a2f7caa633"
},
{
"dataPath": "params_shard_18.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.28.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "32c4d4ebf17d20a2c8da0c11b1e5b080"
},
{
"dataPath": "params_shard_19.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.27.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.27.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.28.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.28.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.28.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.28.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.28.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "cae1134f31044047dc2f91b10dada0af"
},
{
"dataPath": "params_shard_20.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.28.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.28.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.28.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.28.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.29.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "6d7ced2a2ef38b104299aed6adbcd499"
},
{
"dataPath": "params_shard_21.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.29.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "bd190b46e21615db26bc0667d2d028a9"
},
{
"dataPath": "params_shard_22.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.29.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.29.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.29.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.29.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.29.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.29.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "2caa2e559b6e495db6549e1d8f6c95f0"
},
{
"dataPath": "params_shard_23.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.30.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "20d24f9f3cc7a339ad5f99ca9caf4ccc"
},
{
"dataPath": "params_shard_24.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.29.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.29.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.30.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.30.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.30.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.30.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.30.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "cc67d2379662797a94de4732d5d8bf96"
},
{
"dataPath": "params_shard_25.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.30.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.30.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.30.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.30.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.31.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "8271a6efb48390bc676f891ddd9826b3"
},
{
"dataPath": "params_shard_26.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.31.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "e5bf7733ab694fa663ed7fa0a90085ae"
},
{
"dataPath": "params_shard_27.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.31.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.31.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.31.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.31.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.31.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.31.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "bf3eac98211b7772cc2a1e8ba5ef56e3"
},
{
"dataPath": "params_shard_28.bin",
"format": "raw-shard",
"nbytes": 49250304,
"records": [
{
"name": "transformer.embd.q_weight",
"shape": [
32064,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 49250304,
"byteOffset": 0
}
],
"md5sum": "e61f1b06e77de108c4a56240a97fbb3b"
},
{
"dataPath": "params_shard_29.bin",
"format": "raw-shard",
"nbytes": 22093824,
"records": [
{
"name": "transformer.h.31.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.31.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.norm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.embd.q_scale",
"shape": [
32064,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6156288,
"byteOffset": 15931392
},
{
"name": "transformer.h.0.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 22087680
}
],
"md5sum": "46951bed8951c88c54aa1b44217f0d75"
},
{
"dataPath": "params_shard_30.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.0.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "10d39b272ab4c6fd7289cac12ad41611"
},
{
"dataPath": "params_shard_31.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.0.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.0.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.0.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.0.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.0.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.0.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "5f86f0f598cb4ca7b430547f81baf627"
},
{
"dataPath": "params_shard_32.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.1.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "484741977565d1d71f8998a252b4dc36"
},
{
"dataPath": "params_shard_33.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.0.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.0.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.1.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.1.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.1.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.1.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.1.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "499ba4e4630cd92695e34dcce8c0dd9f"
},
{
"dataPath": "params_shard_34.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.1.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.1.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.1.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.1.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.10.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "e61f009ca335976aadee22021b6db74d"
},
{
"dataPath": "params_shard_35.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.10.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "35c62867f472d42255810b0d849c087c"
},
{
"dataPath": "params_shard_36.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.10.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.10.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.10.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.10.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.10.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.10.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "3fffb0aea41e5e1922461774cbf9f8cb"
},
{
"dataPath": "params_shard_37.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.11.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "5921d60f12a98545c09c047ff4c1bdad"
},
{
"dataPath": "params_shard_38.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.10.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.10.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.11.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.11.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.11.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.11.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.11.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "94c24b2d3015d941ad50829f6e417120"
},
{
"dataPath": "params_shard_39.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.11.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.11.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.11.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.11.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.12.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "499f0098e42351c5865b5e0ae71f1aa9"
},
{
"dataPath": "params_shard_40.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.12.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "1ba42809fa7c1f86703078e0f754dff2"
},
{
"dataPath": "params_shard_41.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.12.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.12.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.12.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.12.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.12.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.12.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "3269d7577a33b56e3db0c7c49ac04fe0"
},
{
"dataPath": "params_shard_42.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.13.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "375017006e4bb207100bf697bb972e89"
},
{
"dataPath": "params_shard_43.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.12.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.12.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.13.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.13.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.13.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.13.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.13.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "fbdc2a941157a9dcff0b69ed6a70237e"
},
{
"dataPath": "params_shard_44.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.13.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.13.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.13.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.13.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.14.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "e6d6ad92fdf7aa8239ee58d80c954dd3"
},
{
"dataPath": "params_shard_45.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.14.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "09257112ea5169733328c7131702c779"
},
{
"dataPath": "params_shard_46.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.14.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.14.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.14.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.14.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.14.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.14.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "a73aa60ba509d31774c4dad59d30dedf"
},
{
"dataPath": "params_shard_47.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.15.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "def11cc9fc9c9d7de5b84d11214299c5"
},
{
"dataPath": "params_shard_48.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.14.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.14.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.15.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.15.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.15.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.15.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.15.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "a38ee2ce5e1c4ee55567d12e1bbf81fd"
},
{
"dataPath": "params_shard_49.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.15.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.15.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.15.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.15.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.16.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "db0a8f15a63c0f835ad7360537c26012"
},
{
"dataPath": "params_shard_50.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.16.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "f7f896c7b4ff937d23b1e8090180d675"
},
{
"dataPath": "params_shard_51.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.16.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.16.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.16.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.16.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.16.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.16.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "47fa48d1f8ba220049355ff19ee97044"
},
{
"dataPath": "params_shard_52.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.17.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "9a70479627c1d18fbddc1b9603d0de0a"
},
{
"dataPath": "params_shard_53.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.16.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.16.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.17.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.17.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.17.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.17.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.17.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "c6c79f373639b46938388dda16b85282"
},
{
"dataPath": "params_shard_54.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.17.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.17.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.17.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.17.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.18.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "7d67772852c6fbedc3daf501e7be89d0"
},
{
"dataPath": "params_shard_55.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.18.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "fa21743578b6672f11ff4a7372b6f312"
},
{
"dataPath": "params_shard_56.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.18.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.18.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.18.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.18.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.18.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.18.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "f44e7ac43e25bd9a583ef44081ea0495"
},
{
"dataPath": "params_shard_57.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.19.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "80eb979f8d20bd0ba1c636701040cef1"
},
{
"dataPath": "params_shard_58.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.18.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.18.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.19.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.19.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.19.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.19.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.19.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "0593b4734c545448d30e0659a080cd81"
},
{
"dataPath": "params_shard_59.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.19.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.19.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.19.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.19.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.2.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "4fde5f9e7a85b8f453e6e96348247612"
},
{
"dataPath": "params_shard_60.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.2.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "1fb5a4cf8b8e4550422c4392f05f712a"
},
{
"dataPath": "params_shard_61.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.2.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.2.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.2.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.2.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.2.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.2.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "7e328c611048b7752c57246dcecc2fc4"
},
{
"dataPath": "params_shard_62.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.20.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "5c6113b6b67f4fe11a41bb7f94a4cc16"
},
{
"dataPath": "params_shard_63.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.2.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.2.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.20.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.20.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.20.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.20.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.20.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "3da248097cebcbb8405cb9f8dfa8788b"
},
{
"dataPath": "params_shard_64.bin",
"format": "raw-shard",
"nbytes": 26548224,
"records": [
{
"name": "transformer.h.20.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.20.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.20.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.20.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.21.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 21233664
},
{
"name": "transformer.h.21.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 25952256
},
{
"name": "transformer.h.3.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 26542080
}
],
"md5sum": "f9c1f66e7e9a2c7ac2bfc55d89055594"
},
{
"dataPath": "params_shard_65.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.3.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "33e6ae873b2c26c4c206812c3ed06a3f"
},
{
"dataPath": "params_shard_66.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.3.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.3.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.3.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.3.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.3.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.3.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "6ae5ea60b9220a6c430da183826ddb1e"
},
{
"dataPath": "params_shard_67.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.4.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "cb7fad11cdb09239bf032c34615e20c9"
},
{
"dataPath": "params_shard_68.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.3.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.3.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.4.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.4.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.4.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.4.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.4.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "7da0192b24840d143b45f17c540dc508"
},
{
"dataPath": "params_shard_69.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.4.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.4.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.4.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.4.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.5.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "fb46533dda0bd4e575cf265ef9f52848"
},
{
"dataPath": "params_shard_70.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.5.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "590c8f9e35987ceea65b776786893e9e"
},
{
"dataPath": "params_shard_71.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.5.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.5.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.5.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.5.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.5.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.5.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "70958f1d94286459811141897b2c1260"
},
{
"dataPath": "params_shard_72.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.6.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "e2f6c5ac238728e25d739a74641adeb7"
},
{
"dataPath": "params_shard_73.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.5.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.5.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.6.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.6.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.6.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.6.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.6.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "98f665b5b7f17e41d7a1fd1f255651a2"
},
{
"dataPath": "params_shard_74.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.6.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.6.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.6.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.6.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.7.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "64129d5b01aded66f08537ed26077e93"
},
{
"dataPath": "params_shard_75.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.7.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "60ff3dc3cb5a69eab0b6c49302619bb5"
},
{
"dataPath": "params_shard_76.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.7.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.7.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.7.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.7.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.7.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.7.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "56c3809c33e596b25ebe079c967ada3d"
},
{
"dataPath": "params_shard_77.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.8.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "7a08bcb52123b2f5f20fae1bb5f2608f"
},
{
"dataPath": "params_shard_78.bin",
"format": "raw-shard",
"nbytes": 33239040,
"records": [
{
"name": "transformer.h.7.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.7.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
},
{
"name": "transformer.h.8.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15925248
},
{
"name": "transformer.h.8.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 15931392
},
{
"name": "transformer.h.8.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28514304
},
{
"name": "transformer.h.8.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 30087168
},
{
"name": "transformer.h.8.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 33232896
}
],
"md5sum": "6ed6ac9e03d979e2774ffef6d0ca6016"
},
{
"dataPath": "params_shard_79.bin",
"format": "raw-shard",
"nbytes": 21239808,
"records": [
{
"name": "transformer.h.8.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "transformer.h.8.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 4718592
},
{
"name": "transformer.h.8.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 5308416
},
{
"name": "transformer.h.8.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 19464192
},
{
"name": "transformer.h.9.ln.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 21233664
}
],
"md5sum": "ff4f097c28464cf739694531e2526137"
},
{
"dataPath": "params_shard_80.bin",
"format": "raw-shard",
"nbytes": 25165824,
"records": [
{
"name": "transformer.h.9.mlp.gate_up_proj.q_weight",
"shape": [
16384,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25165824,
"byteOffset": 0
}
],
"md5sum": "3d95e88dce2bac245d38de0b9c5f7401"
},
{
"dataPath": "params_shard_81.bin",
"format": "raw-shard",
"nbytes": 22616064,
"records": [
{
"name": "transformer.h.9.mlp.down_proj.q_weight",
"shape": [
3072,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 12582912,
"byteOffset": 0
},
{
"name": "transformer.h.9.mlp.down_proj.q_scale",
"shape": [
3072,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12582912
},
{
"name": "transformer.h.9.mlp.gate_up_proj.q_scale",
"shape": [
16384,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3145728,
"byteOffset": 14155776
},
{
"name": "transformer.h.9.post_attention_layernorm.weight",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 17301504
},
{
"name": "transformer.h.9.mixer.out_proj.q_weight",
"shape": [
3072,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 17307648
},
{
"name": "transformer.h.9.mixer.out_proj.q_scale",
"shape": [
3072,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 589824,
"byteOffset": 22026240
}
],
"md5sum": "04a78074a5a07f4914515ac0729ec0ae"
},
{
"dataPath": "params_shard_82.bin",
"format": "raw-shard",
"nbytes": 15925248,
"records": [
{
"name": "transformer.h.9.mixer.qkv_proj.q_weight",
"shape": [
9216,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 14155776,
"byteOffset": 0
},
{
"name": "transformer.h.9.mixer.qkv_proj.q_scale",
"shape": [
9216,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1769472,
"byteOffset": 14155776
}
],
"md5sum": "e7efa75fa7df03e39f04cd59d087037c"
}
]
}