Hugging Face's logo

junrushao
/
phi-2-q3f16_1-MLC

Model card Files
xet
Community
phi-2-q3f16_1-MLC / ndarray-cache.json
junrushao's picture
junrushao
Initial commit
9fc6a3b
raw
history blame contribute delete
177 kB
 {
"metadata": {
"ParamSize": 455,
"ParamBytes": 1252341760.0,
"BitsPerParam": 3.6042710814191015
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 52428800,
"records": [
{
"name": "lm_head.linear.q_weight",
"shape": [
51200,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 52428800,
"byteOffset": 0
}
],
"md5sum": "8503cdb31752d6d04435cc790e885ef8"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 30305280,
"records": [
{
"name": "lm_head.linear.bias",
"shape": [
51200
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 102400,
"byteOffset": 0
},
{
"name": "lm_head.linear.q_scale",
"shape": [
51200,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6553600,
"byteOffset": 102400
},
{
"name": "lm_head.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 6656000
},
{
"name": "lm_head.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 6661120
},
{
"name": "transformer.h.30.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 6666240
},
{
"name": "transformer.h.30.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 6681600
},
{
"name": "transformer.h.30.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 14545920
},
{
"name": "transformer.h.30.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 15528960
},
{
"name": "transformer.h.30.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 15534080
},
{
"name": "transformer.h.30.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 18155520
},
{
"name": "transformer.h.30.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 18483200
},
{
"name": "transformer.h.30.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 18503680
},
{
"name": "transformer.h.30.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 28989440
},
{
"name": "transformer.h.30.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 30300160
}
],
"md5sum": "51daf22d278290a6ef9e2cb4c372c731"
},
{
"dataPath": "params_shard_2.bin",
"format": "raw-shard",
"nbytes": 23644160,
"records": [
{
"name": "transformer.h.30.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.30.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.31.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.31.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11801600
},
{
"name": "transformer.h.31.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 11806720
},
{
"name": "transformer.h.31.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 11822080
},
{
"name": "transformer.h.31.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 19686400
},
{
"name": "transformer.h.31.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 20669440
},
{
"name": "transformer.h.31.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 20674560
},
{
"name": "transformer.h.31.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 23296000
},
{
"name": "transformer.h.31.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 23623680
}
],
"md5sum": "220009150531dd089a3ff5c991439f8b"
},
{
"dataPath": "params_shard_3.bin",
"format": "raw-shard",
"nbytes": 52428800,
"records": [
{
"name": "transformer.embd.q_weight",
"shape": [
51200,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 52428800,
"byteOffset": 0
}
],
"md5sum": "444a1ce65cfb68d42136ebe38da9381a"
},
{
"dataPath": "params_shard_4.bin",
"format": "raw-shard",
"nbytes": 30177280,
"records": [
{
"name": "transformer.h.31.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.31.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.31.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.31.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11801600
},
{
"name": "transformer.h.31.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22287360
},
{
"name": "transformer.embd.q_scale",
"shape": [
51200,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6553600,
"byteOffset": 23598080
},
{
"name": "transformer.h.0.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 30151680
},
{
"name": "transformer.h.0.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 30156800
},
{
"name": "transformer.h.0.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 30161920
}
],
"md5sum": "3bb78f1b1e593bd98d4217cedf19191c"
},
{
"dataPath": "params_shard_5.bin",
"format": "raw-shard",
"nbytes": 23623680,
"records": [
{
"name": "transformer.h.0.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 0
},
{
"name": "transformer.h.0.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 7864320
},
{
"name": "transformer.h.0.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 8847360
},
{
"name": "transformer.h.0.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 8852480
},
{
"name": "transformer.h.0.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 11473920
},
{
"name": "transformer.h.0.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 11801600
},
{
"name": "transformer.h.0.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11822080
},
{
"name": "transformer.h.0.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22307840
},
{
"name": "transformer.h.0.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23618560
}
],
"md5sum": "820c7648fe62987900bcd969016cf1bb"
},
{
"dataPath": "params_shard_6.bin",
"format": "raw-shard",
"nbytes": 23644160,
"records": [
{
"name": "transformer.h.0.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.0.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.1.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.1.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11801600
},
{
"name": "transformer.h.1.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 11806720
},
{
"name": "transformer.h.1.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 11822080
},
{
"name": "transformer.h.1.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 19686400
},
{
"name": "transformer.h.1.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 20669440
},
{
"name": "transformer.h.1.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 20674560
},
{
"name": "transformer.h.1.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 23296000
},
{
"name": "transformer.h.1.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 23623680
}
],
"md5sum": "1bf257dfcb0dacaa41167ae4feeb5b45"
},
{
"dataPath": "params_shard_7.bin",
"format": "raw-shard",
"nbytes": 32476160,
"records": [
{
"name": "transformer.h.1.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.1.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.1.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.1.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11801600
},
{
"name": "transformer.h.1.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22287360
},
{
"name": "transformer.h.10.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23598080
},
{
"name": "transformer.h.10.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23603200
},
{
"name": "transformer.h.10.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 23608320
},
{
"name": "transformer.h.10.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 23623680
},
{
"name": "transformer.h.10.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 31488000
},
{
"name": "transformer.h.10.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 32471040
}
],
"md5sum": "51a6f838c8734a64a23ef66a2d02fdde"
},
{
"dataPath": "params_shard_8.bin",
"format": "raw-shard",
"nbytes": 26593280,
"records": [
{
"name": "transformer.h.10.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 0
},
{
"name": "transformer.h.10.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 2621440
},
{
"name": "transformer.h.10.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 2949120
},
{
"name": "transformer.h.10.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 2969600
},
{
"name": "transformer.h.10.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 13455360
},
{
"name": "transformer.h.10.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 14766080
},
{
"name": "transformer.h.10.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 14771200
},
{
"name": "transformer.h.10.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 25256960
},
{
"name": "transformer.h.11.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26567680
},
{
"name": "transformer.h.11.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26572800
},
{
"name": "transformer.h.11.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 26577920
}
],
"md5sum": "8b310759583a154460453b4ef06404f2"
},
{
"dataPath": "params_shard_9.bin",
"format": "raw-shard",
"nbytes": 23623680,
"records": [
{
"name": "transformer.h.11.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 0
},
{
"name": "transformer.h.11.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 7864320
},
{
"name": "transformer.h.11.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 8847360
},
{
"name": "transformer.h.11.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 8852480
},
{
"name": "transformer.h.11.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 11473920
},
{
"name": "transformer.h.11.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 11801600
},
{
"name": "transformer.h.11.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11822080
},
{
"name": "transformer.h.11.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22307840
},
{
"name": "transformer.h.11.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23618560
}
],
"md5sum": "89552acc24a7847330313349a139093e"
},
{
"dataPath": "params_shard_10.bin",
"format": "raw-shard",
"nbytes": 23644160,
"records": [
{
"name": "transformer.h.11.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.11.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.12.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.12.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11801600
},
{
"name": "transformer.h.12.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 11806720
},
{
"name": "transformer.h.12.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 11822080
},
{
"name": "transformer.h.12.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 19686400
},
{
"name": "transformer.h.12.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 20669440
},
{
"name": "transformer.h.12.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 20674560
},
{
"name": "transformer.h.12.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 23296000
},
{
"name": "transformer.h.12.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 23623680
}
],
"md5sum": "0a41aa89ea18701784e6ac93c7a83ab5"
},
{
"dataPath": "params_shard_11.bin",
"format": "raw-shard",
"nbytes": 32476160,
"records": [
{
"name": "transformer.h.12.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.12.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.12.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.12.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11801600
},
{
"name": "transformer.h.12.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22287360
},
{
"name": "transformer.h.13.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23598080
},
{
"name": "transformer.h.13.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23603200
},
{
"name": "transformer.h.13.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 23608320
},
{
"name": "transformer.h.13.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 23623680
},
{
"name": "transformer.h.13.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 31488000
},
{
"name": "transformer.h.13.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 32471040
}
],
"md5sum": "43faab711830b9ea49e3194e846bf628"
},
{
"dataPath": "params_shard_12.bin",
"format": "raw-shard",
"nbytes": 26593280,
"records": [
{
"name": "transformer.h.13.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 0
},
{
"name": "transformer.h.13.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 2621440
},
{
"name": "transformer.h.13.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 2949120
},
{
"name": "transformer.h.13.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 2969600
},
{
"name": "transformer.h.13.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 13455360
},
{
"name": "transformer.h.13.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 14766080
},
{
"name": "transformer.h.13.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 14771200
},
{
"name": "transformer.h.13.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 25256960
},
{
"name": "transformer.h.14.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26567680
},
{
"name": "transformer.h.14.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26572800
},
{
"name": "transformer.h.14.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 26577920
}
],
"md5sum": "80bf75db9e545dc2169d2ccf624da9ef"
},
{
"dataPath": "params_shard_13.bin",
"format": "raw-shard",
"nbytes": 23623680,
"records": [
{
"name": "transformer.h.14.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 0
},
{
"name": "transformer.h.14.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 7864320
},
{
"name": "transformer.h.14.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 8847360
},
{
"name": "transformer.h.14.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 8852480
},
{
"name": "transformer.h.14.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 11473920
},
{
"name": "transformer.h.14.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 11801600
},
{
"name": "transformer.h.14.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11822080
},
{
"name": "transformer.h.14.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22307840
},
{
"name": "transformer.h.14.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23618560
}
],
"md5sum": "bbaf0debc8f982e8fcd67490b925ef81"
},
{
"dataPath": "params_shard_14.bin",
"format": "raw-shard",
"nbytes": 23644160,
"records": [
{
"name": "transformer.h.14.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.14.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.15.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.15.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11801600
},
{
"name": "transformer.h.15.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 11806720
},
{
"name": "transformer.h.15.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 11822080
},
{
"name": "transformer.h.15.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 19686400
},
{
"name": "transformer.h.15.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 20669440
},
{
"name": "transformer.h.15.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 20674560
},
{
"name": "transformer.h.15.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 23296000
},
{
"name": "transformer.h.15.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 23623680
}
],
"md5sum": "cde2fc0462e3f9b39828dc11295e465f"
},
{
"dataPath": "params_shard_15.bin",
"format": "raw-shard",
"nbytes": 32476160,
"records": [
{
"name": "transformer.h.15.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.15.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.15.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.15.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11801600
},
{
"name": "transformer.h.15.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22287360
},
{
"name": "transformer.h.16.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23598080
},
{
"name": "transformer.h.16.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23603200
},
{
"name": "transformer.h.16.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 23608320
},
{
"name": "transformer.h.16.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 23623680
},
{
"name": "transformer.h.16.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 31488000
},
{
"name": "transformer.h.16.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 32471040
}
],
"md5sum": "9aa5ca0761e73307f100a685632bbeea"
},
{
"dataPath": "params_shard_16.bin",
"format": "raw-shard",
"nbytes": 26593280,
"records": [
{
"name": "transformer.h.16.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 0
},
{
"name": "transformer.h.16.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 2621440
},
{
"name": "transformer.h.16.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 2949120
},
{
"name": "transformer.h.16.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 2969600
},
{
"name": "transformer.h.16.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 13455360
},
{
"name": "transformer.h.16.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 14766080
},
{
"name": "transformer.h.16.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 14771200
},
{
"name": "transformer.h.16.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 25256960
},
{
"name": "transformer.h.17.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26567680
},
{
"name": "transformer.h.17.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26572800
},
{
"name": "transformer.h.17.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 26577920
}
],
"md5sum": "bc821dfd4b18d47e408cc50ab55d988b"
},
{
"dataPath": "params_shard_17.bin",
"format": "raw-shard",
"nbytes": 23623680,
"records": [
{
"name": "transformer.h.17.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 0
},
{
"name": "transformer.h.17.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 7864320
},
{
"name": "transformer.h.17.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 8847360
},
{
"name": "transformer.h.17.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 8852480
},
{
"name": "transformer.h.17.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 11473920
},
{
"name": "transformer.h.17.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 11801600
},
{
"name": "transformer.h.17.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11822080
},
{
"name": "transformer.h.17.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22307840
},
{
"name": "transformer.h.17.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23618560
}
],
"md5sum": "37995a6caca01601cd7efa8cacecf833"
},
{
"dataPath": "params_shard_18.bin",
"format": "raw-shard",
"nbytes": 23644160,
"records": [
{
"name": "transformer.h.17.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.17.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.18.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.18.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11801600
},
{
"name": "transformer.h.18.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 11806720
},
{
"name": "transformer.h.18.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 11822080
},
{
"name": "transformer.h.18.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 19686400
},
{
"name": "transformer.h.18.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 20669440
},
{
"name": "transformer.h.18.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 20674560
},
{
"name": "transformer.h.18.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 23296000
},
{
"name": "transformer.h.18.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 23623680
}
],
"md5sum": "2dfb8cf8657c5e804e79e902beb8874e"
},
{
"dataPath": "params_shard_19.bin",
"format": "raw-shard",
"nbytes": 32476160,
"records": [
{
"name": "transformer.h.18.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.18.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.18.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.18.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11801600
},
{
"name": "transformer.h.18.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22287360
},
{
"name": "transformer.h.19.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23598080
},
{
"name": "transformer.h.19.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23603200
},
{
"name": "transformer.h.19.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 23608320
},
{
"name": "transformer.h.19.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 23623680
},
{
"name": "transformer.h.19.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 31488000
},
{
"name": "transformer.h.19.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 32471040
}
],
"md5sum": "e1828b0dfd194a1304eb64b2a3d6ace7"
},
{
"dataPath": "params_shard_20.bin",
"format": "raw-shard",
"nbytes": 26593280,
"records": [
{
"name": "transformer.h.19.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 0
},
{
"name": "transformer.h.19.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 2621440
},
{
"name": "transformer.h.19.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 2949120
},
{
"name": "transformer.h.19.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 2969600
},
{
"name": "transformer.h.19.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 13455360
},
{
"name": "transformer.h.19.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 14766080
},
{
"name": "transformer.h.19.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 14771200
},
{
"name": "transformer.h.19.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 25256960
},
{
"name": "transformer.h.2.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26567680
},
{
"name": "transformer.h.2.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26572800
},
{
"name": "transformer.h.2.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 26577920
}
],
"md5sum": "756a0bf111183079ff599585e64ec58d"
},
{
"dataPath": "params_shard_21.bin",
"format": "raw-shard",
"nbytes": 23623680,
"records": [
{
"name": "transformer.h.2.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 0
},
{
"name": "transformer.h.2.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 7864320
},
{
"name": "transformer.h.2.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 8847360
},
{
"name": "transformer.h.2.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 8852480
},
{
"name": "transformer.h.2.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 11473920
},
{
"name": "transformer.h.2.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 11801600
},
{
"name": "transformer.h.2.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11822080
},
{
"name": "transformer.h.2.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22307840
},
{
"name": "transformer.h.2.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23618560
}
],
"md5sum": "4d3f6284ab57c7a2b3a2e288e5191019"
},
{
"dataPath": "params_shard_22.bin",
"format": "raw-shard",
"nbytes": 23644160,
"records": [
{
"name": "transformer.h.2.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.2.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.20.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.20.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11801600
},
{
"name": "transformer.h.20.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 11806720
},
{
"name": "transformer.h.20.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 11822080
},
{
"name": "transformer.h.20.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 19686400
},
{
"name": "transformer.h.20.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 20669440
},
{
"name": "transformer.h.20.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 20674560
},
{
"name": "transformer.h.20.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 23296000
},
{
"name": "transformer.h.20.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 23623680
}
],
"md5sum": "db2eab185667fca5e71f35a8193cec85"
},
{
"dataPath": "params_shard_23.bin",
"format": "raw-shard",
"nbytes": 32476160,
"records": [
{
"name": "transformer.h.20.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.20.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.20.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.20.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11801600
},
{
"name": "transformer.h.20.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22287360
},
{
"name": "transformer.h.21.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23598080
},
{
"name": "transformer.h.21.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23603200
},
{
"name": "transformer.h.21.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 23608320
},
{
"name": "transformer.h.21.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 23623680
},
{
"name": "transformer.h.21.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 31488000
},
{
"name": "transformer.h.21.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 32471040
}
],
"md5sum": "f7869a670740d5cf77ae277756a046c9"
},
{
"dataPath": "params_shard_24.bin",
"format": "raw-shard",
"nbytes": 26593280,
"records": [
{
"name": "transformer.h.21.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 0
},
{
"name": "transformer.h.21.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 2621440
},
{
"name": "transformer.h.21.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 2949120
},
{
"name": "transformer.h.21.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 2969600
},
{
"name": "transformer.h.21.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 13455360
},
{
"name": "transformer.h.21.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 14766080
},
{
"name": "transformer.h.21.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 14771200
},
{
"name": "transformer.h.21.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 25256960
},
{
"name": "transformer.h.22.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26567680
},
{
"name": "transformer.h.22.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26572800
},
{
"name": "transformer.h.22.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 26577920
}
],
"md5sum": "ae10f14690d9d0c4af86869a6de6e639"
},
{
"dataPath": "params_shard_25.bin",
"format": "raw-shard",
"nbytes": 23623680,
"records": [
{
"name": "transformer.h.22.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 0
},
{
"name": "transformer.h.22.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 7864320
},
{
"name": "transformer.h.22.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 8847360
},
{
"name": "transformer.h.22.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 8852480
},
{
"name": "transformer.h.22.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 11473920
},
{
"name": "transformer.h.22.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 11801600
},
{
"name": "transformer.h.22.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11822080
},
{
"name": "transformer.h.22.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22307840
},
{
"name": "transformer.h.22.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23618560
}
],
"md5sum": "621b33076777a15eab91006f7fcf9632"
},
{
"dataPath": "params_shard_26.bin",
"format": "raw-shard",
"nbytes": 23644160,
"records": [
{
"name": "transformer.h.22.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.22.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.23.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.23.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11801600
},
{
"name": "transformer.h.23.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 11806720
},
{
"name": "transformer.h.23.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 11822080
},
{
"name": "transformer.h.23.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 19686400
},
{
"name": "transformer.h.23.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 20669440
},
{
"name": "transformer.h.23.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 20674560
},
{
"name": "transformer.h.23.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 23296000
},
{
"name": "transformer.h.23.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 23623680
}
],
"md5sum": "0f56328d078938f242d0ba888614d204"
},
{
"dataPath": "params_shard_27.bin",
"format": "raw-shard",
"nbytes": 32476160,
"records": [
{
"name": "transformer.h.23.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.23.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.23.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.23.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11801600
},
{
"name": "transformer.h.23.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22287360
},
{
"name": "transformer.h.24.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23598080
},
{
"name": "transformer.h.24.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23603200
},
{
"name": "transformer.h.24.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 23608320
},
{
"name": "transformer.h.24.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 23623680
},
{
"name": "transformer.h.24.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 31488000
},
{
"name": "transformer.h.24.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 32471040
}
],
"md5sum": "b42a9fc2fd28fc85c7cc1a168dab6c77"
},
{
"dataPath": "params_shard_28.bin",
"format": "raw-shard",
"nbytes": 26593280,
"records": [
{
"name": "transformer.h.24.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 0
},
{
"name": "transformer.h.24.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 2621440
},
{
"name": "transformer.h.24.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 2949120
},
{
"name": "transformer.h.24.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 2969600
},
{
"name": "transformer.h.24.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 13455360
},
{
"name": "transformer.h.24.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 14766080
},
{
"name": "transformer.h.24.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 14771200
},
{
"name": "transformer.h.24.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 25256960
},
{
"name": "transformer.h.25.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26567680
},
{
"name": "transformer.h.25.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26572800
},
{
"name": "transformer.h.25.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 26577920
}
],
"md5sum": "b1d0dd47a2c2279c4a5b15a8f8bbbe34"
},
{
"dataPath": "params_shard_29.bin",
"format": "raw-shard",
"nbytes": 23623680,
"records": [
{
"name": "transformer.h.25.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 0
},
{
"name": "transformer.h.25.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 7864320
},
{
"name": "transformer.h.25.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 8847360
},
{
"name": "transformer.h.25.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 8852480
},
{
"name": "transformer.h.25.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 11473920
},
{
"name": "transformer.h.25.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 11801600
},
{
"name": "transformer.h.25.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11822080
},
{
"name": "transformer.h.25.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22307840
},
{
"name": "transformer.h.25.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23618560
}
],
"md5sum": "052d166da83b518f0e07b2ca99e11ede"
},
{
"dataPath": "params_shard_30.bin",
"format": "raw-shard",
"nbytes": 23644160,
"records": [
{
"name": "transformer.h.25.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.25.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.26.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.26.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11801600
},
{
"name": "transformer.h.26.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 11806720
},
{
"name": "transformer.h.26.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 11822080
},
{
"name": "transformer.h.26.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 19686400
},
{
"name": "transformer.h.26.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 20669440
},
{
"name": "transformer.h.26.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 20674560
},
{
"name": "transformer.h.26.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 23296000
},
{
"name": "transformer.h.26.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 23623680
}
],
"md5sum": "c66734017f96495525e5eddfd68e1e32"
},
{
"dataPath": "params_shard_31.bin",
"format": "raw-shard",
"nbytes": 32476160,
"records": [
{
"name": "transformer.h.26.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.26.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.26.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.26.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11801600
},
{
"name": "transformer.h.26.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22287360
},
{
"name": "transformer.h.27.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23598080
},
{
"name": "transformer.h.27.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23603200
},
{
"name": "transformer.h.27.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 23608320
},
{
"name": "transformer.h.27.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 23623680
},
{
"name": "transformer.h.27.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 31488000
},
{
"name": "transformer.h.27.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 32471040
}
],
"md5sum": "7d903cb9a923785d82c1a2e554482782"
},
{
"dataPath": "params_shard_32.bin",
"format": "raw-shard",
"nbytes": 26593280,
"records": [
{
"name": "transformer.h.27.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 0
},
{
"name": "transformer.h.27.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 2621440
},
{
"name": "transformer.h.27.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 2949120
},
{
"name": "transformer.h.27.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 2969600
},
{
"name": "transformer.h.27.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 13455360
},
{
"name": "transformer.h.27.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 14766080
},
{
"name": "transformer.h.27.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 14771200
},
{
"name": "transformer.h.27.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 25256960
},
{
"name": "transformer.h.28.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26567680
},
{
"name": "transformer.h.28.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26572800
},
{
"name": "transformer.h.28.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 26577920
}
],
"md5sum": "00491905181e6300d3d6246f91054b51"
},
{
"dataPath": "params_shard_33.bin",
"format": "raw-shard",
"nbytes": 23623680,
"records": [
{
"name": "transformer.h.28.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 0
},
{
"name": "transformer.h.28.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 7864320
},
{
"name": "transformer.h.28.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 8847360
},
{
"name": "transformer.h.28.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 8852480
},
{
"name": "transformer.h.28.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 11473920
},
{
"name": "transformer.h.28.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 11801600
},
{
"name": "transformer.h.28.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11822080
},
{
"name": "transformer.h.28.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22307840
},
{
"name": "transformer.h.28.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23618560
}
],
"md5sum": "60b2dd4314c97fd4662491932d5fbb74"
},
{
"dataPath": "params_shard_34.bin",
"format": "raw-shard",
"nbytes": 23644160,
"records": [
{
"name": "transformer.h.28.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.28.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.29.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.29.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11801600
},
{
"name": "transformer.h.29.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 11806720
},
{
"name": "transformer.h.29.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 11822080
},
{
"name": "transformer.h.29.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 19686400
},
{
"name": "transformer.h.29.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 20669440
},
{
"name": "transformer.h.29.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 20674560
},
{
"name": "transformer.h.29.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 23296000
},
{
"name": "transformer.h.29.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 23623680
}
],
"md5sum": "3648d31c36900309d4465fa22f092467"
},
{
"dataPath": "params_shard_35.bin",
"format": "raw-shard",
"nbytes": 32476160,
"records": [
{
"name": "transformer.h.29.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.29.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.29.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.29.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11801600
},
{
"name": "transformer.h.29.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22287360
},
{
"name": "transformer.h.3.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23598080
},
{
"name": "transformer.h.3.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23603200
},
{
"name": "transformer.h.3.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 23608320
},
{
"name": "transformer.h.3.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 23623680
},
{
"name": "transformer.h.3.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 31488000
},
{
"name": "transformer.h.3.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 32471040
}
],
"md5sum": "b4d733d59ac113c63b3feabdd09bf4d1"
},
{
"dataPath": "params_shard_36.bin",
"format": "raw-shard",
"nbytes": 26603520,
"records": [
{
"name": "transformer.h.3.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 0
},
{
"name": "transformer.h.3.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 2621440
},
{
"name": "transformer.h.3.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 2949120
},
{
"name": "transformer.h.3.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 2969600
},
{
"name": "transformer.h.3.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 13455360
},
{
"name": "transformer.h.3.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 14766080
},
{
"name": "transformer.h.3.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 14771200
},
{
"name": "transformer.h.3.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 25256960
},
{
"name": "transformer.h.30.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26567680
},
{
"name": "transformer.h.30.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26572800
},
{
"name": "transformer.h.4.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26577920
},
{
"name": "transformer.h.4.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26583040
},
{
"name": "transformer.h.4.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 26588160
}
],
"md5sum": "e7737b15ba5f7da730a2bb6cab8c66dc"
},
{
"dataPath": "params_shard_37.bin",
"format": "raw-shard",
"nbytes": 23623680,
"records": [
{
"name": "transformer.h.4.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 0
},
{
"name": "transformer.h.4.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 7864320
},
{
"name": "transformer.h.4.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 8847360
},
{
"name": "transformer.h.4.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 8852480
},
{
"name": "transformer.h.4.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 11473920
},
{
"name": "transformer.h.4.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 11801600
},
{
"name": "transformer.h.4.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11822080
},
{
"name": "transformer.h.4.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22307840
},
{
"name": "transformer.h.4.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23618560
}
],
"md5sum": "0597d18af46a583ff35fc0e0a4b0ae06"
},
{
"dataPath": "params_shard_38.bin",
"format": "raw-shard",
"nbytes": 23644160,
"records": [
{
"name": "transformer.h.4.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.4.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.5.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.5.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11801600
},
{
"name": "transformer.h.5.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 11806720
},
{
"name": "transformer.h.5.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 11822080
},
{
"name": "transformer.h.5.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 19686400
},
{
"name": "transformer.h.5.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 20669440
},
{
"name": "transformer.h.5.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 20674560
},
{
"name": "transformer.h.5.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 23296000
},
{
"name": "transformer.h.5.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 23623680
}
],
"md5sum": "77adda8104cb0b0582896e5344a655e2"
},
{
"dataPath": "params_shard_39.bin",
"format": "raw-shard",
"nbytes": 32476160,
"records": [
{
"name": "transformer.h.5.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.5.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.5.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.5.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11801600
},
{
"name": "transformer.h.5.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22287360
},
{
"name": "transformer.h.6.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23598080
},
{
"name": "transformer.h.6.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23603200
},
{
"name": "transformer.h.6.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 23608320
},
{
"name": "transformer.h.6.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 23623680
},
{
"name": "transformer.h.6.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 31488000
},
{
"name": "transformer.h.6.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 32471040
}
],
"md5sum": "8ac89b1b02094e0a34e890e478e520e6"
},
{
"dataPath": "params_shard_40.bin",
"format": "raw-shard",
"nbytes": 26593280,
"records": [
{
"name": "transformer.h.6.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 0
},
{
"name": "transformer.h.6.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 2621440
},
{
"name": "transformer.h.6.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 2949120
},
{
"name": "transformer.h.6.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 2969600
},
{
"name": "transformer.h.6.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 13455360
},
{
"name": "transformer.h.6.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 14766080
},
{
"name": "transformer.h.6.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 14771200
},
{
"name": "transformer.h.6.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 25256960
},
{
"name": "transformer.h.7.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26567680
},
{
"name": "transformer.h.7.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 26572800
},
{
"name": "transformer.h.7.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 26577920
}
],
"md5sum": "241b692a71cb3d5ada05a33c5b1e8717"
},
{
"dataPath": "params_shard_41.bin",
"format": "raw-shard",
"nbytes": 23623680,
"records": [
{
"name": "transformer.h.7.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 0
},
{
"name": "transformer.h.7.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 7864320
},
{
"name": "transformer.h.7.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 8847360
},
{
"name": "transformer.h.7.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 8852480
},
{
"name": "transformer.h.7.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 11473920
},
{
"name": "transformer.h.7.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 11801600
},
{
"name": "transformer.h.7.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11822080
},
{
"name": "transformer.h.7.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22307840
},
{
"name": "transformer.h.7.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23618560
}
],
"md5sum": "28de01ca76153d1044460eb214ef2e76"
},
{
"dataPath": "params_shard_42.bin",
"format": "raw-shard",
"nbytes": 23644160,
"records": [
{
"name": "transformer.h.7.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.7.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.8.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.8.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11801600
},
{
"name": "transformer.h.8.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 11806720
},
{
"name": "transformer.h.8.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 11822080
},
{
"name": "transformer.h.8.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 19686400
},
{
"name": "transformer.h.8.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 20669440
},
{
"name": "transformer.h.8.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 20674560
},
{
"name": "transformer.h.8.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 23296000
},
{
"name": "transformer.h.8.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 23623680
}
],
"md5sum": "9a90bb2f472a31878b452fa00b045506"
},
{
"dataPath": "params_shard_43.bin",
"format": "raw-shard",
"nbytes": 32476160,
"records": [
{
"name": "transformer.h.8.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 0
},
{
"name": "transformer.h.8.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 10485760
},
{
"name": "transformer.h.8.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 11796480
},
{
"name": "transformer.h.8.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 11801600
},
{
"name": "transformer.h.8.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 22287360
},
{
"name": "transformer.h.9.ln.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23598080
},
{
"name": "transformer.h.9.ln.weight",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 23603200
},
{
"name": "transformer.h.9.mixer.Wqkv.bias",
"shape": [
7680
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 15360,
"byteOffset": 23608320
},
{
"name": "transformer.h.9.mixer.Wqkv.q_weight",
"shape": [
7680,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 7864320,
"byteOffset": 23623680
},
{
"name": "transformer.h.9.mixer.Wqkv.q_scale",
"shape": [
7680,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 983040,
"byteOffset": 31488000
},
{
"name": "transformer.h.9.mixer.out_proj.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 32471040
}
],
"md5sum": "8f8fba896332a98cba11fd5581b8c681"
},
{
"dataPath": "params_shard_44.bin",
"format": "raw-shard",
"nbytes": 26567680,
"records": [
{
"name": "transformer.h.9.mixer.out_proj.q_weight",
"shape": [
2560,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2621440,
"byteOffset": 0
},
{
"name": "transformer.h.9.mixer.out_proj.q_scale",
"shape": [
2560,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 327680,
"byteOffset": 2621440
},
{
"name": "transformer.h.9.mlp.fc1.bias",
"shape": [
10240
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 20480,
"byteOffset": 2949120
},
{
"name": "transformer.h.9.mlp.fc1.q_weight",
"shape": [
10240,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 2969600
},
{
"name": "transformer.h.9.mlp.fc1.q_scale",
"shape": [
10240,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 13455360
},
{
"name": "transformer.h.9.mlp.fc2.bias",
"shape": [
2560
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 5120,
"byteOffset": 14766080
},
{
"name": "transformer.h.9.mlp.fc2.q_weight",
"shape": [
2560,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 10485760,
"byteOffset": 14771200
},
{
"name": "transformer.h.9.mlp.fc2.q_scale",
"shape": [
2560,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1310720,
"byteOffset": 25256960
}
],
"md5sum": "3e99440d37f01206f3f1272cb82b84aa"
}
]
}