| { |
| "config": { |
| "num_hidden_layers": 20, |
| "hidden_size": 2048, |
| "intermediate_size": 5120, |
| "moe_intermediate_size": 512, |
| "num_experts": 256, |
| "num_experts_per_tok": 8, |
| "expert_capacity": 48, |
| "block_size": 32, |
| "num_attention_heads": 16, |
| "num_key_value_heads": 4, |
| "head_dim": 128, |
| "rotary_dim": 64, |
| "rope_theta": 3000000, |
| "rms_norm_eps": 1e-06, |
| "routed_scaling_factor": 2.5, |
| "vocab_size": 157184, |
| "first_k_dense_replace": 1 |
| }, |
| "mask_id": 156895, |
| "nonexpert_bits": 8, |
| "lm_bits": 8, |
| "kv_format": "4c/4", |
| "rotation": { |
| "residual_seed": 20261024, |
| "kv_seed": 20261025, |
| "v_folded": true |
| }, |
| "files": { |
| "nonexpert_L0.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 21463040, |
| "i8tiles8", |
| [ |
| 10240, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 32194560, |
| 10731520, |
| "i8tiles8", |
| [ |
| 2048, |
| 5120 |
| ] |
| ] |
| }, |
| "nonexpert_L1.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L2.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L3.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L4.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L5.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L6.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L7.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L8.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L9.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L10.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L11.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L12.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L13.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L14.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L15.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L16.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L17.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L18.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "nonexpert_L19.bin": { |
| "qkv": [ |
| 0, |
| 6438912, |
| "i8tiles8", |
| [ |
| 3072, |
| 2048 |
| ] |
| ], |
| "dense": [ |
| 6438912, |
| 4292608, |
| "i8tiles8", |
| [ |
| 2048, |
| 2048 |
| ] |
| ], |
| "gate_up": [ |
| 10731520, |
| 2146304, |
| "i8tiles8", |
| [ |
| 1024, |
| 2048 |
| ] |
| ], |
| "down": [ |
| 12877824, |
| 1073152, |
| "i8tiles8", |
| [ |
| 2048, |
| 512 |
| ] |
| ] |
| }, |
| "lm_head.bin": { |
| "lm_head": [ |
| 0, |
| 329457664, |
| "i8tiles8", |
| [ |
| 157184, |
| 2048 |
| ] |
| ] |
| }, |
| "small.bin": { |
| "L0.input_layernorm.weight": [ |
| 0, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L0.post_attention_layernorm.weight": [ |
| 8192, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L0.attention.query_layernorm.weight": [ |
| 16384, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L0.attention.key_layernorm.weight": [ |
| 20480, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L1.input_layernorm.weight": [ |
| 24576, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L1.post_attention_layernorm.weight": [ |
| 32768, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L1.attention.query_layernorm.weight": [ |
| 40960, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L1.attention.key_layernorm.weight": [ |
| 45056, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L1.router": [ |
| 49152, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L1.expert_bias": [ |
| 1097728, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L2.input_layernorm.weight": [ |
| 1101824, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L2.post_attention_layernorm.weight": [ |
| 1110016, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L2.attention.query_layernorm.weight": [ |
| 1118208, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L2.attention.key_layernorm.weight": [ |
| 1122304, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L2.router": [ |
| 1126400, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L2.expert_bias": [ |
| 2174976, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L3.input_layernorm.weight": [ |
| 2179072, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L3.post_attention_layernorm.weight": [ |
| 2187264, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L3.attention.query_layernorm.weight": [ |
| 2195456, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L3.attention.key_layernorm.weight": [ |
| 2199552, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L3.router": [ |
| 2203648, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L3.expert_bias": [ |
| 3252224, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L4.input_layernorm.weight": [ |
| 3256320, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L4.post_attention_layernorm.weight": [ |
| 3264512, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L4.attention.query_layernorm.weight": [ |
| 3272704, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L4.attention.key_layernorm.weight": [ |
| 3276800, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L4.router": [ |
| 3280896, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L4.expert_bias": [ |
| 4329472, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L5.input_layernorm.weight": [ |
| 4333568, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L5.post_attention_layernorm.weight": [ |
| 4341760, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L5.attention.query_layernorm.weight": [ |
| 4349952, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L5.attention.key_layernorm.weight": [ |
| 4354048, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L5.router": [ |
| 4358144, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L5.expert_bias": [ |
| 5406720, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L6.input_layernorm.weight": [ |
| 5410816, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L6.post_attention_layernorm.weight": [ |
| 5419008, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L6.attention.query_layernorm.weight": [ |
| 5427200, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L6.attention.key_layernorm.weight": [ |
| 5431296, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L6.router": [ |
| 5435392, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L6.expert_bias": [ |
| 6483968, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L7.input_layernorm.weight": [ |
| 6488064, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L7.post_attention_layernorm.weight": [ |
| 6496256, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L7.attention.query_layernorm.weight": [ |
| 6504448, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L7.attention.key_layernorm.weight": [ |
| 6508544, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L7.router": [ |
| 6512640, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L7.expert_bias": [ |
| 7561216, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L8.input_layernorm.weight": [ |
| 7565312, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L8.post_attention_layernorm.weight": [ |
| 7573504, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L8.attention.query_layernorm.weight": [ |
| 7581696, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L8.attention.key_layernorm.weight": [ |
| 7585792, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L8.router": [ |
| 7589888, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L8.expert_bias": [ |
| 8638464, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L9.input_layernorm.weight": [ |
| 8642560, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L9.post_attention_layernorm.weight": [ |
| 8650752, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L9.attention.query_layernorm.weight": [ |
| 8658944, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L9.attention.key_layernorm.weight": [ |
| 8663040, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L9.router": [ |
| 8667136, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L9.expert_bias": [ |
| 9715712, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L10.input_layernorm.weight": [ |
| 9719808, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L10.post_attention_layernorm.weight": [ |
| 9728000, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L10.attention.query_layernorm.weight": [ |
| 9736192, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L10.attention.key_layernorm.weight": [ |
| 9740288, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L10.router": [ |
| 9744384, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L10.expert_bias": [ |
| 10792960, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L11.input_layernorm.weight": [ |
| 10797056, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L11.post_attention_layernorm.weight": [ |
| 10805248, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L11.attention.query_layernorm.weight": [ |
| 10813440, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L11.attention.key_layernorm.weight": [ |
| 10817536, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L11.router": [ |
| 10821632, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L11.expert_bias": [ |
| 11870208, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L12.input_layernorm.weight": [ |
| 11874304, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L12.post_attention_layernorm.weight": [ |
| 11882496, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L12.attention.query_layernorm.weight": [ |
| 11890688, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L12.attention.key_layernorm.weight": [ |
| 11894784, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L12.router": [ |
| 11898880, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L12.expert_bias": [ |
| 12947456, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L13.input_layernorm.weight": [ |
| 12951552, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L13.post_attention_layernorm.weight": [ |
| 12959744, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L13.attention.query_layernorm.weight": [ |
| 12967936, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L13.attention.key_layernorm.weight": [ |
| 12972032, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L13.router": [ |
| 12976128, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L13.expert_bias": [ |
| 14024704, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L14.input_layernorm.weight": [ |
| 14028800, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L14.post_attention_layernorm.weight": [ |
| 14036992, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L14.attention.query_layernorm.weight": [ |
| 14045184, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L14.attention.key_layernorm.weight": [ |
| 14049280, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L14.router": [ |
| 14053376, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L14.expert_bias": [ |
| 15101952, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L15.input_layernorm.weight": [ |
| 15106048, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L15.post_attention_layernorm.weight": [ |
| 15114240, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L15.attention.query_layernorm.weight": [ |
| 15122432, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L15.attention.key_layernorm.weight": [ |
| 15126528, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L15.router": [ |
| 15130624, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L15.expert_bias": [ |
| 16179200, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L16.input_layernorm.weight": [ |
| 16183296, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L16.post_attention_layernorm.weight": [ |
| 16191488, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L16.attention.query_layernorm.weight": [ |
| 16199680, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L16.attention.key_layernorm.weight": [ |
| 16203776, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L16.router": [ |
| 16207872, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L16.expert_bias": [ |
| 17256448, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L17.input_layernorm.weight": [ |
| 17260544, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L17.post_attention_layernorm.weight": [ |
| 17268736, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L17.attention.query_layernorm.weight": [ |
| 17276928, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L17.attention.key_layernorm.weight": [ |
| 17281024, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L17.router": [ |
| 17285120, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L17.expert_bias": [ |
| 18333696, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L18.input_layernorm.weight": [ |
| 18337792, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L18.post_attention_layernorm.weight": [ |
| 18345984, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L18.attention.query_layernorm.weight": [ |
| 18354176, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L18.attention.key_layernorm.weight": [ |
| 18358272, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L18.router": [ |
| 18362368, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L18.expert_bias": [ |
| 19410944, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "L19.input_layernorm.weight": [ |
| 19415040, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L19.post_attention_layernorm.weight": [ |
| 19423232, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "L19.attention.query_layernorm.weight": [ |
| 19431424, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L19.attention.key_layernorm.weight": [ |
| 19435520, |
| 512, |
| "f32", |
| [ |
| 128 |
| ] |
| ], |
| "L19.router": [ |
| 19439616, |
| 1048576, |
| "bf16", |
| [ |
| 256, |
| 2048 |
| ] |
| ], |
| "L19.expert_bias": [ |
| 20488192, |
| 1024, |
| "f32", |
| [ |
| 256 |
| ] |
| ], |
| "final_norm": [ |
| 20492288, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ], |
| "signs": [ |
| 20500480, |
| 8192, |
| "f32", |
| [ |
| 2048 |
| ] |
| ] |
| }, |
| "embed.bin": { |
| "embed": [ |
| 0, |
| 643825664, |
| "f16", |
| [ |
| 157184, |
| 2048 |
| ] |
| ] |
| } |
| } |
| } |
