{
 "config": {
  "num_hidden_layers": 20,
  "hidden_size": 2048,
  "intermediate_size": 5120,
  "moe_intermediate_size": 512,
  "num_experts": 256,
  "num_experts_per_tok": 8,
  "expert_capacity": 48,
  "block_size": 32,
  "num_attention_heads": 16,
  "num_key_value_heads": 4,
  "head_dim": 128,
  "rotary_dim": 64,
  "rope_theta": 3000000,
  "rms_norm_eps": 1e-06,
  "routed_scaling_factor": 2.5,
  "vocab_size": 157184,
  "first_k_dense_replace": 1
 },
 "mask_id": 156895,
 "nonexpert_bits": 8,
 "lm_bits": 8,
 "kv_format": "4c/4",
 "rotation": {
  "residual_seed": 20261024,
  "kv_seed": 20261025,
  "v_folded": true
 },
 "files": {
  "nonexpert_L0.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    21463040,
    "i8tiles8",
    [
     10240,
     2048
    ]
   ],
   "down": [
    32194560,
    10731520,
    "i8tiles8",
    [
     2048,
     5120
    ]
   ]
  },
  "nonexpert_L1.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L2.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L3.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L4.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L5.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L6.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L7.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L8.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L9.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L10.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L11.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L12.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L13.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L14.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L15.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L16.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L17.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L18.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "nonexpert_L19.bin": {
   "qkv": [
    0,
    6438912,
    "i8tiles8",
    [
     3072,
     2048
    ]
   ],
   "dense": [
    6438912,
    4292608,
    "i8tiles8",
    [
     2048,
     2048
    ]
   ],
   "gate_up": [
    10731520,
    2146304,
    "i8tiles8",
    [
     1024,
     2048
    ]
   ],
   "down": [
    12877824,
    1073152,
    "i8tiles8",
    [
     2048,
     512
    ]
   ]
  },
  "lm_head.bin": {
   "lm_head": [
    0,
    329457664,
    "i8tiles8",
    [
     157184,
     2048
    ]
   ]
  },
  "small.bin": {
   "L0.input_layernorm.weight": [
    0,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L0.post_attention_layernorm.weight": [
    8192,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L0.attention.query_layernorm.weight": [
    16384,
    512,
    "f32",
    [
     128
    ]
   ],
   "L0.attention.key_layernorm.weight": [
    20480,
    512,
    "f32",
    [
     128
    ]
   ],
   "L1.input_layernorm.weight": [
    24576,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L1.post_attention_layernorm.weight": [
    32768,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L1.attention.query_layernorm.weight": [
    40960,
    512,
    "f32",
    [
     128
    ]
   ],
   "L1.attention.key_layernorm.weight": [
    45056,
    512,
    "f32",
    [
     128
    ]
   ],
   "L1.router": [
    49152,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L1.expert_bias": [
    1097728,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L2.input_layernorm.weight": [
    1101824,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L2.post_attention_layernorm.weight": [
    1110016,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L2.attention.query_layernorm.weight": [
    1118208,
    512,
    "f32",
    [
     128
    ]
   ],
   "L2.attention.key_layernorm.weight": [
    1122304,
    512,
    "f32",
    [
     128
    ]
   ],
   "L2.router": [
    1126400,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L2.expert_bias": [
    2174976,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L3.input_layernorm.weight": [
    2179072,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L3.post_attention_layernorm.weight": [
    2187264,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L3.attention.query_layernorm.weight": [
    2195456,
    512,
    "f32",
    [
     128
    ]
   ],
   "L3.attention.key_layernorm.weight": [
    2199552,
    512,
    "f32",
    [
     128
    ]
   ],
   "L3.router": [
    2203648,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L3.expert_bias": [
    3252224,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L4.input_layernorm.weight": [
    3256320,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L4.post_attention_layernorm.weight": [
    3264512,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L4.attention.query_layernorm.weight": [
    3272704,
    512,
    "f32",
    [
     128
    ]
   ],
   "L4.attention.key_layernorm.weight": [
    3276800,
    512,
    "f32",
    [
     128
    ]
   ],
   "L4.router": [
    3280896,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L4.expert_bias": [
    4329472,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L5.input_layernorm.weight": [
    4333568,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L5.post_attention_layernorm.weight": [
    4341760,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L5.attention.query_layernorm.weight": [
    4349952,
    512,
    "f32",
    [
     128
    ]
   ],
   "L5.attention.key_layernorm.weight": [
    4354048,
    512,
    "f32",
    [
     128
    ]
   ],
   "L5.router": [
    4358144,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L5.expert_bias": [
    5406720,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L6.input_layernorm.weight": [
    5410816,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L6.post_attention_layernorm.weight": [
    5419008,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L6.attention.query_layernorm.weight": [
    5427200,
    512,
    "f32",
    [
     128
    ]
   ],
   "L6.attention.key_layernorm.weight": [
    5431296,
    512,
    "f32",
    [
     128
    ]
   ],
   "L6.router": [
    5435392,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L6.expert_bias": [
    6483968,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L7.input_layernorm.weight": [
    6488064,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L7.post_attention_layernorm.weight": [
    6496256,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L7.attention.query_layernorm.weight": [
    6504448,
    512,
    "f32",
    [
     128
    ]
   ],
   "L7.attention.key_layernorm.weight": [
    6508544,
    512,
    "f32",
    [
     128
    ]
   ],
   "L7.router": [
    6512640,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L7.expert_bias": [
    7561216,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L8.input_layernorm.weight": [
    7565312,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L8.post_attention_layernorm.weight": [
    7573504,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L8.attention.query_layernorm.weight": [
    7581696,
    512,
    "f32",
    [
     128
    ]
   ],
   "L8.attention.key_layernorm.weight": [
    7585792,
    512,
    "f32",
    [
     128
    ]
   ],
   "L8.router": [
    7589888,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L8.expert_bias": [
    8638464,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L9.input_layernorm.weight": [
    8642560,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L9.post_attention_layernorm.weight": [
    8650752,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L9.attention.query_layernorm.weight": [
    8658944,
    512,
    "f32",
    [
     128
    ]
   ],
   "L9.attention.key_layernorm.weight": [
    8663040,
    512,
    "f32",
    [
     128
    ]
   ],
   "L9.router": [
    8667136,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L9.expert_bias": [
    9715712,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L10.input_layernorm.weight": [
    9719808,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L10.post_attention_layernorm.weight": [
    9728000,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L10.attention.query_layernorm.weight": [
    9736192,
    512,
    "f32",
    [
     128
    ]
   ],
   "L10.attention.key_layernorm.weight": [
    9740288,
    512,
    "f32",
    [
     128
    ]
   ],
   "L10.router": [
    9744384,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L10.expert_bias": [
    10792960,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L11.input_layernorm.weight": [
    10797056,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L11.post_attention_layernorm.weight": [
    10805248,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L11.attention.query_layernorm.weight": [
    10813440,
    512,
    "f32",
    [
     128
    ]
   ],
   "L11.attention.key_layernorm.weight": [
    10817536,
    512,
    "f32",
    [
     128
    ]
   ],
   "L11.router": [
    10821632,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L11.expert_bias": [
    11870208,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L12.input_layernorm.weight": [
    11874304,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L12.post_attention_layernorm.weight": [
    11882496,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L12.attention.query_layernorm.weight": [
    11890688,
    512,
    "f32",
    [
     128
    ]
   ],
   "L12.attention.key_layernorm.weight": [
    11894784,
    512,
    "f32",
    [
     128
    ]
   ],
   "L12.router": [
    11898880,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L12.expert_bias": [
    12947456,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L13.input_layernorm.weight": [
    12951552,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L13.post_attention_layernorm.weight": [
    12959744,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L13.attention.query_layernorm.weight": [
    12967936,
    512,
    "f32",
    [
     128
    ]
   ],
   "L13.attention.key_layernorm.weight": [
    12972032,
    512,
    "f32",
    [
     128
    ]
   ],
   "L13.router": [
    12976128,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L13.expert_bias": [
    14024704,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L14.input_layernorm.weight": [
    14028800,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L14.post_attention_layernorm.weight": [
    14036992,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L14.attention.query_layernorm.weight": [
    14045184,
    512,
    "f32",
    [
     128
    ]
   ],
   "L14.attention.key_layernorm.weight": [
    14049280,
    512,
    "f32",
    [
     128
    ]
   ],
   "L14.router": [
    14053376,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L14.expert_bias": [
    15101952,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L15.input_layernorm.weight": [
    15106048,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L15.post_attention_layernorm.weight": [
    15114240,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L15.attention.query_layernorm.weight": [
    15122432,
    512,
    "f32",
    [
     128
    ]
   ],
   "L15.attention.key_layernorm.weight": [
    15126528,
    512,
    "f32",
    [
     128
    ]
   ],
   "L15.router": [
    15130624,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L15.expert_bias": [
    16179200,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L16.input_layernorm.weight": [
    16183296,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L16.post_attention_layernorm.weight": [
    16191488,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L16.attention.query_layernorm.weight": [
    16199680,
    512,
    "f32",
    [
     128
    ]
   ],
   "L16.attention.key_layernorm.weight": [
    16203776,
    512,
    "f32",
    [
     128
    ]
   ],
   "L16.router": [
    16207872,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L16.expert_bias": [
    17256448,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L17.input_layernorm.weight": [
    17260544,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L17.post_attention_layernorm.weight": [
    17268736,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L17.attention.query_layernorm.weight": [
    17276928,
    512,
    "f32",
    [
     128
    ]
   ],
   "L17.attention.key_layernorm.weight": [
    17281024,
    512,
    "f32",
    [
     128
    ]
   ],
   "L17.router": [
    17285120,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L17.expert_bias": [
    18333696,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L18.input_layernorm.weight": [
    18337792,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L18.post_attention_layernorm.weight": [
    18345984,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L18.attention.query_layernorm.weight": [
    18354176,
    512,
    "f32",
    [
     128
    ]
   ],
   "L18.attention.key_layernorm.weight": [
    18358272,
    512,
    "f32",
    [
     128
    ]
   ],
   "L18.router": [
    18362368,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L18.expert_bias": [
    19410944,
    1024,
    "f32",
    [
     256
    ]
   ],
   "L19.input_layernorm.weight": [
    19415040,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L19.post_attention_layernorm.weight": [
    19423232,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "L19.attention.query_layernorm.weight": [
    19431424,
    512,
    "f32",
    [
     128
    ]
   ],
   "L19.attention.key_layernorm.weight": [
    19435520,
    512,
    "f32",
    [
     128
    ]
   ],
   "L19.router": [
    19439616,
    1048576,
    "bf16",
    [
     256,
     2048
    ]
   ],
   "L19.expert_bias": [
    20488192,
    1024,
    "f32",
    [
     256
    ]
   ],
   "final_norm": [
    20492288,
    8192,
    "f32",
    [
     2048
    ]
   ],
   "signs": [
    20500480,
    8192,
    "f32",
    [
     2048
    ]
   ]
  },
  "embed.bin": {
   "embed": [
    0,
    643825664,
    "f16",
    [
     157184,
     2048
    ]
   ]
  }
 }
}