Upload Steerling-8B weights (bf16)

config.json

@@ -0,0 +1,52 @@
+{
+  "model_type": "causal_diffusion",
+  "interpretable": true,
+  "n_layers": 32,
+  "n_head": 32,
+  "n_embd": 4096,
+  "block_size": 4096,
+  "n_kv_heads": 4,
+  "diff_block_size": 64,
+  "use_rms_norm": true,
+  "norm_eps": 1e-05,
+  "norm_order": "post",
+  "use_qk_norm": true,
+  "use_rope": true,
+  "rope_base": 500000.0,
+  "rope_full_precision": true,
+  "mlp_type": "swiglu",
+  "activation": "gelu",
+  "mlp_ratio": 4,
+  "intermediate_size": null,
+  "use_bias": false,
+  "clip_qkv": 10.0,
+  "weight_sharing": true,
+  "pad_token_id": 100277,
+  "bos_token_id": 100278,
+  "eos_token_id": 100257,
+  "endofchunk_token_id": 100279,
+  "mask_token_id": 100280,
+  "vocab_size": 100281,
+  "concept": {
+    "n_concepts": 33732,
+    "n_unknown_concepts": 101196,
+    "max_concepts": 16,
+    "concept_dim": 4096,
+    "use_attention_known": false,
+    "use_attention_unknown": false,
+    "topk_known": 16,
+    "topk_known_features": 32,
+    "unknown_topk": 128,
+    "use_unknown": true,
+    "apply_topk_to_unknown": true,
+    "topk_on_logits": false,
+    "factorize_unknown": true,
+    "factorize_rank": 256,
+    "use_epsilon_correction": true,
+    "block_size": 4096,
+    "pad_multiple": 16,
+    "store_unknown_weights": false,
+    "inject_layer": 16,
+    "inject_alpha": 1.0
+  }
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c273e89844fece6f6b8aa8fb00a1a6d2f180bf6e207c823e96d9531e405fbd0
+size 5200533176

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:767d142d8ef12520891a8cc229e4e9fd845984e7daf5208f5dcb0276e3a6d2b1
+size 5259853080

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:141fd4be20aba9caf76534d16b1cc81037d348e6f3a29a4358a93414c2545035
+size 5259853096

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29136b4ed062ab4d9c24b081f27000a5eff35806aa5de47d73208629ba9498e9
+size 1063347184

model.safetensors.index.json

@@ -0,0 +1,271 @@
+{
+  "metadata": {
+    "total_size": 16783556608
+  },
+  "weight_map": {
+    "transformer.tok_emb.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.0.ln_1.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.0.ln_2.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.0.attn.c_attn.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.0.attn.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.0.attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.0.attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.0.mlp.c_fc.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.0.mlp.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.1.ln_1.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.1.ln_2.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.1.attn.c_attn.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.1.attn.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.1.attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.1.attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.1.mlp.c_fc.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.1.mlp.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.2.ln_1.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.2.ln_2.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.2.attn.c_attn.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.2.attn.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.2.attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.2.attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.2.mlp.c_fc.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.2.mlp.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.3.ln_1.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.3.ln_2.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.3.attn.c_attn.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.3.attn.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.3.attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.3.attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.3.mlp.c_fc.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.3.mlp.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.4.ln_1.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.4.ln_2.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.4.attn.c_attn.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.4.attn.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.4.attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.4.attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.4.mlp.c_fc.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.4.mlp.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.5.ln_1.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.5.ln_2.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.5.attn.c_attn.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.5.attn.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.5.attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.5.attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.5.mlp.c_fc.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.5.mlp.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.6.ln_1.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.6.ln_2.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.6.attn.c_attn.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.6.attn.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.6.attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.6.attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.6.mlp.c_fc.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.6.mlp.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.7.ln_1.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.7.ln_2.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.7.attn.c_attn.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.7.attn.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.7.attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.7.attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.7.mlp.c_fc.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.7.mlp.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.8.ln_1.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.8.ln_2.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.8.attn.c_attn.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.8.attn.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.8.attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.8.attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.8.mlp.c_fc.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.8.mlp.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.9.ln_1.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.9.ln_2.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.9.attn.c_attn.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.9.attn.c_proj.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.9.attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.9.attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "transformer.blocks.9.mlp.c_fc.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.9.mlp.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.10.ln_1.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.10.ln_2.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.10.attn.c_attn.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.10.attn.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.10.attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.10.attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.10.mlp.c_fc.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.10.mlp.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.11.ln_1.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.11.ln_2.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.11.attn.c_attn.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.11.attn.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.11.attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.11.attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.11.mlp.c_fc.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.11.mlp.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.12.ln_1.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.12.ln_2.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.12.attn.c_attn.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.12.attn.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.12.attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.12.attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.12.mlp.c_fc.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.12.mlp.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.13.ln_1.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.13.ln_2.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.13.attn.c_attn.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.13.attn.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.13.attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.13.attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.13.mlp.c_fc.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.13.mlp.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.14.ln_1.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.14.ln_2.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.14.attn.c_attn.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.14.attn.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.14.attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.14.attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.14.mlp.c_fc.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.14.mlp.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.15.ln_1.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.15.ln_2.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.15.attn.c_attn.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.15.attn.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.15.attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.15.attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.15.mlp.c_fc.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.15.mlp.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.16.ln_1.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.16.ln_2.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.16.attn.c_attn.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.16.attn.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.16.attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.16.attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.16.mlp.c_fc.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.16.mlp.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.17.ln_1.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.17.ln_2.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.17.attn.c_attn.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.17.attn.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.17.attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.17.attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.17.mlp.c_fc.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.17.mlp.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.18.ln_1.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.18.ln_2.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.18.attn.c_attn.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.18.attn.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.18.attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.18.attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.18.mlp.c_fc.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.18.mlp.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.19.ln_1.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.19.ln_2.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.19.attn.c_attn.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.19.attn.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.19.attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.19.attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.19.mlp.c_fc.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.19.mlp.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.20.ln_1.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.20.ln_2.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.20.attn.c_attn.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.20.attn.c_proj.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.20.attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.20.attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "transformer.blocks.20.mlp.c_fc.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.20.mlp.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.21.ln_1.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.21.ln_2.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.21.attn.c_attn.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.21.attn.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.21.attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.21.attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.21.mlp.c_fc.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.21.mlp.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.22.ln_1.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.22.ln_2.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.22.attn.c_attn.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.22.attn.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.22.attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.22.attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.22.mlp.c_fc.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.22.mlp.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.23.ln_1.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.23.ln_2.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.23.attn.c_attn.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.23.attn.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.23.attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.23.attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.23.mlp.c_fc.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.23.mlp.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.24.ln_1.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.24.ln_2.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.24.attn.c_attn.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.24.attn.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.24.attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.24.attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.24.mlp.c_fc.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.24.mlp.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.25.ln_1.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.25.ln_2.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.25.attn.c_attn.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.25.attn.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.25.attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.25.attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.25.mlp.c_fc.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.25.mlp.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.26.ln_1.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.26.ln_2.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.26.attn.c_attn.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.26.attn.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.26.attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.26.attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.26.mlp.c_fc.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.26.mlp.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.27.ln_1.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.27.ln_2.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.27.attn.c_attn.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.27.attn.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.27.attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.27.attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.27.mlp.c_fc.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.27.mlp.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.28.ln_1.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.28.ln_2.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.28.attn.c_attn.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.28.attn.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.28.attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.28.attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.28.mlp.c_fc.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.28.mlp.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.29.ln_1.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.29.ln_2.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.29.attn.c_attn.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.29.attn.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.29.attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.29.attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.29.mlp.c_fc.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.29.mlp.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.30.ln_1.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.30.ln_2.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.30.attn.c_attn.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.30.attn.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.30.attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.30.attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.30.mlp.c_fc.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.30.mlp.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.31.ln_1.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.31.ln_2.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.31.attn.c_attn.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.31.attn.c_proj.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.31.attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.31.attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "transformer.blocks.31.mlp.c_fc.weight": "model-00004-of-00004.safetensors",
+    "transformer.blocks.31.mlp.c_proj.weight": "model-00004-of-00004.safetensors",
+    "transformer.ln_f.weight": "model-00004-of-00004.safetensors",
+    "known_head.concept_embedding.weight": "model-00004-of-00004.safetensors",
+    "known_head.concept_predictor.weight": "model-00004-of-00004.safetensors",
+    "unknown_head.embedding_coef.weight": "model-00004-of-00004.safetensors",
+    "unknown_head.embedding_basis.weight": "model-00004-of-00004.safetensors",
+    "unknown_head.predictor_down.weight": "model-00004-of-00004.safetensors",
+    "unknown_head.predictor_up.weight": "model-00004-of-00004.safetensors"
+  }
+}