NSA 117M initial export

logs/logs_extra_keys.txt

@@ -1 +1,48 @@
-norm_f.weight
+blocks.0.attn.gate.fc1.bias
+blocks.0.attn.gate.fc1.weight
+blocks.0.attn.gate.fc2.bias
+blocks.0.attn.gate.fc2.weight
+blocks.1.attn.gate.fc1.bias
+blocks.1.attn.gate.fc1.weight
+blocks.1.attn.gate.fc2.bias
+blocks.1.attn.gate.fc2.weight
+blocks.10.attn.gate.fc1.bias
+blocks.10.attn.gate.fc1.weight
+blocks.10.attn.gate.fc2.bias
+blocks.10.attn.gate.fc2.weight
+blocks.11.attn.gate.fc1.bias
+blocks.11.attn.gate.fc1.weight
+blocks.11.attn.gate.fc2.bias
+blocks.11.attn.gate.fc2.weight
+blocks.2.attn.gate.fc1.bias
+blocks.2.attn.gate.fc1.weight
+blocks.2.attn.gate.fc2.bias
+blocks.2.attn.gate.fc2.weight
+blocks.3.attn.gate.fc1.bias
+blocks.3.attn.gate.fc1.weight
+blocks.3.attn.gate.fc2.bias
+blocks.3.attn.gate.fc2.weight
+blocks.4.attn.gate.fc1.bias
+blocks.4.attn.gate.fc1.weight
+blocks.4.attn.gate.fc2.bias
+blocks.4.attn.gate.fc2.weight
+blocks.5.attn.gate.fc1.bias
+blocks.5.attn.gate.fc1.weight
+blocks.5.attn.gate.fc2.bias
+blocks.5.attn.gate.fc2.weight
+blocks.6.attn.gate.fc1.bias
+blocks.6.attn.gate.fc1.weight
+blocks.6.attn.gate.fc2.bias
+blocks.6.attn.gate.fc2.weight
+blocks.7.attn.gate.fc1.bias
+blocks.7.attn.gate.fc1.weight
+blocks.7.attn.gate.fc2.bias
+blocks.7.attn.gate.fc2.weight
+blocks.8.attn.gate.fc1.bias
+blocks.8.attn.gate.fc1.weight
+blocks.8.attn.gate.fc2.bias
+blocks.8.attn.gate.fc2.weight
+blocks.9.attn.gate.fc1.bias
+blocks.9.attn.gate.fc1.weight
+blocks.9.attn.gate.fc2.bias
+blocks.9.attn.gate.fc2.weight

logs/logs_mapping.json

@@ -7,10 +7,6 @@
     "model.blocks.0.attn.W_V_cmp.weight",
     "model.blocks.0.attn.W_V_sel.weight",
     "model.blocks.0.attn.W_V_win.weight",
-    "model.blocks.0.attn.gate.fc1.bias",
-    "model.blocks.0.attn.gate.fc1.weight",
-    "model.blocks.0.attn.gate.fc2.bias",
-    "model.blocks.0.attn.gate.fc2.weight",
     "model.blocks.0.attn.out.weight",
     "model.blocks.0.mlp.fc1.weight",
     "model.blocks.0.mlp.fc2.weight",
@@ -23,10 +19,6 @@
     "model.blocks.1.attn.W_V_cmp.weight",
     "model.blocks.1.attn.W_V_sel.weight",
     "model.blocks.1.attn.W_V_win.weight",
-    "model.blocks.1.attn.gate.fc1.bias",
-    "model.blocks.1.attn.gate.fc1.weight",
-    "model.blocks.1.attn.gate.fc2.bias",
-    "model.blocks.1.attn.gate.fc2.weight",
     "model.blocks.1.attn.out.weight",
     "model.blocks.1.mlp.fc1.weight",
     "model.blocks.1.mlp.fc2.weight",
@@ -39,10 +31,6 @@
     "model.blocks.10.attn.W_V_cmp.weight",
     "model.blocks.10.attn.W_V_sel.weight",
     "model.blocks.10.attn.W_V_win.weight",
-    "model.blocks.10.attn.gate.fc1.bias",
-    "model.blocks.10.attn.gate.fc1.weight",
-    "model.blocks.10.attn.gate.fc2.bias",
-    "model.blocks.10.attn.gate.fc2.weight",
     "model.blocks.10.attn.out.weight",
     "model.blocks.10.mlp.fc1.weight",
     "model.blocks.10.mlp.fc2.weight",
@@ -55,10 +43,6 @@
     "model.blocks.11.attn.W_V_cmp.weight",
     "model.blocks.11.attn.W_V_sel.weight",
     "model.blocks.11.attn.W_V_win.weight",
-    "model.blocks.11.attn.gate.fc1.bias",
-    "model.blocks.11.attn.gate.fc1.weight",
-    "model.blocks.11.attn.gate.fc2.bias",
-    "model.blocks.11.attn.gate.fc2.weight",
     "model.blocks.11.attn.out.weight",
     "model.blocks.11.mlp.fc1.weight",
     "model.blocks.11.mlp.fc2.weight",
@@ -71,10 +55,6 @@
     "model.blocks.2.attn.W_V_cmp.weight",
     "model.blocks.2.attn.W_V_sel.weight",
     "model.blocks.2.attn.W_V_win.weight",
-    "model.blocks.2.attn.gate.fc1.bias",
-    "model.blocks.2.attn.gate.fc1.weight",
-    "model.blocks.2.attn.gate.fc2.bias",
-    "model.blocks.2.attn.gate.fc2.weight",
     "model.blocks.2.attn.out.weight",
     "model.blocks.2.mlp.fc1.weight",
     "model.blocks.2.mlp.fc2.weight",
@@ -87,10 +67,6 @@
     "model.blocks.3.attn.W_V_cmp.weight",
     "model.blocks.3.attn.W_V_sel.weight",
     "model.blocks.3.attn.W_V_win.weight",
-    "model.blocks.3.attn.gate.fc1.bias",
-    "model.blocks.3.attn.gate.fc1.weight",
-    "model.blocks.3.attn.gate.fc2.bias",
-    "model.blocks.3.attn.gate.fc2.weight",
     "model.blocks.3.attn.out.weight",
     "model.blocks.3.mlp.fc1.weight",
     "model.blocks.3.mlp.fc2.weight",
@@ -103,10 +79,6 @@
     "model.blocks.4.attn.W_V_cmp.weight",
     "model.blocks.4.attn.W_V_sel.weight",
     "model.blocks.4.attn.W_V_win.weight",
-    "model.blocks.4.attn.gate.fc1.bias",
-    "model.blocks.4.attn.gate.fc1.weight",
-    "model.blocks.4.attn.gate.fc2.bias",
-    "model.blocks.4.attn.gate.fc2.weight",
     "model.blocks.4.attn.out.weight",
     "model.blocks.4.mlp.fc1.weight",
     "model.blocks.4.mlp.fc2.weight",
@@ -119,10 +91,6 @@
     "model.blocks.5.attn.W_V_cmp.weight",
     "model.blocks.5.attn.W_V_sel.weight",
     "model.blocks.5.attn.W_V_win.weight",
-    "model.blocks.5.attn.gate.fc1.bias",
-    "model.blocks.5.attn.gate.fc1.weight",
-    "model.blocks.5.attn.gate.fc2.bias",
-    "model.blocks.5.attn.gate.fc2.weight",
     "model.blocks.5.attn.out.weight",
     "model.blocks.5.mlp.fc1.weight",
     "model.blocks.5.mlp.fc2.weight",
@@ -135,10 +103,6 @@
     "model.blocks.6.attn.W_V_cmp.weight",
     "model.blocks.6.attn.W_V_sel.weight",
     "model.blocks.6.attn.W_V_win.weight",
-    "model.blocks.6.attn.gate.fc1.bias",
-    "model.blocks.6.attn.gate.fc1.weight",
-    "model.blocks.6.attn.gate.fc2.bias",
-    "model.blocks.6.attn.gate.fc2.weight",
     "model.blocks.6.attn.out.weight",
     "model.blocks.6.mlp.fc1.weight",
     "model.blocks.6.mlp.fc2.weight",
@@ -151,10 +115,6 @@
     "model.blocks.7.attn.W_V_cmp.weight",
     "model.blocks.7.attn.W_V_sel.weight",
     "model.blocks.7.attn.W_V_win.weight",
-    "model.blocks.7.attn.gate.fc1.bias",
-    "model.blocks.7.attn.gate.fc1.weight",
-    "model.blocks.7.attn.gate.fc2.bias",
-    "model.blocks.7.attn.gate.fc2.weight",
     "model.blocks.7.attn.out.weight",
     "model.blocks.7.mlp.fc1.weight",
     "model.blocks.7.mlp.fc2.weight",
@@ -167,10 +127,6 @@
     "model.blocks.8.attn.W_V_cmp.weight",
     "model.blocks.8.attn.W_V_sel.weight",
     "model.blocks.8.attn.W_V_win.weight",
-    "model.blocks.8.attn.gate.fc1.bias",
-    "model.blocks.8.attn.gate.fc1.weight",
-    "model.blocks.8.attn.gate.fc2.bias",
-    "model.blocks.8.attn.gate.fc2.weight",
     "model.blocks.8.attn.out.weight",
     "model.blocks.8.mlp.fc1.weight",
     "model.blocks.8.mlp.fc2.weight",
@@ -183,23 +139,114 @@
     "model.blocks.9.attn.W_V_cmp.weight",
     "model.blocks.9.attn.W_V_sel.weight",
     "model.blocks.9.attn.W_V_win.weight",
-    "model.blocks.9.attn.gate.fc1.bias",
-    "model.blocks.9.attn.gate.fc1.weight",
-    "model.blocks.9.attn.gate.fc2.bias",
-    "model.blocks.9.attn.gate.fc2.weight",
     "model.blocks.9.attn.out.weight",
     "model.blocks.9.mlp.fc1.weight",
     "model.blocks.9.mlp.fc2.weight",
     "model.blocks.9.norm1.weight",
     "model.blocks.9.norm2.weight",
     "model.embed.weight",
-    "model.lm_head.weight"
+    "model.lm_head.weight",
+    "model.norm.weight"
   ],
   "missing": [
-    "model.norm.bias",
-    "model.norm.weight"
+    "model.blocks.0.attn.gate_fc1.bias",
+    "model.blocks.0.attn.gate_fc1.weight",
+    "model.blocks.0.attn.gate_fc2.bias",
+    "model.blocks.0.attn.gate_fc2.weight",
+    "model.blocks.1.attn.gate_fc1.bias",
+    "model.blocks.1.attn.gate_fc1.weight",
+    "model.blocks.1.attn.gate_fc2.bias",
+    "model.blocks.1.attn.gate_fc2.weight",
+    "model.blocks.10.attn.gate_fc1.bias",
+    "model.blocks.10.attn.gate_fc1.weight",
+    "model.blocks.10.attn.gate_fc2.bias",
+    "model.blocks.10.attn.gate_fc2.weight",
+    "model.blocks.11.attn.gate_fc1.bias",
+    "model.blocks.11.attn.gate_fc1.weight",
+    "model.blocks.11.attn.gate_fc2.bias",
+    "model.blocks.11.attn.gate_fc2.weight",
+    "model.blocks.2.attn.gate_fc1.bias",
+    "model.blocks.2.attn.gate_fc1.weight",
+    "model.blocks.2.attn.gate_fc2.bias",
+    "model.blocks.2.attn.gate_fc2.weight",
+    "model.blocks.3.attn.gate_fc1.bias",
+    "model.blocks.3.attn.gate_fc1.weight",
+    "model.blocks.3.attn.gate_fc2.bias",
+    "model.blocks.3.attn.gate_fc2.weight",
+    "model.blocks.4.attn.gate_fc1.bias",
+    "model.blocks.4.attn.gate_fc1.weight",
+    "model.blocks.4.attn.gate_fc2.bias",
+    "model.blocks.4.attn.gate_fc2.weight",
+    "model.blocks.5.attn.gate_fc1.bias",
+    "model.blocks.5.attn.gate_fc1.weight",
+    "model.blocks.5.attn.gate_fc2.bias",
+    "model.blocks.5.attn.gate_fc2.weight",
+    "model.blocks.6.attn.gate_fc1.bias",
+    "model.blocks.6.attn.gate_fc1.weight",
+    "model.blocks.6.attn.gate_fc2.bias",
+    "model.blocks.6.attn.gate_fc2.weight",
+    "model.blocks.7.attn.gate_fc1.bias",
+    "model.blocks.7.attn.gate_fc1.weight",
+    "model.blocks.7.attn.gate_fc2.bias",
+    "model.blocks.7.attn.gate_fc2.weight",
+    "model.blocks.8.attn.gate_fc1.bias",
+    "model.blocks.8.attn.gate_fc1.weight",
+    "model.blocks.8.attn.gate_fc2.bias",
+    "model.blocks.8.attn.gate_fc2.weight",
+    "model.blocks.9.attn.gate_fc1.bias",
+    "model.blocks.9.attn.gate_fc1.weight",
+    "model.blocks.9.attn.gate_fc2.bias",
+    "model.blocks.9.attn.gate_fc2.weight",
+    "model.norm.bias"
   ],
   "extra": [
-    "norm_f.weight"
+    "blocks.0.attn.gate.fc1.bias",
+    "blocks.0.attn.gate.fc1.weight",
+    "blocks.0.attn.gate.fc2.bias",
+    "blocks.0.attn.gate.fc2.weight",
+    "blocks.1.attn.gate.fc1.bias",
+    "blocks.1.attn.gate.fc1.weight",
+    "blocks.1.attn.gate.fc2.bias",
+    "blocks.1.attn.gate.fc2.weight",
+    "blocks.10.attn.gate.fc1.bias",
+    "blocks.10.attn.gate.fc1.weight",
+    "blocks.10.attn.gate.fc2.bias",
+    "blocks.10.attn.gate.fc2.weight",
+    "blocks.11.attn.gate.fc1.bias",
+    "blocks.11.attn.gate.fc1.weight",
+    "blocks.11.attn.gate.fc2.bias",
+    "blocks.11.attn.gate.fc2.weight",
+    "blocks.2.attn.gate.fc1.bias",
+    "blocks.2.attn.gate.fc1.weight",
+    "blocks.2.attn.gate.fc2.bias",
+    "blocks.2.attn.gate.fc2.weight",
+    "blocks.3.attn.gate.fc1.bias",
+    "blocks.3.attn.gate.fc1.weight",
+    "blocks.3.attn.gate.fc2.bias",
+    "blocks.3.attn.gate.fc2.weight",
+    "blocks.4.attn.gate.fc1.bias",
+    "blocks.4.attn.gate.fc1.weight",
+    "blocks.4.attn.gate.fc2.bias",
+    "blocks.4.attn.gate.fc2.weight",
+    "blocks.5.attn.gate.fc1.bias",
+    "blocks.5.attn.gate.fc1.weight",
+    "blocks.5.attn.gate.fc2.bias",
+    "blocks.5.attn.gate.fc2.weight",
+    "blocks.6.attn.gate.fc1.bias",
+    "blocks.6.attn.gate.fc1.weight",
+    "blocks.6.attn.gate.fc2.bias",
+    "blocks.6.attn.gate.fc2.weight",
+    "blocks.7.attn.gate.fc1.bias",
+    "blocks.7.attn.gate.fc1.weight",
+    "blocks.7.attn.gate.fc2.bias",
+    "blocks.7.attn.gate.fc2.weight",
+    "blocks.8.attn.gate.fc1.bias",
+    "blocks.8.attn.gate.fc1.weight",
+    "blocks.8.attn.gate.fc2.bias",
+    "blocks.8.attn.gate.fc2.weight",
+    "blocks.9.attn.gate.fc1.bias",
+    "blocks.9.attn.gate.fc1.weight",
+    "blocks.9.attn.gate.fc2.bias",
+    "blocks.9.attn.gate.fc2.weight"
   ]
 }

logs/logs_missing_keys.txt

@@ -1,2 +1,49 @@
-model.norm.bias
-model.norm.weight
+model.blocks.0.attn.gate_fc1.bias
+model.blocks.0.attn.gate_fc1.weight
+model.blocks.0.attn.gate_fc2.bias
+model.blocks.0.attn.gate_fc2.weight
+model.blocks.1.attn.gate_fc1.bias
+model.blocks.1.attn.gate_fc1.weight
+model.blocks.1.attn.gate_fc2.bias
+model.blocks.1.attn.gate_fc2.weight
+model.blocks.10.attn.gate_fc1.bias
+model.blocks.10.attn.gate_fc1.weight
+model.blocks.10.attn.gate_fc2.bias
+model.blocks.10.attn.gate_fc2.weight
+model.blocks.11.attn.gate_fc1.bias
+model.blocks.11.attn.gate_fc1.weight
+model.blocks.11.attn.gate_fc2.bias
+model.blocks.11.attn.gate_fc2.weight
+model.blocks.2.attn.gate_fc1.bias
+model.blocks.2.attn.gate_fc1.weight
+model.blocks.2.attn.gate_fc2.bias
+model.blocks.2.attn.gate_fc2.weight
+model.blocks.3.attn.gate_fc1.bias
+model.blocks.3.attn.gate_fc1.weight
+model.blocks.3.attn.gate_fc2.bias
+model.blocks.3.attn.gate_fc2.weight
+model.blocks.4.attn.gate_fc1.bias
+model.blocks.4.attn.gate_fc1.weight
+model.blocks.4.attn.gate_fc2.bias
+model.blocks.4.attn.gate_fc2.weight
+model.blocks.5.attn.gate_fc1.bias
+model.blocks.5.attn.gate_fc1.weight
+model.blocks.5.attn.gate_fc2.bias
+model.blocks.5.attn.gate_fc2.weight
+model.blocks.6.attn.gate_fc1.bias
+model.blocks.6.attn.gate_fc1.weight
+model.blocks.6.attn.gate_fc2.bias
+model.blocks.6.attn.gate_fc2.weight
+model.blocks.7.attn.gate_fc1.bias
+model.blocks.7.attn.gate_fc1.weight
+model.blocks.7.attn.gate_fc2.bias
+model.blocks.7.attn.gate_fc2.weight
+model.blocks.8.attn.gate_fc1.bias
+model.blocks.8.attn.gate_fc1.weight
+model.blocks.8.attn.gate_fc2.bias
+model.blocks.8.attn.gate_fc2.weight
+model.blocks.9.attn.gate_fc1.bias
+model.blocks.9.attn.gate_fc1.weight
+model.blocks.9.attn.gate_fc2.bias
+model.blocks.9.attn.gate_fc2.weight
+model.norm.bias

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21d3ac54cadd49cc11ea0e88d37874aa3a9391e7b47a2704b6557a0e9229640c
+oid sha256:973adf1381893bfed4aec91d2329a3f1548afc8e235f4858115d4cf75bc996c5
 size 313204736

modeling_nsa.py

@@ -9,7 +9,7 @@ from transformers.generation.utils import GenerationMixin
 from transformers.modeling_outputs import CausalLMOutput
 
 from .configuration_nsa import NSAConfig
-_HAS_NSA = False  # avoid nested vendor imports in HF dynamic loader
+_HAS_NSA = False  # Do not attempt nested vendor import in HF dynamic loader
 
 
 class RMSNorm(nn.Module):