Upload TransformerForCausalLM

README.md

@@ -8,4 +8,4 @@ license: mit
 datasets:
 - cerebras/SlimPajama-627B
 library_name: fla
----
+---

config.json

@@ -5,9 +5,11 @@
   ],
   "attention_bias": false,
   "bos_token_id": 1,
+  "elementwise_affine": true,
   "eos_token_id": 2,
   "fuse_cross_entropy": true,
   "fuse_norm": true,
+  "fuse_swiglu": true,
   "hidden_act": "swish",
   "hidden_ratio": 4,
   "hidden_size": 2560,
@@ -15,12 +17,16 @@
   "intermediate_size": null,
   "max_position_embeddings": 2048,
   "model_type": "transformer",
+  "norm_eps": 1e-06,
   "num_heads": 20,
   "num_hidden_layers": 32,
+  "num_kv_heads": null,
   "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.0.dev0",
+  "transformers_version": "4.48.2",
   "use_cache": true,
-  "vocab_size": 32000
+  "vocab_size": 32000,
+  "window_size": null
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.40.0.dev0"
+  "transformers_version": "4.48.2"
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42d813182eba43748db525a7a505ff1a649a9e478e83c68a5370a65cad7ae62a
-size 4974527296
+oid sha256:19c08e94c9d4c4a985d07b57823fa7aa589becb294af61a454b69f18f35ce360
+size 4974530632

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:44b900ebf5fb40a51fcec941fcc7badad77b9cf8be2d3ef056f7adf30b54ded3
-size 428622120
+oid sha256:1960431ae037a4a71d147ca442c04d3810bbe9f2f2f564b286616c0b77235c72
+size 428622344

model.safetensors.index.json

@@ -12,6 +12,7 @@
     "model.layers.0.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -20,6 +21,7 @@
     "model.layers.1.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -28,6 +30,7 @@
     "model.layers.10.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -36,6 +39,7 @@
     "model.layers.11.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -44,6 +48,7 @@
     "model.layers.12.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -52,6 +57,7 @@
     "model.layers.13.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -60,6 +66,7 @@
     "model.layers.14.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -68,6 +75,7 @@
     "model.layers.15.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -76,6 +84,7 @@
     "model.layers.16.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -84,6 +93,7 @@
     "model.layers.17.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -92,6 +102,7 @@
     "model.layers.18.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -100,6 +111,7 @@
     "model.layers.19.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -108,6 +120,7 @@
     "model.layers.2.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -116,6 +129,7 @@
     "model.layers.20.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -124,6 +138,7 @@
     "model.layers.21.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -132,6 +147,7 @@
     "model.layers.22.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -140,6 +156,7 @@
     "model.layers.23.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -148,6 +165,7 @@
     "model.layers.24.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.25.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.25.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -156,6 +174,7 @@
     "model.layers.25.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.25.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.25.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.26.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.26.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -164,6 +183,7 @@
     "model.layers.26.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.26.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.26.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.26.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.27.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.27.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -172,6 +192,7 @@
     "model.layers.27.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.27.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.27.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.27.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.28.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.28.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -180,6 +201,7 @@
     "model.layers.28.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.28.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.28.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.28.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.29.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.29.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -188,6 +210,7 @@
     "model.layers.29.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.29.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.29.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.29.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -196,6 +219,7 @@
     "model.layers.3.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.30.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.30.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -204,6 +228,7 @@
     "model.layers.30.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.30.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.31.attn.k_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.31.attn.o_proj.weight": "model-00002-of-00002.safetensors",
@@ -212,6 +237,7 @@
     "model.layers.31.attn_norm.weight": "model-00002-of-00002.safetensors",
     "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.31.mlp_norm.weight": "model-00002-of-00002.safetensors",
     "model.layers.4.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -220,6 +246,7 @@
     "model.layers.4.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -228,6 +255,7 @@
     "model.layers.5.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -236,6 +264,7 @@
     "model.layers.6.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -244,6 +273,7 @@
     "model.layers.7.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -252,6 +282,7 @@
     "model.layers.8.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -260,6 +291,7 @@
     "model.layers.9.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.norm.weight": "model-00002-of-00002.safetensors"
   }