Upload GLAForCausalLM

README.md

@@ -8,4 +8,4 @@ license: mit
 datasets:
 - cerebras/SlimPajama-627B
 library_name: fla
----
+---

config.json

@@ -3,13 +3,16 @@
   "architectures": [
     "GLAForCausalLM"
   ],
+  "attn": null,
   "attn_mode": "chunk",
   "bos_token_id": 1,
   "clamp_min": null,
   "conv_size": 4,
+  "elementwise_affine": true,
   "eos_token_id": 2,
   "expand_k": 0.5,
   "expand_v": 1,
+  "feature_map": null,
   "fuse_cross_entropy": true,
   "fuse_norm": true,
   "hidden_act": "swish",
@@ -19,16 +22,19 @@
   "intermediate_size": null,
   "max_position_embeddings": 2048,
   "model_type": "gla",
+  "norm_eps": 1e-06,
   "num_heads": 4,
   "num_hidden_layers": 24,
+  "num_kv_heads": null,
   "rms_norm_eps": 1e-06,
   "share_conv_kernel": true,
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.39.1",
+  "transformers_version": "4.48.2",
   "use_cache": true,
   "use_gk": true,
   "use_gv": false,
+  "use_output_gate": true,
   "use_short_conv": false,
   "vocab_size": 32000
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.39.1"
+  "transformers_version": "4.48.2"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da7b708c66f1b9d1ef7d86684a97f4605f014deac8962d2b91dba8cef907f597
-size 683449944
+oid sha256:07637f325a73991dcc31936e43ab9da2eb5c1a79135c477c3013bf3118a05a4a
+size 683452592