RyanJT
/

moa-attention-gpt2

Text Generation

text-generation-inference

Model card Files Files and versions

RyanJT commited on Jul 22, 2024

Commit

a7436af

·

verified ·

1 Parent(s): 0ba138d

Upload folder using huggingface_hub

Files changed (2) hide show

config.json +4 -4
pytorch_model.bin +1 -1

config.json CHANGED Viewed

@@ -11,12 +11,12 @@
     "initializer_range": 0.02,
     "layer_norm_epsilon": 1e-05,
     "model_type": "gpt2",
-    "n_ctx": 256,
     "n_embd": 256,
-    "n_head": 4,
     "n_inner": null,
-    "n_layer": 3,
-    "n_positions": 256,
     "reorder_and_upcast_attn": false,
     "resid_pdrop": 0.1,
     "scale_attn_by_inverse_layer_idx": false,

     "initializer_range": 0.02,
     "layer_norm_epsilon": 1e-05,
     "model_type": "gpt2",
+    "n_ctx": 1024,
     "n_embd": 256,
+    "n_head": 8,
     "n_inner": null,
+    "n_layer": 6,
+    "n_positions": 1024,
     "reorder_and_upcast_attn": false,
     "resid_pdrop": 0.1,
     "scale_attn_by_inverse_layer_idx": false,

pytorch_model.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe4b9052cbedb3ac22cb84a5e0d969863b0dd3a598d34d71a26e822344989bc3
 size 112692713

 version https://git-lfs.github.com/spec/v1
+oid sha256:b95c7315c2495c5bbe34b30fd70a44bed88d0bd1354b215ec87e2d086a2a074b
 size 112692713