Upload model - 35000 iterations, loss: 3.4640

config.json

@@ -9,7 +9,7 @@
   "n_layer": 8,
   "n_head": 8,
   "n_inner": 1536,
-  "activation_function": "gelu_new",
+  "activation_function": "gelu",
   "resid_pdrop": 0.1,
   "embd_pdrop": 0.1,
   "attn_pdrop": 0.1,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:18842830ccfaf6638a8c8d8a8fd363fdf2e7512134dbc64c61b99f0a92c19885
+oid sha256:5af8ee2b8124c95ea8667396f3d525176f9e40b8d85f6331034a8bba7245c3e3
 size 211972024