Model V7

Loss: 2.866, Perplexity: 17.567

max_seq_len = 740
batch_size = 256
n_vocab = 50257

n_layers = 24
n_heads = 16
embed_dim = 1024
ffn_dim = embed_dim*4

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24500587a8138ab301ca2ef5f93837a44c350b86ecfc9302fe69ece5f499e3ae
+oid sha256:383f20e84b8249c44167931f1b0ea4a66c52237d4f663e2122f60271959a531e
 size 1418170960