config.json

{
  "vocab_size": 8192,
  "context_len": 12000,
  "n_embd": 768,
  "n_head": 12,
  "n_layer": 12,
  "attn_dropout": 0.1,
  "resid_dropout": 0.1,
  "embd_dropout": 0.1,
  "mlp_dropout": 0.1
}