config.json

{
    "model_type": "custom",
    "architectures": ["APE"],
    "bias" : "False", 
    "use_rope" : "True", 
    "use_ape" : "True",
    "use_alibi" : "False",
    "n_layer": 6,
    "n_head": 6, 
    "n_embd": 384, 
    "block_size": 64,
    "vocab_size": 50304, 
    "rope_base": 10000.0,
    "dropout" : 0
  }