bytegpt-byte-85M / config.json
ShadeEngine's picture
Upload folder using huggingface_hub
1839975 verified
Raw History Blame Contribute Delete
660 Bytes
{
"architecture": "ByteGPT",
"config": {
"vocab_size": 256,
"block_size": 8192,
"n_layer": 12,
"n_head": 12,
"n_embd": 768,
"dropout": 0.0,
"rope_theta": 100000.0
},
"iter": 19999,
"n_params": 85150464,
"tied_weights": {
"lm_head.weight": "embed.weight"
},
"training_args": {
"preset": "small",
"micro_bs": 4,
"grad_accum": 8,
"max_iters": 20000,
"lr": 0.0006,
"min_lr_frac": 0.1,
"warmup_iters": 500,
"weight_decay": 0.1,
"grad_clip": 1.0,
"eval_every": 500,
"eval_windows": 64,
"ckpt_every": 2000,
"log_every": 20,
"compile": true,
"seed": 1337
}
}