File size: 660 Bytes
1839975 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | {
"architecture": "ByteGPT",
"config": {
"vocab_size": 256,
"block_size": 8192,
"n_layer": 12,
"n_head": 12,
"n_embd": 768,
"dropout": 0.0,
"rope_theta": 100000.0
},
"iter": 19999,
"n_params": 85150464,
"tied_weights": {
"lm_head.weight": "embed.weight"
},
"training_args": {
"preset": "small",
"micro_bs": 4,
"grad_accum": 8,
"max_iters": 20000,
"lr": 0.0006,
"min_lr_frac": 0.1,
"warmup_iters": 500,
"weight_decay": 0.1,
"grad_clip": 1.0,
"eval_every": 500,
"eval_windows": 64,
"ckpt_every": 2000,
"log_every": 20,
"compile": true,
"seed": 1337
}
} |