| { | |
| "batch_size": 16, | |
| "grad_accum_steps": 4, | |
| "max_iters": 40000, | |
| "warmup_iters": 2000, | |
| "max_lr": 3e-4, | |
| "weight_decay": 0.1, | |
| "beta1": 0.9, | |
| "beta2": 0.95, | |
| "eps": 1e-8, | |
| "eval_interval": 500, | |
| "eval_iters": 100, | |
| "grad_clip": 1.0, | |
| "device": "cuda", | |
| "seed": 42, | |
| "n_vocab": 50257, | |
| "seq_len": 128, | |
| "n_layers": 4, | |
| "n_head": 4, | |
| "d_model": 256, | |
| "dropout": 0.1, | |
| "bias": false | |
| } |