full_s0/config.json

{
  "variant": "full",
  "seed": 0,
  "out": "C:\\Users\\Bhorn\\Desktop\\SMATest\\runs\\full_s0",
  "max_steps": 12000,
  "warmup": 200,
  "lr_max": 0.0006,
  "lr_min": 6e-05,
  "wd": 0.1,
  "batch_size": 16,
  "grad_accum": 2,
  "block_size": 256,
  "n_layer": 24,
  "n_head": 6,
  "n_embd": 384,
  "vocab_size": 50257,
  "data_seed": 1337,
  "eval_interval": 500,
  "eval_batches": 40,
  "log_interval": 20
}