File size: 660 Bytes
1839975
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
{
  "architecture": "ByteGPT",
  "config": {
    "vocab_size": 256,
    "block_size": 8192,
    "n_layer": 12,
    "n_head": 12,
    "n_embd": 768,
    "dropout": 0.0,
    "rope_theta": 100000.0
  },
  "iter": 19999,
  "n_params": 85150464,
  "tied_weights": {
    "lm_head.weight": "embed.weight"
  },
  "training_args": {
    "preset": "small",
    "micro_bs": 4,
    "grad_accum": 8,
    "max_iters": 20000,
    "lr": 0.0006,
    "min_lr_frac": 0.1,
    "warmup_iters": 500,
    "weight_decay": 0.1,
    "grad_clip": 1.0,
    "eval_every": 500,
    "eval_windows": 64,
    "ckpt_every": 2000,
    "log_every": 20,
    "compile": true,
    "seed": 1337
  }
}