gptxl_100M.meta.json

{
  "meta": {
    "model": "gptxl",
    "size_m": 100,
    "steps": 4000,
    "seq_len": 1024,
    "warmup": 1000,
    "seed": 0,
    "config": {
      "arch": "gptxl",
      "size_m": 100,
      "n_layer": 15,
      "n_embd": 768,
      "n_head": 8,
      "ff": 3072,
      "drop": 0.02
    },
    "data": "europarl en-pl+en-cs byte stream 398.6MB",
    "optimizer": "AdamW lr=1e-3 wd=0.1 cos->1e-4 warmup1000",
    "clip": "ZClip"
  }
}