config.json

{
  "attn_bias": 0,
  "block": 4096,
  "ckpt_sha256": "7d092d4d0bf838da1115a5369bf0a6729fe4c170e0d5ad303873f7320d6c1866",
  "data_seed": 20261023,
  "ffn": "swiglu",
  "ffn_mult": 2.667,
  "logit_cap": 0.0,
  "n_embd": 2048,
  "n_head": 16,
  "n_layer": 20,
  "n_params": 1073912851,
  "norm": "rmsnorm",
  "norm_eps": 1e-06,
  "pos": "rope",
  "qk_norm": true,
  "rope_theta": 500000.0,
  "step": 1430,
  "tied_weights": {
    "tok.weight": "head.weight"
  },
  "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd",
  "trainer": "train_gpt_ref_r23.py",
  "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b",
  "value_residual": true,
  "vocab": 32768,
  "z_loss": 0.0
}