config.json

{
  "architectures": [
    "NanoGPTForCausalLM"
  ],
  "attn_res_block_size": 8,
  "auto_map": {
    "AutoConfig": "modeling_nanogpt.NanoGPTConfig",
    "AutoModel": "modeling_nanogpt.NanoGPTModel",
    "AutoModelForCausalLM": "modeling_nanogpt.NanoGPTForCausalLM",
    "AutoModelForMaskedLM": "modeling_nanogpt.NanoGPTForMaskedLM"
  },
  "bias": false,
  "bidirectional": false,
  "block_size": 512,
  "bos_token_id": 0,
  "dropout": 0.0,
  "eos_token_id": 0,
  "layer_norm_epsilon": 1e-05,
  "model_type": "nanogpt",
  "n_embd": 512,
  "n_head": 8,
  "n_layer": 32,
  "nanogpt_checkpoint": {
    "filename": "ckpt_final-w0910M-i004797.pt",
    "git_sha": "71b982e505ea3f804c54a4b4c0834bd1eee54da8",
    "iter_num": 4797,
    "labels": [
      {
        "actual": 910196864,
        "name": "words_final",
        "revision": "chck_910M",
        "series": "words",
        "target": 910196864
      },
      {
        "actual": 1257504768,
        "name": "tokens_final",
        "revision": "chck_1258M",
        "series": "tokens",
        "target": 1257504768
      }
    ],
    "num_updates": 4797,
    "role": "final",
    "sha256": "dfaff6e50e0759fed5469af2022205228068aa12354537de22ebcde769b32995",
    "tokens_seen": 1257504768,
    "words_seen": 910196864
  },
  "pad_token_id": 0,
  "rope_theta": 10000.0,
  "swiglu_mult": 2.6666666666666665,
  "torch_dtype": "float16",
  "transformers_version": "4.51.3",
  "use_attn_gate": true,
  "use_attn_res": true,
  "use_rmsnorm": true,
  "use_rope": true,
  "use_swiglu": true,
  "vocab_size": 16000
}