{
  "architectures": [
    "NanoGPTForCausalLM"
  ],
  "attn_res_block_size": 8,
  "auto_map": {
    "AutoConfig": "modeling_nanogpt.NanoGPTConfig",
    "AutoModel": "modeling_nanogpt.NanoGPTModel",
    "AutoModelForCausalLM": "modeling_nanogpt.NanoGPTForCausalLM",
    "AutoModelForMaskedLM": "modeling_nanogpt.NanoGPTForMaskedLM"
  },
  "bias": false,
  "bidirectional": false,
  "block_size": 512,
  "bos_token_id": 0,
  "dropout": 0.0,
  "eos_token_id": 0,
  "layer_norm_epsilon": 1e-05,
  "model_type": "nanogpt",
  "n_embd": 512,
  "n_head": 8,
  "n_layer": 32,
  "nanogpt_checkpoint": {
    "filename": "ckpt_final-w0090M-i000471.pt",
    "git_sha": "11a6860211f85a96fcb0cfd30a0e763eda931d2a",
    "iter_num": 471,
    "labels": [
      {
        "actual": 90343884,
        "name": "words_final",
        "revision": "chck_90M",
        "series": "words",
        "target": 90343884
      },
      {
        "actual": 123469824,
        "name": "tokens_final",
        "revision": "chck_123M",
        "series": "tokens",
        "target": 123469824
      }
    ],
    "num_updates": 471,
    "role": "final",
    "sha256": "5e1e936ce311c6535526e52d1a97059b587010c9b4e2dc3e5ee1b94fa742d512",
    "tokens_seen": 123469824,
    "words_seen": 90343884
  },
  "pad_token_id": 0,
  "rope_theta": 10000.0,
  "swiglu_mult": 2.6666666666666665,
  "torch_dtype": "float16",
  "transformers_version": "4.51.3",
  "use_attn_gate": true,
  "use_attn_res": true,
  "use_rmsnorm": true,
  "use_rope": true,
  "use_swiglu": true,
  "vocab_size": 16000
}