config.json

{
  "architectures": ["TinyStoriesGPT"],
  "model_type": "tinystories-gpt",
  "D": 384,
  "L": 12,
  "H": 8,
  "FFN": 1536,
  "vocab_size": 8192,
  "max_position_embeddings": 512,
  "n_params": 24585600,
  "tie_word_embeddings": true,
  "norm": "RMSNorm",
  "ffn": "GELU",
  "attn": "causal SDPA (fused qkv)",
  "trained_on": "roneneldan/TinyStories",
  "step": 6000,
  "val_nats": 2.1618,
  "val_ppl": 8.76,
  "dtype": "float32"
}