{
  "version": "v11",
  "core": "tiny-model-v11",
  "vocab_size": 71260,
  "dim": 512,
  "n_layers": 20,
  "ffn_dim": 2048,
  "n_heads": 8,
  "n_kv_heads": 4,
  "max_seq": 256,
  "rope_theta": 10000.0,
  "tie_embeddings": true,
  "tokenizer": "v11",
  "training": {
    "corpus": "TinyStories",
    "phase2_tokens": 16000000,
    "phase3_tokens": 8000000,
    "phase2_lr": 0.0003,
    "phase3_lr": 0.00015,
    "phase3_strategy": "frozen_ffn_retrain_attention"
  },
  "_note": "Not a transformers architecture. Build TinyModel from the tiny_model_v11/ package shipped in this repo -- see the model card.",
  "pretrain_run": {
    "step": 15625,
    "tokens": 16000000,
    "param_count": 115149312,
    "seed": 42,
    "batch_size": 4,
    "lr": 0.0003,
    "phase": "phase 1 only -- no frozen-FFN phase, no maths mid-train",
    "corpus": "roneneldan/TinyStories @ f54c09fd23315a6f9c86f9dc80f725de7d8f9c64",
    "run_config": "configs/colab.json"
  }
}