training_summary.json

{
  "size": "5M",
  "params": 5115456,
  "config": {
    "vocab_size": 8192,
    "hidden_size": 192,
    "intermediate_size": 512,
    "num_hidden_layers": 9,
    "num_attention_heads": 6,
    "num_key_value_heads": 2,
    "max_position_embeddings": 2048,
    "rope_theta": 10000.0,
    "rms_norm_eps": 1e-05,
    "initializer_range": 0.02
  },
  "tokens": 2999975936,
  "steps": 11444,
  "batch_tokens": 262144,
  "seq_len": 2048,
  "opt": "muon",
  "lr": 0.01,
  "wd": 0.1,
  "warmup_frac": 0.01,
  "decay_frac": 0.2,
  "gpu": "Tesla T4",
  "train_minutes": 339.8,
  "final_val": {
    "dclm": 3.4559,
    "finemath": 2.5755,
    "fineweb": 3.2282,
    "stack": 1.7677,
    "avg": 2.7568
  },
  "samples": [
    "The meaning of life is the same as the meaning of life. The meaning of life is the same as the meaning of life.\nThe meaning of life is the same as the meaning of life. It means that we know what we are doing and how we can do",
    "Once upon a time, the world was not as much of an issue. The world had to be more than just a single day and a week.\nThe world was very different from what we have today. We are all in our own homes. It is a great",
    "def is_prime(n): 10\n# 2. The number of digits in the first digit is 3, and the last digit is 4.\n# 3. The number of digits in the second digit is 5, and the last digit",
    "The derivative of x^2 is the product of its value. The derivative of y^2 is the product of its value.\n\n## 1. What are the derivatives?\n\n### 3. How do you find the derivative of x^2?\n\n### "
  ],
  "parity_max_diff": 1.621246337890625e-05
}