{
  "parameters": 9968128,
  "architecture": {
    "vocab_size": 8192,
    "hidden_size": 256,
    "num_hidden_layers": 10,
    "num_attention_heads": 4,
    "num_key_value_heads": 2,
    "head_dim": 64,
    "intermediate_size": 768,
    "max_position_embeddings": 4096,
    "rope_theta": 100000.0,
    "rms_norm_eps": 1e-06,
    "tie_word_embeddings": true
  },
  "dataset_id": "HuggingFaceFW/fineweb-edu",
  "dataset_config": "sample-100BT",
  "dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
  "tokenizer_repo": "BananaMind/BananaMind-2-Nano",
  "tokenizer_revision": "c8564d1bd3f6177221ed7e4f63ae5f281a677a1c",
  "optimizer": "AdamW",
  "peak_lr": 0.003,
  "global_batch": 132,
  "tokens_per_step": 540672,
  "steps": 46238,
  "tokens_seen": 24999591936,
  "target_tokens": 25000000000,
  "world_size": 8,
  "gpu_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
  "elapsed_seconds": 3753.9637784957886,
  "average_tokens_per_second": 6659518.687742193,
  "estimated_training_flops": 4640871447860871168,
  "seed": 1337
}