configs/pilot.json

{
  "name": "sml-v2-150m-pilot-v1",
  "data_mode": "hf_stream",
  "stream": {"prefetch_batches": 4, "read_attempts": 5, "dedup_window": 10000},
  "model": {"vocab_size": 32000, "max_seq_len": 2048, "d_model": 768, "n_layers": 20, "n_heads": 12, "n_kv_heads": 4, "mlp_ratio": 4.0, "mlp_multiple_of": 256, "bias": false, "qk_norm": true, "attention_impl": "fast", "loss_dtype": "float32", "ce_impl": "metal", "ffn_impl": "packed-metal"},
  "seed": 7337,
  "batches": [4, 3, 3, 3],
  "grad_accum": 4,
  "target_tokens": 500000000,
  "peak_lr": 0.0003,
  "warmup_fraction": 0.01,
  "decay_start_fraction": 0.8,
  "min_lr_ratio": 0.1,
  "betas": [0.9, 0.95],
  "eps": 1e-8,
  "weight_decay": 0.1,
  "grad_clip": 1.0,
  "eval_every_tokens": 25000000,
  "sample_every_tokens": 50000000,
  "save_every_tokens": 10000000,
  "eval_batches_per_source": 16,
  "eval_batch_size": 2,
  "keep_checkpoints": 2,
  "reserve_gib": 8,
  "log_every": 10,
  "prompts": [
    "The library was closed for repairs, so Maya decided to",
    "Plants need sunlight because",
    "A useful way to remember a new word is to",
    "The train arrived late because of heavy snow. When the passengers stepped outside,",
    "Clean drinking water is important because"
  ]
}