| { |
| "depth": 24, |
| "aspect_ratio": 64, |
| "head_dim": 128, |
| "max_seq_len": 2048, |
| "vocab_size": 32768, |
| "params_total": 1384122122, |
| "params_scaling": 729810624, |
| "flops_per_token": 5284833552, |
| "unique_train_tokens": 940460590, |
| "target_tokens_ratio": 7663011552, |
| "target_tokens_epochs": 2821381770, |
| "horizon": "epochs", |
| "num_iterations": 4829, |
| "total_batch_size": 524288, |
| "device_batch_size": 16, |
| "world_size": 4, |
| "device_batch_size_source": "ladder.json device_batch_size", |
| "grad_accum_steps": 4, |
| "train_tokens": 2531786752, |
| "epochs": 2.69, |
| "tokens_per_param": 3.47, |
| "train_flops": 13380071573478703104, |
| "window_pattern": "L", |
| "attention": "sdpa", |
| "fp8": true, |
| "eval_tokens": 10485760, |
| "eval_every": 241, |
| "save_every": 897, |
| "keep_checkpoints": true, |
| "base_overrides": {}, |
| "warmdown_ratio": 0.199755, |
| "stream": true, |
| "decay_at": 3864, |
| "plain_at": 4346, |
| "loader_tails": true, |
| "mask_source_line": true, |
| "regularization": { |
| "wd_schedule": "floor", |
| "wd_floor": 0.5, |
| "table_wd_mult": 1.0, |
| "dropout": 0.0, |
| "dropout_from": 0.5, |
| "wd_lr_compensate": true |
| }, |
| "dropout_from_step": null, |
| "weight_decay": { |
| "horizon": "trained", |
| "base": 0.28, |
| "nanochat_scaled": 0.042241, |
| "scaled": 0.042241, |
| "decay_per_step": 0.000845, |
| "ema_tokens": 620584830 |
| }, |
| "wd_horizon": null, |
| "save_steps": [], |
| "seed": null, |
| "arm": null, |
| "device_type": "cuda", |
| "frozen_at": "2026-10-24T22:01:41" |
| } |
