| { |
| "data_dir": "/dev/shm/data/fineweb-edu-25B", |
| "batch_size": 32, |
| "seq_len": 1024, |
| "max_iters": 12716, |
| "gradient_accumulation_steps": 48, |
| "seed": 1337, |
| "optimizer": "muon", |
| "learning_rate": 0.0006, |
| "min_lr": 6e-05, |
| "weight_decay": 0.1, |
| "beta1": 0.9, |
| "beta2": 0.95, |
| "grad_clip": 1.0, |
| "warmup_iters": 200, |
| "lr_decay_iters": 12716, |
| "muon_lr": 0.012, |
| "muon_min_lr": 0.00015, |
| "adamw_lr": 0.012, |
| "adamw_min_lr": 0.00015, |
| "muon_momentum": 0.95, |
| "muon_nesterov": true, |
| "muon_ns_steps": 5, |
| "muon_weight_decay": 0.0, |
| "lr_schedule": "linear_warmdown", |
| "warmdown_frac": 0.9, |
| "parity_target": 0.0, |
| "parity_margin": 0.03, |
| "parity_warmdown_iters": 1500, |
| "parity_trigger_iter": -1, |
| "retain_from_iter": -1, |
| "save_at_val": [ |
| 3.1584, |
| 3.1484, |
| 3.133, |
| 3.123 |
| ], |
| "device": "cuda", |
| "dtype": "bfloat16", |
| "compile": true, |
| "n_layer": 12, |
| "n_head": 8, |
| "n_embd": 1024, |
| "block_size": 1024, |
| "vocab_size": 50304, |
| "pos_encoding": "learned", |
| "rope_theta": 10000.0, |
| "sparsity_mode": "none", |
| "rmsnorm_affine": true, |
| "cayley_levels": [], |
| "cayley_per_parent_budget": false, |
| "cayley_seed": 42, |
| "cayley_backend": "auto", |
| "cayley_locations": [ |
| "resid_mid" |
| ], |
| "cayley_score_standardize": false, |
| "cayley_forward_standardized": false, |
| "cayley_score_std_ema_beta": 0.99, |
| "cayley_replaces_pre_rmsnorm": false, |
| "cayley_richardson_refine": false, |
| "cayley_matching_pursuit": false, |
| "tied_block_init": false, |
| "dead_threshold_c": 0.1, |
| "resume": true, |
| "out_dir": "/dev/shm/out/vanilla-small-20B", |
| "eval_interval": 100, |
| "eval_iters": 150, |
| "log_interval": 10, |
| "wandb_project": "sparse-nanogpt", |
| "wandb_run_name": "vanilla-small-20B", |
| "wandb_step_offset": 0, |
| "quick_eval_interval": 2500, |
| "quick_eval_timeout": 900, |
| "hierarchy_eval_enabled": true, |
| "hierarchy_eval_iters": null |
| } |
