| { |
| "babylm_checkpoints": true, |
| "babylm_ckpt_dir": null, |
| "beta1": 0.9, |
| "beta2": 0.98, |
| "causal_per_masked": 1, |
| "dev_dir": "/home/sid/sidBabyLM/nanochat/data/dev", |
| "eps": 1e-08, |
| "full_dev": false, |
| "grad_clip": 2.0, |
| "hub_repo": "svsatheesh/BabySteps_ChadBert-100M-mixed", |
| "keep_ratio": 0.1, |
| "lr": 0.007, |
| "mask_end": 0.15, |
| "mask_start": 0.3, |
| "masked_per_causal": 0, |
| "micro_batch": 8, |
| "muon_aux_lr": 0.007, |
| "muon_aux_weight_decay": null, |
| "muon_batch_updates": true, |
| "muon_beta2": 0.95, |
| "muon_eps2": 1e-08, |
| "muon_lr": 0.005, |
| "muon_momentum": 0.95, |
| "muon_nesterov": true, |
| "muon_ns_steps": 5, |
| "muon_split_qkv": true, |
| "muon_weight_decay": 0.1, |
| "optimizer": "muon", |
| "output_dir": "/home/sid/sidBabyLM/gptbert/runs/sweep100m/muon-100m-legacyCK-mixed-lr0.005-aux0.007-s42", |
| "phase1_frac": 0.6, |
| "phase2_frac": 0.8, |
| "precision": "bf16", |
| "push_to_hub": false, |
| "random_ratio": 0.1, |
| "repo": "BabyLM-community/babylm-baseline-100m-gpt-bert-mixed", |
| "retokenize": false, |
| "save_every": 0, |
| "seed": 42, |
| "tail_avg_every": 10, |
| "tail_avg_frac": 0.0, |
| "tokens_per_step": 131072, |
| "total_steps": 12330, |
| "track": "strict", |
| "train_dir": "/home/sid/sidBabyLM/nanochat/data/strict", |
| "val_every": 1000, |
| "val_seq_len": 128, |
| "val_words": 1000000, |
| "warmup_frac": 0.016, |
| "weight_decay": 0.1, |
| "z_loss": 0.0001 |
| } |
