File size: 1,362 Bytes
6fa00d6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | {
"babylm_checkpoints": true,
"babylm_ckpt_dir": null,
"beta1": 0.9,
"beta2": 0.98,
"causal_per_masked": 1,
"dev_dir": "/home/sid/sidBabyLM/nanochat/data/dev",
"eps": 1e-08,
"full_dev": false,
"grad_clip": 2.0,
"hub_repo": null,
"keep_ratio": 0.1,
"lr": 0.007,
"mask_end": 0.15,
"mask_start": 0.3,
"masked_per_causal": 0,
"micro_batch": 8,
"muon_aux_lr": 0.006,
"muon_aux_weight_decay": null,
"muon_batch_updates": true,
"muon_beta2": 0.95,
"muon_eps2": 1e-08,
"muon_lr": 0.02,
"muon_momentum": 0.95,
"muon_nesterov": true,
"muon_ns_steps": 5,
"muon_split_qkv": true,
"muon_weight_decay": 0.1,
"optimizer": "adamuon",
"output_dir": "/home/sid/sidBabyLM/gptbert/runs/sweep_20260713_134513_3036991/adamuon-10m-c2026ck-mixed-lr0.02-aux0.006-s42",
"phase1_frac": 0.6,
"phase2_frac": 0.8,
"precision": "bf16",
"push_to_hub": false,
"random_ratio": 0.1,
"repo": "BabyLM-community/babylm-baseline-10m-gpt-bert-causal-focus",
"retokenize": false,
"save_every": 0,
"seed": 42,
"tail_avg_every": 10,
"tail_avg_frac": 0.2,
"tokens_per_step": 16384,
"total_steps": 9914,
"track": "strict-small",
"train_dir": "/home/sid/sidBabyLM/nanochat/data/small",
"val_every": 1000,
"val_seq_len": 128,
"val_words": 1000000,
"warmup_frac": 0.016,
"weight_decay": 0.1,
"z_loss": 0.0001
} |