c2026ck tail-avg as main
Browse files- args.json +52 -0
- pytorch_model.bin +1 -1
args.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"babylm_checkpoints": true,
|
| 3 |
+
"babylm_ckpt_dir": null,
|
| 4 |
+
"beta1": 0.9,
|
| 5 |
+
"beta2": 0.98,
|
| 6 |
+
"causal_per_masked": 1,
|
| 7 |
+
"dev_dir": "/home/sid/sidBabyLM/nanochat/data/dev",
|
| 8 |
+
"eps": 1e-08,
|
| 9 |
+
"full_dev": false,
|
| 10 |
+
"grad_clip": 2.0,
|
| 11 |
+
"hub_repo": null,
|
| 12 |
+
"keep_ratio": 0.1,
|
| 13 |
+
"lr": 0.007,
|
| 14 |
+
"mask_end": 0.15,
|
| 15 |
+
"mask_start": 0.3,
|
| 16 |
+
"masked_per_causal": 0,
|
| 17 |
+
"micro_batch": 8,
|
| 18 |
+
"muon_aux_lr": 0.006,
|
| 19 |
+
"muon_aux_weight_decay": null,
|
| 20 |
+
"muon_batch_updates": true,
|
| 21 |
+
"muon_beta2": 0.95,
|
| 22 |
+
"muon_eps2": 1e-08,
|
| 23 |
+
"muon_lr": 0.02,
|
| 24 |
+
"muon_momentum": 0.95,
|
| 25 |
+
"muon_nesterov": true,
|
| 26 |
+
"muon_ns_steps": 5,
|
| 27 |
+
"muon_split_qkv": true,
|
| 28 |
+
"muon_weight_decay": 0.1,
|
| 29 |
+
"optimizer": "adamuon",
|
| 30 |
+
"output_dir": "/home/sid/sidBabyLM/gptbert/runs/sweep_20260713_134513_3036991/adamuon-10m-c2026ck-mixed-lr0.02-aux0.006-s42",
|
| 31 |
+
"phase1_frac": 0.6,
|
| 32 |
+
"phase2_frac": 0.8,
|
| 33 |
+
"precision": "bf16",
|
| 34 |
+
"push_to_hub": false,
|
| 35 |
+
"random_ratio": 0.1,
|
| 36 |
+
"repo": "BabyLM-community/babylm-baseline-10m-gpt-bert-causal-focus",
|
| 37 |
+
"retokenize": false,
|
| 38 |
+
"save_every": 0,
|
| 39 |
+
"seed": 42,
|
| 40 |
+
"tail_avg_every": 10,
|
| 41 |
+
"tail_avg_frac": 0.2,
|
| 42 |
+
"tokens_per_step": 16384,
|
| 43 |
+
"total_steps": 9914,
|
| 44 |
+
"track": "strict-small",
|
| 45 |
+
"train_dir": "/home/sid/sidBabyLM/nanochat/data/small",
|
| 46 |
+
"val_every": 1000,
|
| 47 |
+
"val_seq_len": 128,
|
| 48 |
+
"val_words": 1000000,
|
| 49 |
+
"warmup_frac": 0.016,
|
| 50 |
+
"weight_decay": 0.1,
|
| 51 |
+
"z_loss": 0.0001
|
| 52 |
+
}
|
pytorch_model.bin
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 132236895
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0338d54a6628980c6217d2bc39cc58d0c9bb8c6fda776ce765be75706b832812
|
| 3 |
size 132236895
|