Upload folder using huggingface_hub

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1340/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "two_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5.5e-5_61440_1340", "n_embd": 256}

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1340/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1340/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.2942
+0 val perplexity 80354.4531
+0 train 11.297427 (lr=1.1000e-07) (hash(x)=164406924)
+100 val loss 9.9969
+100 val perplexity 21958.2188
+100 train 9.959201 (lr=1.1110e-05) (hash(x)=144903932)
+200 val loss 9.4387
+200 val perplexity 12565.9189
+200 train 9.507050 (lr=2.2110e-05) (hash(x)=167734596)
+300 val loss 8.2846
+300 val perplexity 3962.3276
+300 train 8.291656 (lr=3.3110e-05) (hash(x)=149619098)
+400 val loss 7.7081
+400 val perplexity 2226.1997
+400 train 7.599453 (lr=4.4110e-05) (hash(x)=153710890)
+500 val loss 7.5311
+500 val perplexity 1865.1405
+500 train 7.618016 (lr=5.5000e-05) (hash(x)=145450636)
+600 val loss 7.4343
+600 val perplexity 1693.1133
+600 train 7.293822 (lr=5.4982e-05) (hash(x)=145249251)
+700 val loss 7.3717
+700 val perplexity 1590.2749
+700 train 7.483765 (lr=5.4928e-05) (hash(x)=150475545)
+800 val loss 7.3158
+800 val perplexity 1503.8374
+800 train 7.285135 (lr=5.4839e-05) (hash(x)=144483776)
+900 val loss 7.2436
+900 val perplexity 1399.1897
+900 train 7.350143 (lr=5.4713e-05) (hash(x)=157916369)
+1000 val loss 7.1803
+1000 val perplexity 1313.3057
+1000 train 7.206147 (lr=5.4553e-05) (hash(x)=154856891)
+1100 val loss 7.1115
+1100 val perplexity 1226.0339
+1100 train 6.834246 (lr=5.4357e-05) (hash(x)=136191502)
+1200 val loss 7.0380
+1200 val perplexity 1139.0703
+1200 train 6.990097 (lr=5.4126e-05) (hash(x)=148021541)
+1300 val loss 6.9692
+1300 val perplexity 1063.3638
+1300 train 6.906658 (lr=5.3860e-05) (hash(x)=146005217)
+1400 val loss 6.9033
+1400 val perplexity 995.5350
+1400 train 6.807381 (lr=5.3561e-05) (hash(x)=146019502)
+1500 val loss 6.8342
+1500 val perplexity 929.0643
+1500 train 6.772642 (lr=5.3227e-05) (hash(x)=150127281)
+1600 val loss 6.7654
+1600 val perplexity 867.3138
+1600 train 6.739873 (lr=5.2860e-05) (hash(x)=154120875)
+1700 val loss 6.7085
+1700 val perplexity 819.3484
+1700 train 6.931113 (lr=5.2461e-05) (hash(x)=155797680)
+1800 val loss 6.6364
+1800 val perplexity 762.3098
+1800 train 6.632697 (lr=5.2029e-05) (hash(x)=156809396)
+1900 val loss 6.5833
+1900 val perplexity 722.9105
+1900 train 6.475692 (lr=5.1565e-05) (hash(x)=144640294)
+2000 val loss 6.5380
+2000 val perplexity 690.9203
+2000 train 6.401452 (lr=5.1071e-05) (hash(x)=162831106)
+2100 val loss 6.5124
+2100 val perplexity 673.4485
+2100 train 6.494294 (lr=5.0547e-05) (hash(x)=158239484)
+2200 val loss 6.4541
+2200 val perplexity 635.3251
+2200 train 6.327179 (lr=4.9993e-05) (hash(x)=140504180)
+2300 val loss 6.4105
+2300 val perplexity 608.1702
+2300 train 6.398090 (lr=4.9410e-05) (hash(x)=142234024)
+2400 val loss 6.3766
+2400 val perplexity 587.9339
+2400 train 6.370748 (lr=4.8800e-05) (hash(x)=143091562)
+2500 val loss 6.3607
+2500 val perplexity 578.6287
+2500 train 6.265017 (lr=4.8162e-05) (hash(x)=149857456)
+2600 val loss 6.3018
+2600 val perplexity 545.5706
+2600 train 6.342639 (lr=4.7499e-05) (hash(x)=146191551)
+2700 val loss 6.2641
+2700 val perplexity 525.3575
+2700 train 6.180813 (lr=4.6811e-05) (hash(x)=145375752)
+2800 val loss 6.2381
+2800 val perplexity 511.8631
+2800 train 6.120321 (lr=4.6099e-05) (hash(x)=151568014)
+2900 val loss 6.2250
+2900 val perplexity 505.2483
+2900 train 6.053262 (lr=4.5364e-05) (hash(x)=149366597)
+3000 val loss 6.1816
+3000 val perplexity 483.7887
+3000 train 6.115553 (lr=4.4606e-05) (hash(x)=150464442)
+3100 val loss 6.1649
+3100 val perplexity 475.7505
+3100 train 6.289092 (lr=4.3828e-05) (hash(x)=182449036)
+3200 val loss 6.1437
+3200 val perplexity 465.7634
+3200 train 6.095327 (lr=4.3031e-05) (hash(x)=140141286)
+3300 val loss 6.1290
+3300 val perplexity 458.9869
+3300 train 6.098000 (lr=4.2215e-05) (hash(x)=148099414)
+3400 val loss 6.1030
+3400 val perplexity 447.2079
+3400 train 5.959933 (lr=4.1381e-05) (hash(x)=142633951)
+3500 val loss 6.0916
+3500 val perplexity 442.1234
+3500 train 5.942972 (lr=4.0532e-05) (hash(x)=148368965)
+3600 val loss 6.0609
+3600 val perplexity 428.7545
+3600 train 6.077836 (lr=3.9667e-05) (hash(x)=152372067)
+3700 val loss 6.0433
+3700 val perplexity 421.2732
+3700 train 6.128188 (lr=3.8789e-05) (hash(x)=168885609)
+3800 val loss 6.0292
+3800 val perplexity 415.3896
+3800 train 5.840792 (lr=3.7898e-05) (hash(x)=141633734)
+3900 val loss 6.0000
+3900 val perplexity 403.4280
+3900 train 5.892871 (lr=3.6996e-05) (hash(x)=153141007)
+4000 val loss 5.9856
+4000 val perplexity 397.6561
+4000 train 5.914608 (lr=3.6085e-05) (hash(x)=160577202)
+4100 val loss 5.9754
+4100 val perplexity 393.6075
+4100 train 6.036517 (lr=3.5165e-05) (hash(x)=153858169)
+4200 val loss 5.9518
+4200 val perplexity 384.4639
+4200 train 5.961702 (lr=3.4238e-05) (hash(x)=155889149)
+4300 val loss 5.9410
+4300 val perplexity 380.3158
+4300 train 5.755029 (lr=3.3305e-05) (hash(x)=152294662)
+4400 val loss 5.9276
+4400 val perplexity 375.2413
+4400 train 5.817752 (lr=3.2368e-05) (hash(x)=141804386)
+4500 val loss 5.9067
+4500 val perplexity 367.5023
+4500 train 5.802132 (lr=3.1428e-05) (hash(x)=151095242)
+4600 val loss 5.9146
+4600 val perplexity 370.4135
+4600 train 5.852379 (lr=3.0486e-05) (hash(x)=156414699)
+4700 val loss 5.8803
+4700 val perplexity 357.9089
+4700 train 5.903855 (lr=2.9543e-05) (hash(x)=161556686)
+4800 val loss 5.8648
+4800 val perplexity 352.3964
+4800 train 5.767755 (lr=2.8602e-05) (hash(x)=149000293)
+4900 val loss 5.8552
+4900 val perplexity 349.0420
+4900 train 5.649243 (lr=2.7663e-05) (hash(x)=154349989)
+5000 val loss 5.8402
+5000 val perplexity 343.8578
+5000 train 5.672950 (lr=2.6728e-05) (hash(x)=131475967)
+5100 val loss 5.8295
+5100 val perplexity 340.1834
+5100 train 5.734379 (lr=2.5798e-05) (hash(x)=149717902)
+5200 val loss 5.8185
+5200 val perplexity 336.4688
+5200 train 5.686492 (lr=2.4874e-05) (hash(x)=151407999)
+5300 val loss 5.8054
+5300 val perplexity 332.1041
+5300 train 5.912195 (lr=2.3958e-05) (hash(x)=168602728)
+5400 val loss 5.7925
+5400 val perplexity 327.8458
+5400 train 5.755012 (lr=2.3051e-05) (hash(x)=158344511)
+5500 val loss 5.7864
+5500 val perplexity 325.8506
+5500 train 5.744815 (lr=2.2155e-05) (hash(x)=148350057)
+5600 val loss 5.7736
+5600 val perplexity 321.6869
+5600 train 5.836986 (lr=2.1271e-05) (hash(x)=153847323)
+5700 val loss 5.7646
+5700 val perplexity 318.8127
+5700 train 5.712889 (lr=2.0399e-05) (hash(x)=156607405)
+5800 val loss 5.7552
+5800 val perplexity 315.8275
+5800 train 5.521281 (lr=1.9542e-05) (hash(x)=145115031)
+5900 val loss 5.7465
+5900 val perplexity 313.0825
+5900 train 5.620213 (lr=1.8700e-05) (hash(x)=141584622)
+6000 val loss 5.7369
+6000 val perplexity 310.1079
+6000 train 5.664099 (lr=1.7875e-05) (hash(x)=146613857)
+6100 val loss 5.7318
+6100 val perplexity 308.5253
+6100 train 5.667512 (lr=1.7068e-05) (hash(x)=144621768)
+6200 val loss 5.7259
+6200 val perplexity 306.6945
+6200 train 5.655423 (lr=1.6280e-05) (hash(x)=146521760)
+6300 val loss 5.7146
+6300 val perplexity 303.2649
+6300 train 5.547922 (lr=1.5512e-05) (hash(x)=161378136)
+6400 val loss 5.7088
+6400 val perplexity 301.5195
+6400 train 5.475447 (lr=1.4766e-05) (hash(x)=141624235)
+6500 val loss 5.7019
+6500 val perplexity 299.4456
+6500 train 5.626784 (lr=1.4042e-05) (hash(x)=151197095)
+6600 val loss 5.6994
+6600 val perplexity 298.6746
+6600 train 5.711471 (lr=1.3342e-05) (hash(x)=153269571)
+6700 val loss 5.6925
+6700 val perplexity 296.6243
+6700 train 5.658686 (lr=1.2666e-05) (hash(x)=146111181)
+6800 val loss 5.6823
+6800 val perplexity 293.6195
+6800 train 5.714785 (lr=1.2016e-05) (hash(x)=147269760)
+6900 val loss 5.6803
+6900 val perplexity 293.0303
+6900 train 5.755060 (lr=1.1392e-05) (hash(x)=152912762)
+7000 val loss 5.6743
+7000 val perplexity 291.2797
+7000 train 5.743410 (lr=1.0795e-05) (hash(x)=165412343)
+7100 val loss 5.6656
+7100 val perplexity 288.7697
+7100 train 5.727796 (lr=1.0227e-05) (hash(x)=162866028)
+7200 val loss 5.6604
+7200 val perplexity 287.2653
+7200 train 5.631989 (lr=9.6875e-06) (hash(x)=142998115)
+7300 val loss 5.6586
+7300 val perplexity 286.7529
+7300 train 5.425860 (lr=9.1780e-06) (hash(x)=145486999)
+7400 val loss 5.6518
+7400 val perplexity 284.7945
+7400 train 5.621400 (lr=8.6990e-06) (hash(x)=155325873)
+7500 val loss 5.6477
+7500 val perplexity 283.6333
+7500 train 5.613243 (lr=8.2513e-06) (hash(x)=145131256)
+7600 val loss 5.6528
+7600 val perplexity 285.0802
+7600 train 5.578363 (lr=7.8355e-06) (hash(x)=144008365)
+7700 val loss 5.6407
+7700 val perplexity 281.6557
+7700 train 5.733212 (lr=7.4522e-06) (hash(x)=148848532)
+7800 val loss 5.6377
+7800 val perplexity 280.8100
+7800 train 5.630808 (lr=7.1019e-06) (hash(x)=150391642)
+7900 val loss 5.6370
+7900 val perplexity 280.6281
+7900 train 5.393972 (lr=6.7852e-06) (hash(x)=152191414)
+8000 val loss 5.6318
+8000 val perplexity 279.1779
+8000 train 5.712001 (lr=6.5025e-06) (hash(x)=159755587)
+8100 val loss 5.6283
+8100 val perplexity 278.1980
+8100 train 5.644426 (lr=6.2543e-06) (hash(x)=156664468)
+8200 val loss 5.6266
+8200 val perplexity 277.7228
+8200 train 5.468357 (lr=6.0408e-06) (hash(x)=139457379)
+8300 val loss 5.6308
+8300 val perplexity 278.8773
+8300 train 5.397281 (lr=5.8625e-06) (hash(x)=145478564)
+8400 val loss 5.6214
+8400 val perplexity 276.2874
+8400 train 5.557308 (lr=5.7195e-06) (hash(x)=154982769)
+8500 val loss 5.6189
+8500 val perplexity 275.5889
+8500 train 5.576357 (lr=5.6121e-06) (hash(x)=145798118)
+8600 val loss 5.6180
+8600 val perplexity 275.3282
+8600 train 5.526694 (lr=5.5404e-06) (hash(x)=143231551)
+8700 val loss 5.6215
+8700 val perplexity 276.3135
+8700 train 5.511778 (lr=5.5045e-06) (hash(x)=154780112)
+8749 val loss 5.6129
+8749 val perplexity 273.9432

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1340/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:304b8de6c1cdc4aef4175ea679f9868e1a7aae51e29bbb39390bf4fbec1dd75c
+size 97580418

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1340/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9540f3f7c1f28c9ad2a890d22a9b55d0ac0f17bd1b41c57d7efdd23d669b1fd
+size 188880262