Upload folder using huggingface_hub

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1341/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "two_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5.5e-5_61440_1341", "n_embd": 256}

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1341/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1341/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.3043
+0 val perplexity 81171.1797
+0 train 11.302660 (lr=1.1000e-07) (hash(x)=145079536)
+100 val loss 9.9997
+100 val perplexity 22020.4590
+100 train 10.000200 (lr=1.1110e-05) (hash(x)=155800595)
+200 val loss 9.4117
+200 val perplexity 12230.0889
+200 train 9.391574 (lr=2.2110e-05) (hash(x)=145606733)
+300 val loss 8.3578
+300 val perplexity 4263.4536
+300 train 8.346644 (lr=3.3110e-05) (hash(x)=150367139)
+400 val loss 7.6713
+400 val perplexity 2145.8896
+400 train 7.988379 (lr=4.4110e-05) (hash(x)=155747374)
+500 val loss 7.4494
+500 val perplexity 1718.8815
+500 train 7.329481 (lr=5.5000e-05) (hash(x)=140604760)
+600 val loss 7.3330
+600 val perplexity 1529.9447
+600 train 7.270112 (lr=5.4982e-05) (hash(x)=148404734)
+700 val loss 7.2228
+700 val perplexity 1370.3870
+700 train 7.168687 (lr=5.4928e-05) (hash(x)=148115934)
+800 val loss 7.0991
+800 val perplexity 1210.8527
+800 train 6.975582 (lr=5.4839e-05) (hash(x)=137464699)
+900 val loss 6.9914
+900 val perplexity 1087.2299
+900 train 6.803150 (lr=5.4713e-05) (hash(x)=143886042)
+1000 val loss 6.9016
+1000 val perplexity 993.8161
+1000 train 6.911317 (lr=5.4553e-05) (hash(x)=163799796)
+1100 val loss 6.7969
+1100 val perplexity 895.0783
+1100 train 6.763620 (lr=5.4357e-05) (hash(x)=144592844)
+1200 val loss 6.7134
+1200 val perplexity 823.3582
+1200 train 7.044964 (lr=5.4126e-05) (hash(x)=204706354)
+1300 val loss 6.6361
+1300 val perplexity 762.0994
+1300 train 6.583230 (lr=5.3860e-05) (hash(x)=150862210)
+1400 val loss 6.5738
+1400 val perplexity 716.0992
+1400 train 6.574629 (lr=5.3561e-05) (hash(x)=147766811)
+1500 val loss 6.5000
+1500 val perplexity 665.1137
+1500 train 6.327939 (lr=5.3227e-05) (hash(x)=135925327)
+1600 val loss 6.4667
+1600 val perplexity 643.3799
+1600 train 6.441556 (lr=5.2860e-05) (hash(x)=160440642)
+1700 val loss 6.4074
+1700 val perplexity 606.3422
+1700 train 6.532154 (lr=5.2461e-05) (hash(x)=151184106)
+1800 val loss 6.3630
+1800 val perplexity 579.9720
+1800 train 6.258941 (lr=5.2029e-05) (hash(x)=148100580)
+1900 val loss 6.3471
+1900 val perplexity 570.8587
+1900 train 6.174709 (lr=5.1565e-05) (hash(x)=149434659)
+2000 val loss 6.2835
+2000 val perplexity 535.6703
+2000 train 6.185684 (lr=5.1071e-05) (hash(x)=152285486)
+2100 val loss 6.2462
+2100 val perplexity 516.0637
+2100 train 5.989036 (lr=5.0547e-05) (hash(x)=144294295)
+2200 val loss 6.2195
+2200 val perplexity 502.4321
+2200 train 6.226321 (lr=4.9993e-05) (hash(x)=175030215)
+2300 val loss 6.1890
+2300 val perplexity 487.3704
+2300 train 6.009117 (lr=4.9410e-05) (hash(x)=150831428)
+2400 val loss 6.1505
+2400 val perplexity 468.9300
+2400 train 6.332338 (lr=4.8800e-05) (hash(x)=140808297)
+2500 val loss 6.1267
+2500 val perplexity 457.9323
+2500 train 6.101779 (lr=4.8162e-05) (hash(x)=153160275)
+2600 val loss 6.1051
+2600 val perplexity 448.1566
+2600 train 5.892901 (lr=4.7499e-05) (hash(x)=133990623)
+2700 val loss 6.0845
+2700 val perplexity 438.9897
+2700 train 6.051739 (lr=4.6811e-05) (hash(x)=142860944)
+2800 val loss 6.0576
+2800 val perplexity 427.3339
+2800 train 6.067057 (lr=4.6099e-05) (hash(x)=137959511)
+2900 val loss 6.0341
+2900 val perplexity 417.4136
+2900 train 5.988535 (lr=4.5364e-05) (hash(x)=147009873)
+3000 val loss 6.0131
+3000 val perplexity 408.7382
+3000 train 5.862369 (lr=4.4606e-05) (hash(x)=158264841)
+3100 val loss 6.0030
+3100 val perplexity 404.6213
+3100 train 5.850473 (lr=4.3828e-05) (hash(x)=139232251)
+3200 val loss 5.9732
+3200 val perplexity 392.7657
+3200 train 6.020325 (lr=4.3031e-05) (hash(x)=153436104)
+3300 val loss 5.9533
+3300 val perplexity 385.0064
+3300 train 5.781939 (lr=4.2215e-05) (hash(x)=149681831)
+3400 val loss 5.9414
+3400 val perplexity 380.4658
+3400 train 5.815576 (lr=4.1381e-05) (hash(x)=168425516)
+3500 val loss 5.9152
+3500 val perplexity 370.6419
+3500 train 5.948214 (lr=4.0532e-05) (hash(x)=163104338)
+3600 val loss 5.8957
+3600 val perplexity 363.4672
+3600 train 5.865705 (lr=3.9667e-05) (hash(x)=165109772)
+3700 val loss 5.8880
+3700 val perplexity 360.7011
+3700 train 5.615150 (lr=3.8789e-05) (hash(x)=153420306)
+3800 val loss 5.8597
+3800 val perplexity 350.6320
+3800 train 5.860766 (lr=3.7898e-05) (hash(x)=160168863)
+3900 val loss 5.8453
+3900 val perplexity 345.6214
+3900 train 5.757235 (lr=3.6996e-05) (hash(x)=153906073)
+4000 val loss 5.8365
+4000 val perplexity 342.5663
+4000 train 5.781537 (lr=3.6085e-05) (hash(x)=151055067)
+4100 val loss 5.8201
+4100 val perplexity 337.0215
+4100 train 5.794862 (lr=3.5165e-05) (hash(x)=149629830)
+4200 val loss 5.8028
+4200 val perplexity 331.2238
+4200 train 5.690864 (lr=3.4238e-05) (hash(x)=143101381)
+4300 val loss 5.7912
+4300 val perplexity 327.3950
+4300 train 5.698771 (lr=3.3305e-05) (hash(x)=149712044)
+4400 val loss 5.7770
+4400 val perplexity 322.7938
+4400 train 5.697539 (lr=3.2368e-05) (hash(x)=153446449)
+4500 val loss 5.7614
+4500 val perplexity 317.7894
+4500 train 5.706662 (lr=3.1428e-05) (hash(x)=146086947)
+4600 val loss 5.7539
+4600 val perplexity 315.4029
+4600 train 5.601283 (lr=3.0486e-05) (hash(x)=153800173)
+4700 val loss 5.7392
+4700 val perplexity 310.8164
+4700 train 5.821584 (lr=2.9543e-05) (hash(x)=155962726)
+4800 val loss 5.7287
+4800 val perplexity 307.5836
+4800 train 5.964058 (lr=2.8602e-05) (hash(x)=142045616)
+4900 val loss 5.7200
+4900 val perplexity 304.9108
+4900 train 5.688114 (lr=2.7663e-05) (hash(x)=143418248)
+5000 val loss 5.7074
+5000 val perplexity 301.0942
+5000 train 5.633245 (lr=2.6728e-05) (hash(x)=145789790)
+5100 val loss 5.7028
+5100 val perplexity 299.7068
+5100 train 5.682884 (lr=2.5798e-05) (hash(x)=137795633)
+5200 val loss 5.6910
+5200 val perplexity 296.1953
+5200 train 5.829770 (lr=2.4874e-05) (hash(x)=148907132)
+5300 val loss 5.6825
+5300 val perplexity 293.6928
+5300 train 5.551336 (lr=2.3958e-05) (hash(x)=152343580)
+5400 val loss 5.6728
+5400 val perplexity 290.8553
+5400 train 5.530852 (lr=2.3051e-05) (hash(x)=148578264)
+5500 val loss 5.6600
+5500 val perplexity 287.1486
+5500 train 5.817604 (lr=2.2155e-05) (hash(x)=145635833)
+5600 val loss 5.6543
+5600 val perplexity 285.5133
+5600 train 5.642316 (lr=2.1271e-05) (hash(x)=156337844)
+5700 val loss 5.6500
+5700 val perplexity 284.2942
+5700 train 5.515101 (lr=2.0399e-05) (hash(x)=147168506)
+5800 val loss 5.6385
+5800 val perplexity 281.0499
+5800 train 5.681671 (lr=1.9542e-05) (hash(x)=159566920)
+5900 val loss 5.6326
+5900 val perplexity 279.3899
+5900 train 5.677158 (lr=1.8700e-05) (hash(x)=158273929)
+6000 val loss 5.6294
+6000 val perplexity 278.5004
+6000 train 5.591680 (lr=1.7875e-05) (hash(x)=156649749)
+6100 val loss 5.6193
+6100 val perplexity 275.6945
+6100 train 5.539531 (lr=1.7068e-05) (hash(x)=146812388)
+6200 val loss 5.6159
+6200 val perplexity 274.7614
+6200 train 5.451974 (lr=1.6280e-05) (hash(x)=143522146)
+6300 val loss 5.6099
+6300 val perplexity 273.1064
+6300 train 5.423518 (lr=1.5512e-05) (hash(x)=150124474)
+6400 val loss 5.6025
+6400 val perplexity 271.1142
+6400 train 5.639264 (lr=1.4766e-05) (hash(x)=141242117)
+6500 val loss 5.5959
+6500 val perplexity 269.3148
+6500 train 5.473709 (lr=1.4042e-05) (hash(x)=143529762)
+6600 val loss 5.5897
+6600 val perplexity 267.6462
+6600 train 5.381795 (lr=1.3342e-05) (hash(x)=136948374)
+6700 val loss 5.5893
+6700 val perplexity 267.5611
+6700 train 5.384252 (lr=1.2666e-05) (hash(x)=146268592)
+6800 val loss 5.5810
+6800 val perplexity 265.3392
+6800 train 5.520535 (lr=1.2016e-05) (hash(x)=152676836)
+6900 val loss 5.5745
+6900 val perplexity 263.6298
+6900 train 5.500252 (lr=1.1392e-05) (hash(x)=134657776)
+7000 val loss 5.5703
+7000 val perplexity 262.5054
+7000 train 5.518115 (lr=1.0795e-05) (hash(x)=166721861)
+7100 val loss 5.5693
+7100 val perplexity 262.2430
+7100 train 5.371016 (lr=1.0227e-05) (hash(x)=135496702)
+7200 val loss 5.5653
+7200 val perplexity 261.2061
+7200 train 5.675698 (lr=9.6875e-06) (hash(x)=155567461)
+7300 val loss 5.5581
+7300 val perplexity 259.3283
+7300 train 5.367023 (lr=9.1780e-06) (hash(x)=142803829)
+7400 val loss 5.5576
+7400 val perplexity 259.2104
+7400 train 5.376502 (lr=8.6990e-06) (hash(x)=145294178)
+7500 val loss 5.5544
+7500 val perplexity 258.3814
+7500 train 5.314266 (lr=8.2513e-06) (hash(x)=150573713)
+7600 val loss 5.5520
+7600 val perplexity 257.7412
+7600 train 5.564483 (lr=7.8355e-06) (hash(x)=142771511)
+7700 val loss 5.5452
+7700 val perplexity 256.0127
+7700 train 5.456575 (lr=7.4522e-06) (hash(x)=143602175)
+7800 val loss 5.5427
+7800 val perplexity 255.3568
+7800 train 5.606891 (lr=7.1019e-06) (hash(x)=152379862)
+7900 val loss 5.5415
+7900 val perplexity 255.0660
+7900 train 5.354670 (lr=6.7852e-06) (hash(x)=146655921)
+8000 val loss 5.5422
+8000 val perplexity 255.2316
+8000 train 5.674486 (lr=6.5025e-06) (hash(x)=148262482)
+8100 val loss 5.5344
+8100 val perplexity 253.2485
+8100 train 5.484853 (lr=6.2543e-06) (hash(x)=147683655)
+8200 val loss 5.5338
+8200 val perplexity 253.1054
+8200 train 5.655976 (lr=6.0408e-06) (hash(x)=157312987)
+8300 val loss 5.5314
+8300 val perplexity 252.4995
+8300 train 5.486342 (lr=5.8625e-06) (hash(x)=141107543)
+8400 val loss 5.5264
+8400 val perplexity 251.2418
+8400 train 5.533653 (lr=5.7195e-06) (hash(x)=141323024)
+8500 val loss 5.5261
+8500 val perplexity 251.1534
+8500 train 5.498310 (lr=5.6121e-06) (hash(x)=150696521)
+8600 val loss 5.5246
+8600 val perplexity 250.7867
+8600 train 5.456114 (lr=5.5404e-06) (hash(x)=162288191)
+8700 val loss 5.5204
+8700 val perplexity 249.7345
+8700 train 5.384398 (lr=5.5045e-06) (hash(x)=152860941)
+8749 val loss 5.5192
+8749 val perplexity 249.4464

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1341/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc779e11c745420cfda6f166a52ee83f0c0615f875661a0086abb216482a7ec8
+size 97580418

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1341/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c61ce66acda3961bebd58e366ea15582ec0ebdf3c96e7fdf38b8c53016f2b222
+size 188880262