Upload folder using huggingface_hub

lr5.5e-5_total_batch_size61440_baseline_seed1341/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr5.5e-5_total_batch_size61440_baseline_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5.5e-5_61440_1341", "n_embd": 256}

lr5.5e-5_total_batch_size61440_baseline_seed1341/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr5.5e-5_total_batch_size61440_baseline_seed1341/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.3095
+0 val perplexity 81594.2500
+0 train 11.313040 (lr=1.1000e-07) (hash(x)=145079536)
+100 val loss 10.0281
+100 val perplexity 22654.5977
+100 train 10.021935 (lr=1.1110e-05) (hash(x)=155800595)
+200 val loss 9.2340
+200 val perplexity 10239.2734
+200 train 9.181242 (lr=2.2110e-05) (hash(x)=145606733)
+300 val loss 7.8993
+300 val perplexity 2695.4788
+300 train 7.857963 (lr=3.3110e-05) (hash(x)=150367139)
+400 val loss 7.5089
+400 val perplexity 1824.1461
+400 train 7.830506 (lr=4.4110e-05) (hash(x)=155747374)
+500 val loss 7.3606
+500 val perplexity 1572.7935
+500 train 7.249780 (lr=5.5000e-05) (hash(x)=140604760)
+600 val loss 7.2492
+600 val perplexity 1406.9397
+600 train 7.179748 (lr=5.4982e-05) (hash(x)=148404734)
+700 val loss 7.1529
+700 val perplexity 1277.7548
+700 train 7.100831 (lr=5.4928e-05) (hash(x)=148115934)
+800 val loss 7.0450
+800 val perplexity 1147.0859
+800 train 6.919345 (lr=5.4839e-05) (hash(x)=137464699)
+900 val loss 6.9599
+900 val perplexity 1053.4799
+900 train 6.769871 (lr=5.4713e-05) (hash(x)=143886042)
+1000 val loss 6.8799
+1000 val perplexity 972.5411
+1000 train 6.887217 (lr=5.4553e-05) (hash(x)=163799796)
+1100 val loss 6.7896
+1100 val perplexity 888.5198
+1100 train 6.758375 (lr=5.4357e-05) (hash(x)=144592844)
+1200 val loss 6.7126
+1200 val perplexity 822.6738
+1200 train 7.038452 (lr=5.4126e-05) (hash(x)=204706354)
+1300 val loss 6.6432
+1300 val perplexity 767.5157
+1300 train 6.583486 (lr=5.3860e-05) (hash(x)=150862210)
+1400 val loss 6.5772
+1400 val perplexity 718.5507
+1400 train 6.576581 (lr=5.3561e-05) (hash(x)=147766811)
+1500 val loss 6.5137
+1500 val perplexity 674.2840
+1500 train 6.333205 (lr=5.3227e-05) (hash(x)=135925327)
+1600 val loss 6.4813
+1600 val perplexity 652.7991
+1600 train 6.451069 (lr=5.2860e-05) (hash(x)=160440642)
+1700 val loss 6.4206
+1700 val perplexity 614.3727
+1700 train 6.542951 (lr=5.2461e-05) (hash(x)=151184106)
+1800 val loss 6.3817
+1800 val perplexity 590.9430
+1800 train 6.276783 (lr=5.2029e-05) (hash(x)=148100580)
+1900 val loss 6.3631
+1900 val perplexity 580.0589
+1900 train 6.194247 (lr=5.1565e-05) (hash(x)=149434659)
+2000 val loss 6.3010
+2000 val perplexity 545.0958
+2000 train 6.203556 (lr=5.1071e-05) (hash(x)=152285486)
+2100 val loss 6.2720
+2100 val perplexity 529.5370
+2100 train 6.015169 (lr=5.0547e-05) (hash(x)=144294295)
+2200 val loss 6.2430
+2200 val perplexity 514.3810
+2200 train 6.247200 (lr=4.9993e-05) (hash(x)=175030215)
+2300 val loss 6.2245
+2300 val perplexity 504.9677
+2300 train 6.036710 (lr=4.9410e-05) (hash(x)=150831428)
+2400 val loss 6.1823
+2400 val perplexity 484.0933
+2400 train 6.363623 (lr=4.8800e-05) (hash(x)=140808297)
+2500 val loss 6.1569
+2500 val perplexity 471.9669
+2500 train 6.133573 (lr=4.8162e-05) (hash(x)=153160275)
+2600 val loss 6.1396
+2600 val perplexity 463.8688
+2600 train 5.938820 (lr=4.7499e-05) (hash(x)=133990623)
+2700 val loss 6.1149
+2700 val perplexity 452.5539
+2700 train 6.082070 (lr=4.6811e-05) (hash(x)=142860944)
+2800 val loss 6.0839
+2800 val perplexity 438.7309
+2800 train 6.094187 (lr=4.6099e-05) (hash(x)=137959511)
+2900 val loss 6.0627
+2900 val perplexity 429.5366
+2900 train 6.017261 (lr=4.5364e-05) (hash(x)=147009873)
+3000 val loss 6.0425
+3000 val perplexity 420.9427
+3000 train 5.893528 (lr=4.4606e-05) (hash(x)=158264841)
+3100 val loss 6.0331
+3100 val perplexity 416.9911
+3100 train 5.870986 (lr=4.3828e-05) (hash(x)=139232251)
+3200 val loss 5.9960
+3200 val perplexity 401.8355
+3200 train 6.045129 (lr=4.3031e-05) (hash(x)=153436104)
+3300 val loss 5.9811
+3300 val perplexity 395.8732
+3300 train 5.804980 (lr=4.2215e-05) (hash(x)=149681831)
+3400 val loss 5.9669
+3400 val perplexity 390.2879
+3400 train 5.846797 (lr=4.1381e-05) (hash(x)=168425516)
+3500 val loss 5.9406
+3500 val perplexity 380.1740
+3500 train 5.973686 (lr=4.0532e-05) (hash(x)=163104338)
+3600 val loss 5.9261
+3600 val perplexity 374.6831
+3600 train 5.895655 (lr=3.9667e-05) (hash(x)=165109772)
+3700 val loss 5.9238
+3700 val perplexity 373.8369
+3700 train 5.659553 (lr=3.8789e-05) (hash(x)=153420306)
+3800 val loss 5.8912
+3800 val perplexity 361.8286
+3800 train 5.905637 (lr=3.7898e-05) (hash(x)=160168863)
+3900 val loss 5.8747
+3900 val perplexity 355.9306
+3900 train 5.789441 (lr=3.6996e-05) (hash(x)=153906073)
+4000 val loss 5.8653
+4000 val perplexity 352.5826
+4000 train 5.812453 (lr=3.6085e-05) (hash(x)=151055067)
+4100 val loss 5.8489
+4100 val perplexity 346.8618
+4100 train 5.827378 (lr=3.5165e-05) (hash(x)=149629830)
+4200 val loss 5.8343
+4200 val perplexity 341.8114
+4200 train 5.713896 (lr=3.4238e-05) (hash(x)=143101381)
+4300 val loss 5.8245
+4300 val perplexity 338.4835
+4300 train 5.722774 (lr=3.3305e-05) (hash(x)=149712044)
+4400 val loss 5.8080
+4400 val perplexity 332.9639
+4400 train 5.730769 (lr=3.2368e-05) (hash(x)=153446449)
+4500 val loss 5.7901
+4500 val perplexity 327.0311
+4500 train 5.730693 (lr=3.1428e-05) (hash(x)=146086947)
+4600 val loss 5.7839
+4600 val perplexity 325.0396
+4600 train 5.626281 (lr=3.0486e-05) (hash(x)=153800173)
+4700 val loss 5.7714
+4700 val perplexity 320.9735
+4700 train 5.845865 (lr=2.9543e-05) (hash(x)=155962726)
+4800 val loss 5.7581
+4800 val perplexity 316.7545
+4800 train 5.999410 (lr=2.8602e-05) (hash(x)=142045616)
+4900 val loss 5.7437
+4900 val perplexity 312.2062
+4900 train 5.717124 (lr=2.7663e-05) (hash(x)=143418248)
+5000 val loss 5.7358
+5000 val perplexity 309.7615
+5000 train 5.652305 (lr=2.6728e-05) (hash(x)=145789790)
+5100 val loss 5.7302
+5100 val perplexity 308.0321
+5100 train 5.710855 (lr=2.5798e-05) (hash(x)=137795633)
+5200 val loss 5.7175
+5200 val perplexity 304.1429
+5200 train 5.868896 (lr=2.4874e-05) (hash(x)=148907132)
+5300 val loss 5.7107
+5300 val perplexity 302.0871
+5300 train 5.576705 (lr=2.3958e-05) (hash(x)=152343580)
+5400 val loss 5.6998
+5400 val perplexity 298.8186
+5400 train 5.548367 (lr=2.3051e-05) (hash(x)=148578264)
+5500 val loss 5.6860
+5500 val perplexity 294.7006
+5500 train 5.845662 (lr=2.2155e-05) (hash(x)=145635833)
+5600 val loss 5.6812
+5600 val perplexity 293.3118
+5600 train 5.673640 (lr=2.1271e-05) (hash(x)=156337844)
+5700 val loss 5.6762
+5700 val perplexity 291.8508
+5700 train 5.547314 (lr=2.0399e-05) (hash(x)=147168506)
+5800 val loss 5.6645
+5800 val perplexity 288.4523
+5800 train 5.716799 (lr=1.9542e-05) (hash(x)=159566920)
+5900 val loss 5.6574
+5900 val perplexity 286.4145
+5900 train 5.712609 (lr=1.8700e-05) (hash(x)=158273929)
+6000 val loss 5.6534
+6000 val perplexity 285.2582
+6000 train 5.622340 (lr=1.7875e-05) (hash(x)=156649749)
+6100 val loss 5.6451
+6100 val perplexity 282.9034
+6100 train 5.563344 (lr=1.7068e-05) (hash(x)=146812388)
+6200 val loss 5.6412
+6200 val perplexity 281.8145
+6200 train 5.472699 (lr=1.6280e-05) (hash(x)=143522146)
+6300 val loss 5.6326
+6300 val perplexity 279.3959
+6300 train 5.446299 (lr=1.5512e-05) (hash(x)=150124474)
+6400 val loss 5.6261
+6400 val perplexity 277.5742
+6400 train 5.676538 (lr=1.4766e-05) (hash(x)=141242117)
+6500 val loss 5.6186
+6500 val perplexity 275.5167
+6500 train 5.493915 (lr=1.4042e-05) (hash(x)=143529762)
+6600 val loss 5.6132
+6600 val perplexity 274.0192
+6600 train 5.401798 (lr=1.3342e-05) (hash(x)=136948374)
+6700 val loss 5.6124
+6700 val perplexity 273.7912
+6700 train 5.403396 (lr=1.2666e-05) (hash(x)=146268592)
+6800 val loss 5.6039
+6800 val perplexity 271.4809
+6800 train 5.549529 (lr=1.2016e-05) (hash(x)=152676836)
+6900 val loss 5.5959
+6900 val perplexity 269.3203
+6900 train 5.519081 (lr=1.1392e-05) (hash(x)=134657776)
+7000 val loss 5.5939
+7000 val perplexity 268.7895
+7000 train 5.540829 (lr=1.0795e-05) (hash(x)=166721861)
+7100 val loss 5.5922
+7100 val perplexity 268.3279
+7100 train 5.385216 (lr=1.0227e-05) (hash(x)=135496702)
+7200 val loss 5.5877
+7200 val perplexity 267.1325
+7200 train 5.696855 (lr=9.6875e-06) (hash(x)=155567461)
+7300 val loss 5.5802
+7300 val perplexity 265.1262
+7300 train 5.382809 (lr=9.1780e-06) (hash(x)=142803829)
+7400 val loss 5.5792
+7400 val perplexity 264.8464
+7400 train 5.404070 (lr=8.6990e-06) (hash(x)=145294178)
+7500 val loss 5.5760
+7500 val perplexity 264.0256
+7500 train 5.334823 (lr=8.2513e-06) (hash(x)=150573713)
+7600 val loss 5.5734
+7600 val perplexity 263.3184
+7600 train 5.588169 (lr=7.8355e-06) (hash(x)=142771511)
+7700 val loss 5.5665
+7700 val perplexity 261.5170
+7700 train 5.472699 (lr=7.4522e-06) (hash(x)=143602175)
+7800 val loss 5.5624
+7800 val perplexity 260.4586
+7800 train 5.624176 (lr=7.1019e-06) (hash(x)=152379862)
+7900 val loss 5.5623
+7900 val perplexity 260.4119
+7900 train 5.373997 (lr=6.7852e-06) (hash(x)=146655921)
+8000 val loss 5.5639
+8000 val perplexity 260.8366
+8000 train 5.705983 (lr=6.5025e-06) (hash(x)=148262482)
+8100 val loss 5.5539
+8100 val perplexity 258.2548
+8100 train 5.499075 (lr=6.2543e-06) (hash(x)=147683655)
+8200 val loss 5.5539
+8200 val perplexity 258.2498
+8200 train 5.672094 (lr=6.0408e-06) (hash(x)=157312987)
+8300 val loss 5.5513
+8300 val perplexity 257.5673
+8300 train 5.507140 (lr=5.8625e-06) (hash(x)=141107543)
+8400 val loss 5.5464
+8400 val perplexity 256.3211
+8400 train 5.549830 (lr=5.7195e-06) (hash(x)=141323024)
+8500 val loss 5.5445
+8500 val perplexity 255.8297
+8500 train 5.517342 (lr=5.6121e-06) (hash(x)=150696521)
+8600 val loss 5.5445
+8600 val perplexity 255.8344
+8600 train 5.475835 (lr=5.5404e-06) (hash(x)=162288191)
+8700 val loss 5.5402
+8700 val perplexity 254.7260
+8700 train 5.413832 (lr=5.5045e-06) (hash(x)=152860941)
+8749 val loss 5.5389
+8749 val perplexity 254.3993

lr5.5e-5_total_batch_size61440_baseline_seed1341/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bb426cb4f545242fbc5ffc51066c1ee7b0c29599b1cee4dd6e106b62d23adaa
+size 92843394

lr5.5e-5_total_batch_size61440_baseline_seed1341/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8306044646d618ad0f657c8bbf44566740f34ea13fa5da7bd2f93fbb4d5493c9
+size 179406214