Upload folder using huggingface_hub

lr6e-5_total_batch_size61440_two_masks_4_heads_seed1341/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr6e-5_total_batch_size61440_two_masks_4_heads_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "two_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_1341", "n_embd": 256}

lr6e-5_total_batch_size61440_two_masks_4_heads_seed1341/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr6e-5_total_batch_size61440_two_masks_4_heads_seed1341/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.3043
+0 val perplexity 81171.0312
+0 train 11.302660 (lr=1.2000e-07) (hash(x)=145079536)
+100 val loss 9.9838
+100 val perplexity 21672.7793
+100 train 9.985286 (lr=1.2120e-05) (hash(x)=155800595)
+200 val loss 9.3144
+200 val perplexity 11096.6631
+200 train 9.291759 (lr=2.4120e-05) (hash(x)=145606733)
+300 val loss 8.3288
+300 val perplexity 4141.4258
+300 train 8.317444 (lr=3.6120e-05) (hash(x)=150367139)
+400 val loss 7.6928
+400 val perplexity 2192.6128
+400 train 8.009436 (lr=4.8120e-05) (hash(x)=155747374)
+500 val loss 7.4712
+500 val perplexity 1756.7528
+500 train 7.349464 (lr=6.0000e-05) (hash(x)=140604760)
+600 val loss 7.3842
+600 val perplexity 1610.2777
+600 train 7.327204 (lr=5.9980e-05) (hash(x)=148404734)
+700 val loss 7.3115
+700 val perplexity 1497.3582
+700 train 7.258036 (lr=5.9922e-05) (hash(x)=148115934)
+800 val loss 7.2215
+800 val perplexity 1368.6002
+800 train 7.106967 (lr=5.9824e-05) (hash(x)=137464699)
+900 val loss 7.1318
+900 val perplexity 1251.0938
+900 train 6.950951 (lr=5.9687e-05) (hash(x)=143886042)
+1000 val loss 7.0443
+1000 val perplexity 1146.3440
+1000 train 7.053555 (lr=5.9512e-05) (hash(x)=163799796)
+1100 val loss 6.9622
+1100 val perplexity 1055.9250
+1100 train 6.923687 (lr=5.9298e-05) (hash(x)=144592844)
+1200 val loss 6.9055
+1200 val perplexity 997.7053
+1200 train 7.242189 (lr=5.9046e-05) (hash(x)=204706354)
+1300 val loss 6.8401
+1300 val perplexity 934.5864
+1300 train 6.786232 (lr=5.8757e-05) (hash(x)=150862210)
+1400 val loss 6.7698
+1400 val perplexity 871.1760
+1400 train 6.761644 (lr=5.8430e-05) (hash(x)=147766811)
+1500 val loss 6.7039
+1500 val perplexity 815.5410
+1500 train 6.527036 (lr=5.8066e-05) (hash(x)=135925327)
+1600 val loss 6.6584
+1600 val perplexity 779.2678
+1600 train 6.634343 (lr=5.7666e-05) (hash(x)=160440642)
+1700 val loss 6.5860
+1700 val perplexity 724.8856
+1700 train 6.696508 (lr=5.7230e-05) (hash(x)=151184106)
+1800 val loss 6.5488
+1800 val perplexity 698.4402
+1800 train 6.435990 (lr=5.6759e-05) (hash(x)=148100580)
+1900 val loss 6.5196
+1900 val perplexity 678.3179
+1900 train 6.348972 (lr=5.6253e-05) (hash(x)=149434659)
+2000 val loss 6.4690
+2000 val perplexity 644.8517
+2000 train 6.367797 (lr=5.5714e-05) (hash(x)=152285486)
+2100 val loss 6.4287
+2100 val perplexity 619.3555
+2100 train 6.175550 (lr=5.5142e-05) (hash(x)=144294295)
+2200 val loss 6.3964
+2200 val perplexity 599.6668
+2200 train 6.417540 (lr=5.4537e-05) (hash(x)=175030215)
+2300 val loss 6.3745
+2300 val perplexity 586.6714
+2300 train 6.187569 (lr=5.3902e-05) (hash(x)=150831428)
+2400 val loss 6.3353
+2400 val perplexity 564.1491
+2400 train 6.498894 (lr=5.3236e-05) (hash(x)=140808297)
+2500 val loss 6.3016
+2500 val perplexity 545.4457
+2500 train 6.277270 (lr=5.2541e-05) (hash(x)=153160275)
+2600 val loss 6.2740
+2600 val perplexity 530.6191
+2600 train 6.054704 (lr=5.1817e-05) (hash(x)=133990623)
+2700 val loss 6.2522
+2700 val perplexity 519.1785
+2700 train 6.228147 (lr=5.1067e-05) (hash(x)=142860944)
+2800 val loss 6.2198
+2800 val perplexity 502.5888
+2800 train 6.218305 (lr=5.0290e-05) (hash(x)=137959511)
+2900 val loss 6.1949
+2900 val perplexity 490.2450
+2900 train 6.142346 (lr=4.9487e-05) (hash(x)=147009873)
+3000 val loss 6.1730
+3000 val perplexity 479.6227
+3000 train 6.028874 (lr=4.8662e-05) (hash(x)=158264841)
+3100 val loss 6.1614
+3100 val perplexity 474.0899
+3100 train 6.007837 (lr=4.7813e-05) (hash(x)=139232251)
+3200 val loss 6.1283
+3200 val perplexity 458.6627
+3200 train 6.170417 (lr=4.6943e-05) (hash(x)=153436104)
+3300 val loss 6.1100
+3300 val perplexity 450.3429
+3300 train 5.938949 (lr=4.6052e-05) (hash(x)=149681831)
+3400 val loss 6.0974
+3400 val perplexity 444.7103
+3400 train 5.969666 (lr=4.5143e-05) (hash(x)=168425516)
+3500 val loss 6.0658
+3500 val perplexity 430.8599
+3500 train 6.096596 (lr=4.4216e-05) (hash(x)=163104338)
+3600 val loss 6.0555
+3600 val perplexity 426.4641
+3600 train 6.031762 (lr=4.3273e-05) (hash(x)=165109772)
+3700 val loss 6.0395
+3700 val perplexity 419.6728
+3700 train 5.774659 (lr=4.2315e-05) (hash(x)=153420306)
+3800 val loss 6.0160
+3800 val perplexity 409.9394
+3800 train 6.033447 (lr=4.1343e-05) (hash(x)=160168863)
+3900 val loss 5.9987
+3900 val perplexity 402.8871
+3900 train 5.910958 (lr=4.0360e-05) (hash(x)=153906073)
+4000 val loss 5.9883
+4000 val perplexity 398.7350
+4000 train 5.928736 (lr=3.9365e-05) (hash(x)=151055067)
+4100 val loss 5.9867
+4100 val perplexity 398.0942
+4100 train 5.962761 (lr=3.8362e-05) (hash(x)=149629830)
+4200 val loss 5.9605
+4200 val perplexity 387.8028
+4200 train 5.844605 (lr=3.7351e-05) (hash(x)=143101381)
+4300 val loss 5.9515
+4300 val perplexity 384.3391
+4300 train 5.859135 (lr=3.6333e-05) (hash(x)=149712044)
+4400 val loss 5.9377
+4400 val perplexity 379.0528
+4400 train 5.858887 (lr=3.5311e-05) (hash(x)=153446449)
+4500 val loss 5.9258
+4500 val perplexity 374.5926
+4500 train 5.872770 (lr=3.4285e-05) (hash(x)=146086947)
+4600 val loss 5.9163
+4600 val perplexity 371.0494
+4600 train 5.766239 (lr=3.3257e-05) (hash(x)=153800173)
+4700 val loss 5.9012
+4700 val perplexity 365.4588
+4700 train 5.965096 (lr=3.2229e-05) (hash(x)=155962726)
+4800 val loss 5.8879
+4800 val perplexity 360.6517
+4800 train 6.112103 (lr=3.1202e-05) (hash(x)=142045616)
+4900 val loss 5.8776
+4900 val perplexity 356.9393
+4900 train 5.844268 (lr=3.0178e-05) (hash(x)=143418248)
+5000 val loss 5.8673
+5000 val perplexity 353.2874
+5000 train 5.789495 (lr=2.9157e-05) (hash(x)=145789790)
+5100 val loss 5.8593
+5100 val perplexity 350.4766
+5100 train 5.833439 (lr=2.8143e-05) (hash(x)=137795633)
+5200 val loss 5.8460
+5200 val perplexity 345.8391
+5200 train 5.993758 (lr=2.7135e-05) (hash(x)=148907132)
+5300 val loss 5.8364
+5300 val perplexity 342.5482
+5300 train 5.707291 (lr=2.6136e-05) (hash(x)=152343580)
+5400 val loss 5.8270
+5400 val perplexity 339.3429
+5400 train 5.687649 (lr=2.5147e-05) (hash(x)=148578264)
+5500 val loss 5.8131
+5500 val perplexity 334.6594
+5500 train 5.971906 (lr=2.4169e-05) (hash(x)=145635833)
+5600 val loss 5.8059
+5600 val perplexity 332.2624
+5600 train 5.785269 (lr=2.3204e-05) (hash(x)=156337844)
+5700 val loss 5.7986
+5700 val perplexity 329.8301
+5700 train 5.663870 (lr=2.2253e-05) (hash(x)=147168506)
+5800 val loss 5.7869
+5800 val perplexity 326.0069
+5800 train 5.836335 (lr=2.1318e-05) (hash(x)=159566920)
+5900 val loss 5.7791
+5900 val perplexity 323.4568
+5900 train 5.835177 (lr=2.0400e-05) (hash(x)=158273929)
+6000 val loss 5.7745
+6000 val perplexity 321.9735
+6000 train 5.745675 (lr=1.9500e-05) (hash(x)=156649749)
+6100 val loss 5.7637
+6100 val perplexity 318.5394
+6100 train 5.675130 (lr=1.8620e-05) (hash(x)=146812388)
+6200 val loss 5.7586
+6200 val perplexity 316.9120
+6200 train 5.593012 (lr=1.7760e-05) (hash(x)=143522146)
+6300 val loss 5.7516
+6300 val perplexity 314.6978
+6300 train 5.575982 (lr=1.6923e-05) (hash(x)=150124474)
+6400 val loss 5.7452
+6400 val perplexity 312.6990
+6400 train 5.772494 (lr=1.6108e-05) (hash(x)=141242117)
+6500 val loss 5.7347
+6500 val perplexity 309.4119
+6500 train 5.617639 (lr=1.5319e-05) (hash(x)=143529762)
+6600 val loss 5.7308
+6600 val perplexity 308.2250
+6600 train 5.522328 (lr=1.4555e-05) (hash(x)=136948374)
+6700 val loss 5.7272
+6700 val perplexity 307.1098
+6700 train 5.534031 (lr=1.3817e-05) (hash(x)=146268592)
+6800 val loss 5.7185
+6800 val perplexity 304.4343
+6800 train 5.660174 (lr=1.3108e-05) (hash(x)=152676836)
+6900 val loss 5.7107
+6900 val perplexity 302.0908
+6900 train 5.631036 (lr=1.2427e-05) (hash(x)=134657776)
+7000 val loss 5.7071
+7000 val perplexity 301.0015
+7000 train 5.656062 (lr=1.1777e-05) (hash(x)=166721861)
+7100 val loss 5.7038
+7100 val perplexity 299.9973
+7100 train 5.509681 (lr=1.1157e-05) (hash(x)=135496702)
+7200 val loss 5.7008
+7200 val perplexity 299.1043
+7200 train 5.805583 (lr=1.0568e-05) (hash(x)=155567461)
+7300 val loss 5.6914
+7300 val perplexity 296.3212
+7300 train 5.498210 (lr=1.0012e-05) (hash(x)=142803829)
+7400 val loss 5.6898
+7400 val perplexity 295.8382
+7400 train 5.508755 (lr=9.4899e-06) (hash(x)=145294178)
+7500 val loss 5.6879
+7500 val perplexity 295.2716
+7500 train 5.449854 (lr=9.0014e-06) (hash(x)=150573713)
+7600 val loss 5.6845
+7600 val perplexity 294.2585
+7600 train 5.693716 (lr=8.5478e-06) (hash(x)=142771511)
+7700 val loss 5.6776
+7700 val perplexity 292.2551
+7700 train 5.581897 (lr=8.1297e-06) (hash(x)=143602175)
+7800 val loss 5.6738
+7800 val perplexity 291.1272
+7800 train 5.722617 (lr=7.7476e-06) (hash(x)=152379862)
+7900 val loss 5.6718
+7900 val perplexity 290.5568
+7900 train 5.502820 (lr=7.4021e-06) (hash(x)=146655921)
+8000 val loss 5.6729
+8000 val perplexity 290.8633
+8000 train 5.796638 (lr=7.0937e-06) (hash(x)=148262482)
+8100 val loss 5.6640
+8100 val perplexity 288.2979
+8100 train 5.606502 (lr=6.8229e-06) (hash(x)=147683655)
+8200 val loss 5.6642
+8200 val perplexity 288.3626
+8200 train 5.772038 (lr=6.5900e-06) (hash(x)=157312987)
+8300 val loss 5.6608
+8300 val perplexity 287.3732
+8300 train 5.617775 (lr=6.3954e-06) (hash(x)=141107543)
+8400 val loss 5.6560
+8400 val perplexity 285.9885
+8400 train 5.649451 (lr=6.2395e-06) (hash(x)=141323024)
+8500 val loss 5.6542
+8500 val perplexity 285.4882
+8500 train 5.619328 (lr=6.1223e-06) (hash(x)=150696521)
+8600 val loss 5.6523
+8600 val perplexity 284.9490
+8600 train 5.589996 (lr=6.0440e-06) (hash(x)=162288191)
+8700 val loss 5.6496
+8700 val perplexity 284.1770
+8700 train 5.519243 (lr=6.0049e-06) (hash(x)=152860941)
+8749 val loss 5.6470
+8749 val perplexity 283.4382

lr6e-5_total_batch_size61440_two_masks_4_heads_seed1341/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:000d5748f12c02b3709f248205aa93eef39b233e03c87a744f1d8e7f39840884
+size 97580418

lr6e-5_total_batch_size61440_two_masks_4_heads_seed1341/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f34de396e35f1f0a215b3118b2d82296e876bfcf113533ae071381df9edd5178
+size 188880262