Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1341/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1341", "n_embd": 128}

attention_kindselective_n_heads2_seed1341/dataloader_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
+size 964

attention_kindselective_n_heads2_seed1341/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+size 964

attention_kindselective_n_heads2_seed1341/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+size 964

attention_kindselective_n_heads2_seed1341/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+size 964

attention_kindselective_n_heads2_seed1341/log2.txt

@@ -0,0 +1,303 @@
+max_steps: 10000
+0 val loss 11.7722
+0 val perplexity 129595.2031
+0 train 11.774795 (lr=7.5000e-07) (hash(x)=145079536)
+100 val loss 9.3037
+100 val perplexity 10979.0723
+100 train 9.244246 (lr=7.5750e-05) (hash(x)=154745873)
+200 val loss 7.7460
+200 val perplexity 2312.2744
+200 train 7.946997 (lr=1.5000e-04) (hash(x)=155800595)
+300 val loss 7.5960
+300 val perplexity 1990.2469
+300 train 7.554796 (lr=1.4997e-04) (hash(x)=148595389)
+400 val loss 7.5968
+400 val perplexity 1991.7423
+400 train 7.505088 (lr=1.4986e-04) (hash(x)=145606733)
+500 val loss 7.5520
+500 val perplexity 1904.6262
+500 train 7.350472 (lr=1.4969e-04) (hash(x)=138221231)
+600 val loss 7.5281
+600 val perplexity 1859.5831
+600 train 7.521947 (lr=1.4945e-04) (hash(x)=150367139)
+700 val loss 7.5113
+700 val perplexity 1828.5997
+700 train 7.653578 (lr=1.4913e-04) (hash(x)=155579314)
+800 val loss 7.5076
+800 val perplexity 1821.8347
+800 train 7.586863 (lr=1.4876e-04) (hash(x)=155747374)
+900 val loss 7.4474
+900 val perplexity 1715.4008
+900 train 7.648261 (lr=1.4831e-04) (hash(x)=159334575)
+1000 val loss 7.4414
+1000 val perplexity 1705.1790
+1000 train 7.342261 (lr=1.4779e-04) (hash(x)=140604760)
+1100 val loss 7.3735
+1100 val perplexity 1593.1841
+1100 train 7.223920 (lr=1.4721e-04) (hash(x)=146208052)
+1200 val loss 7.3531
+1200 val perplexity 1560.9684
+1200 train 7.281145 (lr=1.4656e-04) (hash(x)=148404734)
+1300 val loss 7.2945
+1300 val perplexity 1472.1868
+1300 train 7.225808 (lr=1.4585e-04) (hash(x)=155681970)
+1400 val loss 7.2784
+1400 val perplexity 1448.7200
+1400 train 7.236976 (lr=1.4507e-04) (hash(x)=148115934)
+1500 val loss 7.2136
+1500 val perplexity 1357.7581
+1500 train 7.340710 (lr=1.4422e-04) (hash(x)=157074034)
+1600 val loss 7.1754
+1600 val perplexity 1306.8774
+1600 train 6.969009 (lr=1.4332e-04) (hash(x)=137464699)
+1700 val loss 7.1214
+1700 val perplexity 1238.1949
+1700 train 7.288368 (lr=1.4235e-04) (hash(x)=166955614)
+1800 val loss 7.0839
+1800 val perplexity 1192.5714
+1800 train 6.921106 (lr=1.4131e-04) (hash(x)=143886042)
+1900 val loss 7.0449
+1900 val perplexity 1146.9973
+1900 train 7.622898 (lr=1.4022e-04) (hash(x)=193610391)
+2000 val loss 6.9766
+2000 val perplexity 1071.3042
+2000 train 7.168832 (lr=1.3907e-04) (hash(x)=163799796)
+2100 val loss 6.9072
+2100 val perplexity 999.4552
+2100 train 6.886005 (lr=1.3786e-04) (hash(x)=154107339)
+2200 val loss 6.8528
+2200 val perplexity 946.5522
+2200 train 6.839452 (lr=1.3660e-04) (hash(x)=144592844)
+2300 val loss 6.8093
+2300 val perplexity 906.2578
+2300 train 6.876617 (lr=1.3527e-04) (hash(x)=154535861)
+2400 val loss 6.7855
+2400 val perplexity 884.9128
+2400 train 7.465554 (lr=1.3390e-04) (hash(x)=204706354)
+2500 val loss 6.7782
+2500 val perplexity 878.4916
+2500 train 6.612565 (lr=1.3247e-04) (hash(x)=143783202)
+2600 val loss 6.7483
+2600 val perplexity 852.6260
+2600 train 6.657750 (lr=1.3099e-04) (hash(x)=150862210)
+2700 val loss 6.7156
+2700 val perplexity 825.1681
+2700 train 6.771245 (lr=1.2946e-04) (hash(x)=150753426)
+2800 val loss 6.6992
+2800 val perplexity 811.7590
+2800 train 6.700248 (lr=1.2788e-04) (hash(x)=147766811)
+2900 val loss 6.6682
+2900 val perplexity 786.9391
+2900 train 6.603729 (lr=1.2626e-04) (hash(x)=151177814)
+3000 val loss 6.6481
+3000 val perplexity 771.3284
+3000 train 6.530660 (lr=1.2459e-04) (hash(x)=135925327)
+3100 val loss 6.6459
+3100 val perplexity 769.5980
+3100 train 6.306259 (lr=1.2287e-04) (hash(x)=124842353)
+3200 val loss 6.6309
+3200 val perplexity 758.1389
+3200 train 6.726900 (lr=1.2112e-04) (hash(x)=160440642)
+3300 val loss 6.6013
+3300 val perplexity 736.0597
+3300 train 6.514308 (lr=1.1932e-04) (hash(x)=143017131)
+3400 val loss 6.5759
+3400 val perplexity 717.5570
+3400 train 6.911897 (lr=1.1749e-04) (hash(x)=148143474)
+3500 val loss 6.5747
+3500 val perplexity 716.7311
+3500 train 6.394977 (lr=1.1562e-04) (hash(x)=148732639)
+3600 val loss 6.5536
+3600 val perplexity 701.7585
+3600 train 6.547552 (lr=1.1372e-04) (hash(x)=146333803)
+3700 val loss 6.5360
+3700 val perplexity 689.5156
+3700 train 6.359855 (lr=1.1179e-04) (hash(x)=143611445)
+3800 val loss 6.5232
+3800 val perplexity 680.7549
+3800 train 6.323216 (lr=1.0982e-04) (hash(x)=141272115)
+3900 val loss 6.5171
+3900 val perplexity 676.5939
+3900 train 6.454644 (lr=1.0783e-04) (hash(x)=149161848)
+4000 val loss 6.4992
+4000 val perplexity 664.5805
+4000 train 6.553896 (lr=1.0581e-04) (hash(x)=152036955)
+4100 val loss 6.4889
+4100 val perplexity 657.7863
+4100 train 6.545446 (lr=1.0377e-04) (hash(x)=153260968)
+4200 val loss 6.4873
+4200 val perplexity 656.7577
+4200 train 6.374079 (lr=1.0171e-04) (hash(x)=147629478)
+4300 val loss 6.4743
+4300 val perplexity 648.2596
+4300 train 6.486774 (lr=9.9622e-05) (hash(x)=161701080)
+4400 val loss 6.4651
+4400 val perplexity 642.3095
+4400 train 6.521642 (lr=9.7520e-05) (hash(x)=155290524)
+4500 val loss 6.4584
+4500 val perplexity 638.0176
+4500 train 6.259203 (lr=9.5403e-05) (hash(x)=144726593)
+4600 val loss 6.4602
+4600 val perplexity 639.1897
+4600 train 6.407053 (lr=9.3273e-05) (hash(x)=149517202)
+4700 val loss 6.4416
+4700 val perplexity 627.4361
+4700 train 6.434956 (lr=9.1132e-05) (hash(x)=139403008)
+4800 val loss 6.4160
+4800 val perplexity 611.5692
+4800 train 6.517553 (lr=8.8982e-05) (hash(x)=153387238)
+4900 val loss 6.4117
+4900 val perplexity 608.9391
+4900 train 6.209703 (lr=8.6825e-05) (hash(x)=139881569)
+5000 val loss 6.3954
+5000 val perplexity 599.0758
+5000 train 6.397029 (lr=8.4663e-05) (hash(x)=152602580)
+5100 val loss 6.4324
+5100 val perplexity 621.6495
+5100 train 6.492046 (lr=8.2500e-05) (hash(x)=150598460)
+5200 val loss 6.4197
+5200 val perplexity 613.7912
+5200 train 6.316842 (lr=8.0337e-05) (hash(x)=147660496)
+5300 val loss 6.3985
+5300 val perplexity 600.9280
+5300 train 6.150064 (lr=7.8175e-05) (hash(x)=147693848)
+5400 val loss 6.4180
+5400 val perplexity 612.7855
+5400 train 6.179458 (lr=7.6018e-05) (hash(x)=136048145)
+5500 val loss 6.3870
+5500 val perplexity 594.0621
+5500 train 6.369116 (lr=7.3868e-05) (hash(x)=140313123)
+5600 val loss 6.3757
+5600 val perplexity 587.4236
+5600 train 6.401426 (lr=7.1727e-05) (hash(x)=156997993)
+5700 val loss 6.3713
+5700 val perplexity 584.8242
+5700 train 6.384679 (lr=6.9597e-05) (hash(x)=143513880)
+5800 val loss 6.3648
+5800 val perplexity 580.9998
+5800 train 6.479723 (lr=6.7480e-05) (hash(x)=153736628)
+5900 val loss 6.3639
+5900 val perplexity 580.4908
+5900 train 6.517274 (lr=6.5378e-05) (hash(x)=166385417)
+6000 val loss 6.3563
+6000 val perplexity 576.1237
+6000 train 6.592653 (lr=6.3294e-05) (hash(x)=153216517)
+6100 val loss 6.3545
+6100 val perplexity 575.0741
+6100 train 6.244929 (lr=6.1230e-05) (hash(x)=145387545)
+6200 val loss 6.3445
+6200 val perplexity 569.3646
+6200 train 6.456944 (lr=5.9188e-05) (hash(x)=156306460)
+6300 val loss 6.3371
+6300 val perplexity 565.1736
+6300 train 6.238484 (lr=5.7169e-05) (hash(x)=148610447)
+6400 val loss 6.3339
+6400 val perplexity 563.3510
+6400 train 6.519849 (lr=5.5177e-05) (hash(x)=158343535)
+6500 val loss 6.3335
+6500 val perplexity 563.1171
+6500 train 6.496376 (lr=5.3213e-05) (hash(x)=157742557)
+6600 val loss 6.3244
+6600 val perplexity 558.0327
+6600 train 6.256868 (lr=5.1279e-05) (hash(x)=150457308)
+6700 val loss 6.3217
+6700 val perplexity 556.5438
+6700 train 6.202847 (lr=4.9377e-05) (hash(x)=142426205)
+6800 val loss 6.3153
+6800 val perplexity 552.9587
+6800 train 6.127337 (lr=4.7509e-05) (hash(x)=151522525)
+6900 val loss 6.3105
+6900 val perplexity 550.3406
+6900 train 6.256704 (lr=4.5676e-05) (hash(x)=152029630)
+7000 val loss 6.2970
+7000 val perplexity 542.9250
+7000 train 6.171764 (lr=4.3882e-05) (hash(x)=142978954)
+7100 val loss 6.2922
+7100 val perplexity 540.3380
+7100 train 6.182000 (lr=4.2128e-05) (hash(x)=148064359)
+7200 val loss 6.2844
+7200 val perplexity 536.1356
+7200 train 6.192896 (lr=4.0414e-05) (hash(x)=155771845)
+7300 val loss 6.2826
+7300 val perplexity 535.1908
+7300 train 6.097363 (lr=3.8745e-05) (hash(x)=141639844)
+7400 val loss 6.2840
+7400 val perplexity 535.9132
+7400 train 6.340342 (lr=3.7120e-05) (hash(x)=153466021)
+7500 val loss 6.2818
+7500 val perplexity 534.7566
+7500 train 6.249422 (lr=3.5541e-05) (hash(x)=148843587)
+7600 val loss 6.2678
+7600 val perplexity 527.3170
+7600 train 6.355673 (lr=3.4011e-05) (hash(x)=152053933)
+7700 val loss 6.2628
+7700 val perplexity 524.6606
+7700 train 6.090435 (lr=3.2531e-05) (hash(x)=141564003)
+7800 val loss 6.2601
+7800 val perplexity 523.2859
+7800 train 6.137829 (lr=3.1102e-05) (hash(x)=148617843)
+7900 val loss 6.2531
+7900 val perplexity 519.6293
+7900 train 6.211982 (lr=2.9726e-05) (hash(x)=146417668)
+8000 val loss 6.2608
+8000 val perplexity 523.6473
+8000 train 6.075680 (lr=2.8405e-05) (hash(x)=150165488)
+8100 val loss 6.2577
+8100 val perplexity 522.0248
+8100 train 6.238003 (lr=2.7138e-05) (hash(x)=144818855)
+8200 val loss 6.2450
+8200 val perplexity 515.4420
+8200 train 6.336446 (lr=2.5929e-05) (hash(x)=143074954)
+8300 val loss 6.2406
+8300 val perplexity 513.1754
+8300 train 6.135967 (lr=2.4778e-05) (hash(x)=145172494)
+8400 val loss 6.2397
+8400 val perplexity 512.6929
+8400 train 6.257989 (lr=2.3686e-05) (hash(x)=152284393)
+8500 val loss 6.2402
+8500 val perplexity 512.9636
+8500 train 6.090525 (lr=2.2655e-05) (hash(x)=147044705)
+8600 val loss 6.2379
+8600 val perplexity 511.7964
+8600 train 6.147628 (lr=2.1685e-05) (hash(x)=147762608)
+8700 val loss 6.2356
+8700 val perplexity 510.5859
+8700 train 6.242692 (lr=2.0777e-05) (hash(x)=141994567)
+8800 val loss 6.2281
+8800 val perplexity 506.7967
+8800 train 6.295036 (lr=1.9933e-05) (hash(x)=152552896)
+8900 val loss 6.2227
+8900 val perplexity 504.0461
+8900 train 6.078781 (lr=1.9153e-05) (hash(x)=139714454)
+9000 val loss 6.2215
+9000 val perplexity 503.4492
+9000 train 6.259016 (lr=1.8439e-05) (hash(x)=152274419)
+9100 val loss 6.2211
+9100 val perplexity 503.2584
+9100 train 6.327313 (lr=1.7790e-05) (hash(x)=160536885)
+9200 val loss 6.2220
+9200 val perplexity 503.6907
+9200 train 6.066282 (lr=1.7208e-05) (hash(x)=138386899)
+9300 val loss 6.2184
+9300 val perplexity 501.9156
+9300 train 6.252558 (lr=1.6692e-05) (hash(x)=149957457)
+9400 val loss 6.2129
+9400 val perplexity 499.1484
+9400 train 6.262452 (lr=1.6245e-05) (hash(x)=161671548)
+9500 val loss 6.2120
+9500 val perplexity 498.6978
+9500 train 6.317385 (lr=1.5865e-05) (hash(x)=171857400)
+9600 val loss 6.2096
+9600 val perplexity 497.4986
+9600 train 5.995666 (lr=1.5554e-05) (hash(x)=135378471)
+9700 val loss 6.2061
+9700 val perplexity 495.7690
+9700 train 6.124307 (lr=1.5312e-05) (hash(x)=149514591)
+9800 val loss 6.2041
+9800 val perplexity 494.7568
+9800 train 6.190481 (lr=1.5139e-05) (hash(x)=152679072)
+9900 val loss 6.2029
+9900 val perplexity 494.1768
+9900 train 6.138339 (lr=1.5035e-05) (hash(x)=150415193)
+9999 val loss 6.2036
+9999 val perplexity 494.5280

attention_kindselective_n_heads2_seed1341/model_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea5a9ffee08c2df305cd63f255c66725128c7bb6e3bf9c755dd82a9739aedaf0
+size 38587970

attention_kindselective_n_heads2_seed1341/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79dea9083038faa3d1bcc51cc54cd099869a120fb53489c85b8689ea587902fb
+size 38587970

attention_kindselective_n_heads2_seed1341/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30822baa6d6425e32309da513b9d559dcf796e031b8d56a03bb4e4064848c217
+size 38587970

attention_kindselective_n_heads2_seed1341/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08a2dc8c7d7d2e896fb7212afcaf7421c83e9b52e452c7dd25ea0b5c7c65d081
+size 38587970

attention_kindselective_n_heads2_seed1341/optimizer_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe779aeeffb6e9af4d60ed5745e01d01e441578cacff5bb2e370062f5a5795f0
+size 70895430

attention_kindselective_n_heads2_seed1341/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0a3d427e1b44a10639415b4a0efddb95c41c9007926359e498279a780f501be
+size 70895430

attention_kindselective_n_heads2_seed1341/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:208721297887e24ac34e9421700768e117449cd455e38f2d1f66cadb629b12b3
+size 70895430

attention_kindselective_n_heads2_seed1341/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56ba808dfb6d1a1a89f577bed7918728e41144a989442de66b83e5b73d171f3c
+size 70895430