Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 100, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 80, "total_batch_size": 20480, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_20480_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_3/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_3", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1340", "n_embd": 256}

attention_kindselective_n_heads4_seed1340/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
+oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
 size 964

attention_kindselective_n_heads4_seed1340/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+size 964

attention_kindselective_n_heads4_seed1340/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+size 964

attention_kindselective_n_heads4_seed1340/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+size 964

attention_kindselective_n_heads4_seed1340/log2.txt

@@ -1,8 +1,303 @@
-max_steps: 100
-0 val loss 11.2697
-0 val perplexity 78410.2188
-0 train 11.276736 (lr=7.5000e-07) (hash(x)=108792797)
-50 val loss 9.9169
-50 val perplexity 20269.3145
-99 val loss 9.1682
-99 val perplexity 9587.0547
+max_steps: 10000
+0 val loss 11.2714
+0 val perplexity 78545.6094
+0 train 11.276529 (lr=7.5000e-07) (hash(x)=164406924)
+100 val loss 8.8150
+100 val perplexity 6734.2607
+100 train 9.008026 (lr=7.5750e-05) (hash(x)=177407419)
+200 val loss 7.5896
+200 val perplexity 1977.4829
+200 train 7.496974 (lr=1.5000e-04) (hash(x)=144903932)
+300 val loss 7.3940
+300 val perplexity 1626.2600
+300 train 7.754192 (lr=1.4997e-04) (hash(x)=173839165)
+400 val loss 7.2907
+400 val perplexity 1466.5542
+400 train 7.382928 (lr=1.4986e-04) (hash(x)=167734596)
+500 val loss 7.2478
+500 val perplexity 1404.9714
+500 train 7.273375 (lr=1.4969e-04) (hash(x)=153224076)
+600 val loss 7.1863
+600 val perplexity 1321.2104
+600 train 7.091864 (lr=1.4945e-04) (hash(x)=149619098)
+700 val loss 7.1670
+700 val perplexity 1295.9884
+700 train 7.079926 (lr=1.4913e-04) (hash(x)=146539909)
+800 val loss 7.0517
+800 val perplexity 1154.7979
+800 train 7.019431 (lr=1.4876e-04) (hash(x)=153710890)
+900 val loss 7.0787
+900 val perplexity 1186.3789
+900 train 7.007246 (lr=1.4831e-04) (hash(x)=155873620)
+1000 val loss 7.0019
+1000 val perplexity 1098.6797
+1000 train 6.921768 (lr=1.4779e-04) (hash(x)=145450636)
+1100 val loss 6.9365
+1100 val perplexity 1029.1884
+1100 train 7.008905 (lr=1.4721e-04) (hash(x)=154123388)
+1200 val loss 6.8752
+1200 val perplexity 967.9332
+1200 train 6.743853 (lr=1.4656e-04) (hash(x)=145249251)
+1300 val loss 6.8314
+1300 val perplexity 926.5063
+1300 train 6.742863 (lr=1.4585e-04) (hash(x)=148937127)
+1400 val loss 6.8084
+1400 val perplexity 905.3779
+1400 train 6.911214 (lr=1.4507e-04) (hash(x)=150475545)
+1500 val loss 6.7983
+1500 val perplexity 896.3302
+1500 train 6.726580 (lr=1.4422e-04) (hash(x)=154653428)
+1600 val loss 6.7646
+1600 val perplexity 866.6557
+1600 train 6.755625 (lr=1.4332e-04) (hash(x)=144483776)
+1700 val loss 6.7377
+1700 val perplexity 843.6576
+1700 train 6.890347 (lr=1.4235e-04) (hash(x)=157395496)
+1800 val loss 6.7153
+1800 val perplexity 824.9490
+1800 train 6.730232 (lr=1.4131e-04) (hash(x)=157916369)
+1900 val loss 6.6871
+1900 val perplexity 801.9601
+1900 train 6.886857 (lr=1.4022e-04) (hash(x)=166073923)
+2000 val loss 6.6997
+2000 val perplexity 812.1252
+2000 train 6.739951 (lr=1.3907e-04) (hash(x)=154856891)
+2100 val loss 6.6662
+2100 val perplexity 785.4100
+2100 train 6.646859 (lr=1.3786e-04) (hash(x)=151925203)
+2200 val loss 6.6335
+2200 val perplexity 760.1653
+2200 train 6.329777 (lr=1.3660e-04) (hash(x)=136191502)
+2300 val loss 6.6140
+2300 val perplexity 745.4225
+2300 train 6.765590 (lr=1.3527e-04) (hash(x)=153273362)
+2400 val loss 6.6332
+2400 val perplexity 759.9003
+2400 train 6.583218 (lr=1.3390e-04) (hash(x)=148021541)
+2500 val loss 6.5995
+2500 val perplexity 734.7002
+2500 train 6.545035 (lr=1.3247e-04) (hash(x)=141356608)
+2600 val loss 6.5794
+2600 val perplexity 720.0811
+2600 train 6.530703 (lr=1.3099e-04) (hash(x)=146005217)
+2700 val loss 6.5640
+2700 val perplexity 709.1022
+2700 train 6.412026 (lr=1.2946e-04) (hash(x)=144511718)
+2800 val loss 6.5768
+2800 val perplexity 718.2365
+2800 train 6.447824 (lr=1.2788e-04) (hash(x)=146019502)
+2900 val loss 6.5444
+2900 val perplexity 695.3627
+2900 train 6.449443 (lr=1.2626e-04) (hash(x)=146496200)
+3000 val loss 6.5497
+3000 val perplexity 699.0353
+3000 train 6.473566 (lr=1.2459e-04) (hash(x)=150127281)
+3100 val loss 6.5133
+3100 val perplexity 674.0693
+3100 train 6.502546 (lr=1.2287e-04) (hash(x)=142022255)
+3200 val loss 6.5119
+3200 val perplexity 673.0822
+3200 train 6.562176 (lr=1.2112e-04) (hash(x)=154120875)
+3300 val loss 6.5126
+3300 val perplexity 673.5568
+3300 train 6.590917 (lr=1.1932e-04) (hash(x)=153999717)
+3400 val loss 6.5166
+3400 val perplexity 676.2653
+3400 train 6.303998 (lr=1.1749e-04) (hash(x)=139694097)
+3500 val loss 6.4932
+3500 val perplexity 660.6465
+3500 train 6.682529 (lr=1.1562e-04) (hash(x)=162992732)
+3600 val loss 6.4755
+3600 val perplexity 649.0635
+3600 train 6.450060 (lr=1.1372e-04) (hash(x)=147574101)
+3700 val loss 6.4528
+3700 val perplexity 634.4659
+3700 train 6.548763 (lr=1.1179e-04) (hash(x)=157763099)
+3800 val loss 6.4608
+3800 val perplexity 639.5983
+3800 train 6.602811 (lr=1.0982e-04) (hash(x)=170800034)
+3900 val loss 6.4688
+3900 val perplexity 644.6869
+3900 train 6.549332 (lr=1.0783e-04) (hash(x)=164984528)
+4000 val loss 6.4359
+4000 val perplexity 623.8695
+4000 train 6.299426 (lr=1.0581e-04) (hash(x)=141743323)
+4100 val loss 6.4323
+4100 val perplexity 621.5958
+4100 train 6.459531 (lr=1.0377e-04) (hash(x)=153392872)
+4200 val loss 6.4372
+4200 val perplexity 624.6572
+4200 train 6.306248 (lr=1.0171e-04) (hash(x)=149074933)
+4300 val loss 6.4231
+4300 val perplexity 615.9368
+4300 train 6.761147 (lr=9.9622e-05) (hash(x)=167823423)
+4400 val loss 6.4058
+4400 val perplexity 605.3599
+4400 train 6.181880 (lr=9.7520e-05) (hash(x)=141203114)
+4500 val loss 6.3847
+4500 val perplexity 592.6842
+4500 train 6.381039 (lr=9.5403e-05) (hash(x)=146284780)
+4600 val loss 6.3872
+4600 val perplexity 594.1828
+4600 train 6.197599 (lr=9.3273e-05) (hash(x)=141126464)
+4700 val loss 6.3703
+4700 val perplexity 584.2452
+4700 train 6.402414 (lr=9.1132e-05) (hash(x)=154751926)
+4800 val loss 6.3828
+4800 val perplexity 591.6098
+4800 train 6.453866 (lr=8.8982e-05) (hash(x)=154793198)
+4900 val loss 6.3816
+4900 val perplexity 590.8748
+4900 train 6.118082 (lr=8.6825e-05) (hash(x)=139406392)
+5000 val loss 6.3803
+5000 val perplexity 590.1230
+5000 train 6.104129 (lr=8.4663e-05) (hash(x)=153548741)
+5100 val loss 6.3450
+5100 val perplexity 569.6191
+5100 train 6.405098 (lr=8.2500e-05) (hash(x)=160488568)
+5200 val loss 6.3360
+5200 val perplexity 564.5164
+5200 train 6.349102 (lr=8.0337e-05) (hash(x)=149645053)
+5300 val loss 6.3192
+5300 val perplexity 555.1131
+5300 train 6.399147 (lr=7.8175e-05) (hash(x)=155820556)
+5400 val loss 6.3165
+5400 val perplexity 553.6335
+5400 train 6.258524 (lr=7.6018e-05) (hash(x)=147538134)
+5500 val loss 6.3046
+5500 val perplexity 547.1053
+5500 train 6.505478 (lr=7.3868e-05) (hash(x)=166889307)
+5600 val loss 6.2934
+5600 val perplexity 541.0081
+5600 train 6.022371 (lr=7.1727e-05) (hash(x)=139516699)
+5700 val loss 6.2970
+5700 val perplexity 542.9310
+5700 train 5.996075 (lr=6.9597e-05) (hash(x)=140453511)
+5800 val loss 6.3013
+5800 val perplexity 545.3030
+5800 train 6.238603 (lr=6.7480e-05) (hash(x)=162964847)
+5900 val loss 6.2847
+5900 val perplexity 536.3085
+5900 train 6.292588 (lr=6.5378e-05) (hash(x)=150606634)
+6000 val loss 6.2641
+6000 val perplexity 525.3836
+6000 train 6.354014 (lr=6.3294e-05) (hash(x)=149890857)
+6100 val loss 6.2563
+6100 val perplexity 521.2612
+6100 train 6.352149 (lr=6.1230e-05) (hash(x)=173884145)
+6200 val loss 6.2540
+6200 val perplexity 520.1025
+6200 train 6.313532 (lr=5.9188e-05) (hash(x)=151987098)
+6300 val loss 6.2369
+6300 val perplexity 511.2752
+6300 train 6.189893 (lr=5.7169e-05) (hash(x)=148853562)
+6400 val loss 6.2421
+6400 val perplexity 513.9292
+6400 train 5.996774 (lr=5.5177e-05) (hash(x)=141530101)
+6500 val loss 6.2371
+6500 val perplexity 511.3586
+6500 train 6.104002 (lr=5.3213e-05) (hash(x)=142297809)
+6600 val loss 6.2206
+6600 val perplexity 502.9856
+6600 train 6.123561 (lr=5.1279e-05) (hash(x)=142447782)
+6700 val loss 6.2118
+6700 val perplexity 498.5954
+6700 train 6.167513 (lr=4.9377e-05) (hash(x)=147004686)
+6800 val loss 6.2079
+6800 val perplexity 496.6617
+6800 train 5.954415 (lr=4.7509e-05) (hash(x)=133438702)
+6900 val loss 6.2137
+6900 val perplexity 499.5584
+6900 train 6.254116 (lr=4.5676e-05) (hash(x)=157085143)
+7000 val loss 6.2020
+7000 val perplexity 493.7236
+7000 train 6.080952 (lr=4.3882e-05) (hash(x)=139437666)
+7100 val loss 6.1982
+7100 val perplexity 491.8668
+7100 train 6.245137 (lr=4.2128e-05) (hash(x)=159792986)
+7200 val loss 6.1857
+7200 val perplexity 485.7494
+7200 train 6.146164 (lr=4.0414e-05) (hash(x)=144930687)
+7300 val loss 6.1878
+7300 val perplexity 486.7650
+7300 train 6.239649 (lr=3.8745e-05) (hash(x)=156242690)
+7400 val loss 6.1793
+7400 val perplexity 482.6702
+7400 train 6.021890 (lr=3.7120e-05) (hash(x)=148183719)
+7500 val loss 6.1775
+7500 val perplexity 481.8043
+7500 train 6.192832 (lr=3.5541e-05) (hash(x)=152494758)
+7600 val loss 6.1712
+7600 val perplexity 478.7643
+7600 train 5.871748 (lr=3.4011e-05) (hash(x)=142485027)
+7700 val loss 6.1611
+7700 val perplexity 473.9326
+7700 train 6.010452 (lr=3.2531e-05) (hash(x)=147512165)
+7800 val loss 6.1533
+7800 val perplexity 470.2879
+7800 train 6.045797 (lr=3.1102e-05) (hash(x)=160346994)
+7900 val loss 6.1551
+7900 val perplexity 471.1004
+7900 train 6.012663 (lr=2.9726e-05) (hash(x)=144488254)
+8000 val loss 6.1499
+8000 val perplexity 468.6790
+8000 train 5.967876 (lr=2.8405e-05) (hash(x)=147637019)
+8100 val loss 6.1483
+8100 val perplexity 467.9341
+8100 train 6.004601 (lr=2.7138e-05) (hash(x)=147340534)
+8200 val loss 6.1459
+8200 val perplexity 466.7805
+8200 train 6.177722 (lr=2.5929e-05) (hash(x)=151630665)
+8300 val loss 6.1351
+8300 val perplexity 461.7711
+8300 train 6.219531 (lr=2.4778e-05) (hash(x)=149747064)
+8400 val loss 6.1336
+8400 val perplexity 461.0908
+8400 train 6.320711 (lr=2.3686e-05) (hash(x)=154245770)
+8500 val loss 6.1307
+8500 val perplexity 459.7752
+8500 train 6.014756 (lr=2.2655e-05) (hash(x)=152559100)
+8600 val loss 6.1278
+8600 val perplexity 458.4228
+8600 train 6.624016 (lr=2.1685e-05) (hash(x)=181365926)
+8700 val loss 6.1307
+8700 val perplexity 459.7448
+8700 train 5.961389 (lr=2.0777e-05) (hash(x)=154405991)
+8800 val loss 6.1234
+8800 val perplexity 456.4016
+8800 train 6.065418 (lr=1.9933e-05) (hash(x)=153755904)
+8900 val loss 6.1210
+8900 val perplexity 455.2997
+8900 train 6.049755 (lr=1.9153e-05) (hash(x)=152120568)
+9000 val loss 6.1119
+9000 val perplexity 451.1904
+9000 train 5.915850 (lr=1.8439e-05) (hash(x)=142797279)
+9100 val loss 6.1117
+9100 val perplexity 451.1129
+9100 train 5.974414 (lr=1.7790e-05) (hash(x)=143037503)
+9200 val loss 6.1116
+9200 val perplexity 451.0441
+9200 train 6.015065 (lr=1.7208e-05) (hash(x)=113690273)
+9300 val loss 6.1111
+9300 val perplexity 450.8334
+9300 train 6.035745 (lr=1.6692e-05) (hash(x)=158025077)
+9400 val loss 6.1085
+9400 val perplexity 449.6438
+9400 train 6.165466 (lr=1.6245e-05) (hash(x)=158251718)
+9500 val loss 6.1043
+9500 val perplexity 447.7663
+9500 train 6.131958 (lr=1.5865e-05) (hash(x)=154752610)
+9600 val loss 6.1010
+9600 val perplexity 446.3106
+9600 train 5.971730 (lr=1.5554e-05) (hash(x)=146889093)
+9700 val loss 6.0977
+9700 val perplexity 444.8401
+9700 train 6.104051 (lr=1.5312e-05) (hash(x)=156906516)
+9800 val loss 6.0975
+9800 val perplexity 444.7247
+9800 train 5.942422 (lr=1.5139e-05) (hash(x)=153841927)
+9900 val loss 6.0961
+9900 val perplexity 444.1263
+9900 train 6.294320 (lr=1.5035e-05) (hash(x)=163514334)
+9999 val loss 6.0919
+9999 val perplexity 442.2750

attention_kindselective_n_heads4_seed1340/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4bd8e6e8e4b100755ee0de1110fab412d7a6df313956bf2aa1b9eb064bbf32b3
+oid sha256:f77a0e86af837e96ddb73bcf1787b783ad3319e4b7779f945478abab4e76e4f9
 size 92843394

attention_kindselective_n_heads4_seed1340/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e7e75dbf141b37b031861e58726b1c6227e8d371b6136bb3f597ea05775f710
+size 92843394

attention_kindselective_n_heads4_seed1340/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8c999d26ff45c1412e6520045612b33258b6ff6547a6a4b6e054e2afd81f84b
+size 92843394

attention_kindselective_n_heads4_seed1340/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:505e431b92125e3353dbe2fe8f73f190410499e8fa0aae75dd9b4e6148abf833
+size 92843394

attention_kindselective_n_heads4_seed1340/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8cc612538e25bd455d70cedc9288e443fba764646caaac52ef4ab34493ae45bc
+oid sha256:1839e8234b3588e61b65c80b0816353d4dbf9956962b809e16783bd005d8e8c7
 size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8c3254c5328b3a1eae82aa7bbcfae1269f5902ef429b7426fe60a02c5cdb78e
+size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da5e88ac26e1e0c3c72ce92eb60bd8311d31396b4c5509b1d6a424a6bec8127c
+size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5853170d79d7583bbccfae317f503078a42902ade708ab27df68638a7bbfd6f5
+size 179406214