Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1339", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_3/attention_kindselective_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_3", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1339", "n_embd": 128}

attention_kindselective_n_heads2_seed1339/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
+oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
 size 964

attention_kindselective_n_heads2_seed1339/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+size 964

attention_kindselective_n_heads2_seed1339/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+size 964

attention_kindselective_n_heads2_seed1339/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+size 964

attention_kindselective_n_heads2_seed1339/log2.txt

@@ -1,33 +1,303 @@
-max_steps: 1000
+max_steps: 10000
 0 val loss 11.7467
 0 val perplexity 126331.5391
 0 train 11.740535 (lr=7.5000e-07) (hash(x)=150724848)
-100 val loss 9.5441
-100 val perplexity 13962.1650
-100 train 9.588784 (lr=7.5750e-05) (hash(x)=149217061)
-200 val loss 7.7467
-200 val perplexity 2313.9080
-200 train 7.728883 (lr=1.5000e-04) (hash(x)=149910534)
-300 val loss 7.6659
-300 val perplexity 2134.2258
-300 train 7.542049 (lr=1.4486e-04) (hash(x)=142185643)
-400 val loss 7.6514
-400 val perplexity 2103.6157
-400 train 7.689255 (lr=1.3023e-04) (hash(x)=148123706)
-500 val loss 7.6465
-500 val perplexity 2093.3313
-500 train 7.502706 (lr=1.0833e-04) (hash(x)=149952383)
-600 val loss 7.6388
-600 val perplexity 2077.1519
-600 train 7.447033 (lr=8.2500e-05) (hash(x)=146678221)
-700 val loss 7.5877
-700 val perplexity 1973.6790
-700 train 7.608317 (lr=5.6669e-05) (hash(x)=156180736)
-800 val loss 7.5683
-800 val perplexity 1935.7953
-800 train 7.559825 (lr=3.4770e-05) (hash(x)=151700982)
-900 val loss 7.5518
-900 val perplexity 1904.2003
-900 train 7.377645 (lr=2.0138e-05) (hash(x)=147288467)
-999 val loss 7.5496
-999 val perplexity 1899.9149
+100 val loss 9.4365
+100 val perplexity 12538.0400
+100 train 9.480801 (lr=7.5750e-05) (hash(x)=149217061)
+200 val loss 7.6992
+200 val perplexity 2206.4814
+200 train 7.687409 (lr=1.5000e-04) (hash(x)=149910534)
+300 val loss 7.6357
+300 val perplexity 2070.8967
+300 train 7.504811 (lr=1.4997e-04) (hash(x)=142185643)
+400 val loss 7.6326
+400 val perplexity 2064.4209
+400 train 7.661612 (lr=1.4986e-04) (hash(x)=148123706)
+500 val loss 7.6077
+500 val perplexity 2013.7218
+500 train 7.464101 (lr=1.4969e-04) (hash(x)=149952383)
+600 val loss 7.6349
+600 val perplexity 2069.1643
+600 train 7.453125 (lr=1.4945e-04) (hash(x)=146678221)
+700 val loss 7.5944
+700 val perplexity 1987.0646
+700 train 7.617968 (lr=1.4913e-04) (hash(x)=156180736)
+800 val loss 7.5767
+800 val perplexity 1952.2250
+800 train 7.557334 (lr=1.4876e-04) (hash(x)=151700982)
+900 val loss 7.5475
+900 val perplexity 1895.9889
+900 train 7.363899 (lr=1.4831e-04) (hash(x)=147288467)
+1000 val loss 7.5335
+1000 val perplexity 1869.6943
+1000 train 7.592796 (lr=1.4779e-04) (hash(x)=156182087)
+1100 val loss 7.5065
+1100 val perplexity 1819.7588
+1100 train 7.376440 (lr=1.4721e-04) (hash(x)=147861550)
+1200 val loss 7.4818
+1200 val perplexity 1775.3508
+1200 train 7.343384 (lr=1.4656e-04) (hash(x)=149318660)
+1300 val loss 7.4929
+1300 val perplexity 1795.2754
+1300 train 7.324528 (lr=1.4585e-04) (hash(x)=151631103)
+1400 val loss 7.4838
+1400 val perplexity 1779.0669
+1400 train 7.515356 (lr=1.4507e-04) (hash(x)=150482428)
+1500 val loss 7.4626
+1500 val perplexity 1741.7150
+1500 train 7.344044 (lr=1.4422e-04) (hash(x)=144458617)
+1600 val loss 7.3651
+1600 val perplexity 1579.8989
+1600 train 7.253219 (lr=1.4332e-04) (hash(x)=143268605)
+1700 val loss 7.3347
+1700 val perplexity 1532.5645
+1700 train 7.271320 (lr=1.4235e-04) (hash(x)=150817011)
+1800 val loss 7.3513
+1800 val perplexity 1558.1730
+1800 train 7.360166 (lr=1.4131e-04) (hash(x)=152322423)
+1900 val loss 7.2976
+1900 val perplexity 1476.7399
+1900 train 7.127239 (lr=1.4022e-04) (hash(x)=143574126)
+2000 val loss 7.3026
+2000 val perplexity 1484.1338
+2000 train 7.090799 (lr=1.3907e-04) (hash(x)=147904298)
+2100 val loss 7.2743
+2100 val perplexity 1442.7886
+2100 train 7.011063 (lr=1.3786e-04) (hash(x)=144719400)
+2200 val loss 7.2155
+2200 val perplexity 1360.3833
+2200 train 7.452095 (lr=1.3660e-04) (hash(x)=154343147)
+2300 val loss 7.1752
+2300 val perplexity 1306.6619
+2300 train 7.215823 (lr=1.3527e-04) (hash(x)=154372635)
+2400 val loss 7.1679
+2400 val perplexity 1297.0833
+2400 train 7.051396 (lr=1.3390e-04) (hash(x)=141843115)
+2500 val loss 7.1449
+2500 val perplexity 1267.6400
+2500 train 7.062555 (lr=1.3247e-04) (hash(x)=149417679)
+2600 val loss 7.1157
+2600 val perplexity 1231.1683
+2600 train 7.002046 (lr=1.3099e-04) (hash(x)=145279030)
+2700 val loss 7.1132
+2700 val perplexity 1228.1211
+2700 train 7.009352 (lr=1.2946e-04) (hash(x)=146363911)
+2800 val loss 7.1103
+2800 val perplexity 1224.4899
+2800 train 7.086044 (lr=1.2788e-04) (hash(x)=152507639)
+2900 val loss 7.1079
+2900 val perplexity 1221.6112
+2900 train 6.923614 (lr=1.2626e-04) (hash(x)=139296191)
+3000 val loss 7.1002
+3000 val perplexity 1212.2461
+3000 train 7.111192 (lr=1.2459e-04) (hash(x)=148473774)
+3100 val loss 7.0391
+3100 val perplexity 1140.3469
+3100 train 7.136829 (lr=1.2287e-04) (hash(x)=154130527)
+3200 val loss 6.9963
+3200 val perplexity 1092.5740
+3200 train 6.936981 (lr=1.2112e-04) (hash(x)=151117002)
+3300 val loss 7.0011
+3300 val perplexity 1097.8837
+3300 train 7.113845 (lr=1.1932e-04) (hash(x)=151035383)
+3400 val loss 6.9671
+3400 val perplexity 1061.1619
+3400 train 7.074393 (lr=1.1749e-04) (hash(x)=150614749)
+3500 val loss 6.9428
+3500 val perplexity 1035.6863
+3500 train 6.973127 (lr=1.1562e-04) (hash(x)=151431391)
+3600 val loss 6.9251
+3600 val perplexity 1017.4580
+3600 train 6.882280 (lr=1.1372e-04) (hash(x)=151140797)
+3700 val loss 6.9142
+3700 val perplexity 1006.4932
+3700 train 7.091825 (lr=1.1179e-04) (hash(x)=154608340)
+3800 val loss 6.8872
+3800 val perplexity 979.6236
+3800 train 6.868873 (lr=1.0982e-04) (hash(x)=169641309)
+3900 val loss 6.8553
+3900 val perplexity 948.8849
+3900 train 6.805549 (lr=1.0783e-04) (hash(x)=150592479)
+4000 val loss 6.8401
+4000 val perplexity 934.5740
+4000 train 6.819444 (lr=1.0581e-04) (hash(x)=155495317)
+4100 val loss 6.8482
+4100 val perplexity 942.1501
+4100 train 7.033608 (lr=1.0377e-04) (hash(x)=170323970)
+4200 val loss 6.8270
+4200 val perplexity 922.4595
+4200 train 6.649766 (lr=1.0171e-04) (hash(x)=140854360)
+4300 val loss 6.8013
+4300 val perplexity 899.0114
+4300 train 6.802259 (lr=9.9622e-05) (hash(x)=148926644)
+4400 val loss 6.7754
+4400 val perplexity 875.9948
+4400 train 6.811431 (lr=9.7520e-05) (hash(x)=149997871)
+4500 val loss 6.7574
+4500 val perplexity 860.4335
+4500 train 6.723810 (lr=9.5403e-05) (hash(x)=149219289)
+4600 val loss 6.7387
+4600 val perplexity 844.4553
+4600 train 6.559234 (lr=9.3273e-05) (hash(x)=139836197)
+4700 val loss 6.7162
+4700 val perplexity 825.6404
+4700 train 6.670391 (lr=9.1132e-05) (hash(x)=150722883)
+4800 val loss 6.7031
+4800 val perplexity 814.8918
+4800 train 6.871788 (lr=8.8982e-05) (hash(x)=166864648)
+4900 val loss 6.6805
+4900 val perplexity 796.7543
+4900 train 6.777838 (lr=8.6825e-05) (hash(x)=157746743)
+5000 val loss 6.6579
+5000 val perplexity 778.9282
+5000 train 6.616292 (lr=8.4663e-05) (hash(x)=153032257)
+5100 val loss 6.6439
+5100 val perplexity 768.1103
+5100 train 6.713952 (lr=8.2500e-05) (hash(x)=151862371)
+5200 val loss 6.6381
+5200 val perplexity 763.6665
+5200 train 6.914250 (lr=8.0337e-05) (hash(x)=168862995)
+5300 val loss 6.6273
+5300 val perplexity 755.4161
+5300 train 6.587457 (lr=7.8175e-05) (hash(x)=148243567)
+5400 val loss 6.6006
+5400 val perplexity 735.5362
+5400 train 6.533762 (lr=7.6018e-05) (hash(x)=145381742)
+5500 val loss 6.5832
+5500 val perplexity 722.8611
+5500 train 6.719530 (lr=7.3868e-05) (hash(x)=155602174)
+5600 val loss 6.5691
+5600 val perplexity 712.7457
+5600 train 6.656135 (lr=7.1727e-05) (hash(x)=160637672)
+5700 val loss 6.5469
+5700 val perplexity 697.0565
+5700 train 6.475524 (lr=6.9597e-05) (hash(x)=152101131)
+5800 val loss 6.5437
+5800 val perplexity 694.8675
+5800 train 6.523787 (lr=6.7480e-05) (hash(x)=152327089)
+5900 val loss 6.5318
+5900 val perplexity 686.6349
+5900 train 6.474981 (lr=6.5378e-05) (hash(x)=151094543)
+6000 val loss 6.5252
+6000 val perplexity 682.1018
+6000 train 6.609112 (lr=6.3294e-05) (hash(x)=160233315)
+6100 val loss 6.5045
+6100 val perplexity 668.1599
+6100 train 6.587837 (lr=6.1230e-05) (hash(x)=158295337)
+6200 val loss 6.4956
+6200 val perplexity 662.2494
+6200 train 6.458873 (lr=5.9188e-05) (hash(x)=151269817)
+6300 val loss 6.4957
+6300 val perplexity 662.2895
+6300 train 6.622801 (lr=5.7169e-05) (hash(x)=156912756)
+6400 val loss 6.4735
+6400 val perplexity 647.7662
+6400 train 6.468271 (lr=5.5177e-05) (hash(x)=125779637)
+6500 val loss 6.4767
+6500 val perplexity 649.8300
+6500 train 6.383432 (lr=5.3213e-05) (hash(x)=148157661)
+6600 val loss 6.4666
+6600 val perplexity 643.2645
+6600 train 6.497171 (lr=5.1279e-05) (hash(x)=152820049)
+6700 val loss 6.4568
+6700 val perplexity 637.0014
+6700 train 6.417848 (lr=4.9377e-05) (hash(x)=155818717)
+6800 val loss 6.4443
+6800 val perplexity 629.1221
+6800 train 6.465776 (lr=4.7509e-05) (hash(x)=158833881)
+6900 val loss 6.4343
+6900 val perplexity 622.8764
+6900 train 6.397180 (lr=4.5676e-05) (hash(x)=164984909)
+7000 val loss 6.4265
+7000 val perplexity 617.9967
+7000 train 6.319517 (lr=4.3882e-05) (hash(x)=151172493)
+7100 val loss 6.4253
+7100 val perplexity 617.2631
+7100 train 6.444489 (lr=4.2128e-05) (hash(x)=154023633)
+7200 val loss 6.4220
+7200 val perplexity 615.2341
+7200 train 6.309972 (lr=4.0414e-05) (hash(x)=139351250)
+7300 val loss 6.4162
+7300 val perplexity 611.6689
+7300 train 6.344171 (lr=3.8745e-05) (hash(x)=158719458)
+7400 val loss 6.3990
+7400 val perplexity 601.2144
+7400 train 6.475732 (lr=3.7120e-05) (hash(x)=162250810)
+7500 val loss 6.3922
+7500 val perplexity 597.1456
+7500 train 6.276504 (lr=3.5541e-05) (hash(x)=142827557)
+7600 val loss 6.3901
+7600 val perplexity 595.9185
+7600 train 6.312879 (lr=3.4011e-05) (hash(x)=147830801)
+7700 val loss 6.3845
+7700 val perplexity 592.5839
+7700 train 6.500862 (lr=3.2531e-05) (hash(x)=165233314)
+7800 val loss 6.3826
+7800 val perplexity 591.4781
+7800 train 6.380383 (lr=3.1102e-05) (hash(x)=160654717)
+7900 val loss 6.3789
+7900 val perplexity 589.2634
+7900 train 6.249607 (lr=2.9726e-05) (hash(x)=140712586)
+8000 val loss 6.3717
+8000 val perplexity 585.0411
+8000 train 6.513039 (lr=2.8405e-05) (hash(x)=150309061)
+8100 val loss 6.3597
+8100 val perplexity 578.0466
+8100 train 6.361465 (lr=2.7138e-05) (hash(x)=158805466)
+8200 val loss 6.3576
+8200 val perplexity 576.8763
+8200 train 6.229097 (lr=2.5929e-05) (hash(x)=142814078)
+8300 val loss 6.3545
+8300 val perplexity 575.0615
+8300 train 6.310745 (lr=2.4778e-05) (hash(x)=148340417)
+8400 val loss 6.3521
+8400 val perplexity 573.7128
+8400 train 6.309781 (lr=2.3686e-05) (hash(x)=157132639)
+8500 val loss 6.3498
+8500 val perplexity 572.3974
+8500 train 6.310510 (lr=2.2655e-05) (hash(x)=152994240)
+8600 val loss 6.3441
+8600 val perplexity 569.1282
+8600 train 6.290955 (lr=2.1685e-05) (hash(x)=152210154)
+8700 val loss 6.3343
+8700 val perplexity 563.5939
+8700 train 6.391054 (lr=2.0777e-05) (hash(x)=149631247)
+8800 val loss 6.3324
+8800 val perplexity 562.4875
+8800 train 6.204415 (lr=1.9933e-05) (hash(x)=145617784)
+8900 val loss 6.3294
+8900 val perplexity 560.8342
+8900 train 6.412525 (lr=1.9153e-05) (hash(x)=155984970)
+9000 val loss 6.3313
+9000 val perplexity 561.8929
+9000 train 6.085052 (lr=1.8439e-05) (hash(x)=143433013)
+9100 val loss 6.3293
+9100 val perplexity 560.7858
+9100 train 6.209786 (lr=1.7790e-05) (hash(x)=152507533)
+9200 val loss 6.3209
+9200 val perplexity 556.0552
+9200 train 6.429985 (lr=1.7208e-05) (hash(x)=162400650)
+9300 val loss 6.3179
+9300 val perplexity 554.4227
+9300 train 6.355889 (lr=1.6692e-05) (hash(x)=148419068)
+9400 val loss 6.3182
+9400 val perplexity 554.5536
+9400 train 6.285955 (lr=1.6245e-05) (hash(x)=145025169)
+9500 val loss 6.3124
+9500 val perplexity 551.3729
+9500 train 6.280643 (lr=1.5865e-05) (hash(x)=145104235)
+9600 val loss 6.3172
+9600 val perplexity 554.0310
+9600 train 6.251455 (lr=1.5554e-05) (hash(x)=150499797)
+9700 val loss 6.3145
+9700 val perplexity 552.5436
+9700 train 6.136825 (lr=1.5312e-05) (hash(x)=139690022)
+9800 val loss 6.3089
+9800 val perplexity 549.4202
+9800 train 6.298676 (lr=1.5139e-05) (hash(x)=145645279)
+9900 val loss 6.3025
+9900 val perplexity 545.9294
+9900 train 6.201912 (lr=1.5035e-05) (hash(x)=150103432)
+9999 val loss 6.3028
+9999 val perplexity 546.0888

attention_kindselective_n_heads2_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:055f28a767ebaf6c839d2130ee16aeef730806567c75ef9cf23ee35882a9f8ea
+oid sha256:fb1443cb2223a3202b6db20d87fc5c06325ae65ec82e291e8b6a510e727775c1
 size 38587970

attention_kindselective_n_heads2_seed1339/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96a1976960dfa6e5f5541b027d8b1326f833d19d32c42d74bed201967fd5a0b3
+size 38587970

attention_kindselective_n_heads2_seed1339/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c9e4fe573fb5956acb8a1b31c6ee7bbff7b8ed970c371503fa40301f3a0651a
+size 38587970

attention_kindselective_n_heads2_seed1339/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:172ad7c2469fd0b7aa14daef98d1cb52b19cc8e90bd5095af623e814bb489078
+size 38587970

attention_kindselective_n_heads2_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e7ce65a06f433f178e882350c1fd5ecbc5278caa1c92d70ea3c30e44c222263
+oid sha256:fff3493189e40fabdd0eb47a94f5ab767303f05484bcae36576e6fc71ba90972
 size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae20372cd86ef92af86587c9501cc81592410a5df6ce190e3560e17a9616f28e
+size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efa3f6adbdeaac2e96b04670e11f4081965c41854ca90eb5be8a00fb3cbbe147
+size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54de0acaec2b5b040fdf1208efb00cdf6fb5b9b0a6ce755aad2926c9707faa27
+size 70895430