Upload folder using huggingface_hub

attention_kindself_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_0/attention_kindself_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10, "warmup_steps": 200, "group": "wider_is_better_0", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_10240_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1339", "n_embd": 256}

attention_kindself_n_heads4_seed1339/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
+oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
 size 964

attention_kindself_n_heads4_seed1339/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+size 964

attention_kindself_n_heads4_seed1339/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+size 964

attention_kindself_n_heads4_seed1339/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+size 964

attention_kindself_n_heads4_seed1339/log2.txt

@@ -1,15 +1,303 @@
-max_steps: 10
-0 val loss 11.2731
-0 val perplexity 78673.3594
-0 train 11.304537 (lr=7.5000e-07) (hash(x)=55241167)
-1 train 11.289993 (lr=1.5000e-06) (hash(x)=51881965)
-2 train 11.279264 (lr=2.2500e-06) (hash(x)=43601716)
-3 train 11.258142 (lr=3.0000e-06) (hash(x)=58724315)
-4 train 11.291172 (lr=3.7500e-06) (hash(x)=51093844)
-5 train 11.250208 (lr=4.5000e-06) (hash(x)=49464291)
-6 train 11.222798 (lr=5.2500e-06) (hash(x)=48476334)
-7 train 11.243051 (lr=6.0000e-06) (hash(x)=60510992)
-8 train 11.218795 (lr=6.7500e-06) (hash(x)=60540681)
-9 val loss 11.1978
-9 val perplexity 72972.0781
-9 train 11.212037 (lr=7.5000e-06) (hash(x)=52990419)
+max_steps: 10000
+0 val loss 11.2796
+0 val perplexity 79186.1172
+0 train 11.291462 (lr=7.5000e-07) (hash(x)=150724848)
+100 val loss 9.3316
+100 val perplexity 11289.2031
+100 train 9.386396 (lr=7.5750e-05) (hash(x)=149217061)
+200 val loss 7.7580
+200 val perplexity 2340.2415
+200 train 7.766219 (lr=1.5000e-04) (hash(x)=149910534)
+300 val loss 7.5387
+300 val perplexity 1879.3298
+300 train 7.404105 (lr=1.4997e-04) (hash(x)=142185643)
+400 val loss 7.3849
+400 val perplexity 1611.4607
+400 train 7.409924 (lr=1.4986e-04) (hash(x)=148123706)
+500 val loss 7.2848
+500 val perplexity 1457.9835
+500 train 7.114595 (lr=1.4969e-04) (hash(x)=149952383)
+600 val loss 7.2275
+600 val perplexity 1376.8268
+600 train 7.018831 (lr=1.4945e-04) (hash(x)=146678221)
+700 val loss 7.1586
+700 val perplexity 1285.1116
+700 train 7.201500 (lr=1.4913e-04) (hash(x)=156180736)
+800 val loss 7.1339
+800 val perplexity 1253.7417
+800 train 7.115716 (lr=1.4876e-04) (hash(x)=151700982)
+900 val loss 7.0960
+900 val perplexity 1207.1274
+900 train 6.877567 (lr=1.4831e-04) (hash(x)=147288467)
+1000 val loss 7.0511
+1000 val perplexity 1154.1218
+1000 train 7.116826 (lr=1.4779e-04) (hash(x)=156182087)
+1100 val loss 7.0467
+1100 val perplexity 1149.0145
+1100 train 6.910080 (lr=1.4721e-04) (hash(x)=147861550)
+1200 val loss 7.0220
+1200 val perplexity 1121.0123
+1200 train 6.883329 (lr=1.4656e-04) (hash(x)=149318660)
+1300 val loss 7.0377
+1300 val perplexity 1138.7271
+1300 train 6.882957 (lr=1.4585e-04) (hash(x)=151631103)
+1400 val loss 7.0220
+1400 val perplexity 1121.0166
+1400 train 7.034495 (lr=1.4507e-04) (hash(x)=150482428)
+1500 val loss 6.9852
+1500 val perplexity 1080.5740
+1500 train 6.833119 (lr=1.4422e-04) (hash(x)=144458617)
+1600 val loss 6.9992
+1600 val perplexity 1095.8010
+1600 train 6.893284 (lr=1.4332e-04) (hash(x)=143268605)
+1700 val loss 6.9816
+1700 val perplexity 1076.6697
+1700 train 6.925849 (lr=1.4235e-04) (hash(x)=150817011)
+1800 val loss 6.9799
+1800 val perplexity 1074.7789
+1800 train 6.975879 (lr=1.4131e-04) (hash(x)=152322423)
+1900 val loss 6.9624
+1900 val perplexity 1056.1371
+1900 train 6.774517 (lr=1.4022e-04) (hash(x)=143574126)
+2000 val loss 6.9909
+2000 val perplexity 1086.6913
+2000 train 6.782327 (lr=1.3907e-04) (hash(x)=147904298)
+2100 val loss 6.9824
+2100 val perplexity 1077.4812
+2100 train 6.763468 (lr=1.3786e-04) (hash(x)=144719400)
+2200 val loss 6.9828
+2200 val perplexity 1077.9159
+2200 train 7.221006 (lr=1.3660e-04) (hash(x)=154343147)
+2300 val loss 6.9937
+2300 val perplexity 1089.7175
+2300 train 7.024708 (lr=1.3527e-04) (hash(x)=154372635)
+2400 val loss 6.9764
+2400 val perplexity 1071.0907
+2400 train 6.818256 (lr=1.3390e-04) (hash(x)=141843115)
+2500 val loss 6.9814
+2500 val perplexity 1076.4762
+2500 train 6.891120 (lr=1.3247e-04) (hash(x)=149417679)
+2600 val loss 6.9879
+2600 val perplexity 1083.4302
+2600 train 6.861284 (lr=1.3099e-04) (hash(x)=145279030)
+2700 val loss 7.0230
+2700 val perplexity 1122.1450
+2700 train 6.928087 (lr=1.2946e-04) (hash(x)=146363911)
+2800 val loss 7.0102
+2800 val perplexity 1107.8602
+2800 train 6.996984 (lr=1.2788e-04) (hash(x)=152507639)
+2900 val loss 7.0251
+2900 val perplexity 1124.5314
+2900 train 6.853853 (lr=1.2626e-04) (hash(x)=139296191)
+3000 val loss 7.0815
+3000 val perplexity 1189.8097
+3000 train 7.084819 (lr=1.2459e-04) (hash(x)=148473774)
+3100 val loss 7.0437
+3100 val perplexity 1145.5751
+3100 train 7.125661 (lr=1.2287e-04) (hash(x)=154130527)
+3200 val loss 7.0794
+3200 val perplexity 1187.2012
+3200 train 7.033405 (lr=1.2112e-04) (hash(x)=151117002)
+3300 val loss 7.0905
+3300 val perplexity 1200.5089
+3300 train 7.205884 (lr=1.1932e-04) (hash(x)=151035383)
+3400 val loss 7.0632
+3400 val perplexity 1168.2102
+3400 train 7.170041 (lr=1.1749e-04) (hash(x)=150614749)
+3500 val loss 7.0934
+3500 val perplexity 1204.0105
+3500 train 7.124344 (lr=1.1562e-04) (hash(x)=151431391)
+3600 val loss 7.1095
+3600 val perplexity 1223.4929
+3600 train 7.066493 (lr=1.1372e-04) (hash(x)=151140797)
+3700 val loss 7.1210
+3700 val perplexity 1237.6566
+3700 train 7.303615 (lr=1.1179e-04) (hash(x)=154608340)
+3800 val loss 7.1782
+3800 val perplexity 1310.5081
+3800 train 7.108573 (lr=1.0982e-04) (hash(x)=169641309)
+3900 val loss 7.1296
+3900 val perplexity 1248.3175
+3900 train 7.071731 (lr=1.0783e-04) (hash(x)=150592479)
+4000 val loss 7.1371
+4000 val perplexity 1257.7638
+4000 train 7.126672 (lr=1.0581e-04) (hash(x)=155495317)
+4100 val loss 7.1221
+4100 val perplexity 1239.0602
+4100 train 7.290395 (lr=1.0377e-04) (hash(x)=170323970)
+4200 val loss 7.1540
+4200 val perplexity 1279.2599
+4200 train 6.982704 (lr=1.0171e-04) (hash(x)=140854360)
+4300 val loss 7.1362
+4300 val perplexity 1256.6296
+4300 train 7.135914 (lr=9.9622e-05) (hash(x)=148926644)
+4400 val loss 7.1230
+4400 val perplexity 1240.2017
+4400 train 7.164248 (lr=9.7520e-05) (hash(x)=149997871)
+4500 val loss 7.1416
+4500 val perplexity 1263.4196
+4500 train 7.111341 (lr=9.5403e-05) (hash(x)=149219289)
+4600 val loss 7.1663
+4600 val perplexity 1295.0135
+4600 train 6.972812 (lr=9.3273e-05) (hash(x)=139836197)
+4700 val loss 7.1523
+4700 val perplexity 1277.0450
+4700 train 7.122142 (lr=9.1132e-05) (hash(x)=150722883)
+4800 val loss 7.1510
+4800 val perplexity 1275.3193
+4800 train 7.313559 (lr=8.8982e-05) (hash(x)=166864648)
+4900 val loss 7.1670
+4900 val perplexity 1295.9025
+4900 train 7.252068 (lr=8.6825e-05) (hash(x)=157746743)
+5000 val loss 7.1829
+5000 val perplexity 1316.6577
+5000 train 7.156727 (lr=8.4663e-05) (hash(x)=153032257)
+5100 val loss 7.1774
+5100 val perplexity 1309.5005
+5100 train 7.255030 (lr=8.2500e-05) (hash(x)=151862371)
+5200 val loss 7.1758
+5200 val perplexity 1307.4421
+5200 train 7.425144 (lr=8.0337e-05) (hash(x)=168862995)
+5300 val loss 7.1714
+5300 val perplexity 1301.7012
+5300 train 7.149970 (lr=7.8175e-05) (hash(x)=148243567)
+5400 val loss 7.1838
+5400 val perplexity 1317.8958
+5400 train 7.125934 (lr=7.6018e-05) (hash(x)=145381742)
+5500 val loss 7.2053
+5500 val perplexity 1346.5536
+5500 train 7.331578 (lr=7.3868e-05) (hash(x)=155602174)
+5600 val loss 7.1589
+5600 val perplexity 1285.4670
+5600 train 7.253695 (lr=7.1727e-05) (hash(x)=160637672)
+5700 val loss 7.1751
+5700 val perplexity 1306.5392
+5700 train 7.153136 (lr=6.9597e-05) (hash(x)=152101131)
+5800 val loss 7.1774
+5800 val perplexity 1309.5411
+5800 train 7.159649 (lr=6.7480e-05) (hash(x)=152327089)
+5900 val loss 7.1646
+5900 val perplexity 1292.8016
+5900 train 7.118360 (lr=6.5378e-05) (hash(x)=151094543)
+6000 val loss 7.1787
+6000 val perplexity 1311.2225
+6000 train 7.286202 (lr=6.3294e-05) (hash(x)=160233315)
+6100 val loss 7.1632
+6100 val perplexity 1290.9937
+6100 train 7.260243 (lr=6.1230e-05) (hash(x)=158295337)
+6200 val loss 7.1683
+6200 val perplexity 1297.5900
+6200 train 7.161770 (lr=5.9188e-05) (hash(x)=151269817)
+6300 val loss 7.1598
+6300 val perplexity 1286.6083
+6300 train 7.258369 (lr=5.7169e-05) (hash(x)=156912756)
+6400 val loss 7.1634
+6400 val perplexity 1291.2910
+6400 train 7.076042 (lr=5.5177e-05) (hash(x)=125779637)
+6500 val loss 7.1931
+6500 val perplexity 1330.2091
+6500 train 7.120435 (lr=5.3213e-05) (hash(x)=148157661)
+6600 val loss 7.1875
+6600 val perplexity 1322.8066
+6600 train 7.224742 (lr=5.1279e-05) (hash(x)=152820049)
+6700 val loss 7.1680
+6700 val perplexity 1297.1934
+6700 train 7.127453 (lr=4.9377e-05) (hash(x)=155818717)
+6800 val loss 7.1549
+6800 val perplexity 1280.3846
+6800 train 7.204819 (lr=4.7509e-05) (hash(x)=158833881)
+6900 val loss 7.1392
+6900 val perplexity 1260.3671
+6900 train 7.168673 (lr=4.5676e-05) (hash(x)=164984909)
+7000 val loss 7.1415
+7000 val perplexity 1263.3184
+7000 train 7.052586 (lr=4.3882e-05) (hash(x)=151172493)
+7100 val loss 7.1452
+7100 val perplexity 1268.0675
+7100 train 7.115786 (lr=4.2128e-05) (hash(x)=154023633)
+7200 val loss 7.1357
+7200 val perplexity 1256.0227
+7200 train 7.033777 (lr=4.0414e-05) (hash(x)=139351250)
+7300 val loss 7.1326
+7300 val perplexity 1252.1747
+7300 train 7.107747 (lr=3.8745e-05) (hash(x)=158719458)
+7400 val loss 7.1271
+7400 val perplexity 1245.2498
+7400 train 7.238004 (lr=3.7120e-05) (hash(x)=162250810)
+7500 val loss 7.1292
+7500 val perplexity 1247.8699
+7500 train 7.009450 (lr=3.5541e-05) (hash(x)=142827557)
+7600 val loss 7.1253
+7600 val perplexity 1243.0115
+7600 train 7.086011 (lr=3.4011e-05) (hash(x)=147830801)
+7700 val loss 7.1211
+7700 val perplexity 1237.7563
+7700 train 7.250870 (lr=3.2531e-05) (hash(x)=165233314)
+7800 val loss 7.1121
+7800 val perplexity 1226.6871
+7800 train 7.139935 (lr=3.1102e-05) (hash(x)=160654717)
+7900 val loss 7.1114
+7900 val perplexity 1225.8656
+7900 train 6.983757 (lr=2.9726e-05) (hash(x)=140712586)
+8000 val loss 7.1024
+8000 val perplexity 1214.8588
+8000 train 7.194554 (lr=2.8405e-05) (hash(x)=150309061)
+8100 val loss 7.1089
+8100 val perplexity 1222.7528
+8100 train 7.132548 (lr=2.7138e-05) (hash(x)=158805466)
+8200 val loss 7.1218
+8200 val perplexity 1238.6584
+8200 train 7.008895 (lr=2.5929e-05) (hash(x)=142814078)
+8300 val loss 7.1024
+8300 val perplexity 1214.8744
+8300 train 7.062787 (lr=2.4778e-05) (hash(x)=148340417)
+8400 val loss 7.1228
+8400 val perplexity 1239.8574
+8400 train 7.071526 (lr=2.3686e-05) (hash(x)=157132639)
+8500 val loss 7.1088
+8500 val perplexity 1222.6788
+8500 train 7.080158 (lr=2.2655e-05) (hash(x)=152994240)
+8600 val loss 7.0931
+8600 val perplexity 1203.6345
+8600 train 7.056999 (lr=2.1685e-05) (hash(x)=152210154)
+8700 val loss 7.0910
+8700 val perplexity 1201.0780
+8700 train 7.111240 (lr=2.0777e-05) (hash(x)=149631247)
+8800 val loss 7.0898
+8800 val perplexity 1199.6757
+8800 train 6.979906 (lr=1.9933e-05) (hash(x)=145617784)
+8900 val loss 7.0862
+8900 val perplexity 1195.3822
+8900 train 7.186379 (lr=1.9153e-05) (hash(x)=155984970)
+9000 val loss 7.0871
+9000 val perplexity 1196.4508
+9000 train 6.891949 (lr=1.8439e-05) (hash(x)=143433013)
+9100 val loss 7.0887
+9100 val perplexity 1198.3727
+9100 train 7.001738 (lr=1.7790e-05) (hash(x)=152507533)
+9200 val loss 7.0833
+9200 val perplexity 1191.8619
+9200 train 7.180696 (lr=1.7208e-05) (hash(x)=162400650)
+9300 val loss 7.0836
+9300 val perplexity 1192.3075
+9300 train 7.095459 (lr=1.6692e-05) (hash(x)=148419068)
+9400 val loss 7.0818
+9400 val perplexity 1190.0691
+9400 train 7.052043 (lr=1.6245e-05) (hash(x)=145025169)
+9500 val loss 7.0796
+9500 val perplexity 1187.4729
+9500 train 7.045411 (lr=1.5865e-05) (hash(x)=145104235)
+9600 val loss 7.0753
+9600 val perplexity 1182.3990
+9600 train 7.012111 (lr=1.5554e-05) (hash(x)=150499797)
+9700 val loss 7.0802
+9700 val perplexity 1188.2228
+9700 train 6.903148 (lr=1.5312e-05) (hash(x)=139690022)
+9800 val loss 7.0721
+9800 val perplexity 1178.6555
+9800 train 7.046154 (lr=1.5139e-05) (hash(x)=145645279)
+9900 val loss 7.0681
+9900 val perplexity 1173.9249
+9900 train 7.007585 (lr=1.5035e-05) (hash(x)=150103432)
+9999 val loss 7.0719
+9999 val perplexity 1178.4133

attention_kindself_n_heads4_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77dde6c131c05f2beecb9d04c782f5c0da41a20fb349a4536a9066f758d3a232
+oid sha256:2ba11bae78a2338db851c51b584bdbb28e92c38db347f007eb25550ca636a5f2
 size 92843394

attention_kindself_n_heads4_seed1339/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e60a13c58b3b32a653aff34dc26a7de2f820000fb6a45b551f251151a5e24f1c
+size 92843394

attention_kindself_n_heads4_seed1339/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9569b576688f5ed68359c15b1252b68c654b9916fd3e04372fd2a8c76fc8ce3b
+size 92843394

attention_kindself_n_heads4_seed1339/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6e86c352cf79dc3191b93412d0de0c33ee277ff7d480cf26f37ebb1eac08792
+size 92843394

attention_kindself_n_heads4_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48dfc8d8fb0e47ca34e9f8566a682d36dcda6708f124cdd32b39bbf7b0bf439d
+oid sha256:c5e423b3da9278fb38f4bb6d9d37eaa0dce092cbc9b59be54e83223283d8bc77
 size 179406214

attention_kindself_n_heads4_seed1339/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c19594c4ee6043c95ad102a7ff571eda191ed6f3aeb274b9417977c9a2242d44
+size 179406214

attention_kindself_n_heads4_seed1339/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be4742da3573b50a7ab496bebc83225e31d357a635d1f6fc9507877675197280
+size 179406214

attention_kindself_n_heads4_seed1339/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2bac616ceb7bc96fb3c0c662e9f3bffd6b79eb542cfae6fa36a9638b3c32075
+size 179406214