Upload folder using huggingface_hub

attention_kindself_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_0/attention_kindself_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10, "warmup_steps": 200, "group": "wider_is_better_0", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_10240_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1338", "n_embd": 256}

attention_kindself_n_heads4_seed1338/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
+oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
 size 964

attention_kindself_n_heads4_seed1338/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+size 964

attention_kindself_n_heads4_seed1338/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+size 964

attention_kindself_n_heads4_seed1338/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+size 964

attention_kindself_n_heads4_seed1338/log2.txt

@@ -1,15 +1,303 @@
-max_steps: 10
-0 val loss 11.2876
-0 val perplexity 79826.5938
-0 train 11.276918 (lr=7.5000e-07) (hash(x)=50671684)
-1 train 11.299708 (lr=1.5000e-06) (hash(x)=48334907)
-2 train 11.279386 (lr=2.2500e-06) (hash(x)=51320861)
-3 train 11.293447 (lr=3.0000e-06) (hash(x)=46712336)
-4 train 11.271300 (lr=3.7500e-06) (hash(x)=51050552)
-5 train 11.284151 (lr=4.5000e-06) (hash(x)=53176793)
-6 train 11.232478 (lr=5.2500e-06) (hash(x)=56077270)
-7 train 11.211646 (lr=6.0000e-06) (hash(x)=50171817)
-8 train 11.223553 (lr=6.7500e-06) (hash(x)=54201336)
-9 val loss 11.2091
-9 val perplexity 73795.9219
-9 train 11.213384 (lr=7.5000e-06) (hash(x)=49973223)
+max_steps: 10000
+0 val loss 11.2886
+0 val perplexity 79902.0703
+0 train 11.283466 (lr=7.5000e-07) (hash(x)=150327452)
+100 val loss 9.3785
+100 val perplexity 11831.5137
+100 train 9.383852 (lr=7.5750e-05) (hash(x)=166441190)
+200 val loss 7.8808
+200 val perplexity 2645.9348
+200 train 8.070142 (lr=1.5000e-04) (hash(x)=166780046)
+300 val loss 7.5206
+300 val perplexity 1845.6257
+300 train 7.483925 (lr=1.4997e-04) (hash(x)=159835303)
+400 val loss 7.4219
+400 val perplexity 1672.2504
+400 train 7.168677 (lr=1.4986e-04) (hash(x)=155040610)
+500 val loss 7.3280
+500 val perplexity 1522.2717
+500 train 7.020287 (lr=1.4969e-04) (hash(x)=130190460)
+600 val loss 7.2209
+600 val perplexity 1367.6549
+600 train 7.230540 (lr=1.4945e-04) (hash(x)=155504036)
+700 val loss 7.1466
+700 val perplexity 1269.7841
+700 train 7.021465 (lr=1.4913e-04) (hash(x)=137347213)
+800 val loss 7.0969
+800 val perplexity 1208.2078
+800 train 6.952953 (lr=1.4876e-04) (hash(x)=143823248)
+900 val loss 7.0507
+900 val perplexity 1153.6415
+900 train 7.114755 (lr=1.4831e-04) (hash(x)=156260416)
+1000 val loss 7.0423
+1000 val perplexity 1143.9812
+1000 train 7.001041 (lr=1.4779e-04) (hash(x)=143734685)
+1100 val loss 7.0197
+1100 val perplexity 1118.3988
+1100 train 6.961253 (lr=1.4721e-04) (hash(x)=160013925)
+1200 val loss 7.0138
+1200 val perplexity 1111.9210
+1200 train 6.752813 (lr=1.4656e-04) (hash(x)=150678249)
+1300 val loss 7.0305
+1300 val perplexity 1130.6281
+1300 train 6.818944 (lr=1.4585e-04) (hash(x)=149073315)
+1400 val loss 7.0169
+1400 val perplexity 1115.3191
+1400 train 7.227762 (lr=1.4507e-04) (hash(x)=175802021)
+1500 val loss 6.9904
+1500 val perplexity 1086.1665
+1500 train 7.242711 (lr=1.4422e-04) (hash(x)=171034639)
+1600 val loss 7.0097
+1600 val perplexity 1107.2745
+1600 train 7.185740 (lr=1.4332e-04) (hash(x)=158681215)
+1700 val loss 7.0072
+1700 val perplexity 1104.5930
+1700 train 6.966218 (lr=1.4235e-04) (hash(x)=152116061)
+1800 val loss 7.0236
+1800 val perplexity 1122.8741
+1800 train 6.965474 (lr=1.4131e-04) (hash(x)=146108145)
+1900 val loss 7.0468
+1900 val perplexity 1149.1926
+1900 train 6.898216 (lr=1.4022e-04) (hash(x)=147598108)
+2000 val loss 7.0514
+2000 val perplexity 1154.5193
+2000 train 6.938709 (lr=1.3907e-04) (hash(x)=154996086)
+2100 val loss 7.0655
+2100 val perplexity 1170.8207
+2100 train 6.890048 (lr=1.3786e-04) (hash(x)=153396183)
+2200 val loss 7.0841
+2200 val perplexity 1192.7903
+2200 train 7.076140 (lr=1.3660e-04) (hash(x)=153885445)
+2300 val loss 7.0796
+2300 val perplexity 1187.5176
+2300 train 7.058900 (lr=1.3527e-04) (hash(x)=159666385)
+2400 val loss 7.0250
+2400 val perplexity 1124.3641
+2400 train 7.010871 (lr=1.3390e-04) (hash(x)=142353087)
+2500 val loss 7.0845
+2500 val perplexity 1193.3342
+2500 train 7.069442 (lr=1.3247e-04) (hash(x)=146491718)
+2600 val loss 7.0627
+2600 val perplexity 1167.5798
+2600 train 7.007412 (lr=1.3099e-04) (hash(x)=150750353)
+2700 val loss 7.0971
+2700 val perplexity 1208.4486
+2700 train 6.841060 (lr=1.2946e-04) (hash(x)=129849193)
+2800 val loss 7.1062
+2800 val perplexity 1219.4514
+2800 train 6.993633 (lr=1.2788e-04) (hash(x)=152767913)
+2900 val loss 7.1069
+2900 val perplexity 1220.3513
+2900 train 6.923536 (lr=1.2626e-04) (hash(x)=146531140)
+3000 val loss 7.0982
+3000 val perplexity 1209.7919
+3000 train 7.153556 (lr=1.2459e-04) (hash(x)=151562048)
+3100 val loss 7.1035
+3100 val perplexity 1216.1699
+3100 train 7.088020 (lr=1.2287e-04) (hash(x)=146001424)
+3200 val loss 7.1201
+3200 val perplexity 1236.5682
+3200 train 7.156642 (lr=1.2112e-04) (hash(x)=166486165)
+3300 val loss 7.0978
+3300 val perplexity 1209.3202
+3300 train 7.028473 (lr=1.1932e-04) (hash(x)=150866680)
+3400 val loss 7.1455
+3400 val perplexity 1268.3693
+3400 train 7.064568 (lr=1.1749e-04) (hash(x)=143900419)
+3500 val loss 7.1583
+3500 val perplexity 1284.7684
+3500 train 7.038066 (lr=1.1562e-04) (hash(x)=148845794)
+3600 val loss 7.2640
+3600 val perplexity 1427.9215
+3600 train 7.125070 (lr=1.1372e-04) (hash(x)=145667796)
+3700 val loss 7.1962
+3700 val perplexity 1334.3002
+3700 train 7.212631 (lr=1.1179e-04) (hash(x)=163563851)
+3800 val loss 7.2033
+3800 val perplexity 1343.8755
+3800 train 7.165361 (lr=1.0982e-04) (hash(x)=147488689)
+3900 val loss 7.2326
+3900 val perplexity 1383.8372
+3900 train 7.216290 (lr=1.0783e-04) (hash(x)=148186608)
+4000 val loss 7.2666
+4000 val perplexity 1431.7081
+4000 train 7.116256 (lr=1.0581e-04) (hash(x)=142970187)
+4100 val loss 7.2117
+4100 val perplexity 1355.1702
+4100 train 7.177159 (lr=1.0377e-04) (hash(x)=141584883)
+4200 val loss 7.2359
+4200 val perplexity 1388.4315
+4200 train 7.076993 (lr=1.0171e-04) (hash(x)=145664585)
+4300 val loss 7.2608
+4300 val perplexity 1423.4381
+4300 train 7.147635 (lr=9.9622e-05) (hash(x)=143736499)
+4400 val loss 7.2374
+4400 val perplexity 1390.4741
+4400 train 7.086786 (lr=9.7520e-05) (hash(x)=151883322)
+4500 val loss 7.2709
+4500 val perplexity 1437.8669
+4500 train 7.208516 (lr=9.5403e-05) (hash(x)=153904871)
+4600 val loss 7.2542
+4600 val perplexity 1413.9868
+4600 train 7.287719 (lr=9.3273e-05) (hash(x)=154893521)
+4700 val loss 7.2740
+4700 val perplexity 1442.3077
+4700 train 7.333047 (lr=9.1132e-05) (hash(x)=152323949)
+4800 val loss 7.2486
+4800 val perplexity 1406.1262
+4800 train 7.187540 (lr=8.8982e-05) (hash(x)=154104619)
+4900 val loss 7.2405
+4900 val perplexity 1394.7579
+4900 train 7.216898 (lr=8.6825e-05) (hash(x)=146311426)
+5000 val loss 7.2337
+5000 val perplexity 1385.3900
+5000 train 7.289016 (lr=8.4663e-05) (hash(x)=156741847)
+5100 val loss 7.2136
+5100 val perplexity 1357.7510
+5100 train 7.028937 (lr=8.2500e-05) (hash(x)=142086346)
+5200 val loss 7.2310
+5200 val perplexity 1381.6060
+5200 train 7.083183 (lr=8.0337e-05) (hash(x)=150265428)
+5300 val loss 7.2254
+5300 val perplexity 1373.9017
+5300 train 7.169150 (lr=7.8175e-05) (hash(x)=151339108)
+5400 val loss 7.2208
+5400 val perplexity 1367.5669
+5400 train 7.210334 (lr=7.6018e-05) (hash(x)=154654372)
+5500 val loss 7.2389
+5500 val perplexity 1392.4932
+5500 train 7.223175 (lr=7.3868e-05) (hash(x)=150575051)
+5600 val loss 7.2404
+5600 val perplexity 1394.5990
+5600 train 7.149243 (lr=7.1727e-05) (hash(x)=140396423)
+5700 val loss 7.2114
+5700 val perplexity 1354.8497
+5700 train 7.126038 (lr=6.9597e-05) (hash(x)=144678758)
+5800 val loss 7.2017
+5800 val perplexity 1341.6639
+5800 train 7.318895 (lr=6.7480e-05) (hash(x)=151992743)
+5900 val loss 7.2180
+5900 val perplexity 1363.7925
+5900 train 7.045027 (lr=6.5378e-05) (hash(x)=144396927)
+6000 val loss 7.2203
+6000 val perplexity 1366.9260
+6000 train 7.175666 (lr=6.3294e-05) (hash(x)=165478625)
+6100 val loss 7.1996
+6100 val perplexity 1338.8456
+6100 train 7.033978 (lr=6.1230e-05) (hash(x)=147088621)
+6200 val loss 7.2209
+6200 val perplexity 1367.7762
+6200 train 7.069060 (lr=5.9188e-05) (hash(x)=140794994)
+6300 val loss 7.2201
+6300 val perplexity 1366.5735
+6300 train 7.120542 (lr=5.7169e-05) (hash(x)=134780906)
+6400 val loss 7.2234
+6400 val perplexity 1371.1732
+6400 train 7.226955 (lr=5.5177e-05) (hash(x)=149023655)
+6500 val loss 7.2028
+6500 val perplexity 1343.2477
+6500 train 7.115757 (lr=5.3213e-05) (hash(x)=147497796)
+6600 val loss 7.2079
+6600 val perplexity 1350.1128
+6600 train 7.102191 (lr=5.1279e-05) (hash(x)=152902689)
+6700 val loss 7.2061
+6700 val perplexity 1347.5677
+6700 train 7.246368 (lr=4.9377e-05) (hash(x)=153846046)
+6800 val loss 7.2093
+6800 val perplexity 1351.9836
+6800 train 7.330764 (lr=4.7509e-05) (hash(x)=158512738)
+6900 val loss 7.1865
+6900 val perplexity 1321.5305
+6900 train 7.424375 (lr=4.5676e-05) (hash(x)=156849968)
+7000 val loss 7.1959
+7000 val perplexity 1333.9860
+7000 train 7.043914 (lr=4.3882e-05) (hash(x)=142395855)
+7100 val loss 7.1989
+7100 val perplexity 1337.9470
+7100 train 7.106159 (lr=4.2128e-05) (hash(x)=147114884)
+7200 val loss 7.1802
+7200 val perplexity 1313.1328
+7200 train 7.203697 (lr=4.0414e-05) (hash(x)=156979839)
+7300 val loss 7.1783
+7300 val perplexity 1310.6949
+7300 train 6.993206 (lr=3.8745e-05) (hash(x)=145584373)
+7400 val loss 7.1934
+7400 val perplexity 1330.6176
+7400 train 6.996182 (lr=3.7120e-05) (hash(x)=141508204)
+7500 val loss 7.1840
+7500 val perplexity 1318.2195
+7500 train 7.227539 (lr=3.5541e-05) (hash(x)=148803965)
+7600 val loss 7.1778
+7600 val perplexity 1309.9590
+7600 train 7.204811 (lr=3.4011e-05) (hash(x)=151019676)
+7700 val loss 7.1746
+7700 val perplexity 1305.8422
+7700 train 7.141462 (lr=3.2531e-05) (hash(x)=143155750)
+7800 val loss 7.1695
+7800 val perplexity 1299.1667
+7800 train 7.170794 (lr=3.1102e-05) (hash(x)=152569653)
+7900 val loss 7.1715
+7900 val perplexity 1301.7706
+7900 train 7.044502 (lr=2.9726e-05) (hash(x)=143519455)
+8000 val loss 7.1761
+8000 val perplexity 1307.8531
+8000 train 7.372725 (lr=2.8405e-05) (hash(x)=161180944)
+8100 val loss 7.1799
+8100 val perplexity 1312.8022
+8100 train 7.221662 (lr=2.7138e-05) (hash(x)=154107345)
+8200 val loss 7.1785
+8200 val perplexity 1310.8956
+8200 train 7.181197 (lr=2.5929e-05) (hash(x)=152486517)
+8300 val loss 7.1800
+8300 val perplexity 1312.8931
+8300 train 7.233239 (lr=2.4778e-05) (hash(x)=156167749)
+8400 val loss 7.1772
+8400 val perplexity 1309.2814
+8400 train 7.235495 (lr=2.3686e-05) (hash(x)=149155006)
+8500 val loss 7.1779
+8500 val perplexity 1310.1251
+8500 train 7.165373 (lr=2.2655e-05) (hash(x)=147844390)
+8600 val loss 7.1768
+8600 val perplexity 1308.7402
+8600 train 7.409989 (lr=2.1685e-05) (hash(x)=165753320)
+8700 val loss 7.1742
+8700 val perplexity 1305.2993
+8700 train 7.002594 (lr=2.0777e-05) (hash(x)=146079979)
+8800 val loss 7.1709
+8800 val perplexity 1301.0707
+8800 train 7.373499 (lr=1.9933e-05) (hash(x)=172259509)
+8900 val loss 7.1748
+8900 val perplexity 1306.0651
+8900 train 6.947332 (lr=1.9153e-05) (hash(x)=145148314)
+9000 val loss 7.1684
+9000 val perplexity 1297.7539
+9000 train 7.104209 (lr=1.8439e-05) (hash(x)=144250633)
+9100 val loss 7.1666
+9100 val perplexity 1295.3718
+9100 train 7.257204 (lr=1.7790e-05) (hash(x)=157219797)
+9200 val loss 7.1579
+9200 val perplexity 1284.2423
+9200 train 7.106236 (lr=1.7208e-05) (hash(x)=142743778)
+9300 val loss 7.1560
+9300 val perplexity 1281.8220
+9300 train 7.048266 (lr=1.6692e-05) (hash(x)=139669771)
+9400 val loss 7.1577
+9400 val perplexity 1283.9049
+9400 train 7.100833 (lr=1.6245e-05) (hash(x)=145916843)
+9500 val loss 7.1547
+9500 val perplexity 1280.1686
+9500 train 7.092788 (lr=1.5865e-05) (hash(x)=150196125)
+9600 val loss 7.1533
+9600 val perplexity 1278.3093
+9600 train 7.405205 (lr=1.5554e-05) (hash(x)=160041419)
+9700 val loss 7.1543
+9700 val perplexity 1279.5973
+9700 train 6.997491 (lr=1.5312e-05) (hash(x)=139931627)
+9800 val loss 7.1590
+9800 val perplexity 1285.5835
+9800 train 7.096377 (lr=1.5139e-05) (hash(x)=150370792)
+9900 val loss 7.1522
+9900 val perplexity 1276.9653
+9900 train 7.073775 (lr=1.5035e-05) (hash(x)=153014886)
+9999 val loss 7.1555
+9999 val perplexity 1281.0717

attention_kindself_n_heads4_seed1338/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:502140dfff20494de53b4d0e0b0ffa82d46145afad79a936c45b209687b6490f
+oid sha256:33bdec72102977a017b95e7077a1451305c54765f3fa6b004684c155327bb47c
 size 92843394

attention_kindself_n_heads4_seed1338/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12402700c7a26ef863238483371d9b995f727a734ab1ff749b8d18d0d8bebaa9
+size 92843394

attention_kindself_n_heads4_seed1338/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24f20598da322e545f87f6fa88c75cbfec505ceff9533f5c7766ec27a408ca7f
+size 92843394

attention_kindself_n_heads4_seed1338/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7448aeea0d1362ae86e4b6bff0478150b2bd54d5c8b583cbaeeacc8009dfa09
+size 92843394

attention_kindself_n_heads4_seed1338/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:176987c91384b218fd32193467183a6aab8df570526bfdcf2a4ffd89de67d4db
+oid sha256:2802f18e8205bc7cd54bcb31128f6d0712c66a6e3c7508a5cfe7d8d63bf555a9
 size 179406214

attention_kindself_n_heads4_seed1338/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87fbc6041be854a980be6242dd41115b4453abd3fdfeac7f017ba6abefdb95d5
+size 179406214

attention_kindself_n_heads4_seed1338/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3f51e26281eabb0d381d4ef920fbe96b8b9cbebf74f381f373063581908ef48
+size 179406214

attention_kindself_n_heads4_seed1338/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:342bd7719f449fc97ae54978c6e499bcb8f0ee33bc90551016c56b3b69d65d1e
+size 179406214