Upload folder using huggingface_hub

attention_kindself_n_heads4_seed1341/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1341", "n_embd": 256}

attention_kindself_n_heads4_seed1341/dataloader_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
+size 964

attention_kindself_n_heads4_seed1341/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+size 964

attention_kindself_n_heads4_seed1341/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+size 964

attention_kindself_n_heads4_seed1341/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+size 964

attention_kindself_n_heads4_seed1341/log2.txt

@@ -0,0 +1,303 @@
+max_steps: 10000
+0 val loss 11.3001
+0 val perplexity 80832.1328
+0 train 11.306698 (lr=7.5000e-07) (hash(x)=145079536)
+100 val loss 9.3955
+100 val perplexity 12033.8779
+100 train 9.328828 (lr=7.5750e-05) (hash(x)=154745873)
+200 val loss 7.8945
+200 val perplexity 2682.5410
+200 train 8.046517 (lr=1.5000e-04) (hash(x)=155800595)
+300 val loss 7.5646
+300 val perplexity 1928.6428
+300 train 7.518878 (lr=1.4997e-04) (hash(x)=148595389)
+400 val loss 7.4628
+400 val perplexity 1741.9973
+400 train 7.364924 (lr=1.4986e-04) (hash(x)=145606733)
+500 val loss 7.3461
+500 val perplexity 1550.1819
+500 train 7.149421 (lr=1.4969e-04) (hash(x)=138221231)
+600 val loss 7.2483
+600 val perplexity 1405.6442
+600 train 7.251230 (lr=1.4945e-04) (hash(x)=150367139)
+700 val loss 7.1820
+700 val perplexity 1315.5784
+700 train 7.341902 (lr=1.4913e-04) (hash(x)=155579314)
+800 val loss 7.1344
+800 val perplexity 1254.3445
+800 train 7.203811 (lr=1.4876e-04) (hash(x)=155747374)
+900 val loss 7.0829
+900 val perplexity 1191.3647
+900 train 7.300850 (lr=1.4831e-04) (hash(x)=159334575)
+1000 val loss 7.0469
+1000 val perplexity 1149.2343
+1000 train 6.926498 (lr=1.4779e-04) (hash(x)=140604760)
+1100 val loss 7.0195
+1100 val perplexity 1118.2042
+1100 train 6.859845 (lr=1.4721e-04) (hash(x)=146208052)
+1200 val loss 6.9986
+1200 val perplexity 1095.0535
+1200 train 6.893322 (lr=1.4656e-04) (hash(x)=148404734)
+1300 val loss 6.9836
+1300 val perplexity 1078.7654
+1300 train 6.927601 (lr=1.4585e-04) (hash(x)=155681970)
+1400 val loss 6.9791
+1400 val perplexity 1073.9409
+1400 train 6.923635 (lr=1.4507e-04) (hash(x)=148115934)
+1500 val loss 6.9565
+1500 val perplexity 1049.9689
+1500 train 7.091454 (lr=1.4422e-04) (hash(x)=157074034)
+1600 val loss 6.9471
+1600 val perplexity 1040.1470
+1600 train 6.729518 (lr=1.4332e-04) (hash(x)=137464699)
+1700 val loss 6.9357
+1700 val perplexity 1028.3800
+1700 train 7.057977 (lr=1.4235e-04) (hash(x)=166955614)
+1800 val loss 6.9376
+1800 val perplexity 1030.3394
+1800 train 6.764245 (lr=1.4131e-04) (hash(x)=143886042)
+1900 val loss 6.9336
+1900 val perplexity 1026.2212
+1900 train 7.465428 (lr=1.4022e-04) (hash(x)=193610391)
+2000 val loss 6.9376
+2000 val perplexity 1030.2677
+2000 train 7.123946 (lr=1.3907e-04) (hash(x)=163799796)
+2100 val loss 6.9394
+2100 val perplexity 1032.1489
+2100 train 6.917353 (lr=1.3786e-04) (hash(x)=154107339)
+2200 val loss 6.9471
+2200 val perplexity 1040.1465
+2200 train 6.929567 (lr=1.3660e-04) (hash(x)=144592844)
+2300 val loss 6.9317
+2300 val perplexity 1024.1855
+2300 train 7.003430 (lr=1.3527e-04) (hash(x)=154535861)
+2400 val loss 6.9294
+2400 val perplexity 1021.9094
+2400 train 7.594524 (lr=1.3390e-04) (hash(x)=204706354)
+2500 val loss 6.9464
+2500 val perplexity 1039.4346
+2500 train 6.789571 (lr=1.3247e-04) (hash(x)=143783202)
+2600 val loss 6.9908
+2600 val perplexity 1086.5525
+2600 train 6.898761 (lr=1.3099e-04) (hash(x)=150862210)
+2700 val loss 7.0879
+2700 val perplexity 1197.4342
+2700 train 7.115728 (lr=1.2946e-04) (hash(x)=150753426)
+2800 val loss 7.0544
+2800 val perplexity 1157.9237
+2800 train 7.033858 (lr=1.2788e-04) (hash(x)=147766811)
+2900 val loss 7.0444
+2900 val perplexity 1146.4587
+2900 train 6.995884 (lr=1.2626e-04) (hash(x)=151177814)
+3000 val loss 7.0758
+3000 val perplexity 1182.9889
+3000 train 6.961751 (lr=1.2459e-04) (hash(x)=135925327)
+3100 val loss 7.0452
+3100 val perplexity 1147.3901
+3100 train 6.727612 (lr=1.2287e-04) (hash(x)=124842353)
+3200 val loss 7.0620
+3200 val perplexity 1166.8268
+3200 train 7.186998 (lr=1.2112e-04) (hash(x)=160440642)
+3300 val loss 7.0383
+3300 val perplexity 1139.4587
+3300 train 6.945041 (lr=1.1932e-04) (hash(x)=143017131)
+3400 val loss 7.0683
+3400 val perplexity 1174.1327
+3400 train 7.301206 (lr=1.1749e-04) (hash(x)=148143474)
+3500 val loss 7.0685
+3500 val perplexity 1174.3734
+3500 train 6.894324 (lr=1.1562e-04) (hash(x)=148732639)
+3600 val loss 7.0478
+3600 val perplexity 1150.2919
+3600 train 7.020499 (lr=1.1372e-04) (hash(x)=146333803)
+3700 val loss 7.0632
+3700 val perplexity 1168.2002
+3700 train 6.920694 (lr=1.1179e-04) (hash(x)=143611445)
+3800 val loss 7.0754
+3800 val perplexity 1182.5382
+3800 train 6.906082 (lr=1.0982e-04) (hash(x)=141272115)
+3900 val loss 7.0808
+3900 val perplexity 1188.9431
+3900 train 7.021241 (lr=1.0783e-04) (hash(x)=149161848)
+4000 val loss 7.0733
+4000 val perplexity 1180.0232
+4000 train 7.144450 (lr=1.0581e-04) (hash(x)=152036955)
+4100 val loss 7.0870
+4100 val perplexity 1196.3624
+4100 train 7.139550 (lr=1.0377e-04) (hash(x)=153260968)
+4200 val loss 7.0995
+4200 val perplexity 1211.3430
+4200 train 7.001007 (lr=1.0171e-04) (hash(x)=147629478)
+4300 val loss 7.1032
+4300 val perplexity 1215.8655
+4300 train 7.152019 (lr=9.9622e-05) (hash(x)=161701080)
+4400 val loss 7.1204
+4400 val perplexity 1236.9551
+4400 train 7.195014 (lr=9.7520e-05) (hash(x)=155290524)
+4500 val loss 7.1122
+4500 val perplexity 1226.8528
+4500 train 6.941273 (lr=9.5403e-05) (hash(x)=144726593)
+4600 val loss 7.0994
+4600 val perplexity 1211.2708
+4600 train 7.047345 (lr=9.3273e-05) (hash(x)=149517202)
+4700 val loss 7.1178
+4700 val perplexity 1233.7041
+4700 train 7.075057 (lr=9.1132e-05) (hash(x)=139403008)
+4800 val loss 7.1471
+4800 val perplexity 1270.3921
+4800 train 7.197305 (lr=8.8982e-05) (hash(x)=153387238)
+4900 val loss 7.1528
+4900 val perplexity 1277.6281
+4900 train 6.937476 (lr=8.6825e-05) (hash(x)=139881569)
+5000 val loss 7.1791
+5000 val perplexity 1311.7185
+5000 train 7.149471 (lr=8.4663e-05) (hash(x)=152602580)
+5100 val loss 7.1850
+5100 val perplexity 1319.4634
+5100 train 7.225554 (lr=8.2500e-05) (hash(x)=150598460)
+5200 val loss 7.1622
+5200 val perplexity 1289.7594
+5200 train 7.097391 (lr=8.0337e-05) (hash(x)=147660496)
+5300 val loss 7.2029
+5300 val perplexity 1343.2982
+5300 train 7.020668 (lr=7.8175e-05) (hash(x)=147693848)
+5400 val loss 7.1715
+5400 val perplexity 1301.7905
+5400 train 6.968065 (lr=7.6018e-05) (hash(x)=136048145)
+5500 val loss 7.1732
+5500 val perplexity 1304.0588
+5500 train 7.087708 (lr=7.3868e-05) (hash(x)=140313123)
+5600 val loss 7.1686
+5600 val perplexity 1298.0120
+5600 train 7.184072 (lr=7.1727e-05) (hash(x)=156997993)
+5700 val loss 7.1618
+5700 val perplexity 1289.2294
+5700 train 7.135228 (lr=6.9597e-05) (hash(x)=143513880)
+5800 val loss 7.1865
+5800 val perplexity 1321.4089
+5800 train 7.319064 (lr=6.7480e-05) (hash(x)=153736628)
+5900 val loss 7.1542
+5900 val perplexity 1279.5192
+5900 train 7.347223 (lr=6.5378e-05) (hash(x)=166385417)
+6000 val loss 7.1738
+6000 val perplexity 1304.7692
+6000 train 7.386747 (lr=6.3294e-05) (hash(x)=153216517)
+6100 val loss 7.1957
+6100 val perplexity 1333.7188
+6100 train 7.121440 (lr=6.1230e-05) (hash(x)=145387545)
+6200 val loss 7.1819
+6200 val perplexity 1315.4391
+6200 train 7.279177 (lr=5.9188e-05) (hash(x)=156306460)
+6300 val loss 7.1514
+6300 val perplexity 1275.8540
+6300 train 7.050121 (lr=5.7169e-05) (hash(x)=148610447)
+6400 val loss 7.1558
+6400 val perplexity 1281.5262
+6400 train 7.306859 (lr=5.5177e-05) (hash(x)=158343535)
+6500 val loss 7.1446
+6500 val perplexity 1267.2605
+6500 train 7.276059 (lr=5.3213e-05) (hash(x)=157742557)
+6600 val loss 7.1413
+6600 val perplexity 1263.0376
+6600 train 7.115061 (lr=5.1279e-05) (hash(x)=150457308)
+6700 val loss 7.1304
+6700 val perplexity 1249.4382
+6700 train 7.057846 (lr=4.9377e-05) (hash(x)=142426205)
+6800 val loss 7.1562
+6800 val perplexity 1282.0842
+6800 train 7.020480 (lr=4.7509e-05) (hash(x)=151522525)
+6900 val loss 7.1463
+6900 val perplexity 1269.3373
+6900 train 7.045939 (lr=4.5676e-05) (hash(x)=152029630)
+7000 val loss 7.1282
+7000 val perplexity 1246.6816
+7000 train 7.010334 (lr=4.3882e-05) (hash(x)=142978954)
+7100 val loss 7.1316
+7100 val perplexity 1250.8677
+7100 train 7.048053 (lr=4.2128e-05) (hash(x)=148064359)
+7200 val loss 7.1285
+7200 val perplexity 1247.0454
+7200 train 7.076699 (lr=4.0414e-05) (hash(x)=155771845)
+7300 val loss 7.1311
+7300 val perplexity 1250.2863
+7300 train 6.963120 (lr=3.8745e-05) (hash(x)=141639844)
+7400 val loss 7.1365
+7400 val perplexity 1257.0677
+7400 train 7.203446 (lr=3.7120e-05) (hash(x)=153466021)
+7500 val loss 7.1197
+7500 val perplexity 1236.1337
+7500 train 7.136865 (lr=3.5541e-05) (hash(x)=148843587)
+7600 val loss 7.1116
+7600 val perplexity 1226.1165
+7600 train 7.277462 (lr=3.4011e-05) (hash(x)=152053933)
+7700 val loss 7.1060
+7700 val perplexity 1219.2164
+7700 train 6.897977 (lr=3.2531e-05) (hash(x)=141564003)
+7800 val loss 7.1045
+7800 val perplexity 1217.3755
+7800 train 6.994552 (lr=3.1102e-05) (hash(x)=148617843)
+7900 val loss 7.1060
+7900 val perplexity 1219.3043
+7900 train 7.082467 (lr=2.9726e-05) (hash(x)=146417668)
+8000 val loss 7.1018
+8000 val perplexity 1214.1541
+8000 train 6.974138 (lr=2.8405e-05) (hash(x)=150165488)
+8100 val loss 7.0963
+8100 val perplexity 1207.5344
+8100 train 7.060221 (lr=2.7138e-05) (hash(x)=144818855)
+8200 val loss 7.0909
+8200 val perplexity 1201.0133
+8200 train 7.119874 (lr=2.5929e-05) (hash(x)=143074954)
+8300 val loss 7.0967
+8300 val perplexity 1208.0228
+8300 train 7.036266 (lr=2.4778e-05) (hash(x)=145172494)
+8400 val loss 7.0913
+8400 val perplexity 1201.4298
+8400 train 7.098520 (lr=2.3686e-05) (hash(x)=152284393)
+8500 val loss 7.0969
+8500 val perplexity 1208.2601
+8500 train 6.983650 (lr=2.2655e-05) (hash(x)=147044705)
+8600 val loss 7.0955
+8600 val perplexity 1206.5703
+8600 train 7.039187 (lr=2.1685e-05) (hash(x)=147762608)
+8700 val loss 7.0910
+8700 val perplexity 1201.1680
+8700 train 7.076082 (lr=2.0777e-05) (hash(x)=141994567)
+8800 val loss 7.0858
+8800 val perplexity 1194.8208
+8800 train 7.110641 (lr=1.9933e-05) (hash(x)=152552896)
+8900 val loss 7.0899
+8900 val perplexity 1199.7546
+8900 train 6.966035 (lr=1.9153e-05) (hash(x)=139714454)
+9000 val loss 7.0826
+9000 val perplexity 1191.0267
+9000 train 7.120393 (lr=1.8439e-05) (hash(x)=152274419)
+9100 val loss 7.0831
+9100 val perplexity 1191.6499
+9100 train 7.232574 (lr=1.7790e-05) (hash(x)=160536885)
+9200 val loss 7.0775
+9200 val perplexity 1185.0089
+9200 train 6.920467 (lr=1.7208e-05) (hash(x)=138386899)
+9300 val loss 7.0761
+9300 val perplexity 1183.3505
+9300 train 7.090115 (lr=1.6692e-05) (hash(x)=149957457)
+9400 val loss 7.0705
+9400 val perplexity 1176.6985
+9400 train 7.159608 (lr=1.6245e-05) (hash(x)=161671548)
+9500 val loss 7.0700
+9500 val perplexity 1176.1343
+9500 train 7.194458 (lr=1.5865e-05) (hash(x)=171857400)
+9600 val loss 7.0682
+9600 val perplexity 1173.9771
+9600 train 6.827000 (lr=1.5554e-05) (hash(x)=135378471)
+9700 val loss 7.0748
+9700 val perplexity 1181.8060
+9700 train 7.002652 (lr=1.5312e-05) (hash(x)=149514591)
+9800 val loss 7.0707
+9800 val perplexity 1176.9982
+9800 train 7.059768 (lr=1.5139e-05) (hash(x)=152679072)
+9900 val loss 7.0689
+9900 val perplexity 1174.8854
+9900 train 6.991833 (lr=1.5035e-05) (hash(x)=150415193)
+9999 val loss 7.0655
+9999 val perplexity 1170.8290

attention_kindself_n_heads4_seed1341/model_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c33d853b824a618231b03a5099efc279b9d9e5b64406a41441614ca3495a496
+size 92843394

attention_kindself_n_heads4_seed1341/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7dfce5e2e2490fbc54169a381133a6a7f50c5bd6315114f93ec0d34ab57035b
+size 92843394

attention_kindself_n_heads4_seed1341/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f022e1f318a9ff5f71f8bf03ca4a0be59c7e19aec7499228c3cb84c234b02fbe
+size 92843394

attention_kindself_n_heads4_seed1341/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41a5c62783abbee1a7e70f418aafd4affde384eb53f7607404494b593f2a22c2
+size 92843394

attention_kindself_n_heads4_seed1341/optimizer_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:767fa84c5f70a8e53dbf1419e39982a08fbe6e6fc5ae1b5c775d0feb0f1e7d9a
+size 179406214

attention_kindself_n_heads4_seed1341/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42bd6d830de914e5aabdd77f01551e171e6027880554a54365e1ed34959782d1
+size 179406214

attention_kindself_n_heads4_seed1341/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f52907cfa74b5974aa1b2136db3f96d064bc659e38dd36c475a0c72d2119681f
+size 179406214

attention_kindself_n_heads4_seed1341/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e77d542f2c6b2c551056a5b7ea5c142d14614182cc2e0410bc1f0990e310db03
+size 179406214