Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1341/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1341", "n_embd": 256}

attention_kindselective_n_heads4_seed1341/dataloader_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
+size 964

attention_kindselective_n_heads4_seed1341/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+size 964

attention_kindselective_n_heads4_seed1341/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+size 964

attention_kindselective_n_heads4_seed1341/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+size 964

attention_kindselective_n_heads4_seed1341/log2.txt

@@ -0,0 +1,303 @@
+max_steps: 10000
+0 val loss 11.3107
+0 val perplexity 81693.5234
+0 train 11.308768 (lr=7.5000e-07) (hash(x)=145079536)
+100 val loss 8.8391
+100 val perplexity 6898.6484
+100 train 8.772367 (lr=7.5750e-05) (hash(x)=154745873)
+200 val loss 7.6156
+200 val perplexity 2029.7083
+200 train 7.846752 (lr=1.5000e-04) (hash(x)=155800595)
+300 val loss 7.4963
+300 val perplexity 1801.3328
+300 train 7.453661 (lr=1.4997e-04) (hash(x)=148595389)
+400 val loss 7.4522
+400 val perplexity 1723.7068
+400 train 7.349169 (lr=1.4986e-04) (hash(x)=145606733)
+500 val loss 7.4211
+500 val perplexity 1670.7958
+500 train 7.219723 (lr=1.4969e-04) (hash(x)=138221231)
+600 val loss 7.3785
+600 val perplexity 1601.2350
+600 train 7.374033 (lr=1.4945e-04) (hash(x)=150367139)
+700 val loss 7.3640
+700 val perplexity 1578.1882
+700 train 7.532372 (lr=1.4913e-04) (hash(x)=155579314)
+800 val loss 7.3582
+800 val perplexity 1569.0526
+800 train 7.439426 (lr=1.4876e-04) (hash(x)=155747374)
+900 val loss 7.3403
+900 val perplexity 1541.1282
+900 train 7.548749 (lr=1.4831e-04) (hash(x)=159334575)
+1000 val loss 7.2672
+1000 val perplexity 1432.5939
+1000 train 7.151237 (lr=1.4779e-04) (hash(x)=140604760)
+1100 val loss 7.2486
+1100 val perplexity 1406.1088
+1100 train 7.090343 (lr=1.4721e-04) (hash(x)=146208052)
+1200 val loss 7.2222
+1200 val perplexity 1369.4868
+1200 train 7.149972 (lr=1.4656e-04) (hash(x)=148404734)
+1300 val loss 7.1887
+1300 val perplexity 1324.3914
+1300 train 7.119121 (lr=1.4585e-04) (hash(x)=155681970)
+1400 val loss 7.1846
+1400 val perplexity 1318.9463
+1400 train 7.138198 (lr=1.4507e-04) (hash(x)=148115934)
+1500 val loss 7.2087
+1500 val perplexity 1351.1703
+1500 train 7.343921 (lr=1.4422e-04) (hash(x)=157074034)
+1600 val loss 7.1871
+1600 val perplexity 1322.3191
+1600 train 6.987100 (lr=1.4332e-04) (hash(x)=137464699)
+1700 val loss 7.1538
+1700 val perplexity 1278.8964
+1700 train 7.346094 (lr=1.4235e-04) (hash(x)=166955614)
+1800 val loss 7.1954
+1800 val perplexity 1333.2679
+1800 train 7.040450 (lr=1.4131e-04) (hash(x)=143886042)
+1900 val loss 7.2243
+1900 val perplexity 1372.3500
+1900 train 7.750473 (lr=1.4022e-04) (hash(x)=193610391)
+2000 val loss 7.2533
+2000 val perplexity 1412.8269
+2000 train 7.405033 (lr=1.3907e-04) (hash(x)=163799796)
+2100 val loss 7.2782
+2100 val perplexity 1448.4299
+2100 train 7.240331 (lr=1.3786e-04) (hash(x)=154107339)
+2200 val loss 7.2263
+2200 val perplexity 1375.1130
+2200 train 7.216848 (lr=1.3660e-04) (hash(x)=144592844)
+2300 val loss 7.2623
+2300 val perplexity 1425.5057
+2300 train 7.331630 (lr=1.3527e-04) (hash(x)=154535861)
+2400 val loss 7.2641
+2400 val perplexity 1428.1707
+2400 train 7.981302 (lr=1.3390e-04) (hash(x)=204706354)
+2500 val loss 7.2574
+2500 val perplexity 1418.5338
+2500 train 7.104072 (lr=1.3247e-04) (hash(x)=143783202)
+2600 val loss 7.2129
+2600 val perplexity 1356.7827
+2600 train 7.102783 (lr=1.3099e-04) (hash(x)=150862210)
+2700 val loss 7.1940
+2700 val perplexity 1331.4440
+2700 train 7.248954 (lr=1.2946e-04) (hash(x)=150753426)
+2800 val loss 7.2206
+2800 val perplexity 1367.2833
+2800 train 7.188674 (lr=1.2788e-04) (hash(x)=147766811)
+2900 val loss 7.1182
+2900 val perplexity 1234.2701
+2900 train 7.065569 (lr=1.2626e-04) (hash(x)=151177814)
+3000 val loss 7.0958
+3000 val perplexity 1206.8909
+3000 train 6.964172 (lr=1.2459e-04) (hash(x)=135925327)
+3100 val loss 7.1356
+3100 val perplexity 1255.9456
+3100 train 6.881985 (lr=1.2287e-04) (hash(x)=124842353)
+3200 val loss 7.1582
+3200 val perplexity 1284.6527
+3200 train 7.265501 (lr=1.2112e-04) (hash(x)=160440642)
+3300 val loss 7.1195
+3300 val perplexity 1235.8479
+3300 train 7.041489 (lr=1.1932e-04) (hash(x)=143017131)
+3400 val loss 7.1233
+3400 val perplexity 1240.5908
+3400 train 7.341820 (lr=1.1749e-04) (hash(x)=148143474)
+3500 val loss 7.1400
+3500 val perplexity 1261.4679
+3500 train 6.968966 (lr=1.1562e-04) (hash(x)=148732639)
+3600 val loss 7.1300
+3600 val perplexity 1248.8193
+3600 train 7.117164 (lr=1.1372e-04) (hash(x)=146333803)
+3700 val loss 7.1258
+3700 val perplexity 1243.6559
+3700 train 6.980933 (lr=1.1179e-04) (hash(x)=143611445)
+3800 val loss 7.1099
+3800 val perplexity 1224.0386
+3800 train 6.920319 (lr=1.0982e-04) (hash(x)=141272115)
+3900 val loss 7.0775
+3900 val perplexity 1185.0203
+3900 train 7.024056 (lr=1.0783e-04) (hash(x)=149161848)
+4000 val loss 7.0654
+4000 val perplexity 1170.7074
+4000 train 7.113854 (lr=1.0581e-04) (hash(x)=152036955)
+4100 val loss 7.0592
+4100 val perplexity 1163.5604
+4100 train 7.113382 (lr=1.0377e-04) (hash(x)=153260968)
+4200 val loss 7.0436
+4200 val perplexity 1145.5483
+4200 train 6.939513 (lr=1.0171e-04) (hash(x)=147629478)
+4300 val loss 7.0205
+4300 val perplexity 1119.3015
+4300 train 7.055896 (lr=9.9622e-05) (hash(x)=161701080)
+4400 val loss 7.0205
+4400 val perplexity 1119.4014
+4400 train 7.079413 (lr=9.7520e-05) (hash(x)=155290524)
+4500 val loss 7.0365
+4500 val perplexity 1137.4241
+4500 train 6.863458 (lr=9.5403e-05) (hash(x)=144726593)
+4600 val loss 7.0281
+4600 val perplexity 1127.8313
+4600 train 6.985920 (lr=9.3273e-05) (hash(x)=149517202)
+4700 val loss 6.9983
+4700 val perplexity 1094.7998
+4700 train 7.000892 (lr=9.1132e-05) (hash(x)=139403008)
+4800 val loss 6.9746
+4800 val perplexity 1069.1084
+4800 train 7.027068 (lr=8.8982e-05) (hash(x)=153387238)
+4900 val loss 6.9510
+4900 val perplexity 1044.2091
+4900 train 6.737017 (lr=8.6825e-05) (hash(x)=139881569)
+5000 val loss 6.9486
+5000 val perplexity 1041.6897
+5000 train 6.945867 (lr=8.4663e-05) (hash(x)=152602580)
+5100 val loss 6.9585
+5100 val perplexity 1052.0623
+5100 train 7.015290 (lr=8.2500e-05) (hash(x)=150598460)
+5200 val loss 6.9349
+5200 val perplexity 1027.5084
+5200 train 6.861823 (lr=8.0337e-05) (hash(x)=147660496)
+5300 val loss 6.9434
+5300 val perplexity 1036.3123
+5300 train 6.717848 (lr=7.8175e-05) (hash(x)=147693848)
+5400 val loss 6.9382
+5400 val perplexity 1030.9615
+5400 train 6.709671 (lr=7.6018e-05) (hash(x)=136048145)
+5500 val loss 6.9367
+5500 val perplexity 1029.3636
+5500 train 6.882251 (lr=7.3868e-05) (hash(x)=140313123)
+5600 val loss 6.9212
+5600 val perplexity 1013.5551
+5600 train 6.946658 (lr=7.1727e-05) (hash(x)=156997993)
+5700 val loss 6.9164
+5700 val perplexity 1008.6826
+5700 train 6.920675 (lr=6.9597e-05) (hash(x)=143513880)
+5800 val loss 6.9113
+5800 val perplexity 1003.5991
+5800 train 7.050395 (lr=6.7480e-05) (hash(x)=153736628)
+5900 val loss 6.9003
+5900 val perplexity 992.5403
+5900 train 7.066628 (lr=6.5378e-05) (hash(x)=166385417)
+6000 val loss 6.9046
+6000 val perplexity 996.8475
+6000 train 7.093999 (lr=6.3294e-05) (hash(x)=153216517)
+6100 val loss 6.9056
+6100 val perplexity 997.8129
+6100 train 6.820673 (lr=6.1230e-05) (hash(x)=145387545)
+6200 val loss 6.8980
+6200 val perplexity 990.2825
+6200 train 7.000422 (lr=5.9188e-05) (hash(x)=156306460)
+6300 val loss 6.8904
+6300 val perplexity 982.8094
+6300 train 6.779611 (lr=5.7169e-05) (hash(x)=148610447)
+6400 val loss 6.8853
+6400 val perplexity 977.7564
+6400 train 7.063712 (lr=5.5177e-05) (hash(x)=158343535)
+6500 val loss 6.8739
+6500 val perplexity 966.6749
+6500 train 7.008237 (lr=5.3213e-05) (hash(x)=157742557)
+6600 val loss 6.8612
+6600 val perplexity 954.4707
+6600 train 6.838261 (lr=5.1279e-05) (hash(x)=150457308)
+6700 val loss 6.8735
+6700 val perplexity 966.3348
+6700 train 6.790349 (lr=4.9377e-05) (hash(x)=142426205)
+6800 val loss 6.8622
+6800 val perplexity 955.5117
+6800 train 6.693820 (lr=4.7509e-05) (hash(x)=151522525)
+6900 val loss 6.8624
+6900 val perplexity 955.6771
+6900 train 6.791017 (lr=4.5676e-05) (hash(x)=152029630)
+7000 val loss 6.8435
+7000 val perplexity 937.7366
+7000 train 6.719049 (lr=4.3882e-05) (hash(x)=142978954)
+7100 val loss 6.8394
+7100 val perplexity 933.9645
+7100 train 6.734494 (lr=4.2128e-05) (hash(x)=148064359)
+7200 val loss 6.8358
+7200 val perplexity 930.6148
+7200 train 6.782277 (lr=4.0414e-05) (hash(x)=155771845)
+7300 val loss 6.8361
+7300 val perplexity 930.8500
+7300 train 6.662339 (lr=3.8745e-05) (hash(x)=141639844)
+7400 val loss 6.8391
+7400 val perplexity 933.6702
+7400 train 6.885146 (lr=3.7120e-05) (hash(x)=153466021)
+7500 val loss 6.8325
+7500 val perplexity 927.5389
+7500 train 6.823289 (lr=3.5541e-05) (hash(x)=148843587)
+7600 val loss 6.8233
+7600 val perplexity 919.0372
+7600 train 6.953442 (lr=3.4011e-05) (hash(x)=152053933)
+7700 val loss 6.8180
+7700 val perplexity 914.1565
+7700 train 6.620625 (lr=3.2531e-05) (hash(x)=141564003)
+7800 val loss 6.8160
+7800 val perplexity 912.3558
+7800 train 6.709721 (lr=3.1102e-05) (hash(x)=148617843)
+7900 val loss 6.8102
+7900 val perplexity 907.0121
+7900 train 6.794557 (lr=2.9726e-05) (hash(x)=146417668)
+8000 val loss 6.8158
+8000 val perplexity 912.1326
+8000 train 6.651595 (lr=2.8405e-05) (hash(x)=150165488)
+8100 val loss 6.8004
+8100 val perplexity 898.2380
+8100 train 6.779853 (lr=2.7138e-05) (hash(x)=144818855)
+8200 val loss 6.7972
+8200 val perplexity 895.3443
+8200 train 6.843230 (lr=2.5929e-05) (hash(x)=143074954)
+8300 val loss 6.7966
+8300 val perplexity 894.8095
+8300 train 6.712090 (lr=2.4778e-05) (hash(x)=145172494)
+8400 val loss 6.7914
+8400 val perplexity 890.1903
+8400 train 6.796282 (lr=2.3686e-05) (hash(x)=152284393)
+8500 val loss 6.7894
+8500 val perplexity 888.4059
+8500 train 6.656650 (lr=2.2655e-05) (hash(x)=147044705)
+8600 val loss 6.7893
+8600 val perplexity 888.3284
+8600 train 6.691913 (lr=2.1685e-05) (hash(x)=147762608)
+8700 val loss 6.7876
+8700 val perplexity 886.8052
+8700 train 6.776985 (lr=2.0777e-05) (hash(x)=141994567)
+8800 val loss 6.7824
+8800 val perplexity 882.1815
+8800 train 6.821270 (lr=1.9933e-05) (hash(x)=152552896)
+8900 val loss 6.7805
+8900 val perplexity 880.5118
+8900 train 6.645514 (lr=1.9153e-05) (hash(x)=139714454)
+9000 val loss 6.7779
+9000 val perplexity 878.2654
+9000 train 6.816576 (lr=1.8439e-05) (hash(x)=152274419)
+9100 val loss 6.7756
+9100 val perplexity 876.1732
+9100 train 6.920788 (lr=1.7790e-05) (hash(x)=160536885)
+9200 val loss 6.7732
+9200 val perplexity 874.1129
+9200 train 6.611001 (lr=1.7208e-05) (hash(x)=138386899)
+9300 val loss 6.7720
+9300 val perplexity 873.0236
+9300 train 6.784839 (lr=1.6692e-05) (hash(x)=149957457)
+9400 val loss 6.7659
+9400 val perplexity 867.7105
+9400 train 6.830824 (lr=1.6245e-05) (hash(x)=161671548)
+9500 val loss 6.7652
+9500 val perplexity 867.1352
+9500 train 6.889501 (lr=1.5865e-05) (hash(x)=171857400)
+9600 val loss 6.7605
+9600 val perplexity 863.0999
+9600 train 6.499973 (lr=1.5554e-05) (hash(x)=135378471)
+9700 val loss 6.7598
+9700 val perplexity 862.4960
+9700 train 6.683197 (lr=1.5312e-05) (hash(x)=149514591)
+9800 val loss 6.7579
+9800 val perplexity 860.8607
+9800 train 6.735991 (lr=1.5139e-05) (hash(x)=152679072)
+9900 val loss 6.7552
+9900 val perplexity 858.5027
+9900 train 6.687136 (lr=1.5035e-05) (hash(x)=150415193)
+9999 val loss 6.7556
+9999 val perplexity 858.8172

attention_kindselective_n_heads4_seed1341/model_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1a36134feee8c838ca59f4f4cf424e2fca8a150b9359d0b271c8fcf70e2accd
+size 92843394

attention_kindselective_n_heads4_seed1341/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e6706f1ac208ee2aa2c18be2455c671f7c4625e8295b4a3ad2f06915da9e3a6
+size 92843394

attention_kindselective_n_heads4_seed1341/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeea8582c4be0c746ae7bfb25d547cfa9115d740fb997f5b01c137d6c55be2c4
+size 92843394

attention_kindselective_n_heads4_seed1341/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24392d1319c6233ffb98a24dbd2887950aad5ebd6d687d9b5a14b369e8b2ee0c
+size 92843394

attention_kindselective_n_heads4_seed1341/optimizer_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b3ee680be0c5dba7e1c48deb68b6a24e64a79ea7a8d3a868ed2524130474d23
+size 179406214

attention_kindselective_n_heads4_seed1341/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b4942203e4b5d098edd126d941bf815c57f9948a4c52e565b12e9801fdf55ed
+size 179406214

attention_kindselective_n_heads4_seed1341/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:961e8ef3395d436796afd9a5a48e3c262a1face942572c935f59e061c05f4621
+size 179406214

attention_kindselective_n_heads4_seed1341/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3632ba34851aa4a52a98c6a9cd20d04bbf16ea432d3fccfc412c40d99e2526ef
+size 179406214