Upload folder using huggingface_hub

lr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1338/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "two_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6.5e-5_61440_1338", "n_embd": 256}

lr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1338/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1338/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.3155
+0 val perplexity 82083.8438
+0 train 11.313581 (lr=1.3000e-07) (hash(x)=150327452)
+100 val loss 9.8793
+100 val perplexity 19521.2734
+100 train 9.909836 (lr=1.3130e-05) (hash(x)=166780046)
+200 val loss 9.0197
+200 val perplexity 8264.4717
+200 train 8.975257 (lr=2.6130e-05) (hash(x)=155040610)
+300 val loss 7.9539
+300 val perplexity 2846.5232
+300 train 7.952665 (lr=3.9130e-05) (hash(x)=155504036)
+400 val loss 7.5349
+400 val perplexity 1872.2289
+400 train 7.392513 (lr=5.2130e-05) (hash(x)=143823248)
+500 val loss 7.3964
+500 val perplexity 1630.0316
+500 train 7.359871 (lr=6.5000e-05) (hash(x)=143734685)
+600 val loss 7.3060
+600 val perplexity 1489.1882
+600 train 7.079813 (lr=6.4979e-05) (hash(x)=150678249)
+700 val loss 7.1895
+700 val perplexity 1325.4363
+700 train 7.314929 (lr=6.4915e-05) (hash(x)=175802021)
+800 val loss 7.0547
+800 val perplexity 1158.2384
+800 train 7.065736 (lr=6.4809e-05) (hash(x)=158681215)
+900 val loss 6.9451
+900 val perplexity 1038.0031
+900 train 6.842464 (lr=6.4661e-05) (hash(x)=146108145)
+1000 val loss 6.8589
+1000 val perplexity 952.3304
+1000 train 6.686711 (lr=6.4471e-05) (hash(x)=154996086)
+1100 val loss 6.7491
+1100 val perplexity 853.2959
+1100 train 6.739240 (lr=6.4240e-05) (hash(x)=153885445)
+1200 val loss 6.6558
+1200 val perplexity 777.2594
+1200 train 6.613417 (lr=6.3967e-05) (hash(x)=142353087)
+1300 val loss 6.5957
+1300 val perplexity 731.9092
+1300 train 6.503266 (lr=6.3653e-05) (hash(x)=150750353)
+1400 val loss 6.5492
+1400 val perplexity 698.7020
+1400 train 6.352491 (lr=6.3299e-05) (hash(x)=152767913)
+1500 val loss 6.4932
+1500 val perplexity 660.6160
+1500 train 6.553432 (lr=6.2905e-05) (hash(x)=151562048)
+1600 val loss 6.4316
+1600 val perplexity 621.1786
+1600 train 6.439937 (lr=6.2471e-05) (hash(x)=166486165)
+1700 val loss 6.4092
+1700 val perplexity 607.4033
+1700 train 6.056235 (lr=6.1999e-05) (hash(x)=130835396)
+1800 val loss 6.3676
+1800 val perplexity 582.6727
+1800 train 6.419384 (lr=6.1489e-05) (hash(x)=158851816)
+1900 val loss 6.3446
+1900 val perplexity 569.4151
+1900 train 6.371546 (lr=6.0941e-05) (hash(x)=153313879)
+2000 val loss 6.2853
+2000 val perplexity 536.6510
+2000 train 6.214882 (lr=6.0357e-05) (hash(x)=158245023)
+2100 val loss 6.2548
+2100 val perplexity 520.4918
+2100 train 6.076084 (lr=5.9737e-05) (hash(x)=157204896)
+2200 val loss 6.2306
+2200 val perplexity 508.0652
+2200 train 6.077985 (lr=5.9082e-05) (hash(x)=137541932)
+2300 val loss 6.2063
+2300 val perplexity 495.8385
+2300 train 6.164446 (lr=5.8394e-05) (hash(x)=150149692)
+2400 val loss 6.1749
+2400 val perplexity 480.5267
+2400 train 6.158689 (lr=5.7672e-05) (hash(x)=151730720)
+2500 val loss 6.1479
+2500 val perplexity 467.7471
+2500 train 5.934008 (lr=5.6919e-05) (hash(x)=143406752)
+2600 val loss 6.1457
+2600 val perplexity 466.7149
+2600 train 5.979381 (lr=5.6135e-05) (hash(x)=157272496)
+2700 val loss 6.1146
+2700 val perplexity 452.4108
+2700 train 6.080712 (lr=5.5322e-05) (hash(x)=155342327)
+2800 val loss 6.0893
+2800 val perplexity 441.1259
+2800 train 5.973903 (lr=5.4480e-05) (hash(x)=140626679)
+2900 val loss 6.0617
+2900 val perplexity 429.1196
+2900 train 5.927510 (lr=5.3611e-05) (hash(x)=144953350)
+3000 val loss 6.0464
+3000 val perplexity 422.5970
+3000 train 5.938092 (lr=5.2717e-05) (hash(x)=172449837)
+3100 val loss 6.0343
+3100 val perplexity 417.4877
+3100 train 5.821952 (lr=5.1797e-05) (hash(x)=141710086)
+3200 val loss 6.0100
+3200 val perplexity 407.4644
+3200 train 5.951566 (lr=5.0855e-05) (hash(x)=151299772)
+3300 val loss 5.9896
+3300 val perplexity 399.2491
+3300 train 5.857679 (lr=4.9890e-05) (hash(x)=146473110)
+3400 val loss 5.9616
+3400 val perplexity 388.2283
+3400 train 6.050941 (lr=4.8905e-05) (hash(x)=153954157)
+3500 val loss 5.9475
+3500 val perplexity 382.8080
+3500 train 5.891314 (lr=4.7901e-05) (hash(x)=153717336)
+3600 val loss 5.9317
+3600 val perplexity 376.8052
+3600 train 5.689588 (lr=4.6879e-05) (hash(x)=144965161)
+3700 val loss 5.9218
+3700 val perplexity 373.0699
+3700 train 5.705297 (lr=4.5841e-05) (hash(x)=125969741)
+3800 val loss 5.9031
+3800 val perplexity 366.1742
+3800 train 5.769847 (lr=4.4789e-05) (hash(x)=155070487)
+3900 val loss 5.8825
+3900 val perplexity 358.7109
+3900 train 5.753562 (lr=4.3723e-05) (hash(x)=149444644)
+4000 val loss 5.8647
+4000 val perplexity 352.3839
+4000 train 5.721871 (lr=4.2646e-05) (hash(x)=151663033)
+4100 val loss 5.8583
+4100 val perplexity 350.1156
+4100 train 5.817134 (lr=4.1559e-05) (hash(x)=143688282)
+4200 val loss 5.8360
+4200 val perplexity 342.3985
+4200 train 5.836743 (lr=4.0463e-05) (hash(x)=163361651)
+4300 val loss 5.8160
+4300 val perplexity 335.6259
+4300 train 5.836700 (lr=3.9361e-05) (hash(x)=153619361)
+4400 val loss 5.8052
+4400 val perplexity 332.0158
+4400 train 6.133276 (lr=3.8253e-05) (hash(x)=168527064)
+4500 val loss 5.7975
+4500 val perplexity 329.4859
+4500 train 5.634286 (lr=3.7142e-05) (hash(x)=125588037)
+4600 val loss 5.7717
+4600 val perplexity 321.0944
+4600 train 5.736407 (lr=3.6028e-05) (hash(x)=143710941)
+4700 val loss 5.7606
+4700 val perplexity 317.5525
+4700 train 5.605982 (lr=3.4915e-05) (hash(x)=150952742)
+4800 val loss 5.7518
+4800 val perplexity 314.7671
+4800 train 5.640090 (lr=3.3802e-05) (hash(x)=145323659)
+4900 val loss 5.7444
+4900 val perplexity 312.4257
+4900 train 5.773422 (lr=3.2693e-05) (hash(x)=153151397)
+5000 val loss 5.7364
+5000 val perplexity 309.9592
+5000 train 5.654369 (lr=3.1587e-05) (hash(x)=143182059)
+5100 val loss 5.7180
+5100 val perplexity 304.3066
+5100 train 5.737705 (lr=3.0488e-05) (hash(x)=170083586)
+5200 val loss 5.7081
+5200 val perplexity 301.2913
+5200 train 5.584399 (lr=2.9396e-05) (hash(x)=149363919)
+5300 val loss 5.6978
+5300 val perplexity 298.2038
+5300 train 5.574603 (lr=2.8314e-05) (hash(x)=152033784)
+5400 val loss 5.6836
+5400 val perplexity 294.0122
+5400 train 5.804317 (lr=2.7243e-05) (hash(x)=154614289)
+5500 val loss 5.6773
+5500 val perplexity 292.1574
+5500 train 5.782804 (lr=2.6183e-05) (hash(x)=157745174)
+5600 val loss 5.6687
+5600 val perplexity 289.6526
+5600 train 5.569401 (lr=2.5138e-05) (hash(x)=147693222)
+5700 val loss 5.6577
+5700 val perplexity 286.4858
+5700 train 5.591632 (lr=2.4108e-05) (hash(x)=149784627)
+5800 val loss 5.6579
+5800 val perplexity 286.5462
+5800 train 5.595331 (lr=2.3095e-05) (hash(x)=158620729)
+5900 val loss 5.6426
+5900 val perplexity 282.1931
+5900 train 5.597601 (lr=2.2100e-05) (hash(x)=159763910)
+6000 val loss 5.6332
+6000 val perplexity 279.5502
+6000 train 5.577635 (lr=2.1125e-05) (hash(x)=147640561)
+6100 val loss 5.6259
+6100 val perplexity 277.5132
+6100 train 5.615512 (lr=2.0171e-05) (hash(x)=156613394)
+6200 val loss 5.6196
+6200 val perplexity 275.7928
+6200 train 5.533775 (lr=1.9240e-05) (hash(x)=186221290)
+6300 val loss 5.6085
+6300 val perplexity 272.7408
+6300 train 5.488870 (lr=1.8333e-05) (hash(x)=152081419)
+6400 val loss 5.6033
+6400 val perplexity 271.3312
+6400 train 5.558011 (lr=1.7451e-05) (hash(x)=154808349)
+6500 val loss 5.6007
+6500 val perplexity 270.6288
+6500 train 5.564942 (lr=1.6595e-05) (hash(x)=159437208)
+6600 val loss 5.5962
+6600 val perplexity 269.4092
+6600 train 5.415828 (lr=1.5768e-05) (hash(x)=157933074)
+6700 val loss 5.5885
+6700 val perplexity 267.3249
+6700 train 5.567544 (lr=1.4969e-05) (hash(x)=161560240)
+6800 val loss 5.5834
+6800 val perplexity 265.9746
+6800 train 5.559826 (lr=1.4200e-05) (hash(x)=155424292)
+6900 val loss 5.5823
+6900 val perplexity 265.6790
+6900 train 5.583934 (lr=1.3463e-05) (hash(x)=148561470)
+7000 val loss 5.5731
+7000 val perplexity 263.2437
+7000 train 5.478024 (lr=1.2758e-05) (hash(x)=141527450)
+7100 val loss 5.5681
+7100 val perplexity 261.9442
+7100 train 5.525320 (lr=1.2086e-05) (hash(x)=151066339)
+7200 val loss 5.5682
+7200 val perplexity 261.9723
+7200 train 5.366111 (lr=1.1449e-05) (hash(x)=155231264)
+7300 val loss 5.5588
+7300 val perplexity 259.5035
+7300 train 5.589715 (lr=1.0847e-05) (hash(x)=150281149)
+7400 val loss 5.5553
+7400 val perplexity 258.6158
+7400 train 5.443874 (lr=1.0281e-05) (hash(x)=148421717)
+7500 val loss 5.5552
+7500 val perplexity 258.5861
+7500 train 5.320126 (lr=9.7516e-06) (hash(x)=146921118)
+7600 val loss 5.5487
+7600 val perplexity 256.9053
+7600 train 5.594157 (lr=9.2601e-06) (hash(x)=150660048)
+7700 val loss 5.5455
+7700 val perplexity 256.0896
+7700 train 5.430185 (lr=8.8071e-06) (hash(x)=148059852)
+7800 val loss 5.5453
+7800 val perplexity 256.0200
+7800 train 5.238227 (lr=8.3932e-06) (hash(x)=148331002)
+7900 val loss 5.5385
+7900 val perplexity 254.3064
+7900 train 5.474245 (lr=8.0189e-06) (hash(x)=164923883)
+8000 val loss 5.5358
+8000 val perplexity 253.6200
+8000 train 5.421015 (lr=7.6848e-06) (hash(x)=143545384)
+8100 val loss 5.5358
+8100 val perplexity 253.6193
+8100 train 5.352304 (lr=7.3914e-06) (hash(x)=160686959)
+8200 val loss 5.5324
+8200 val perplexity 252.7419
+8200 train 5.478696 (lr=7.1392e-06) (hash(x)=156501889)
+8300 val loss 5.5302
+8300 val perplexity 252.1862
+8300 train 5.372092 (lr=6.9284e-06) (hash(x)=142716875)
+8400 val loss 5.5248
+8400 val perplexity 250.8459
+8400 train 5.450275 (lr=6.7594e-06) (hash(x)=154436684)
+8500 val loss 5.5237
+8500 val perplexity 250.5624
+8500 train 5.610744 (lr=6.6324e-06) (hash(x)=147965839)
+8600 val loss 5.5223
+8600 val perplexity 250.2127
+8600 train 5.311404 (lr=6.5477e-06) (hash(x)=145228097)
+8700 val loss 5.5175
+8700 val perplexity 249.0127
+8700 train 5.716743 (lr=6.5053e-06) (hash(x)=152910357)
+8749 val loss 5.5181
+8749 val perplexity 249.1618

lr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1338/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c752129a58f6951c594ddd50d5339b17cf5de9e8899e43853a497e0437faa30d
+size 97580418

lr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1338/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fba350c507dc8829b75addb4f4bee55ce23cc374e65ea072d2a8947dd34f58f
+size 188880262