Upload folder using huggingface_hub

lr6.5e-5_total_batch_size61440_baseline_seed1338/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr6.5e-5_total_batch_size61440_baseline_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6.5e-5_61440_1338", "n_embd": 256}

lr6.5e-5_total_batch_size61440_baseline_seed1338/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr6.5e-5_total_batch_size61440_baseline_seed1338/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.2646
+0 val perplexity 78014.1562
+0 train 11.267750 (lr=1.3000e-07) (hash(x)=150327452)
+100 val loss 9.9228
+100 val perplexity 20390.9570
+100 train 9.974588 (lr=1.3130e-05) (hash(x)=166780046)
+200 val loss 8.9873
+200 val perplexity 8000.4355
+200 train 8.917374 (lr=2.6130e-05) (hash(x)=155040610)
+300 val loss 7.9490
+300 val perplexity 2832.6418
+300 train 7.943207 (lr=3.9130e-05) (hash(x)=155504036)
+400 val loss 7.5817
+400 val perplexity 1961.8741
+400 train 7.444796 (lr=5.2130e-05) (hash(x)=143823248)
+500 val loss 7.4548
+500 val perplexity 1728.1913
+500 train 7.420609 (lr=6.5000e-05) (hash(x)=143734685)
+600 val loss 7.3784
+600 val perplexity 1600.9633
+600 train 7.149719 (lr=6.4979e-05) (hash(x)=150678249)
+700 val loss 7.2655
+700 val perplexity 1430.0460
+700 train 7.388148 (lr=6.4915e-05) (hash(x)=175802021)
+800 val loss 7.1429
+800 val perplexity 1265.1154
+800 train 7.159602 (lr=6.4809e-05) (hash(x)=158681215)
+900 val loss 7.0587
+900 val perplexity 1162.9596
+900 train 6.950723 (lr=6.4661e-05) (hash(x)=146108145)
+1000 val loss 6.9905
+1000 val perplexity 1086.3038
+1000 train 6.834347 (lr=6.4471e-05) (hash(x)=154996086)
+1100 val loss 6.9001
+1100 val perplexity 992.3643
+1100 train 6.881894 (lr=6.4240e-05) (hash(x)=153885445)
+1200 val loss 6.8135
+1200 val perplexity 910.0621
+1200 train 6.775256 (lr=6.3967e-05) (hash(x)=142353087)
+1300 val loss 6.7648
+1300 val perplexity 866.7863
+1300 train 6.670671 (lr=6.3653e-05) (hash(x)=150750353)
+1400 val loss 6.7158
+1400 val perplexity 825.3747
+1400 train 6.535036 (lr=6.3299e-05) (hash(x)=152767913)
+1500 val loss 6.6403
+1500 val perplexity 765.3521
+1500 train 6.682803 (lr=6.2905e-05) (hash(x)=151562048)
+1600 val loss 6.5796
+1600 val perplexity 720.2851
+1600 train 6.592773 (lr=6.2471e-05) (hash(x)=166486165)
+1700 val loss 6.5488
+1700 val perplexity 698.3929
+1700 train 6.190404 (lr=6.1999e-05) (hash(x)=130835396)
+1800 val loss 6.5054
+1800 val perplexity 668.7193
+1800 train 6.552690 (lr=6.1489e-05) (hash(x)=158851816)
+1900 val loss 6.4753
+1900 val perplexity 648.9299
+1900 train 6.501315 (lr=6.0941e-05) (hash(x)=153313879)
+2000 val loss 6.4194
+2000 val perplexity 613.6628
+2000 train 6.353259 (lr=6.0357e-05) (hash(x)=158245023)
+2100 val loss 6.3896
+2100 val perplexity 595.5972
+2100 train 6.211575 (lr=5.9737e-05) (hash(x)=157204896)
+2200 val loss 6.3738
+2200 val perplexity 586.2643
+2200 train 6.212805 (lr=5.9082e-05) (hash(x)=137541932)
+2300 val loss 6.3515
+2300 val perplexity 573.3417
+2300 train 6.304212 (lr=5.8394e-05) (hash(x)=150149692)
+2400 val loss 6.3250
+2400 val perplexity 558.3503
+2400 train 6.315724 (lr=5.7672e-05) (hash(x)=151730720)
+2500 val loss 6.2948
+2500 val perplexity 541.7536
+2500 train 6.072569 (lr=5.6919e-05) (hash(x)=143406752)
+2600 val loss 6.2908
+2600 val perplexity 539.6070
+2600 train 6.120640 (lr=5.6135e-05) (hash(x)=157272496)
+2700 val loss 6.2710
+2700 val perplexity 528.9924
+2700 train 6.259467 (lr=5.5322e-05) (hash(x)=155342327)
+2800 val loss 6.2434
+2800 val perplexity 514.5922
+2800 train 6.125520 (lr=5.4480e-05) (hash(x)=140626679)
+2900 val loss 6.2308
+2900 val perplexity 508.1862
+2900 train 6.088692 (lr=5.3611e-05) (hash(x)=144953350)
+3000 val loss 6.2112
+3000 val perplexity 498.3144
+3000 train 6.110980 (lr=5.2717e-05) (hash(x)=172449837)
+3100 val loss 6.1988
+3100 val perplexity 492.1492
+3100 train 5.992530 (lr=5.1797e-05) (hash(x)=141710086)
+3200 val loss 6.1669
+3200 val perplexity 476.6863
+3200 train 6.096434 (lr=5.0855e-05) (hash(x)=151299772)
+3300 val loss 6.1561
+3300 val perplexity 471.5733
+3300 train 6.015643 (lr=4.9890e-05) (hash(x)=146473110)
+3400 val loss 6.1233
+3400 val perplexity 456.3883
+3400 train 6.200891 (lr=4.8905e-05) (hash(x)=153954157)
+3500 val loss 6.1082
+3500 val perplexity 449.5430
+3500 train 6.040827 (lr=4.7901e-05) (hash(x)=153717336)
+3600 val loss 6.0938
+3600 val perplexity 443.1069
+3600 train 5.849350 (lr=4.6879e-05) (hash(x)=144965161)
+3700 val loss 6.0912
+3700 val perplexity 441.9467
+3700 train 5.863019 (lr=4.5841e-05) (hash(x)=125969741)
+3800 val loss 6.0584
+3800 val perplexity 427.6957
+3800 train 5.922933 (lr=4.4789e-05) (hash(x)=155070487)
+3900 val loss 6.0421
+3900 val perplexity 420.7831
+3900 train 5.912959 (lr=4.3723e-05) (hash(x)=149444644)
+4000 val loss 6.0281
+4000 val perplexity 414.9382
+4000 train 5.880025 (lr=4.2646e-05) (hash(x)=151663033)
+4100 val loss 6.0237
+4100 val perplexity 413.1187
+4100 train 5.973774 (lr=4.1559e-05) (hash(x)=143688282)
+4200 val loss 6.0041
+4200 val perplexity 405.0912
+4200 train 6.008862 (lr=4.0463e-05) (hash(x)=163361651)
+4300 val loss 5.9852
+4300 val perplexity 397.5141
+4300 train 6.010225 (lr=3.9361e-05) (hash(x)=153619361)
+4400 val loss 5.9816
+4400 val perplexity 396.0814
+4400 train 6.278453 (lr=3.8253e-05) (hash(x)=168527064)
+4500 val loss 5.9705
+4500 val perplexity 391.7015
+4500 train 5.776382 (lr=3.7142e-05) (hash(x)=125588037)
+4600 val loss 5.9478
+4600 val perplexity 382.8964
+4600 train 5.908142 (lr=3.6028e-05) (hash(x)=143710941)
+4700 val loss 5.9324
+4700 val perplexity 377.0414
+4700 train 5.762898 (lr=3.4915e-05) (hash(x)=150952742)
+4800 val loss 5.9259
+4800 val perplexity 374.6181
+4800 train 5.797978 (lr=3.3802e-05) (hash(x)=145323659)
+4900 val loss 5.9166
+4900 val perplexity 371.1647
+4900 train 5.935788 (lr=3.2693e-05) (hash(x)=153151397)
+5000 val loss 5.9052
+5000 val perplexity 366.9487
+5000 train 5.812093 (lr=3.1587e-05) (hash(x)=143182059)
+5100 val loss 5.8913
+5100 val perplexity 361.8893
+5100 train 5.929032 (lr=3.0488e-05) (hash(x)=170083586)
+5200 val loss 5.8781
+5200 val perplexity 357.1300
+5200 train 5.767028 (lr=2.9396e-05) (hash(x)=149363919)
+5300 val loss 5.8742
+5300 val perplexity 355.7533
+5300 train 5.759417 (lr=2.8314e-05) (hash(x)=152033784)
+5400 val loss 5.8565
+5400 val perplexity 349.5002
+5400 train 5.964608 (lr=2.7243e-05) (hash(x)=154614289)
+5500 val loss 5.8507
+5500 val perplexity 347.4780
+5500 train 5.938869 (lr=2.6183e-05) (hash(x)=157745174)
+5600 val loss 5.8442
+5600 val perplexity 345.2171
+5600 train 5.734215 (lr=2.5138e-05) (hash(x)=147693222)
+5700 val loss 5.8286
+5700 val perplexity 339.8724
+5700 train 5.765426 (lr=2.4108e-05) (hash(x)=149784627)
+5800 val loss 5.8309
+5800 val perplexity 340.6623
+5800 train 5.744448 (lr=2.3095e-05) (hash(x)=158620729)
+5900 val loss 5.8167
+5900 val perplexity 335.8583
+5900 train 5.765829 (lr=2.2100e-05) (hash(x)=159763910)
+6000 val loss 5.8045
+6000 val perplexity 331.7740
+6000 train 5.740259 (lr=2.1125e-05) (hash(x)=147640561)
+6100 val loss 5.7968
+6100 val perplexity 329.2335
+6100 train 5.779181 (lr=2.0171e-05) (hash(x)=156613394)
+6200 val loss 5.7926
+6200 val perplexity 327.8591
+6200 train 5.714095 (lr=1.9240e-05) (hash(x)=186221290)
+6300 val loss 5.7800
+6300 val perplexity 323.7451
+6300 train 5.658869 (lr=1.8333e-05) (hash(x)=152081419)
+6400 val loss 5.7773
+6400 val perplexity 322.9007
+6400 train 5.724511 (lr=1.7451e-05) (hash(x)=154808349)
+6500 val loss 5.7685
+6500 val perplexity 320.0473
+6500 train 5.727124 (lr=1.6595e-05) (hash(x)=159437208)
+6600 val loss 5.7678
+6600 val perplexity 319.8395
+6600 train 5.576637 (lr=1.5768e-05) (hash(x)=157933074)
+6700 val loss 5.7586
+6700 val perplexity 316.9053
+6700 train 5.735898 (lr=1.4969e-05) (hash(x)=161560240)
+6800 val loss 5.7525
+6800 val perplexity 314.9650
+6800 train 5.725802 (lr=1.4200e-05) (hash(x)=155424292)
+6900 val loss 5.7529
+6900 val perplexity 315.1120
+6900 train 5.746572 (lr=1.3463e-05) (hash(x)=148561470)
+7000 val loss 5.7421
+7000 val perplexity 311.7108
+7000 train 5.640338 (lr=1.2758e-05) (hash(x)=141527450)
+7100 val loss 5.7371
+7100 val perplexity 310.1727
+7100 train 5.687416 (lr=1.2086e-05) (hash(x)=151066339)
+7200 val loss 5.7368
+7200 val perplexity 310.0710
+7200 train 5.531171 (lr=1.1449e-05) (hash(x)=155231264)
+7300 val loss 5.7275
+7300 val perplexity 307.1997
+7300 train 5.749755 (lr=1.0847e-05) (hash(x)=150281149)
+7400 val loss 5.7237
+7400 val perplexity 306.0485
+7400 train 5.606393 (lr=1.0281e-05) (hash(x)=148421717)
+7500 val loss 5.7235
+7500 val perplexity 305.9654
+7500 train 5.489408 (lr=9.7516e-06) (hash(x)=146921118)
+7600 val loss 5.7141
+7600 val perplexity 303.1094
+7600 train 5.754773 (lr=9.2601e-06) (hash(x)=150660048)
+7700 val loss 5.7139
+7700 val perplexity 303.0468
+7700 train 5.589794 (lr=8.8071e-06) (hash(x)=148059852)
+7800 val loss 5.7114
+7800 val perplexity 302.2888
+7800 train 5.404067 (lr=8.3932e-06) (hash(x)=148331002)
+7900 val loss 5.7065
+7900 val perplexity 300.8213
+7900 train 5.652740 (lr=8.0189e-06) (hash(x)=164923883)
+8000 val loss 5.7029
+8000 val perplexity 299.7312
+8000 train 5.588421 (lr=7.6848e-06) (hash(x)=143545384)
+8100 val loss 5.7027
+8100 val perplexity 299.6745
+8100 train 5.515050 (lr=7.3914e-06) (hash(x)=160686959)
+8200 val loss 5.6994
+8200 val perplexity 298.6984
+8200 train 5.649620 (lr=7.1392e-06) (hash(x)=156501889)
+8300 val loss 5.6963
+8300 val perplexity 297.7697
+8300 train 5.541341 (lr=6.9284e-06) (hash(x)=142716875)
+8400 val loss 5.6920
+8400 val perplexity 296.4888
+8400 train 5.607490 (lr=6.7594e-06) (hash(x)=154436684)
+8500 val loss 5.6898
+8500 val perplexity 295.8345
+8500 train 5.765838 (lr=6.6324e-06) (hash(x)=147965839)
+8600 val loss 5.6892
+8600 val perplexity 295.6509
+8600 train 5.483855 (lr=6.5477e-06) (hash(x)=145228097)
+8700 val loss 5.6831
+8700 val perplexity 293.8553
+8700 train 5.865552 (lr=6.5053e-06) (hash(x)=152910357)
+8749 val loss 5.6827
+8749 val perplexity 293.7317

lr6.5e-5_total_batch_size61440_baseline_seed1338/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:156508e8cdc7882b8c7dcb6ab89c1aba2c50f9036db8722742173c1ba55fc528
+size 92843394

lr6.5e-5_total_batch_size61440_baseline_seed1338/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e0cb3f11be3caf20210c7c61e5904d7d60d834be2b1b57929a991feb8862199
+size 179406214