Upload folder using huggingface_hub

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "n_latent_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": 2, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6.5e-5_61440_1338", "n_embd": 256}

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.2524
+0 val perplexity 77061.3750
+0 train 11.253548 (lr=1.3000e-07) (hash(x)=150327452)
+100 val loss 10.0352
+100 val perplexity 22816.3691
+100 train 10.068452 (lr=1.3130e-05) (hash(x)=166780046)
+200 val loss 9.8348
+200 val perplexity 18671.6504
+200 train 9.846405 (lr=2.6130e-05) (hash(x)=155040610)
+300 val loss 9.5989
+300 val perplexity 14749.0391
+300 train 9.638331 (lr=3.9130e-05) (hash(x)=155504036)
+400 val loss 9.3482
+400 val perplexity 11477.9893
+400 train 9.342222 (lr=5.2130e-05) (hash(x)=143823248)
+500 val loss 9.1604
+500 val perplexity 9512.5654
+500 train 9.172577 (lr=6.5000e-05) (hash(x)=143734685)
+600 val loss 8.9727
+600 val perplexity 7884.9683
+600 train 8.931612 (lr=6.4979e-05) (hash(x)=150678249)
+700 val loss 8.6956
+700 val perplexity 5976.7144
+700 train 8.782192 (lr=6.4915e-05) (hash(x)=175802021)
+800 val loss 8.5906
+800 val perplexity 5380.8311
+800 train 8.582792 (lr=6.4809e-05) (hash(x)=158681215)
+900 val loss 8.5300
+900 val perplexity 5064.4639
+900 train 8.531790 (lr=6.4661e-05) (hash(x)=146108145)
+1000 val loss 8.3226
+1000 val perplexity 4116.0488
+1000 train 8.303654 (lr=6.4471e-05) (hash(x)=154996086)
+1100 val loss 8.2189
+1100 val perplexity 3710.3682
+1100 train 8.252198 (lr=6.4240e-05) (hash(x)=153885445)
+1200 val loss 8.1503
+1200 val perplexity 3464.3755
+1200 train 8.147595 (lr=6.3967e-05) (hash(x)=142353087)
+1300 val loss 8.1429
+1300 val perplexity 3438.9236
+1300 train 8.144033 (lr=6.3653e-05) (hash(x)=150750353)
+1400 val loss 8.1100
+1400 val perplexity 3327.6880
+1400 train 8.036261 (lr=6.3299e-05) (hash(x)=152767913)
+1500 val loss 8.1184
+1500 val perplexity 3355.6589
+1500 train 8.174913 (lr=6.2905e-05) (hash(x)=151562048)
+1600 val loss 8.0854
+1600 val perplexity 3246.8037
+1600 train 8.080959 (lr=6.2471e-05) (hash(x)=166486165)
+1700 val loss 8.1011
+1700 val perplexity 3298.0278
+1700 train 7.995829 (lr=6.1999e-05) (hash(x)=130835396)
+1800 val loss 8.1464
+1800 val perplexity 3451.0598
+1800 train 8.183622 (lr=6.1489e-05) (hash(x)=158851816)
+1900 val loss 8.3785
+1900 val perplexity 4352.5581
+1900 train 8.391592 (lr=6.0941e-05) (hash(x)=153313879)
+2000 val loss 8.1213
+2000 val perplexity 3365.4434
+2000 train 8.101875 (lr=6.0357e-05) (hash(x)=158245023)
+2100 val loss 8.1160
+2100 val perplexity 3347.4541
+2100 train 8.018796 (lr=5.9737e-05) (hash(x)=157204896)
+2200 val loss 8.2594
+2200 val perplexity 3863.9351
+2200 train 8.242746 (lr=5.9082e-05) (hash(x)=137541932)
+2300 val loss 8.1054
+2300 val perplexity 3312.3035
+2300 train 8.105439 (lr=5.8394e-05) (hash(x)=150149692)
+2400 val loss 8.1099
+2400 val perplexity 3327.3420
+2400 train 8.157024 (lr=5.7672e-05) (hash(x)=151730720)
+2500 val loss 8.1024
+2500 val perplexity 3302.5161
+2500 train 7.872808 (lr=5.6919e-05) (hash(x)=143406752)
+2600 val loss 8.1017
+2600 val perplexity 3300.1357
+2600 train 8.056396 (lr=5.6135e-05) (hash(x)=157272496)
+2700 val loss 8.1024
+2700 val perplexity 3302.2388
+2700 train 8.167391 (lr=5.5322e-05) (hash(x)=155342327)
+2800 val loss 8.0977
+2800 val perplexity 3286.7620
+2800 train 8.009883 (lr=5.4480e-05) (hash(x)=140626679)
+2900 val loss 8.0872
+2900 val perplexity 3252.4565
+2900 train 8.019983 (lr=5.3611e-05) (hash(x)=144953350)
+3000 val loss 8.0163
+3000 val perplexity 3029.8655
+3000 train 8.044260 (lr=5.2717e-05) (hash(x)=172449837)
+3100 val loss 7.9766
+3100 val perplexity 2912.0151
+3100 train 7.923613 (lr=5.1797e-05) (hash(x)=141710086)
+3200 val loss 7.8835
+3200 val perplexity 2653.0972
+3200 train 7.833907 (lr=5.0855e-05) (hash(x)=151299772)
+3300 val loss 7.9947
+3300 val perplexity 2965.3137
+3300 train 7.943204 (lr=4.9890e-05) (hash(x)=146473110)
+3400 val loss 8.4858
+3400 val perplexity 4845.6655
+3400 train 8.512304 (lr=4.8905e-05) (hash(x)=153954157)
+3500 val loss 8.1640
+3500 val perplexity 3512.3433
+3500 train 8.084970 (lr=4.7901e-05) (hash(x)=153717336)
+3600 val loss 7.8892
+3600 val perplexity 2668.4158
+3600 train 7.760597 (lr=4.6879e-05) (hash(x)=144965161)
+3700 val loss 7.8951
+3700 val perplexity 2683.9666
+3700 train 7.739192 (lr=4.5841e-05) (hash(x)=125969741)
+3800 val loss 7.8924
+3800 val perplexity 2676.9570
+3800 train 7.849920 (lr=4.4789e-05) (hash(x)=155070487)
+3900 val loss 7.8898
+3900 val perplexity 2669.9304
+3900 train 7.824447 (lr=4.3723e-05) (hash(x)=149444644)
+4000 val loss 7.8883
+4000 val perplexity 2665.8418
+4000 train 7.867583 (lr=4.2646e-05) (hash(x)=151663033)
+4100 val loss 7.8937
+4100 val perplexity 2680.3496
+4100 train 7.945187 (lr=4.1559e-05) (hash(x)=143688282)
+4200 val loss 7.9070
+4200 val perplexity 2716.1353
+4200 train 7.881064 (lr=4.0463e-05) (hash(x)=163361651)
+4300 val loss 7.9711
+4300 val perplexity 2895.9800
+4300 train 8.173737 (lr=3.9361e-05) (hash(x)=153619361)
+4400 val loss 8.0445
+4400 val perplexity 3116.7263
+4400 train 8.247282 (lr=3.8253e-05) (hash(x)=168527064)
+4500 val loss 8.0043
+4500 val perplexity 2993.6870
+4500 train 7.923067 (lr=3.7142e-05) (hash(x)=125588037)
+4600 val loss 7.9555
+4600 val perplexity 2851.1296
+4600 train 7.884278 (lr=3.6028e-05) (hash(x)=143710941)
+4700 val loss 7.9828
+4700 val perplexity 2930.0696
+4700 train 7.785313 (lr=3.4915e-05) (hash(x)=150952742)
+4800 val loss 7.9951
+4800 val perplexity 2966.5115
+4800 train 7.926626 (lr=3.3802e-05) (hash(x)=145323659)
+4900 val loss 7.9859
+4900 val perplexity 2939.3459
+4900 train 8.019696 (lr=3.2693e-05) (hash(x)=153151397)
+5000 val loss 7.9958
+5000 val perplexity 2968.5151
+5000 train 7.941575 (lr=3.1587e-05) (hash(x)=143182059)
+5100 val loss 8.0145
+5100 val perplexity 3024.3630
+5100 train 8.088722 (lr=3.0488e-05) (hash(x)=170083586)
+5200 val loss 8.0061
+5200 val perplexity 2999.1907
+5200 train 7.966730 (lr=2.9396e-05) (hash(x)=149363919)
+5300 val loss 7.9882
+5300 val perplexity 2945.8972
+5300 train 8.058408 (lr=2.8314e-05) (hash(x)=152033784)
+5400 val loss 7.9946
+5400 val perplexity 2964.8301
+5400 train 7.970897 (lr=2.7243e-05) (hash(x)=154614289)
+5500 val loss 7.9752
+5500 val perplexity 2908.0273
+5500 train 8.044887 (lr=2.6183e-05) (hash(x)=157745174)
+5600 val loss 7.9756
+5600 val perplexity 2909.2229
+5600 train 7.904576 (lr=2.5138e-05) (hash(x)=147693222)
+5700 val loss 7.9916
+5700 val perplexity 2956.0779
+5700 train 7.946126 (lr=2.4108e-05) (hash(x)=149784627)
+5800 val loss 7.9862
+5800 val perplexity 2940.0593
+5800 train 7.966474 (lr=2.3095e-05) (hash(x)=158620729)
+5900 val loss 7.9863
+5900 val perplexity 2940.3369
+5900 train 7.934534 (lr=2.2100e-05) (hash(x)=159763910)
+6000 val loss 7.9842
+6000 val perplexity 2934.0959
+6000 train 7.948881 (lr=2.1125e-05) (hash(x)=147640561)
+6100 val loss 7.9836
+6100 val perplexity 2932.4470
+6100 train 8.000985 (lr=2.0171e-05) (hash(x)=156613394)
+6200 val loss 7.9820
+6200 val perplexity 2927.7441
+6200 train 8.094741 (lr=1.9240e-05) (hash(x)=186221290)
+6300 val loss 7.9726
+6300 val perplexity 2900.4521
+6300 train 7.913179 (lr=1.8333e-05) (hash(x)=152081419)
+6400 val loss 7.9694
+6400 val perplexity 2891.0530
+6400 train 7.934944 (lr=1.7451e-05) (hash(x)=154808349)
+6500 val loss 7.9702
+6500 val perplexity 2893.5176
+6500 train 7.933500 (lr=1.6595e-05) (hash(x)=159437208)
+6600 val loss 7.9693
+6600 val perplexity 2890.9248
+6600 train 7.889696 (lr=1.5768e-05) (hash(x)=157933074)
+6700 val loss 7.9663
+6700 val perplexity 2882.1763
+6700 train 7.967562 (lr=1.4969e-05) (hash(x)=161560240)
+6800 val loss 7.9649
+6800 val perplexity 2878.0974
+6800 train 7.919674 (lr=1.4200e-05) (hash(x)=155424292)
+6900 val loss 7.9680
+6900 val perplexity 2887.1501
+6900 train 7.981733 (lr=1.3463e-05) (hash(x)=148561470)
+7000 val loss 7.9681
+7000 val perplexity 2887.2878
+7000 train 7.919036 (lr=1.2758e-05) (hash(x)=141527450)
+7100 val loss 7.9673
+7100 val perplexity 2885.0984
+7100 train 7.935656 (lr=1.2086e-05) (hash(x)=151066339)
+7200 val loss 7.9660
+7200 val perplexity 2881.3547
+7200 train 7.941572 (lr=1.1449e-05) (hash(x)=155231264)
+7300 val loss 7.9715
+7300 val perplexity 2897.1182
+7300 train 7.985583 (lr=1.0847e-05) (hash(x)=150281149)
+7400 val loss 7.9754
+7400 val perplexity 2908.4988
+7400 train 7.903443 (lr=1.0281e-05) (hash(x)=148421717)
+7500 val loss 7.9762
+7500 val perplexity 2910.7102
+7500 train 7.860454 (lr=9.7516e-06) (hash(x)=146921118)
+7600 val loss 7.9743
+7600 val perplexity 2905.2000
+7600 train 8.032375 (lr=9.2601e-06) (hash(x)=150660048)
+7700 val loss 7.9730
+7700 val perplexity 2901.6653
+7700 train 7.931540 (lr=8.8071e-06) (hash(x)=148059852)
+7800 val loss 7.9703
+7800 val perplexity 2893.7549
+7800 train 7.814187 (lr=8.3932e-06) (hash(x)=148331002)
+7900 val loss 7.9673
+7900 val perplexity 2885.0391
+7900 train 7.950665 (lr=8.0189e-06) (hash(x)=164923883)
+8000 val loss 7.9660
+8000 val perplexity 2881.2859
+8000 train 7.895516 (lr=7.6848e-06) (hash(x)=143545384)
+8100 val loss 7.9662
+8100 val perplexity 2881.8630
+8100 train 7.954533 (lr=7.3914e-06) (hash(x)=160686959)
+8200 val loss 7.9715
+8200 val perplexity 2897.3062
+8200 train 8.036483 (lr=7.1392e-06) (hash(x)=156501889)
+8300 val loss 7.9715
+8300 val perplexity 2897.2632
+8300 train 7.917995 (lr=6.9284e-06) (hash(x)=142716875)
+8400 val loss 7.9713
+8400 val perplexity 2896.5657
+8400 train 7.890598 (lr=6.7594e-06) (hash(x)=154436684)
+8500 val loss 7.9706
+8500 val perplexity 2894.6794
+8500 train 7.980845 (lr=6.6324e-06) (hash(x)=147965839)
+8600 val loss 7.9703
+8600 val perplexity 2893.7949
+8600 train 7.913826 (lr=6.5477e-06) (hash(x)=145228097)
+8700 val loss 7.9707
+8700 val perplexity 2894.9004
+8700 train 8.076840 (lr=6.5053e-06) (hash(x)=152910357)
+8749 val loss 7.9716
+8749 val perplexity 2897.3530

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6adc489c47a357b95515ca78f56668d7b9f54ab9ef59cfbc006a438e3328f915
+size 95220594

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3f514235c7cdbf64a4d13151db52cc1355bf6d8ea84561725791939fd40aab3
+size 184163894