Upload folder using huggingface_hub

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "n_latent_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": 2, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_1338", "n_embd": 256}

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.2524
+0 val perplexity 77061.3750
+0 train 11.253548 (lr=1.2000e-07) (hash(x)=150327452)
+100 val loss 10.0800
+100 val perplexity 23860.5059
+100 train 10.110741 (lr=1.2120e-05) (hash(x)=166780046)
+200 val loss 9.8451
+200 val perplexity 18865.0098
+200 train 9.856663 (lr=2.4120e-05) (hash(x)=155040610)
+300 val loss 9.6589
+300 val perplexity 15660.1348
+300 train 9.699568 (lr=3.6120e-05) (hash(x)=155504036)
+400 val loss 9.3308
+400 val perplexity 11280.6152
+400 train 9.324850 (lr=4.8120e-05) (hash(x)=143823248)
+500 val loss 9.0827
+500 val perplexity 8801.6543
+500 train 9.094957 (lr=6.0000e-05) (hash(x)=143734685)
+600 val loss 8.9223
+600 val perplexity 7497.6660
+600 train 8.886049 (lr=5.9980e-05) (hash(x)=150678249)
+700 val loss 8.6511
+700 val perplexity 5716.1870
+700 train 8.739485 (lr=5.9922e-05) (hash(x)=175802021)
+800 val loss 8.4693
+800 val perplexity 4766.2109
+800 train 8.462464 (lr=5.9824e-05) (hash(x)=158681215)
+900 val loss 8.6006
+900 val perplexity 5435.0195
+900 train 8.579044 (lr=5.9687e-05) (hash(x)=146108145)
+1000 val loss 8.3132
+1000 val perplexity 4077.5107
+1000 train 8.297157 (lr=5.9512e-05) (hash(x)=154996086)
+1100 val loss 8.2820
+1100 val perplexity 3952.1191
+1100 train 8.307036 (lr=5.9298e-05) (hash(x)=153885445)
+1200 val loss 8.2964
+1200 val perplexity 4009.6040
+1200 train 8.308805 (lr=5.9046e-05) (hash(x)=142353087)
+1300 val loss 8.2845
+1300 val perplexity 3961.9912
+1300 train 8.296026 (lr=5.8757e-05) (hash(x)=150750353)
+1400 val loss 8.2955
+1400 val perplexity 4005.9958
+1400 train 8.237262 (lr=5.8430e-05) (hash(x)=152767913)
+1500 val loss 8.2893
+1500 val perplexity 3981.2324
+1500 train 8.343046 (lr=5.8066e-05) (hash(x)=151562048)
+1600 val loss 8.2925
+1600 val perplexity 3993.6597
+1600 train 8.273527 (lr=5.7666e-05) (hash(x)=166486165)
+1700 val loss 8.3093
+1700 val perplexity 4061.3157
+1700 train 8.232641 (lr=5.7230e-05) (hash(x)=130835396)
+1800 val loss 8.2637
+1800 val perplexity 3880.2791
+1800 train 8.302708 (lr=5.6759e-05) (hash(x)=158851816)
+1900 val loss 8.2549
+1900 val perplexity 3846.5444
+1900 train 8.269764 (lr=5.6253e-05) (hash(x)=153313879)
+2000 val loss 8.2642
+2000 val perplexity 3882.2075
+2000 train 8.243088 (lr=5.5714e-05) (hash(x)=158245023)
+2100 val loss 8.2627
+2100 val perplexity 3876.6284
+2100 train 8.180153 (lr=5.5142e-05) (hash(x)=157204896)
+2200 val loss 8.2419
+2200 val perplexity 3796.5608
+2200 train 8.225559 (lr=5.4537e-05) (hash(x)=137541932)
+2300 val loss 8.2622
+2300 val perplexity 3874.6990
+2300 train 8.268251 (lr=5.3902e-05) (hash(x)=150149692)
+2400 val loss 8.3845
+2400 val perplexity 4378.5288
+2400 train 8.434961 (lr=5.3236e-05) (hash(x)=151730720)
+2500 val loss 8.2590
+2500 val perplexity 3862.1814
+2500 train 8.027147 (lr=5.2541e-05) (hash(x)=143406752)
+2600 val loss 8.2304
+2600 val perplexity 3753.4353
+2600 train 8.191346 (lr=5.1817e-05) (hash(x)=157272496)
+2700 val loss 8.2296
+2700 val perplexity 3750.4656
+2700 train 8.293442 (lr=5.1067e-05) (hash(x)=155342327)
+2800 val loss 8.2184
+2800 val perplexity 3708.7090
+2800 train 8.141396 (lr=5.0290e-05) (hash(x)=140626679)
+2900 val loss 8.2138
+2900 val perplexity 3691.7178
+2900 train 8.162292 (lr=4.9487e-05) (hash(x)=144953350)
+3000 val loss 8.2398
+3000 val perplexity 3788.6865
+3000 train 8.280184 (lr=4.8662e-05) (hash(x)=172449837)
+3100 val loss 8.2512
+3100 val perplexity 3832.2061
+3100 train 8.230724 (lr=4.7813e-05) (hash(x)=141710086)
+3200 val loss 8.2153
+3200 val perplexity 3697.0061
+3200 train 8.172878 (lr=4.6943e-05) (hash(x)=151299772)
+3300 val loss 8.2158
+3300 val perplexity 3698.9880
+3300 train 8.188112 (lr=4.6052e-05) (hash(x)=146473110)
+3400 val loss 8.2062
+3400 val perplexity 3663.7258
+3400 train 8.266872 (lr=4.5143e-05) (hash(x)=153954157)
+3500 val loss 8.2016
+3500 val perplexity 3646.8433
+3500 train 8.141716 (lr=4.4216e-05) (hash(x)=153717336)
+3600 val loss 8.2424
+3600 val perplexity 3798.6143
+3600 train 8.138775 (lr=4.3273e-05) (hash(x)=144965161)
+3700 val loss 8.1980
+3700 val perplexity 3633.8105
+3700 train 8.066285 (lr=4.2315e-05) (hash(x)=125969741)
+3800 val loss 8.1957
+3800 val perplexity 3625.3232
+3800 train 8.171384 (lr=4.1343e-05) (hash(x)=155070487)
+3900 val loss 8.1979
+3900 val perplexity 3633.4087
+3900 train 8.142708 (lr=4.0360e-05) (hash(x)=149444644)
+4000 val loss 8.1966
+4000 val perplexity 3628.7200
+4000 train 8.213164 (lr=3.9365e-05) (hash(x)=151663033)
+4100 val loss 8.1984
+4100 val perplexity 3635.1208
+4100 train 8.284185 (lr=3.8362e-05) (hash(x)=143688282)
+4200 val loss 8.2049
+4200 val perplexity 3658.7188
+4200 train 8.179093 (lr=3.7351e-05) (hash(x)=163361651)
+4300 val loss 8.1999
+4300 val perplexity 3640.7378
+4300 train 8.385876 (lr=3.6333e-05) (hash(x)=153619361)
+4400 val loss 8.2212
+4400 val perplexity 3718.9766
+4400 train 8.387505 (lr=3.5311e-05) (hash(x)=168527064)
+4500 val loss 8.2074
+4500 val perplexity 3668.1589
+4500 train 8.171785 (lr=3.4285e-05) (hash(x)=125588037)
+4600 val loss 8.1562
+4600 val perplexity 3484.7673
+4600 train 8.087983 (lr=3.3257e-05) (hash(x)=143710941)
+4700 val loss 8.1115
+4700 val perplexity 3332.6169
+4700 train 7.910356 (lr=3.2229e-05) (hash(x)=150952742)
+4800 val loss 8.0541
+4800 val perplexity 3146.7600
+4800 train 7.988973 (lr=3.1202e-05) (hash(x)=145323659)
+4900 val loss 8.0506
+4900 val perplexity 3135.6790
+4900 train 8.083582 (lr=3.0178e-05) (hash(x)=153151397)
+5000 val loss 8.0310
+5000 val perplexity 3074.7332
+5000 train 7.980440 (lr=2.9157e-05) (hash(x)=143182059)
+5100 val loss 8.0517
+5100 val perplexity 3139.2427
+5100 train 8.127379 (lr=2.8143e-05) (hash(x)=170083586)
+5200 val loss 8.0293
+5200 val perplexity 3069.6060
+5200 train 8.001753 (lr=2.7135e-05) (hash(x)=149363919)
+5300 val loss 8.0128
+5300 val perplexity 3019.3169
+5300 train 8.077737 (lr=2.6136e-05) (hash(x)=152033784)
+5400 val loss 8.0620
+5400 val perplexity 3171.7368
+5400 train 8.036554 (lr=2.5147e-05) (hash(x)=154614289)
+5500 val loss 8.0108
+5500 val perplexity 3013.4399
+5500 train 8.097231 (lr=2.4169e-05) (hash(x)=157745174)
+5600 val loss 8.0236
+5600 val perplexity 3052.2285
+5600 train 7.957674 (lr=2.3204e-05) (hash(x)=147693222)
+5700 val loss 8.0121
+5700 val perplexity 3017.2300
+5700 train 7.969118 (lr=2.2253e-05) (hash(x)=149784627)
+5800 val loss 8.0002
+5800 val perplexity 2981.4414
+5800 train 7.991749 (lr=2.1318e-05) (hash(x)=158620729)
+5900 val loss 7.9887
+5900 val perplexity 2947.3838
+5900 train 7.935953 (lr=2.0400e-05) (hash(x)=159763910)
+6000 val loss 7.9742
+6000 val perplexity 2905.1582
+6000 train 7.939895 (lr=1.9500e-05) (hash(x)=147640561)
+6100 val loss 7.9775
+6100 val perplexity 2914.5393
+6100 train 8.000101 (lr=1.8620e-05) (hash(x)=156613394)
+6200 val loss 7.9554
+6200 val perplexity 2850.8755
+6200 train 8.084249 (lr=1.7760e-05) (hash(x)=186221290)
+6300 val loss 7.9477
+6300 val perplexity 2828.9556
+6300 train 7.887838 (lr=1.6923e-05) (hash(x)=152081419)
+6400 val loss 7.9432
+6400 val perplexity 2816.4341
+6400 train 7.907034 (lr=1.6108e-05) (hash(x)=154808349)
+6500 val loss 7.9465
+6500 val perplexity 2825.6970
+6500 train 7.908285 (lr=1.5319e-05) (hash(x)=159437208)
+6600 val loss 7.9413
+6600 val perplexity 2811.1423
+6600 train 7.853995 (lr=1.4555e-05) (hash(x)=157933074)
+6700 val loss 7.9418
+6700 val perplexity 2812.2805
+6700 train 7.949341 (lr=1.3817e-05) (hash(x)=161560240)
+6800 val loss 7.9350
+6800 val perplexity 2793.3040
+6800 train 7.891088 (lr=1.3108e-05) (hash(x)=155424292)
+6900 val loss 7.9357
+6900 val perplexity 2795.4253
+6900 train 7.958529 (lr=1.2427e-05) (hash(x)=148561470)
+7000 val loss 7.9329
+7000 val perplexity 2787.5876
+7000 train 7.877773 (lr=1.1777e-05) (hash(x)=141527450)
+7100 val loss 7.9321
+7100 val perplexity 2785.2307
+7100 train 7.904116 (lr=1.1157e-05) (hash(x)=151066339)
+7200 val loss 7.9393
+7200 val perplexity 2805.4858
+7200 train 7.909233 (lr=1.0568e-05) (hash(x)=155231264)
+7300 val loss 7.9481
+7300 val perplexity 2830.2332
+7300 train 7.962085 (lr=1.0012e-05) (hash(x)=150281149)
+7400 val loss 7.9688
+7400 val perplexity 2889.2656
+7400 train 7.895494 (lr=9.4899e-06) (hash(x)=148421717)
+7500 val loss 7.9474
+7500 val perplexity 2828.2634
+7500 train 7.826111 (lr=9.0014e-06) (hash(x)=146921118)
+7600 val loss 7.9379
+7600 val perplexity 2801.5449
+7600 train 8.004966 (lr=8.5478e-06) (hash(x)=150660048)
+7700 val loss 7.9298
+7700 val perplexity 2778.9531
+7700 train 7.889890 (lr=8.1297e-06) (hash(x)=148059852)
+7800 val loss 7.9186
+7800 val perplexity 2747.9478
+7800 train 7.761279 (lr=7.7476e-06) (hash(x)=148331002)
+7900 val loss 7.9113
+7900 val perplexity 2728.0078
+7900 train 7.897078 (lr=7.4021e-06) (hash(x)=164923883)
+8000 val loss 7.9089
+8000 val perplexity 2721.4011
+8000 train 7.841110 (lr=7.0937e-06) (hash(x)=143545384)
+8100 val loss 7.9112
+8100 val perplexity 2727.7437
+8100 train 7.907278 (lr=6.8229e-06) (hash(x)=160686959)
+8200 val loss 7.9073
+8200 val perplexity 2717.0005
+8200 train 7.982006 (lr=6.5900e-06) (hash(x)=156501889)
+8300 val loss 7.9087
+8300 val perplexity 2720.9185
+8300 train 7.858608 (lr=6.3954e-06) (hash(x)=142716875)
+8400 val loss 7.9064
+8400 val perplexity 2714.7188
+8400 train 7.816538 (lr=6.2395e-06) (hash(x)=154436684)
+8500 val loss 7.9057
+8500 val perplexity 2712.7000
+8500 train 7.910065 (lr=6.1223e-06) (hash(x)=147965839)
+8600 val loss 7.9059
+8600 val perplexity 2713.3364
+8600 train 7.847117 (lr=6.0440e-06) (hash(x)=145228097)
+8700 val loss 7.9046
+8700 val perplexity 2709.6787
+8700 train 8.005460 (lr=6.0049e-06) (hash(x)=152910357)
+8749 val loss 7.9064
+8749 val perplexity 2714.6890

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72ac3fd834e67725f07b094d453c8dd81bab6d2c285179aeb657cf088dc5ba38
+size 95220594

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1338/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6073bbe3959570287e57667148c646cacc12ab4898e35772d6978b9c47a6086e
+size 184163894