Upload folder using huggingface_hub

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "n_latent_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": 2, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6.5e-5_61440_1340", "n_embd": 256}

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.2917
+0 val perplexity 80150.6406
+0 train 11.290673 (lr=1.3000e-07) (hash(x)=164406924)
+100 val loss 10.0612
+100 val perplexity 23415.5879
+100 train 10.002015 (lr=1.3130e-05) (hash(x)=144903932)
+200 val loss 9.8104
+200 val perplexity 18222.2051
+200 train 9.859276 (lr=2.6130e-05) (hash(x)=167734596)
+300 val loss 9.5317
+300 val perplexity 13789.5029
+300 train 9.566351 (lr=3.9130e-05) (hash(x)=149619098)
+400 val loss 8.9619
+400 val perplexity 7800.1025
+400 train 8.939888 (lr=5.2130e-05) (hash(x)=153710890)
+500 val loss 8.7995
+500 val perplexity 6630.6650
+500 train 8.812294 (lr=6.5000e-05) (hash(x)=145450636)
+600 val loss 8.5685
+600 val perplexity 5263.2920
+600 train 8.455111 (lr=6.4979e-05) (hash(x)=145249251)
+700 val loss 8.2392
+700 val perplexity 3786.5554
+700 train 8.329495 (lr=6.4915e-05) (hash(x)=150475545)
+800 val loss 8.1019
+800 val perplexity 3300.7969
+800 train 8.114813 (lr=6.4809e-05) (hash(x)=144483776)
+900 val loss 8.0912
+900 val perplexity 3265.4976
+900 train 8.169922 (lr=6.4661e-05) (hash(x)=157916369)
+1000 val loss 8.0539
+1000 val perplexity 3146.1748
+1000 train 8.126588 (lr=6.4471e-05) (hash(x)=154856891)
+1100 val loss 8.0576
+1100 val perplexity 3157.7146
+1100 train 7.964921 (lr=6.4240e-05) (hash(x)=136191502)
+1200 val loss 7.9831
+1200 val perplexity 2930.9275
+1200 train 7.937511 (lr=6.3967e-05) (hash(x)=148021541)
+1300 val loss 7.9902
+1300 val perplexity 2951.8706
+1300 train 7.993743 (lr=6.3653e-05) (hash(x)=146005217)
+1400 val loss 8.5616
+1400 val perplexity 5227.2754
+1400 train 8.533630 (lr=6.3299e-05) (hash(x)=146019502)
+1500 val loss 7.9889
+1500 val perplexity 2948.1091
+1500 train 7.941971 (lr=6.2905e-05) (hash(x)=150127281)
+1600 val loss 7.9679
+1600 val perplexity 2886.8596
+1600 train 7.973657 (lr=6.2471e-05) (hash(x)=154120875)
+1700 val loss 7.8902
+1700 val perplexity 2670.8550
+1700 train 8.106688 (lr=6.1999e-05) (hash(x)=155797680)
+1800 val loss 7.9440
+1800 val perplexity 2818.7368
+1800 train 7.975287 (lr=6.1489e-05) (hash(x)=156809396)
+1900 val loss 7.8997
+1900 val perplexity 2696.4197
+1900 train 7.832520 (lr=6.0941e-05) (hash(x)=144640294)
+2000 val loss 7.9872
+2000 val perplexity 2943.0652
+2000 train 8.002056 (lr=6.0357e-05) (hash(x)=162831106)
+2100 val loss 7.8656
+2100 val perplexity 2606.0242
+2100 train 7.849998 (lr=5.9737e-05) (hash(x)=158239484)
+2200 val loss 7.8387
+2200 val perplexity 2536.9612
+2200 train 7.752504 (lr=5.9082e-05) (hash(x)=140504180)
+2300 val loss 7.8613
+2300 val perplexity 2595.0166
+2300 train 7.852221 (lr=5.8394e-05) (hash(x)=142234024)
+2400 val loss 7.8809
+2400 val perplexity 2646.3701
+2400 train 7.923631 (lr=5.7672e-05) (hash(x)=143091562)
+2500 val loss 7.8435
+2500 val perplexity 2549.0459
+2500 train 7.841086 (lr=5.6919e-05) (hash(x)=149857456)
+2600 val loss 7.9218
+2600 val perplexity 2756.6475
+2600 train 7.979512 (lr=5.6135e-05) (hash(x)=146191551)
+2700 val loss 7.8216
+2700 val perplexity 2493.8472
+2700 train 7.787008 (lr=5.5322e-05) (hash(x)=145375752)
+2800 val loss 7.8695
+2800 val perplexity 2616.2588
+2800 train 7.828942 (lr=5.4480e-05) (hash(x)=151568014)
+2900 val loss 7.8925
+2900 val perplexity 2677.0925
+2900 train 7.830199 (lr=5.3611e-05) (hash(x)=149366597)
+3000 val loss 7.9695
+3000 val perplexity 2891.3894
+3000 train 7.955544 (lr=5.2717e-05) (hash(x)=150464442)
+3100 val loss 8.1594
+3100 val perplexity 3495.9316
+3100 train 8.201312 (lr=5.1797e-05) (hash(x)=182449036)
+3200 val loss 7.8876
+3200 val perplexity 2663.9917
+3200 train 7.885420 (lr=5.0855e-05) (hash(x)=140141286)
+3300 val loss 7.8671
+3300 val perplexity 2609.9590
+3300 train 7.851835 (lr=4.9890e-05) (hash(x)=148099414)
+3400 val loss 7.8525
+3400 val perplexity 2572.1321
+3400 train 7.770096 (lr=4.8905e-05) (hash(x)=142633951)
+3500 val loss 7.8538
+3500 val perplexity 2575.4309
+3500 train 7.819212 (lr=4.7901e-05) (hash(x)=148368965)
+3600 val loss 7.8476
+3600 val perplexity 2559.6572
+3600 train 7.861516 (lr=4.6879e-05) (hash(x)=152372067)
+3700 val loss 7.8433
+3700 val perplexity 2548.7153
+3700 train 7.984025 (lr=4.5841e-05) (hash(x)=168885609)
+3800 val loss 7.8526
+3800 val perplexity 2572.4189
+3800 train 7.750323 (lr=4.4789e-05) (hash(x)=141633734)
+3900 val loss 7.8335
+3900 val perplexity 2523.6289
+3900 train 7.798890 (lr=4.3723e-05) (hash(x)=153141007)
+4000 val loss 7.8894
+4000 val perplexity 2668.7644
+4000 train 7.909665 (lr=4.2646e-05) (hash(x)=160577202)
+4100 val loss 7.8610
+4100 val perplexity 2594.2148
+4100 train 7.929330 (lr=4.1559e-05) (hash(x)=153858169)
+4200 val loss 7.8881
+4200 val perplexity 2665.3486
+4200 train 7.894275 (lr=4.0463e-05) (hash(x)=155889149)
+4300 val loss 7.8585
+4300 val perplexity 2587.7163
+4300 train 7.792262 (lr=3.9361e-05) (hash(x)=152294662)
+4400 val loss 7.8614
+4400 val perplexity 2595.1216
+4400 train 7.778205 (lr=3.8253e-05) (hash(x)=141804386)
+4500 val loss 7.8521
+4500 val perplexity 2571.0898
+4500 train 7.855331 (lr=3.7142e-05) (hash(x)=151095242)
+4600 val loss 7.8534
+4600 val perplexity 2574.5100
+4600 train 7.792521 (lr=3.6028e-05) (hash(x)=156414699)
+4700 val loss 7.8527
+4700 val perplexity 2572.7097
+4700 train 7.890131 (lr=3.4915e-05) (hash(x)=161556686)
+4800 val loss 7.8517
+4800 val perplexity 2570.0994
+4800 train 7.906765 (lr=3.3802e-05) (hash(x)=149000293)
+4900 val loss 7.8500
+4900 val perplexity 2565.7976
+4900 train 7.759993 (lr=3.2693e-05) (hash(x)=154349989)
+5000 val loss 7.8443
+5000 val perplexity 2551.0303
+5000 train 7.697754 (lr=3.1587e-05) (hash(x)=131475967)
+5100 val loss 7.8353
+5100 val perplexity 2528.2686
+5100 train 7.814276 (lr=3.0488e-05) (hash(x)=149717902)
+5200 val loss 7.8317
+5200 val perplexity 2519.0830
+5200 train 7.752923 (lr=2.9396e-05) (hash(x)=151407999)
+5300 val loss 7.8376
+5300 val perplexity 2534.0232
+5300 train 7.947274 (lr=2.8314e-05) (hash(x)=168602728)
+5400 val loss 7.8290
+5400 val perplexity 2512.4731
+5400 train 7.835303 (lr=2.7243e-05) (hash(x)=158344511)
+5500 val loss 7.8480
+5500 val perplexity 2560.5068
+5500 train 7.866498 (lr=2.6183e-05) (hash(x)=148350057)
+5600 val loss 7.8369
+5600 val perplexity 2532.3057
+5600 train 7.868742 (lr=2.5138e-05) (hash(x)=153847323)
+5700 val loss 7.8335
+5700 val perplexity 2523.8564
+5700 train 7.789353 (lr=2.4108e-05) (hash(x)=156607405)
+5800 val loss 7.8339
+5800 val perplexity 2524.7627
+5800 train 7.701171 (lr=2.3095e-05) (hash(x)=145115031)
+5900 val loss 7.8317
+5900 val perplexity 2519.2019
+5900 train 7.687846 (lr=2.2100e-05) (hash(x)=141584622)
+6000 val loss 7.8333
+6000 val perplexity 2523.2859
+6000 train 7.774045 (lr=2.1125e-05) (hash(x)=146613857)
+6100 val loss 7.8249
+6100 val perplexity 2502.0540
+6100 train 7.692036 (lr=2.0171e-05) (hash(x)=144621768)
+6200 val loss 7.8198
+6200 val perplexity 2489.4275
+6200 train 7.741303 (lr=1.9240e-05) (hash(x)=146521760)
+6300 val loss 7.8210
+6300 val perplexity 2492.4099
+6300 train 7.744120 (lr=1.8333e-05) (hash(x)=161378136)
+6400 val loss 7.8230
+6400 val perplexity 2497.4231
+6400 train 7.693834 (lr=1.7451e-05) (hash(x)=141624235)
+6500 val loss 7.8205
+6500 val perplexity 2491.0566
+6500 train 7.745716 (lr=1.6595e-05) (hash(x)=151197095)
+6600 val loss 7.8241
+6600 val perplexity 2500.2078
+6600 train 7.792052 (lr=1.5768e-05) (hash(x)=153269571)
+6700 val loss 7.8358
+6700 val perplexity 2529.4888
+6700 train 7.806519 (lr=1.4969e-05) (hash(x)=146111181)
+6800 val loss 7.8188
+6800 val perplexity 2486.9253
+6800 train 7.821423 (lr=1.4200e-05) (hash(x)=147269760)
+6900 val loss 7.8186
+6900 val perplexity 2486.3750
+6900 train 7.832389 (lr=1.3463e-05) (hash(x)=152912762)
+7000 val loss 7.8205
+7000 val perplexity 2491.1897
+7000 train 7.805366 (lr=1.2758e-05) (hash(x)=165412343)
+7100 val loss 7.8147
+7100 val perplexity 2476.7205
+7100 train 7.871611 (lr=1.2086e-05) (hash(x)=162866028)
+7200 val loss 7.8146
+7200 val perplexity 2476.4844
+7200 train 7.786273 (lr=1.1449e-05) (hash(x)=142998115)
+7300 val loss 7.8162
+7300 val perplexity 2480.4150
+7300 train 7.703725 (lr=1.0847e-05) (hash(x)=145486999)
+7400 val loss 7.8152
+7400 val perplexity 2477.8699
+7400 train 7.814345 (lr=1.0281e-05) (hash(x)=155325873)
+7500 val loss 7.8125
+7500 val perplexity 2471.2209
+7500 train 7.739144 (lr=9.7516e-06) (hash(x)=145131256)
+7600 val loss 7.8148
+7600 val perplexity 2476.9639
+7600 train 7.745480 (lr=9.2601e-06) (hash(x)=144008365)
+7700 val loss 7.8145
+7700 val perplexity 2476.2576
+7700 train 7.712640 (lr=8.8071e-06) (hash(x)=148848532)
+7800 val loss 7.8197
+7800 val perplexity 2489.1853
+7800 train 7.799828 (lr=8.3932e-06) (hash(x)=150391642)
+7900 val loss 7.8162
+7900 val perplexity 2480.4729
+7900 train 7.713821 (lr=8.0189e-06) (hash(x)=152191414)
+8000 val loss 7.8112
+8000 val perplexity 2468.1802
+8000 train 7.883222 (lr=7.6848e-06) (hash(x)=159755587)
+8100 val loss 7.8109
+8100 val perplexity 2467.4612
+8100 train 7.849056 (lr=7.3914e-06) (hash(x)=156664468)
+8200 val loss 7.8100
+8200 val perplexity 2465.1514
+8200 train 7.639417 (lr=7.1392e-06) (hash(x)=139457379)
+8300 val loss 7.8133
+8300 val perplexity 2473.2366
+8300 train 7.702578 (lr=6.9284e-06) (hash(x)=145478564)
+8400 val loss 7.8126
+8400 val perplexity 2471.6570
+8400 train 7.812517 (lr=6.7594e-06) (hash(x)=154982769)
+8500 val loss 7.8156
+8500 val perplexity 2478.9004
+8500 train 7.719147 (lr=6.6324e-06) (hash(x)=145798118)
+8600 val loss 7.8100
+8600 val perplexity 2465.0820
+8600 train 7.763993 (lr=6.5477e-06) (hash(x)=143231551)
+8700 val loss 7.8123
+8700 val perplexity 2470.9180
+8700 train 7.753017 (lr=6.5053e-06) (hash(x)=154780112)
+8749 val loss 7.8173
+8749 val perplexity 2483.2791

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe20bdd475b465aa249b6d490c6aca55521db70c5e2b0f0e4ce952b2fad3bcd5
+size 95220594

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b93d1224ea167e84e2b31be9af2c5bd14b30e6e62d656f4f6b52f082715ad9c7
+size 184163894