Upload folder using huggingface_hub

lr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1340/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "two_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6.5e-5_61440_1340", "n_embd": 256}

lr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1340/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1340/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.2942
+0 val perplexity 80354.4531
+0 train 11.297427 (lr=1.3000e-07) (hash(x)=164406924)
+100 val loss 9.9602
+100 val perplexity 21167.5000
+100 train 9.922249 (lr=1.3130e-05) (hash(x)=144903932)
+200 val loss 9.2027
+200 val perplexity 9923.6943
+200 train 9.276415 (lr=2.6130e-05) (hash(x)=167734596)
+300 val loss 8.0725
+300 val perplexity 3205.0442
+300 train 8.064857 (lr=3.9130e-05) (hash(x)=149619098)
+400 val loss 7.6432
+400 val perplexity 2086.3328
+400 train 7.529013 (lr=5.2130e-05) (hash(x)=153710890)
+500 val loss 7.4805
+500 val perplexity 1773.0826
+500 train 7.574828 (lr=6.5000e-05) (hash(x)=145450636)
+600 val loss 7.3561
+600 val perplexity 1565.7214
+600 train 7.219141 (lr=6.4979e-05) (hash(x)=145249251)
+700 val loss 7.2710
+700 val perplexity 1437.9286
+700 train 7.385961 (lr=6.4915e-05) (hash(x)=150475545)
+800 val loss 7.1964
+800 val perplexity 1334.5865
+800 train 7.160237 (lr=6.4809e-05) (hash(x)=144483776)
+900 val loss 7.0939
+900 val perplexity 1204.5492
+900 train 7.199740 (lr=6.4661e-05) (hash(x)=157916369)
+1000 val loss 7.0196
+1000 val perplexity 1118.3572
+1000 train 7.040990 (lr=6.4471e-05) (hash(x)=154856891)
+1100 val loss 6.9414
+1100 val perplexity 1034.1772
+1100 train 6.641282 (lr=6.4240e-05) (hash(x)=136191502)
+1200 val loss 6.8438
+1200 val perplexity 938.0255
+1200 train 6.801001 (lr=6.3967e-05) (hash(x)=148021541)
+1300 val loss 6.7649
+1300 val perplexity 866.8417
+1300 train 6.698213 (lr=6.3653e-05) (hash(x)=146005217)
+1400 val loss 6.6939
+1400 val perplexity 807.4353
+1400 train 6.607660 (lr=6.3299e-05) (hash(x)=146019502)
+1500 val loss 6.6204
+1500 val perplexity 750.2204
+1500 train 6.559786 (lr=6.2905e-05) (hash(x)=150127281)
+1600 val loss 6.5560
+1600 val perplexity 703.4319
+1600 train 6.532626 (lr=6.2471e-05) (hash(x)=154120875)
+1700 val loss 6.5140
+1700 val perplexity 674.4991
+1700 train 6.733201 (lr=6.1999e-05) (hash(x)=155797680)
+1800 val loss 6.4441
+1800 val perplexity 629.0027
+1800 train 6.433759 (lr=6.1489e-05) (hash(x)=156809396)
+1900 val loss 6.3948
+1900 val perplexity 598.7380
+1900 train 6.281144 (lr=6.0941e-05) (hash(x)=144640294)
+2000 val loss 6.3530
+2000 val perplexity 574.2274
+2000 train 6.211150 (lr=6.0357e-05) (hash(x)=162831106)
+2100 val loss 6.3290
+2100 val perplexity 560.5960
+2100 train 6.303570 (lr=5.9737e-05) (hash(x)=158239484)
+2200 val loss 6.2809
+2200 val perplexity 534.2560
+2200 train 6.159258 (lr=5.9082e-05) (hash(x)=140504180)
+2300 val loss 6.2419
+2300 val perplexity 513.8290
+2300 train 6.236554 (lr=5.8394e-05) (hash(x)=142234024)
+2400 val loss 6.2104
+2400 val perplexity 497.8902
+2400 train 6.207381 (lr=5.7672e-05) (hash(x)=143091562)
+2500 val loss 6.2085
+2500 val perplexity 496.9310
+2500 train 6.110419 (lr=5.6919e-05) (hash(x)=149857456)
+2600 val loss 6.1579
+2600 val perplexity 472.4452
+2600 train 6.191381 (lr=5.6135e-05) (hash(x)=146191551)
+2700 val loss 6.1372
+2700 val perplexity 462.7452
+2700 train 6.054928 (lr=5.5322e-05) (hash(x)=145375752)
+2800 val loss 6.1068
+2800 val perplexity 448.8983
+2800 train 5.985631 (lr=5.4480e-05) (hash(x)=151568014)
+2900 val loss 6.0957
+2900 val perplexity 443.9410
+2900 train 5.923908 (lr=5.3611e-05) (hash(x)=149366597)
+3000 val loss 6.0562
+3000 val perplexity 426.7336
+3000 train 5.982791 (lr=5.2717e-05) (hash(x)=150464442)
+3100 val loss 6.0347
+3100 val perplexity 417.6824
+3100 train 6.148521 (lr=5.1797e-05) (hash(x)=182449036)
+3200 val loss 6.0185
+3200 val perplexity 410.9479
+3200 train 5.966038 (lr=5.0855e-05) (hash(x)=140141286)
+3300 val loss 5.9941
+3300 val perplexity 401.0497
+3300 train 5.969191 (lr=4.9890e-05) (hash(x)=148099414)
+3400 val loss 5.9742
+3400 val perplexity 393.1415
+3400 train 5.841685 (lr=4.8905e-05) (hash(x)=142633951)
+3500 val loss 5.9631
+3500 val perplexity 388.8141
+3500 train 5.815297 (lr=4.7901e-05) (hash(x)=148368965)
+3600 val loss 5.9317
+3600 val perplexity 376.7973
+3600 train 5.949997 (lr=4.6879e-05) (hash(x)=152372067)
+3700 val loss 5.9170
+3700 val perplexity 371.3018
+3700 train 5.998743 (lr=4.5841e-05) (hash(x)=168885609)
+3800 val loss 5.9008
+3800 val perplexity 365.3466
+3800 train 5.719118 (lr=4.4789e-05) (hash(x)=141633734)
+3900 val loss 5.8729
+3900 val perplexity 355.2605
+3900 train 5.766616 (lr=4.3723e-05) (hash(x)=153141007)
+4000 val loss 5.8642
+4000 val perplexity 352.2099
+4000 train 5.792525 (lr=4.2646e-05) (hash(x)=160577202)
+4100 val loss 5.8511
+4100 val perplexity 347.6018
+4100 train 5.915715 (lr=4.1559e-05) (hash(x)=153858169)
+4200 val loss 5.8308
+4200 val perplexity 340.6423
+4200 train 5.838648 (lr=4.0463e-05) (hash(x)=155889149)
+4300 val loss 5.8198
+4300 val perplexity 336.9162
+4300 train 5.632066 (lr=3.9361e-05) (hash(x)=152294662)
+4400 val loss 5.8072
+4400 val perplexity 332.6979
+4400 train 5.707955 (lr=3.8253e-05) (hash(x)=141804386)
+4500 val loss 5.7877
+4500 val perplexity 326.2760
+4500 train 5.683947 (lr=3.7142e-05) (hash(x)=151095242)
+4600 val loss 5.7924
+4600 val perplexity 327.8123
+4600 train 5.742775 (lr=3.6028e-05) (hash(x)=156414699)
+4700 val loss 5.7610
+4700 val perplexity 317.6762
+4700 train 5.773689 (lr=3.4915e-05) (hash(x)=161556686)
+4800 val loss 5.7479
+4800 val perplexity 313.5244
+4800 train 5.647160 (lr=3.3802e-05) (hash(x)=149000293)
+4900 val loss 5.7401
+4900 val perplexity 311.1111
+4900 train 5.523637 (lr=3.2693e-05) (hash(x)=154349989)
+5000 val loss 5.7269
+5000 val perplexity 307.0268
+5000 train 5.559866 (lr=3.1587e-05) (hash(x)=131475967)
+5100 val loss 5.7184
+5100 val perplexity 304.4272
+5100 train 5.624811 (lr=3.0488e-05) (hash(x)=149717902)
+5200 val loss 5.7093
+5200 val perplexity 301.6509
+5200 train 5.584415 (lr=2.9396e-05) (hash(x)=151407999)
+5300 val loss 5.6987
+5300 val perplexity 298.4662
+5300 train 5.798537 (lr=2.8314e-05) (hash(x)=168602728)
+5400 val loss 5.6869
+5400 val perplexity 294.9734
+5400 train 5.645269 (lr=2.7243e-05) (hash(x)=158344511)
+5500 val loss 5.6820
+5500 val perplexity 293.5395
+5500 train 5.636278 (lr=2.6183e-05) (hash(x)=148350057)
+5600 val loss 5.6683
+5600 val perplexity 289.5385
+5600 train 5.730073 (lr=2.5138e-05) (hash(x)=153847323)
+5700 val loss 5.6613
+5700 val perplexity 287.5226
+5700 train 5.604956 (lr=2.4108e-05) (hash(x)=156607405)
+5800 val loss 5.6529
+5800 val perplexity 285.1111
+5800 train 5.426610 (lr=2.3095e-05) (hash(x)=145115031)
+5900 val loss 5.6435
+5900 val perplexity 282.4457
+5900 train 5.527198 (lr=2.2100e-05) (hash(x)=141584622)
+6000 val loss 5.6341
+6000 val perplexity 279.8120
+6000 train 5.570864 (lr=2.1125e-05) (hash(x)=146613857)
+6100 val loss 5.6312
+6100 val perplexity 279.0052
+6100 train 5.567684 (lr=2.0171e-05) (hash(x)=144621768)
+6200 val loss 5.6256
+6200 val perplexity 277.4482
+6200 train 5.554934 (lr=1.9240e-05) (hash(x)=146521760)
+6300 val loss 5.6152
+6300 val perplexity 274.5722
+6300 train 5.448624 (lr=1.8333e-05) (hash(x)=161378136)
+6400 val loss 5.6101
+6400 val perplexity 273.1631
+6400 train 5.378306 (lr=1.7451e-05) (hash(x)=141624235)
+6500 val loss 5.6035
+6500 val perplexity 271.3796
+6500 train 5.527589 (lr=1.6595e-05) (hash(x)=151197095)
+6600 val loss 5.6025
+6600 val perplexity 271.1061
+6600 train 5.619163 (lr=1.5768e-05) (hash(x)=153269571)
+6700 val loss 5.5954
+6700 val perplexity 269.1819
+6700 train 5.556411 (lr=1.4969e-05) (hash(x)=146111181)
+6800 val loss 5.5855
+6800 val perplexity 266.5431
+6800 train 5.618752 (lr=1.4200e-05) (hash(x)=147269760)
+6900 val loss 5.5834
+6900 val perplexity 265.9734
+6900 train 5.666532 (lr=1.3463e-05) (hash(x)=152912762)
+7000 val loss 5.5787
+7000 val perplexity 264.7168
+7000 train 5.657790 (lr=1.2758e-05) (hash(x)=165412343)
+7100 val loss 5.5707
+7100 val perplexity 262.6209
+7100 train 5.638406 (lr=1.2086e-05) (hash(x)=162866028)
+7200 val loss 5.5661
+7200 val perplexity 261.4069
+7200 train 5.535567 (lr=1.1449e-05) (hash(x)=142998115)
+7300 val loss 5.5631
+7300 val perplexity 260.6265
+7300 train 5.332856 (lr=1.0847e-05) (hash(x)=145486999)
+7400 val loss 5.5562
+7400 val perplexity 258.8340
+7400 train 5.528593 (lr=1.0281e-05) (hash(x)=155325873)
+7500 val loss 5.5535
+7500 val perplexity 258.1432
+7500 train 5.519760 (lr=9.7516e-06) (hash(x)=145131256)
+7600 val loss 5.5583
+7600 val perplexity 259.3928
+7600 train 5.488729 (lr=9.2601e-06) (hash(x)=144008365)
+7700 val loss 5.5471
+7700 val perplexity 256.5012
+7700 train 5.654284 (lr=8.8071e-06) (hash(x)=148848532)
+7800 val loss 5.5434
+7800 val perplexity 255.5558
+7800 train 5.538763 (lr=8.3932e-06) (hash(x)=150391642)
+7900 val loss 5.5438
+7900 val perplexity 255.6517
+7900 train 5.297364 (lr=8.0189e-06) (hash(x)=152191414)
+8000 val loss 5.5384
+8000 val perplexity 254.2788
+8000 train 5.614222 (lr=7.6848e-06) (hash(x)=159755587)
+8100 val loss 5.5365
+8100 val perplexity 253.7773
+8100 train 5.555556 (lr=7.3914e-06) (hash(x)=156664468)
+8200 val loss 5.5334
+8200 val perplexity 253.0146
+8200 train 5.381723 (lr=7.1392e-06) (hash(x)=139457379)
+8300 val loss 5.5381
+8300 val perplexity 254.2029
+8300 train 5.306790 (lr=6.9284e-06) (hash(x)=145478564)
+8400 val loss 5.5289
+8400 val perplexity 251.8552
+8400 train 5.458697 (lr=6.7594e-06) (hash(x)=154982769)
+8500 val loss 5.5264
+8500 val perplexity 251.2348
+8500 train 5.494715 (lr=6.6324e-06) (hash(x)=145798118)
+8600 val loss 5.5262
+8600 val perplexity 251.1895
+8600 train 5.440418 (lr=6.5477e-06) (hash(x)=143231551)
+8700 val loss 5.5291
+8700 val perplexity 251.9241
+8700 train 5.423745 (lr=6.5053e-06) (hash(x)=154780112)
+8749 val loss 5.5218
+8749 val perplexity 250.0809

lr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1340/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff70950c9c8508f7cecce017a122b3096a6d63ff2fea1626dd1a1904ba4d5a65
+size 97580418

lr6.5e-5_total_batch_size61440_two_masks_4_heads_seed1340/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3101ab1ef78d2720953754bda17553f4d1086ca9af9a7fd71cc45d0d078609b1
+size 188880262