Upload folder using huggingface_hub

lr6e-5_total_batch_size61440_two_masks_4_heads_seed1339/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr6e-5_total_batch_size61440_two_masks_4_heads_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "two_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_1339", "n_embd": 256}

lr6e-5_total_batch_size61440_two_masks_4_heads_seed1339/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr6e-5_total_batch_size61440_two_masks_4_heads_seed1339/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.2561
+0 val perplexity 77350.9609
+0 train 11.256043 (lr=1.2000e-07) (hash(x)=150724848)
+100 val loss 10.0106
+100 val perplexity 22260.3613
+100 train 10.019782 (lr=1.2120e-05) (hash(x)=149910534)
+200 val loss 9.4212
+200 val perplexity 12347.5879
+200 train 9.453641 (lr=2.4120e-05) (hash(x)=148123706)
+300 val loss 8.3326
+300 val perplexity 4157.0444
+300 train 8.238178 (lr=3.6120e-05) (hash(x)=146678221)
+400 val loss 7.7372
+400 val perplexity 2292.0398
+400 train 7.632246 (lr=4.8120e-05) (hash(x)=151700982)
+500 val loss 7.5408
+500 val perplexity 1883.3497
+500 train 7.628691 (lr=6.0000e-05) (hash(x)=156182087)
+600 val loss 7.4409
+600 val perplexity 1704.2441
+600 train 7.437863 (lr=5.9980e-05) (hash(x)=149318660)
+700 val loss 7.3756
+700 val perplexity 1596.6025
+700 train 7.414310 (lr=5.9922e-05) (hash(x)=150482428)
+800 val loss 7.3037
+800 val perplexity 1485.7595
+800 train 7.181281 (lr=5.9824e-05) (hash(x)=143268605)
+900 val loss 7.2288
+900 val perplexity 1378.5511
+900 train 7.195660 (lr=5.9687e-05) (hash(x)=152322423)
+1000 val loss 7.1645
+1000 val perplexity 1292.7592
+1000 train 7.029536 (lr=5.9512e-05) (hash(x)=147904298)
+1100 val loss 7.0876
+1100 val perplexity 1197.0170
+1100 train 7.279038 (lr=5.9298e-05) (hash(x)=154343147)
+1200 val loss 6.9971
+1200 val perplexity 1093.4266
+1200 train 6.981460 (lr=5.9046e-05) (hash(x)=141843115)
+1300 val loss 6.9141
+1300 val perplexity 1006.3934
+1300 train 6.803492 (lr=5.8757e-05) (hash(x)=145279030)
+1400 val loss 6.8364
+1400 val perplexity 931.1666
+1400 train 6.781555 (lr=5.8430e-05) (hash(x)=152507639)
+1500 val loss 6.7377
+1500 val perplexity 843.5795
+1500 train 6.773906 (lr=5.8066e-05) (hash(x)=148473774)
+1600 val loss 6.6373
+1600 val perplexity 763.0455
+1600 train 6.672471 (lr=5.7666e-05) (hash(x)=151117002)
+1700 val loss 6.5729
+1700 val perplexity 715.4456
+1700 train 6.503576 (lr=5.7230e-05) (hash(x)=138011335)
+1800 val loss 6.5051
+1800 val perplexity 668.5574
+1800 train 6.639477 (lr=5.6759e-05) (hash(x)=171180926)
+1900 val loss 6.4382
+1900 val perplexity 625.3024
+1900 train 6.356393 (lr=5.6253e-05) (hash(x)=141769419)
+2000 val loss 6.3844
+2000 val perplexity 592.5333
+2000 train 6.327018 (lr=5.5714e-05) (hash(x)=151963443)
+2100 val loss 6.3660
+2100 val perplexity 581.6981
+2100 train 6.420936 (lr=5.5142e-05) (hash(x)=162947470)
+2200 val loss 6.2967
+2200 val perplexity 542.7556
+2200 train 6.524726 (lr=5.4537e-05) (hash(x)=154954810)
+2300 val loss 6.2573
+2300 val perplexity 521.7907
+2300 train 6.240864 (lr=5.3902e-05) (hash(x)=151878111)
+2400 val loss 6.2313
+2400 val perplexity 508.4060
+2400 train 6.125352 (lr=5.3236e-05) (hash(x)=158661057)
+2500 val loss 6.1914
+2500 val perplexity 488.5492
+2500 train 6.221001 (lr=5.2541e-05) (hash(x)=150925584)
+2600 val loss 6.1655
+2600 val perplexity 476.0305
+2600 train 6.075785 (lr=5.1817e-05) (hash(x)=144515755)
+2700 val loss 6.1455
+2700 val perplexity 466.6103
+2700 train 6.120259 (lr=5.1067e-05) (hash(x)=153109144)
+2800 val loss 6.1122
+2800 val perplexity 451.3339
+2800 train 6.019416 (lr=5.0290e-05) (hash(x)=151152897)
+2900 val loss 6.0876
+2900 val perplexity 440.3468
+2900 train 6.022316 (lr=4.9487e-05) (hash(x)=145800210)
+3000 val loss 6.0684
+3000 val perplexity 432.0015
+3000 train 5.871547 (lr=4.8662e-05) (hash(x)=141997485)
+3100 val loss 6.0350
+3100 val perplexity 417.8127
+3100 train 5.959313 (lr=4.7813e-05) (hash(x)=154049740)
+3200 val loss 6.0143
+3200 val perplexity 409.2258
+3200 train 5.951076 (lr=4.6943e-05) (hash(x)=150471842)
+3300 val loss 6.0058
+3300 val perplexity 405.7905
+3300 train 5.951585 (lr=4.6052e-05) (hash(x)=149048126)
+3400 val loss 5.9749
+3400 val perplexity 393.4469
+3400 train 6.098579 (lr=4.5143e-05) (hash(x)=161261339)
+3500 val loss 5.9550
+3500 val perplexity 385.6769
+3500 train 5.921894 (lr=4.4216e-05) (hash(x)=157495564)
+3600 val loss 5.9388
+3600 val perplexity 379.4659
+3600 train 5.857219 (lr=4.3273e-05) (hash(x)=144352932)
+3700 val loss 5.9208
+3700 val perplexity 372.7189
+3700 train 5.893260 (lr=4.2315e-05) (hash(x)=149389012)
+3800 val loss 5.8982
+3800 val perplexity 364.3758
+3800 train 5.839093 (lr=4.1343e-05) (hash(x)=146607620)
+3900 val loss 5.8898
+3900 val perplexity 361.3354
+3900 train 5.805131 (lr=4.0360e-05) (hash(x)=143754617)
+4000 val loss 5.8702
+4000 val perplexity 354.3158
+4000 train 5.847651 (lr=3.9365e-05) (hash(x)=156930722)
+4100 val loss 5.8546
+4100 val perplexity 348.8230
+4100 train 5.693880 (lr=3.8362e-05) (hash(x)=147500519)
+4200 val loss 5.8416
+4200 val perplexity 344.3155
+4200 train 5.685581 (lr=3.7351e-05) (hash(x)=143232237)
+4300 val loss 5.8238
+4300 val perplexity 338.2632
+4300 train 5.730355 (lr=3.6333e-05) (hash(x)=146811670)
+4400 val loss 5.8094
+4400 val perplexity 333.4224
+4400 train 5.845029 (lr=3.5311e-05) (hash(x)=158418746)
+4500 val loss 5.7963
+4500 val perplexity 329.0671
+4500 train 5.756180 (lr=3.4285e-05) (hash(x)=156695778)
+4600 val loss 5.7852
+4600 val perplexity 325.4496
+4600 train 5.797576 (lr=3.3257e-05) (hash(x)=147791497)
+4700 val loss 5.7695
+4700 val perplexity 320.3735
+4700 train 5.664525 (lr=3.2229e-05) (hash(x)=155533088)
+4800 val loss 5.7604
+4800 val perplexity 317.4795
+4800 train 5.614402 (lr=3.1202e-05) (hash(x)=138350044)
+4900 val loss 5.7464
+4900 val perplexity 313.0471
+4900 train 5.746566 (lr=3.0178e-05) (hash(x)=143735284)
+5000 val loss 5.7341
+5000 val perplexity 309.2351
+5000 train 5.691934 (lr=2.9157e-05) (hash(x)=154976463)
+5100 val loss 5.7297
+5100 val perplexity 307.8734
+5100 train 5.622232 (lr=2.8143e-05) (hash(x)=149894982)
+5200 val loss 5.7155
+5200 val perplexity 303.5452
+5200 train 5.496017 (lr=2.7135e-05) (hash(x)=159326689)
+5300 val loss 5.7035
+5300 val perplexity 299.9124
+5300 train 5.717716 (lr=2.6136e-05) (hash(x)=159484800)
+5400 val loss 5.6889
+5400 val perplexity 295.5559
+5400 train 5.597114 (lr=2.5147e-05) (hash(x)=140385615)
+5500 val loss 5.6804
+5500 val perplexity 293.0521
+5500 train 5.510218 (lr=2.4169e-05) (hash(x)=148498335)
+5600 val loss 5.6792
+5600 val perplexity 292.7039
+5600 train 5.420145 (lr=2.3204e-05) (hash(x)=151907614)
+5700 val loss 5.6702
+5700 val perplexity 290.0860
+5700 train 5.639629 (lr=2.2253e-05) (hash(x)=155192267)
+5800 val loss 5.6541
+5800 val perplexity 285.4713
+5800 train 5.701157 (lr=2.1318e-05) (hash(x)=153132158)
+5900 val loss 5.6533
+5900 val perplexity 285.2193
+5900 train 5.619873 (lr=2.0400e-05) (hash(x)=161446764)
+6000 val loss 5.6477
+6000 val perplexity 283.6525
+6000 train 5.445190 (lr=1.9500e-05) (hash(x)=151512446)
+6100 val loss 5.6351
+6100 val perplexity 280.0993
+6100 train 5.734171 (lr=1.8620e-05) (hash(x)=188094053)
+6200 val loss 5.6253
+6200 val perplexity 277.3429
+6200 train 5.512557 (lr=1.7760e-05) (hash(x)=149389789)
+6300 val loss 5.6203
+6300 val perplexity 275.9817
+6300 train 5.442704 (lr=1.6923e-05) (hash(x)=138212820)
+6400 val loss 5.6165
+6400 val perplexity 274.9195
+6400 train 5.402879 (lr=1.6108e-05) (hash(x)=146535423)
+6500 val loss 5.6079
+6500 val perplexity 272.5826
+6500 train 5.481564 (lr=1.5319e-05) (hash(x)=145950843)
+6600 val loss 5.5999
+6600 val perplexity 270.4070
+6600 train 5.505300 (lr=1.4555e-05) (hash(x)=141162902)
+6700 val loss 5.5991
+6700 val perplexity 270.1852
+6700 train 5.518464 (lr=1.3817e-05) (hash(x)=153018737)
+6800 val loss 5.5907
+6800 val perplexity 267.9099
+6800 train 5.506629 (lr=1.3108e-05) (hash(x)=155640155)
+6900 val loss 5.5868
+6900 val perplexity 266.8924
+6900 train 5.553267 (lr=1.2427e-05) (hash(x)=153722115)
+7000 val loss 5.5800
+7000 val perplexity 265.0749
+7000 train 5.497493 (lr=1.1777e-05) (hash(x)=146953450)
+7100 val loss 5.5797
+7100 val perplexity 264.9916
+7100 train 5.423279 (lr=1.1157e-05) (hash(x)=137663885)
+7200 val loss 5.5679
+7200 val perplexity 261.8954
+7200 train 5.647879 (lr=1.0568e-05) (hash(x)=146172950)
+7300 val loss 5.5648
+7300 val perplexity 261.0784
+7300 train 5.459376 (lr=1.0012e-05) (hash(x)=150018163)
+7400 val loss 5.5609
+7400 val perplexity 260.0634
+7400 train 5.574271 (lr=9.4899e-06) (hash(x)=145351166)
+7500 val loss 5.5609
+7500 val perplexity 260.0666
+7500 train 5.348469 (lr=9.0014e-06) (hash(x)=145292116)
+7600 val loss 5.5526
+7600 val perplexity 257.9185
+7600 train 5.474073 (lr=8.5478e-06) (hash(x)=150235132)
+7700 val loss 5.5492
+7700 val perplexity 257.0341
+7700 train 5.456861 (lr=8.1297e-06) (hash(x)=154543455)
+7800 val loss 5.5473
+7800 val perplexity 256.5529
+7800 train 5.457225 (lr=7.7476e-06) (hash(x)=142456852)
+7900 val loss 5.5468
+7900 val perplexity 256.4154
+7900 train 5.321056 (lr=7.4021e-06) (hash(x)=147363479)
+8000 val loss 5.5394
+8000 val perplexity 254.5357
+8000 train 5.570914 (lr=7.0937e-06) (hash(x)=156122973)
+8100 val loss 5.5377
+8100 val perplexity 254.0970
+8100 train 5.588730 (lr=6.8229e-06) (hash(x)=156153179)
+8200 val loss 5.5352
+8200 val perplexity 253.4494
+8200 train 5.584256 (lr=6.5900e-06) (hash(x)=146430698)
+8300 val loss 5.5342
+8300 val perplexity 253.1967
+8300 train 5.407774 (lr=6.3954e-06) (hash(x)=143507257)
+8400 val loss 5.5294
+8400 val perplexity 251.9823
+8400 train 5.507139 (lr=6.2395e-06) (hash(x)=166272643)
+8500 val loss 5.5269
+8500 val perplexity 251.3694
+8500 train 5.430086 (lr=6.1223e-06) (hash(x)=143887848)
+8600 val loss 5.5259
+8600 val perplexity 251.1117
+8600 train 5.559775 (lr=6.0440e-06) (hash(x)=156900341)
+8700 val loss 5.5208
+8700 val perplexity 249.8283
+8700 train 5.737858 (lr=6.0049e-06) (hash(x)=146417632)
+8749 val loss 5.5203
+8749 val perplexity 249.7142

lr6e-5_total_batch_size61440_two_masks_4_heads_seed1339/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f35da01512e757d31a48acc9aaed16c1291f277c79d2ce7089e269b1af8ecd09
+size 97580418

lr6e-5_total_batch_size61440_two_masks_4_heads_seed1339/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:851db782e4e0fe51974459f9d086e199d7c156049422bba50084505bc5101f08
+size 188880262