Upload folder using huggingface_hub

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "n_latent_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": 2, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6.5e-5_61440_1341", "n_embd": 256}

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.3022
+0 val perplexity 81003.2969
+0 train 11.302774 (lr=1.3000e-07) (hash(x)=145079536)
+100 val loss 10.1329
+100 val perplexity 25158.1328
+100 train 10.114728 (lr=1.3130e-05) (hash(x)=155800595)
+200 val loss 9.8966
+200 val perplexity 19861.7520
+200 train 9.851768 (lr=2.6130e-05) (hash(x)=145606733)
+300 val loss 9.6050
+300 val perplexity 14839.2510
+300 train 9.630487 (lr=3.9130e-05) (hash(x)=150367139)
+400 val loss 9.1535
+400 val perplexity 9447.0752
+400 train 9.323906 (lr=5.2130e-05) (hash(x)=155747374)
+500 val loss 8.7976
+500 val perplexity 6618.0112
+500 train 8.791979 (lr=6.5000e-05) (hash(x)=140604760)
+600 val loss 8.2360
+600 val perplexity 3774.5781
+600 train 8.216065 (lr=6.4979e-05) (hash(x)=148404734)
+700 val loss 8.0356
+700 val perplexity 3088.9790
+700 train 8.013445 (lr=6.4915e-05) (hash(x)=148115934)
+800 val loss 8.0068
+800 val perplexity 3001.2395
+800 train 7.934578 (lr=6.4809e-05) (hash(x)=137464699)
+900 val loss 8.0003
+900 val perplexity 2981.7483
+900 train 7.900009 (lr=6.4661e-05) (hash(x)=143886042)
+1000 val loss 8.0320
+1000 val perplexity 3077.8430
+1000 train 8.066036 (lr=6.4471e-05) (hash(x)=163799796)
+1100 val loss 7.9372
+1100 val perplexity 2799.4990
+1100 train 7.895987 (lr=6.4240e-05) (hash(x)=144592844)
+1200 val loss 7.9211
+1200 val perplexity 2754.7002
+1200 train 8.229635 (lr=6.3967e-05) (hash(x)=204706354)
+1300 val loss 7.9828
+1300 val perplexity 2929.9968
+1300 train 7.965636 (lr=6.3653e-05) (hash(x)=150862210)
+1400 val loss 7.9359
+1400 val perplexity 2795.8518
+1400 train 7.919776 (lr=6.3299e-05) (hash(x)=147766811)
+1500 val loss 7.9422
+1500 val perplexity 2813.6741
+1500 train 7.829128 (lr=6.2905e-05) (hash(x)=135925327)
+1600 val loss 7.9212
+1600 val perplexity 2755.1311
+1600 train 7.944923 (lr=6.2471e-05) (hash(x)=160440642)
+1700 val loss 7.9573
+1700 val perplexity 2856.3972
+1700 train 8.026003 (lr=6.1999e-05) (hash(x)=151184106)
+1800 val loss 8.5006
+1800 val perplexity 4917.7412
+1800 train 8.464700 (lr=6.1489e-05) (hash(x)=148100580)
+1900 val loss 8.8386
+1900 val perplexity 6895.0571
+1900 train 8.792997 (lr=6.0941e-05) (hash(x)=149434659)
+2000 val loss 8.8397
+2000 val perplexity 6902.8013
+2000 train 8.782389 (lr=6.0357e-05) (hash(x)=152285486)
+2100 val loss 9.0314
+2100 val perplexity 8361.7227
+2100 train 8.939817 (lr=5.9737e-05) (hash(x)=144294295)
+2200 val loss 9.0730
+2200 val perplexity 8716.8926
+2200 train 9.108875 (lr=5.9082e-05) (hash(x)=175030215)
+2300 val loss 8.0124
+2300 val perplexity 3018.1626
+2300 train 7.922067 (lr=5.8394e-05) (hash(x)=150831428)
+2400 val loss 8.0316
+2400 val perplexity 3076.5283
+2400 train 8.136871 (lr=5.7672e-05) (hash(x)=140808297)
+2500 val loss 8.0477
+2500 val perplexity 3126.4431
+2500 train 8.047722 (lr=5.6919e-05) (hash(x)=153160275)
+2600 val loss 8.1796
+2600 val perplexity 3567.5422
+2600 train 8.046284 (lr=5.6135e-05) (hash(x)=133990623)
+2700 val loss 8.0016
+2700 val perplexity 2985.5984
+2700 train 8.029459 (lr=5.5322e-05) (hash(x)=142860944)
+2800 val loss 8.0142
+2800 val perplexity 3023.6768
+2800 train 8.094273 (lr=5.4480e-05) (hash(x)=137959511)
+2900 val loss 8.2043
+2900 val perplexity 3656.8071
+2900 train 8.181103 (lr=5.3611e-05) (hash(x)=147009873)
+3000 val loss 8.1098
+3000 val perplexity 3327.0342
+3000 train 8.076117 (lr=5.2717e-05) (hash(x)=158264841)
+3100 val loss 8.0139
+3100 val perplexity 3022.7280
+3100 train 8.042549 (lr=5.1797e-05) (hash(x)=139232251)
+3200 val loss 8.0203
+3200 val perplexity 3042.1533
+3200 train 8.048290 (lr=5.0855e-05) (hash(x)=153436104)
+3300 val loss 8.0054
+3300 val perplexity 2997.1721
+3300 train 7.906382 (lr=4.9890e-05) (hash(x)=149681831)
+3400 val loss 8.0139
+3400 val perplexity 3022.7021
+3400 train 8.047218 (lr=4.8905e-05) (hash(x)=168425516)
+3500 val loss 7.9939
+3500 val perplexity 2962.7427
+3500 train 8.012372 (lr=4.7901e-05) (hash(x)=163104338)
+3600 val loss 7.9897
+3600 val perplexity 2950.5195
+3600 train 8.007005 (lr=4.6879e-05) (hash(x)=165109772)
+3700 val loss 7.9931
+3700 val perplexity 2960.5850
+3700 train 7.885751 (lr=4.5841e-05) (hash(x)=153420306)
+3800 val loss 7.9891
+3800 val perplexity 2948.6912
+3800 train 8.007331 (lr=4.4789e-05) (hash(x)=160168863)
+3900 val loss 7.9952
+3900 val perplexity 2966.7859
+3900 train 8.045807 (lr=4.3723e-05) (hash(x)=153906073)
+4000 val loss 7.9758
+4000 val perplexity 2909.6252
+4000 train 7.945781 (lr=4.2646e-05) (hash(x)=151055067)
+4100 val loss 7.9692
+4100 val perplexity 2890.6079
+4100 train 7.952957 (lr=4.1559e-05) (hash(x)=149629830)
+4200 val loss 7.9691
+4200 val perplexity 2890.1860
+4200 train 7.907584 (lr=4.0463e-05) (hash(x)=143101381)
+4300 val loss 7.9720
+4300 val perplexity 2898.7170
+4300 train 7.923145 (lr=3.9361e-05) (hash(x)=149712044)
+4400 val loss 7.9686
+4400 val perplexity 2888.8730
+4400 train 7.936096 (lr=3.8253e-05) (hash(x)=153446449)
+4500 val loss 7.9661
+4500 val perplexity 2881.5059
+4500 train 7.939479 (lr=3.7142e-05) (hash(x)=146086947)
+4600 val loss 7.9602
+4600 val perplexity 2864.7751
+4600 train 7.898771 (lr=3.6028e-05) (hash(x)=153800173)
+4700 val loss 7.9463
+4700 val perplexity 2825.1445
+4700 train 7.911625 (lr=3.4915e-05) (hash(x)=155962726)
+4800 val loss 7.9498
+4800 val perplexity 2835.0173
+4800 train 8.130209 (lr=3.3802e-05) (hash(x)=142045616)
+4900 val loss 7.9630
+4900 val perplexity 2872.5693
+4900 train 7.916558 (lr=3.2693e-05) (hash(x)=143418248)
+5000 val loss 7.9453
+5000 val perplexity 2822.3789
+5000 train 7.935678 (lr=3.1587e-05) (hash(x)=145789790)
+5100 val loss 7.9463
+5100 val perplexity 2824.9722
+5100 train 7.931813 (lr=3.0488e-05) (hash(x)=137795633)
+5200 val loss 7.9565
+5200 val perplexity 2854.0093
+5200 train 8.094601 (lr=2.9396e-05) (hash(x)=148907132)
+5300 val loss 7.9453
+5300 val perplexity 2822.2820
+5300 train 7.904957 (lr=2.8314e-05) (hash(x)=152343580)
+5400 val loss 8.1075
+5400 val perplexity 3319.3394
+5400 train 8.036669 (lr=2.7243e-05) (hash(x)=148578264)
+5500 val loss 7.9494
+5500 val perplexity 2833.9998
+5500 train 8.057515 (lr=2.6183e-05) (hash(x)=145635833)
+5600 val loss 7.9405
+5600 val perplexity 2808.8296
+5600 train 7.963914 (lr=2.5138e-05) (hash(x)=156337844)
+5700 val loss 7.9356
+5700 val perplexity 2794.9001
+5700 train 7.932220 (lr=2.4108e-05) (hash(x)=147168506)
+5800 val loss 8.0013
+5800 val perplexity 2984.9407
+5800 train 8.063798 (lr=2.3095e-05) (hash(x)=159566920)
+5900 val loss 7.9393
+5900 val perplexity 2805.4297
+5900 train 8.017948 (lr=2.2100e-05) (hash(x)=158273929)
+6000 val loss 7.9231
+6000 val perplexity 2760.2017
+6000 train 7.921714 (lr=2.1125e-05) (hash(x)=156649749)
+6100 val loss 7.9186
+6100 val perplexity 2748.0132
+6100 train 7.903388 (lr=2.0171e-05) (hash(x)=146812388)
+6200 val loss 8.0483
+6200 val perplexity 3128.5608
+6200 train 7.996803 (lr=1.9240e-05) (hash(x)=143522146)
+6300 val loss 7.9783
+6300 val perplexity 2916.9294
+6300 train 7.901447 (lr=1.8333e-05) (hash(x)=150124474)
+6400 val loss 7.9342
+6400 val perplexity 2791.2483
+6400 train 7.904402 (lr=1.7451e-05) (hash(x)=141242117)
+6500 val loss 7.9276
+6500 val perplexity 2772.8618
+6500 train 7.875103 (lr=1.6595e-05) (hash(x)=143529762)
+6600 val loss 8.0651
+6600 val perplexity 3181.5220
+6600 train 7.954253 (lr=1.5768e-05) (hash(x)=136948374)
+6700 val loss 7.9281
+6700 val perplexity 2774.0122
+6700 train 7.888034 (lr=1.4969e-05) (hash(x)=146268592)
+6800 val loss 7.9238
+6800 val perplexity 2762.1265
+6800 train 7.909309 (lr=1.4200e-05) (hash(x)=152676836)
+6900 val loss 7.9202
+6900 val perplexity 2752.3250
+6900 train 7.908381 (lr=1.3463e-05) (hash(x)=134657776)
+7000 val loss 7.9198
+7000 val perplexity 2751.1770
+7000 train 7.998529 (lr=1.2758e-05) (hash(x)=166721861)
+7100 val loss 7.9214
+7100 val perplexity 2755.6001
+7100 train 7.829399 (lr=1.2086e-05) (hash(x)=135496702)
+7200 val loss 8.1521
+7200 val perplexity 3470.5425
+7200 train 8.215370 (lr=1.1449e-05) (hash(x)=155567461)
+7300 val loss 8.3503
+7300 val perplexity 4231.2959
+7300 train 8.260271 (lr=1.0847e-05) (hash(x)=142803829)
+7400 val loss 8.0494
+7400 val perplexity 3132.0476
+7400 train 7.954559 (lr=1.0281e-05) (hash(x)=145294178)
+7500 val loss 7.9910
+7500 val perplexity 2954.1912
+7500 train 7.853759 (lr=9.7516e-06) (hash(x)=150573713)
+7600 val loss 7.9848
+7600 val perplexity 2936.0471
+7600 train 7.983268 (lr=9.2601e-06) (hash(x)=142771511)
+7700 val loss 7.9675
+7700 val perplexity 2885.5549
+7700 train 7.907674 (lr=8.8071e-06) (hash(x)=143602175)
+7800 val loss 7.9711
+7800 val perplexity 2895.9180
+7800 train 8.024320 (lr=8.3932e-06) (hash(x)=152379862)
+7900 val loss 7.9679
+7900 val perplexity 2886.6794
+7900 train 7.947361 (lr=8.0189e-06) (hash(x)=146655921)
+8000 val loss 7.9724
+8000 val perplexity 2899.9060
+8000 train 8.041166 (lr=7.6848e-06) (hash(x)=148262482)
+8100 val loss 7.9770
+8100 val perplexity 2913.1541
+8100 train 7.930525 (lr=7.3914e-06) (hash(x)=147683655)
+8200 val loss 7.9767
+8200 val perplexity 2912.2402
+8200 train 8.003559 (lr=7.1392e-06) (hash(x)=157312987)
+8300 val loss 7.9740
+8300 val perplexity 2904.4160
+8300 train 8.019750 (lr=6.9284e-06) (hash(x)=141107543)
+8400 val loss 7.9678
+8400 val perplexity 2886.3655
+8400 train 7.920293 (lr=6.7594e-06) (hash(x)=141323024)
+8500 val loss 7.9739
+8500 val perplexity 2904.0654
+8500 train 8.042336 (lr=6.6324e-06) (hash(x)=150696521)
+8600 val loss 7.9726
+8600 val perplexity 2900.5227
+8600 train 8.048990 (lr=6.5477e-06) (hash(x)=162288191)
+8700 val loss 7.9750
+8700 val perplexity 2907.4949
+8700 train 7.882213 (lr=6.5053e-06) (hash(x)=152860941)
+8749 val loss 7.9777
+8749 val perplexity 2915.1550

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0427ff4815cd549d1aa4e2517b2e7372734cbb7bf33fcddb8f73fa85b1335f1a
+size 95220594

lr6.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fec9e2a601863ac48295d07d7e5a3f2c47c1ac4473a04f61aa577e6d8904700
+size 184163894