Upload folder using huggingface_hub

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "n_latent_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": 2, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_1341", "n_embd": 256}

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.3022
+0 val perplexity 81003.2969
+0 train 11.302774 (lr=1.2000e-07) (hash(x)=145079536)
+100 val loss 10.1813
+100 val perplexity 26404.1973
+100 train 10.158878 (lr=1.2120e-05) (hash(x)=155800595)
+200 val loss 9.9129
+200 val perplexity 20188.7715
+200 train 9.868517 (lr=2.4120e-05) (hash(x)=145606733)
+300 val loss 9.6447
+300 val perplexity 15439.3730
+300 train 9.670784 (lr=3.6120e-05) (hash(x)=150367139)
+400 val loss 9.3377
+400 val perplexity 11357.7764
+400 train 9.477560 (lr=4.8120e-05) (hash(x)=155747374)
+500 val loss 9.1418
+500 val perplexity 9337.9492
+500 train 9.147530 (lr=6.0000e-05) (hash(x)=140604760)
+600 val loss 8.9356
+600 val perplexity 7597.6377
+600 train 8.935740 (lr=5.9980e-05) (hash(x)=148404734)
+700 val loss 8.7538
+700 val perplexity 6334.8677
+700 train 8.754250 (lr=5.9922e-05) (hash(x)=148115934)
+800 val loss 8.6325
+800 val perplexity 5611.0654
+800 train 8.589380 (lr=5.9824e-05) (hash(x)=137464699)
+900 val loss 8.5862
+900 val perplexity 5357.2109
+900 train 8.529787 (lr=5.9687e-05) (hash(x)=143886042)
+1000 val loss 8.5439
+1000 val perplexity 5135.5269
+1000 train 8.577871 (lr=5.9512e-05) (hash(x)=163799796)
+1100 val loss 8.4062
+1100 val perplexity 4474.9478
+1100 train 8.374664 (lr=5.9298e-05) (hash(x)=144592844)
+1200 val loss 8.4067
+1200 val perplexity 4477.0439
+1200 train 8.628639 (lr=5.9046e-05) (hash(x)=204706354)
+1300 val loss 8.4059
+1300 val perplexity 4473.3350
+1300 train 8.400589 (lr=5.8757e-05) (hash(x)=150862210)
+1400 val loss 8.3872
+1400 val perplexity 4390.5337
+1400 train 8.388836 (lr=5.8430e-05) (hash(x)=147766811)
+1500 val loss 8.3914
+1500 val perplexity 4408.8359
+1500 train 8.313634 (lr=5.8066e-05) (hash(x)=135925327)
+1600 val loss 8.3716
+1600 val perplexity 4322.3784
+1600 train 8.422943 (lr=5.7666e-05) (hash(x)=160440642)
+1700 val loss 8.4504
+1700 val perplexity 4676.9580
+1700 train 8.510411 (lr=5.7230e-05) (hash(x)=151184106)
+1800 val loss 8.3393
+1800 val perplexity 4185.3354
+1800 train 8.295940 (lr=5.6759e-05) (hash(x)=148100580)
+1900 val loss 8.3477
+1900 val perplexity 4220.3062
+1900 train 8.342902 (lr=5.6253e-05) (hash(x)=149434659)
+2000 val loss 8.3434
+2000 val perplexity 4202.2373
+2000 train 8.225173 (lr=5.5714e-05) (hash(x)=152285486)
+2100 val loss 8.3391
+2100 val perplexity 4184.3057
+2100 train 8.232183 (lr=5.5142e-05) (hash(x)=144294295)
+2200 val loss 8.3419
+2200 val perplexity 4196.0781
+2200 train 8.402203 (lr=5.4537e-05) (hash(x)=175030215)
+2300 val loss 8.3638
+2300 val perplexity 4289.0151
+2300 train 8.304497 (lr=5.3902e-05) (hash(x)=150831428)
+2400 val loss 8.3439
+2400 val perplexity 4204.6426
+2400 train 8.434280 (lr=5.3236e-05) (hash(x)=140808297)
+2500 val loss 8.3428
+2500 val perplexity 4199.7212
+2500 train 8.343163 (lr=5.2541e-05) (hash(x)=153160275)
+2600 val loss 8.3510
+2600 val perplexity 4234.5576
+2600 train 8.256983 (lr=5.1817e-05) (hash(x)=133990623)
+2700 val loss 8.3480
+2700 val perplexity 4221.7510
+2700 train 8.404637 (lr=5.1067e-05) (hash(x)=142860944)
+2800 val loss 8.3501
+2800 val perplexity 4230.7393
+2800 train 8.444883 (lr=5.0290e-05) (hash(x)=137959511)
+2900 val loss 8.3519
+2900 val perplexity 4238.2017
+2900 train 8.355026 (lr=4.9487e-05) (hash(x)=147009873)
+3000 val loss 8.3410
+3000 val perplexity 4192.4863
+3000 train 8.334948 (lr=4.8662e-05) (hash(x)=158264841)
+3100 val loss 8.3390
+3100 val perplexity 4183.8027
+3100 train 8.372059 (lr=4.7813e-05) (hash(x)=139232251)
+3200 val loss 8.3396
+3200 val perplexity 4186.4932
+3200 train 8.349333 (lr=4.6943e-05) (hash(x)=153436104)
+3300 val loss 8.3409
+3300 val perplexity 4191.8623
+3300 train 8.275049 (lr=4.6052e-05) (hash(x)=149681831)
+3400 val loss 8.3368
+3400 val perplexity 4174.8350
+3400 train 8.394770 (lr=4.5143e-05) (hash(x)=168425516)
+3500 val loss 8.3488
+3500 val perplexity 4225.0698
+3500 train 8.363139 (lr=4.4216e-05) (hash(x)=163104338)
+3600 val loss 8.3458
+3600 val perplexity 4212.2642
+3600 train 8.373493 (lr=4.3273e-05) (hash(x)=165109772)
+3700 val loss 8.3445
+3700 val perplexity 4206.8804
+3700 train 8.283165 (lr=4.2315e-05) (hash(x)=153420306)
+3800 val loss 8.3340
+3800 val perplexity 4162.9238
+3800 train 8.341437 (lr=4.1343e-05) (hash(x)=160168863)
+3900 val loss 8.3616
+3900 val perplexity 4279.3281
+3900 train 8.420931 (lr=4.0360e-05) (hash(x)=153906073)
+4000 val loss 8.3633
+4000 val perplexity 4286.7129
+4000 train 8.344972 (lr=3.9365e-05) (hash(x)=151055067)
+4100 val loss 8.3393
+4100 val perplexity 4184.9761
+4100 train 8.326389 (lr=3.8362e-05) (hash(x)=149629830)
+4200 val loss 8.3403
+4200 val perplexity 4189.2769
+4200 train 8.299853 (lr=3.7351e-05) (hash(x)=143101381)
+4300 val loss 8.3360
+4300 val perplexity 4171.4883
+4300 train 8.311817 (lr=3.6333e-05) (hash(x)=149712044)
+4400 val loss 8.3316
+4400 val perplexity 4152.9551
+4400 train 8.305692 (lr=3.5311e-05) (hash(x)=153446449)
+4500 val loss 8.3397
+4500 val perplexity 4186.8765
+4500 train 8.340549 (lr=3.4285e-05) (hash(x)=146086947)
+4600 val loss 8.3304
+4600 val perplexity 4147.9678
+4600 train 8.301800 (lr=3.3257e-05) (hash(x)=153800173)
+4700 val loss 8.3333
+4700 val perplexity 4159.9946
+4700 train 8.280998 (lr=3.2229e-05) (hash(x)=155962726)
+4800 val loss 8.3364
+4800 val perplexity 4173.0518
+4800 train 8.531429 (lr=3.1202e-05) (hash(x)=142045616)
+4900 val loss 8.3406
+4900 val perplexity 4190.6636
+4900 train 8.315807 (lr=3.0178e-05) (hash(x)=143418248)
+5000 val loss 8.3419
+5000 val perplexity 4195.9302
+5000 train 8.341086 (lr=2.9157e-05) (hash(x)=145789790)
+5100 val loss 8.3421
+5100 val perplexity 4196.7666
+5100 train 8.367014 (lr=2.8143e-05) (hash(x)=137795633)
+5200 val loss 8.3686
+5200 val perplexity 4309.4624
+5200 train 8.494040 (lr=2.7135e-05) (hash(x)=148907132)
+5300 val loss 8.3396
+5300 val perplexity 4186.5088
+5300 train 8.326537 (lr=2.6136e-05) (hash(x)=152343580)
+5400 val loss 8.3411
+5400 val perplexity 4192.8940
+5400 train 8.303223 (lr=2.5147e-05) (hash(x)=148578264)
+5500 val loss 8.3422
+5500 val perplexity 4197.2549
+5500 train 8.441550 (lr=2.4169e-05) (hash(x)=145635833)
+5600 val loss 8.3577
+5600 val perplexity 4262.8521
+5600 train 8.384826 (lr=2.3204e-05) (hash(x)=156337844)
+5700 val loss 8.3529
+5700 val perplexity 4242.5288
+5700 train 8.379089 (lr=2.2253e-05) (hash(x)=147168506)
+5800 val loss 8.3541
+5800 val perplexity 4247.7271
+5800 train 8.399971 (lr=2.1318e-05) (hash(x)=159566920)
+5900 val loss 8.3549
+5900 val perplexity 4250.8477
+5900 train 8.418188 (lr=2.0400e-05) (hash(x)=158273929)
+6000 val loss 8.3512
+6000 val perplexity 4235.2886
+6000 train 8.370876 (lr=1.9500e-05) (hash(x)=156649749)
+6100 val loss 8.3442
+6100 val perplexity 4205.6851
+6100 train 8.319452 (lr=1.8620e-05) (hash(x)=146812388)
+6200 val loss 8.3447
+6200 val perplexity 4208.0122
+6200 train 8.330069 (lr=1.7760e-05) (hash(x)=143522146)
+6300 val loss 8.3403
+6300 val perplexity 4189.2168
+6300 train 8.282401 (lr=1.6923e-05) (hash(x)=150124474)
+6400 val loss 8.3373
+6400 val perplexity 4176.6748
+6400 train 8.314153 (lr=1.6108e-05) (hash(x)=141242117)
+6500 val loss 8.3378
+6500 val perplexity 4178.8345
+6500 train 8.311840 (lr=1.5319e-05) (hash(x)=143529762)
+6600 val loss 8.3344
+6600 val perplexity 4164.8696
+6600 train 8.261427 (lr=1.4555e-05) (hash(x)=136948374)
+6700 val loss 8.3367
+6700 val perplexity 4174.3452
+6700 train 8.335506 (lr=1.3817e-05) (hash(x)=146268592)
+6800 val loss 8.3317
+6800 val perplexity 4153.5806
+6800 train 8.322599 (lr=1.3108e-05) (hash(x)=152676836)
+6900 val loss 8.3327
+6900 val perplexity 4157.7656
+6900 train 8.323095 (lr=1.2427e-05) (hash(x)=134657776)
+7000 val loss 8.3310
+7000 val perplexity 4150.4053
+7000 train 8.412806 (lr=1.1777e-05) (hash(x)=166721861)
+7100 val loss 8.3285
+7100 val perplexity 4140.0005
+7100 train 8.277332 (lr=1.1157e-05) (hash(x)=135496702)
+7200 val loss 8.3322
+7200 val perplexity 4155.3638
+7200 train 8.379076 (lr=1.0568e-05) (hash(x)=155567461)
+7300 val loss 8.3251
+7300 val perplexity 4126.2163
+7300 train 8.254663 (lr=1.0012e-05) (hash(x)=142803829)
+7400 val loss 8.3322
+7400 val perplexity 4155.6250
+7400 train 8.265259 (lr=9.4899e-06) (hash(x)=145294178)
+7500 val loss 8.3261
+7500 val perplexity 4130.3462
+7500 train 8.223894 (lr=9.0014e-06) (hash(x)=150573713)
+7600 val loss 8.3267
+7600 val perplexity 4132.7104
+7600 train 8.316321 (lr=8.5478e-06) (hash(x)=142771511)
+7700 val loss 8.3275
+7700 val perplexity 4136.1567
+7700 train 8.304243 (lr=8.1297e-06) (hash(x)=143602175)
+7800 val loss 8.3305
+7800 val perplexity 4148.6875
+7800 train 8.387541 (lr=7.7476e-06) (hash(x)=152379862)
+7900 val loss 8.3261
+7900 val perplexity 4130.3384
+7900 train 8.336418 (lr=7.4021e-06) (hash(x)=146655921)
+8000 val loss 8.3267
+8000 val perplexity 4132.8447
+8000 train 8.387903 (lr=7.0937e-06) (hash(x)=148262482)
+8100 val loss 8.3324
+8100 val perplexity 4156.3071
+8100 train 8.308302 (lr=6.8229e-06) (hash(x)=147683655)
+8200 val loss 8.3241
+8200 val perplexity 4122.0786
+8200 train 8.360710 (lr=6.5900e-06) (hash(x)=157312987)
+8300 val loss 8.3282
+8300 val perplexity 4138.7725
+8300 train 8.397158 (lr=6.3954e-06) (hash(x)=141107543)
+8400 val loss 8.3242
+8400 val perplexity 4122.5781
+8400 train 8.281261 (lr=6.2395e-06) (hash(x)=141323024)
+8500 val loss 8.3244
+8500 val perplexity 4123.1050
+8500 train 8.405952 (lr=6.1223e-06) (hash(x)=150696521)
+8600 val loss 8.3265
+8600 val perplexity 4131.8198
+8600 train 8.440543 (lr=6.0440e-06) (hash(x)=162288191)
+8700 val loss 8.3254
+8700 val perplexity 4127.2988
+8700 train 8.242277 (lr=6.0049e-06) (hash(x)=152860941)
+8749 val loss 8.3260
+8749 val perplexity 4130.0156

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f68d7f3c79549fcf734bb47e387a8ee0cc3e5a16cbb39fd4ea83f994e1862720
+size 95220594

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1341/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd1fda4a5fb4baf239617453d02d4937c3a855fb876ebb66b552a5e4fd2d8039
+size 184163894