Upload folder using huggingface_hub

allowing_more_selection_patterns/one_mask_per_head_1_latent_vector_degenerate_seed_1338/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "allowing_more_selection_patterns/one_mask_per_head_1_latent_vector_degenerate_seed_1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 12, "n_embd": 768, "head_dim": 64, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": null, "warmup_steps": null, "group": "allowing_more_selection_patterns", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "n_latent_masks", "selection_head_linear_combo_scale": 0.0, "disable_selection_head_linear_combo_bias": true, "assert_latent_matches_no_head": false, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": null, "batch_size": 4, "total_batch_size": null, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": 1, "init_latent_masks_to_identity": true, "init_latent_masks_to_inverse": false, "latent_mask_scale": null, "latent_mask_sigmoid": false, "S_layernorm": false, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": false, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": null, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "one_mask_per_head_1_latent_vector_degenerate", "latent_mask_precision": "bfloat16"}

allowing_more_selection_patterns/one_mask_per_head_1_latent_vector_degenerate_seed_1338/dataloader_02499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86bf2d12aa2d7d065a4e8fd1660928dd629c2417a5062dd8aecee75fb283886f
+size 964

allowing_more_selection_patterns/one_mask_per_head_1_latent_vector_degenerate_seed_1338/log2.txt

@@ -0,0 +1,303 @@
+max_steps: 2500
+0 val loss 10.9653
+0 val perplexity 57829.3633
+0 train 10.964844 (lr=8.3916e-07) (hash(x)=20472174)
+10 train 10.149137 (lr=9.2308e-06) (hash(x)=18772821)
+20 train 9.635406 (lr=1.7622e-05) (hash(x)=18169834)
+30 train 9.403813 (lr=2.6014e-05) (hash(x)=21772648)
+40 train 9.198076 (lr=3.4406e-05) (hash(x)=19737653)
+50 train 8.879588 (lr=4.2797e-05) (hash(x)=18397495)
+60 train 8.527857 (lr=5.1189e-05) (hash(x)=19313033)
+70 train 8.126386 (lr=5.9580e-05) (hash(x)=19235256)
+80 train 7.880077 (lr=6.7972e-05) (hash(x)=15263116)
+90 train 7.608851 (lr=7.6364e-05) (hash(x)=20660013)
+100 val loss 7.4652
+100 val perplexity 1746.1788
+100 train 7.425018 (lr=8.4755e-05) (hash(x)=19046599)
+110 train 7.326130 (lr=9.3147e-05) (hash(x)=24079550)
+120 train 7.092932 (lr=1.0154e-04) (hash(x)=26797340)
+130 train 7.108392 (lr=1.0993e-04) (hash(x)=20619718)
+140 train 6.925122 (lr=1.1832e-04) (hash(x)=18313305)
+150 train 6.869478 (lr=1.2671e-04) (hash(x)=20825391)
+160 train 6.646358 (lr=1.3510e-04) (hash(x)=20505694)
+170 train 6.525844 (lr=1.4350e-04) (hash(x)=24020301)
+180 train 6.584141 (lr=1.5189e-04) (hash(x)=23124134)
+190 train 6.422416 (lr=1.6028e-04) (hash(x)=17851881)
+200 val loss 6.3819
+200 val perplexity 591.0219
+200 train 6.362373 (lr=1.6867e-04) (hash(x)=21138724)
+210 train 6.190283 (lr=1.7706e-04) (hash(x)=16978198)
+220 train 6.121952 (lr=1.8545e-04) (hash(x)=20812935)
+230 train 6.197385 (lr=1.9385e-04) (hash(x)=20619471)
+240 train 6.132792 (lr=2.0224e-04) (hash(x)=17706413)
+250 train 5.943398 (lr=2.1063e-04) (hash(x)=21557598)
+260 train 5.892084 (lr=2.1902e-04) (hash(x)=18077648)
+270 train 6.004688 (lr=2.2741e-04) (hash(x)=17053957)
+280 train 5.930441 (lr=2.3580e-04) (hash(x)=20526478)
+290 train 5.760366 (lr=2.4420e-04) (hash(x)=22226244)
+300 val loss 5.7749
+300 val perplexity 322.1271
+300 train 5.687343 (lr=2.5259e-04) (hash(x)=19583531)
+310 train 5.543598 (lr=2.6098e-04) (hash(x)=18986313)
+320 train 5.761025 (lr=2.6937e-04) (hash(x)=23629157)
+330 train 5.663108 (lr=2.7776e-04) (hash(x)=21312406)
+340 train 5.482827 (lr=2.8615e-04) (hash(x)=19278398)
+350 train 5.504622 (lr=2.9455e-04) (hash(x)=19370729)
+360 train 5.337851 (lr=3.0294e-04) (hash(x)=20663223)
+370 train 5.504222 (lr=3.1133e-04) (hash(x)=22952038)
+380 train 5.452213 (lr=3.1972e-04) (hash(x)=21902455)
+390 train 5.441855 (lr=3.2811e-04) (hash(x)=21953350)
+400 val loss 5.3378
+400 val perplexity 208.0621
+400 train 5.401706 (lr=3.3650e-04) (hash(x)=22915018)
+410 train 5.319546 (lr=3.4490e-04) (hash(x)=18301453)
+420 train 5.226571 (lr=3.5329e-04) (hash(x)=17422968)
+430 train 5.097789 (lr=3.6168e-04) (hash(x)=20616951)
+440 train 5.279465 (lr=3.7007e-04) (hash(x)=19912399)
+450 train 5.125679 (lr=3.7846e-04) (hash(x)=21154839)
+460 train 5.072629 (lr=3.8685e-04) (hash(x)=18919228)
+470 train 5.048383 (lr=3.9524e-04) (hash(x)=20916162)
+480 train 4.943963 (lr=4.0364e-04) (hash(x)=19136877)
+490 train 5.049657 (lr=4.1203e-04) (hash(x)=22631452)
+500 val loss 4.9411
+500 val perplexity 139.9249
+500 train 4.969503 (lr=4.2042e-04) (hash(x)=21454149)
+510 train 4.809808 (lr=4.2881e-04) (hash(x)=18263355)
+520 train 4.776715 (lr=4.3720e-04) (hash(x)=21540112)
+530 train 4.775882 (lr=4.4559e-04) (hash(x)=20378460)
+540 train 4.856249 (lr=4.5399e-04) (hash(x)=25242015)
+550 train 4.734632 (lr=4.6238e-04) (hash(x)=18067565)
+560 train 4.629473 (lr=4.7077e-04) (hash(x)=18532926)
+570 train 4.525438 (lr=4.7916e-04) (hash(x)=17014430)
+580 train 4.551474 (lr=4.8755e-04) (hash(x)=21587296)
+590 train 4.646731 (lr=4.9594e-04) (hash(x)=19956044)
+600 val loss 4.5973
+600 val perplexity 99.2118
+600 train 4.645379 (lr=5.0434e-04) (hash(x)=20658639)
+610 train 4.542713 (lr=5.1273e-04) (hash(x)=18301842)
+620 train 4.500952 (lr=5.2112e-04) (hash(x)=18675952)
+630 train 4.600066 (lr=5.2951e-04) (hash(x)=17602704)
+640 train 4.523263 (lr=5.3790e-04) (hash(x)=21491802)
+650 train 4.522380 (lr=5.4629e-04) (hash(x)=20557423)
+660 train 4.388939 (lr=5.5469e-04) (hash(x)=21188117)
+670 train 4.503300 (lr=5.6308e-04) (hash(x)=20683583)
+680 train 4.433919 (lr=5.7147e-04) (hash(x)=24879052)
+690 train 4.322607 (lr=5.7986e-04) (hash(x)=19538285)
+700 val loss 4.3354
+700 val perplexity 76.3560
+700 train 4.351713 (lr=5.8825e-04) (hash(x)=18338040)
+710 train 4.347413 (lr=5.9664e-04) (hash(x)=21087427)
+720 train 4.283252 (lr=5.9999e-04) (hash(x)=20546196)
+730 train 4.307380 (lr=5.9991e-04) (hash(x)=19605436)
+740 train 4.272770 (lr=5.9974e-04) (hash(x)=20510264)
+750 train 4.327110 (lr=5.9949e-04) (hash(x)=19782550)
+760 train 4.216037 (lr=5.9915e-04) (hash(x)=20997594)
+770 train 4.088340 (lr=5.9874e-04) (hash(x)=20733511)
+780 train 4.186656 (lr=5.9824e-04) (hash(x)=17584884)
+790 train 4.122427 (lr=5.9765e-04) (hash(x)=20952808)
+800 val loss 4.1762
+800 val perplexity 65.1149
+800 train 4.070551 (lr=5.9698e-04) (hash(x)=16092551)
+810 train 4.112890 (lr=5.9623e-04) (hash(x)=16116155)
+820 train 4.177746 (lr=5.9540e-04) (hash(x)=22196607)
+830 train 4.065981 (lr=5.9449e-04) (hash(x)=21490054)
+840 train 3.967163 (lr=5.9349e-04) (hash(x)=19471853)
+850 train 4.118798 (lr=5.9241e-04) (hash(x)=40176264)
+860 train 4.113213 (lr=5.9126e-04) (hash(x)=19241841)
+870 train 3.951173 (lr=5.9002e-04) (hash(x)=19010654)
+880 train 4.056929 (lr=5.8869e-04) (hash(x)=21812805)
+890 train 4.155994 (lr=5.8729e-04) (hash(x)=23266756)
+900 val loss 4.0417
+900 val perplexity 56.9229
+900 train 4.016778 (lr=5.8581e-04) (hash(x)=17198172)
+910 train 3.915779 (lr=5.8425e-04) (hash(x)=17654181)
+920 train 4.011151 (lr=5.8262e-04) (hash(x)=18651087)
+930 train 3.970577 (lr=5.8090e-04) (hash(x)=18812778)
+940 train 3.877449 (lr=5.7911e-04) (hash(x)=19229059)
+950 train 3.995198 (lr=5.7723e-04) (hash(x)=16244689)
+960 train 3.949887 (lr=5.7529e-04) (hash(x)=14252464)
+970 train 3.863349 (lr=5.7326e-04) (hash(x)=20812699)
+980 train 4.040447 (lr=5.7116e-04) (hash(x)=19274156)
+990 train 3.902785 (lr=5.6899e-04) (hash(x)=23893376)
+1000 val loss 3.9364
+1000 val perplexity 51.2362
+1000 train 3.793566 (lr=5.6674e-04) (hash(x)=20084739)
+1010 train 3.875041 (lr=5.6442e-04) (hash(x)=20537558)
+1020 train 3.964690 (lr=5.6202e-04) (hash(x)=20432294)
+1030 train 3.875535 (lr=5.5956e-04) (hash(x)=22466028)
+1040 train 3.844952 (lr=5.5702e-04) (hash(x)=17894926)
+1050 train 3.941573 (lr=5.5441e-04) (hash(x)=21738864)
+1060 train 3.860307 (lr=5.5174e-04) (hash(x)=18724801)
+1070 train 3.818421 (lr=5.4899e-04) (hash(x)=21845067)
+1080 train 3.775688 (lr=5.4618e-04) (hash(x)=17269002)
+1090 train 3.900982 (lr=5.4330e-04) (hash(x)=20777124)
+1100 val loss 3.8591
+1100 val perplexity 47.4222
+1100 train 3.829301 (lr=5.4035e-04) (hash(x)=21658646)
+1110 train 3.802896 (lr=5.3734e-04) (hash(x)=25217492)
+1120 train 3.866534 (lr=5.3426e-04) (hash(x)=17597309)
+1130 train 3.845275 (lr=5.3113e-04) (hash(x)=15251388)
+1140 train 3.831608 (lr=5.2792e-04) (hash(x)=16526754)
+1150 train 3.791356 (lr=5.2466e-04) (hash(x)=16839845)
+1160 train 3.805662 (lr=5.2134e-04) (hash(x)=19271941)
+1170 train 3.794544 (lr=5.1796e-04) (hash(x)=20159915)
+1180 train 3.760350 (lr=5.1452e-04) (hash(x)=18768657)
+1190 train 3.805496 (lr=5.1102e-04) (hash(x)=19691298)
+1200 val loss 3.7893
+1200 val perplexity 44.2264
+1200 train 3.802570 (lr=5.0746e-04) (hash(x)=20619530)
+1210 train 3.739477 (lr=5.0386e-04) (hash(x)=20327868)
+1220 train 3.728197 (lr=5.0019e-04) (hash(x)=17297005)
+1230 train 3.749378 (lr=4.9648e-04) (hash(x)=22743384)
+1240 train 3.752196 (lr=4.9271e-04) (hash(x)=19764130)
+1250 train 3.657744 (lr=4.8889e-04) (hash(x)=19685029)
+1260 train 3.719430 (lr=4.8503e-04) (hash(x)=21621842)
+1270 train 3.723538 (lr=4.8111e-04) (hash(x)=21024799)
+1280 train 3.727242 (lr=4.7715e-04) (hash(x)=20459305)
+1290 train 3.703695 (lr=4.7315e-04) (hash(x)=21945208)
+1300 val loss 3.7467
+1300 val perplexity 42.3791
+1300 train 3.756041 (lr=4.6909e-04) (hash(x)=19421909)
+1310 train 3.678718 (lr=4.6500e-04) (hash(x)=22065431)
+1320 train 3.651360 (lr=4.6086e-04) (hash(x)=18310987)
+1330 train 3.748827 (lr=4.5669e-04) (hash(x)=18046899)
+1340 train 3.665390 (lr=4.5247e-04) (hash(x)=19877417)
+1350 train 3.804711 (lr=4.4822e-04) (hash(x)=21157293)
+1360 train 3.681175 (lr=4.4393e-04) (hash(x)=20260598)
+1370 train 3.683757 (lr=4.3960e-04) (hash(x)=20189167)
+1380 train 3.673514 (lr=4.3524e-04) (hash(x)=22193383)
+1390 train 3.670562 (lr=4.3085e-04) (hash(x)=16407046)
+1400 val loss 3.6948
+1400 val perplexity 40.2357
+1400 train 3.683640 (lr=4.2643e-04) (hash(x)=22770666)
+1410 train 3.701669 (lr=4.2197e-04) (hash(x)=22418421)
+1420 train 3.766599 (lr=4.1749e-04) (hash(x)=17883057)
+1430 train 3.603860 (lr=4.1298e-04) (hash(x)=18408864)
+1440 train 3.593240 (lr=4.0845e-04) (hash(x)=19699467)
+1450 train 3.656230 (lr=4.0389e-04) (hash(x)=21354581)
+1460 train 3.682720 (lr=3.9931e-04) (hash(x)=20422335)
+1470 train 3.585140 (lr=3.9470e-04) (hash(x)=18953370)
+1480 train 3.695054 (lr=3.9008e-04) (hash(x)=23442747)
+1490 train 3.741323 (lr=3.8544e-04) (hash(x)=20126187)
+1500 val loss 3.6550
+1500 val perplexity 38.6672
+1500 train 3.682213 (lr=3.8078e-04) (hash(x)=20342754)
+1510 train 3.641150 (lr=3.7610e-04) (hash(x)=19036313)
+1520 train 3.687203 (lr=3.7142e-04) (hash(x)=22164007)
+1530 train 3.654878 (lr=3.6671e-04) (hash(x)=24857054)
+1540 train 3.623704 (lr=3.6200e-04) (hash(x)=21281762)
+1550 train 3.582209 (lr=3.5728e-04) (hash(x)=19650272)
+1560 train 3.684850 (lr=3.5255e-04) (hash(x)=20137905)
+1570 train 3.521481 (lr=3.4781e-04) (hash(x)=14416059)
+1580 train 3.653732 (lr=3.4306e-04) (hash(x)=23684747)
+1590 train 3.496739 (lr=3.3831e-04) (hash(x)=20092448)
+1600 val loss 3.6176
+1600 val perplexity 37.2491
+1600 train 3.579444 (lr=3.3356e-04) (hash(x)=18725986)
+1610 train 3.565784 (lr=3.2881e-04) (hash(x)=27357245)
+1620 train 3.479879 (lr=3.2406e-04) (hash(x)=20033464)
+1630 train 3.708945 (lr=3.1931e-04) (hash(x)=20930109)
+1640 train 3.647739 (lr=3.1456e-04) (hash(x)=19398603)
+1650 train 3.683493 (lr=3.0982e-04) (hash(x)=20959448)
+1660 train 3.402952 (lr=3.0509e-04) (hash(x)=19692923)
+1670 train 3.660202 (lr=3.0036e-04) (hash(x)=23448284)
+1680 train 3.650345 (lr=2.9564e-04) (hash(x)=17968371)
+1690 train 3.579404 (lr=2.9093e-04) (hash(x)=15669180)
+1700 val loss 3.5855
+1700 val perplexity 36.0696
+1700 train 3.393665 (lr=2.8624e-04) (hash(x)=17816072)
+1710 train 3.593935 (lr=2.8156e-04) (hash(x)=19307466)
+1720 train 3.637766 (lr=2.7689e-04) (hash(x)=22978699)
+1730 train 3.606547 (lr=2.7224e-04) (hash(x)=23143048)
+1740 train 3.479164 (lr=2.6761e-04) (hash(x)=18722687)
+1750 train 3.409755 (lr=2.6299e-04) (hash(x)=18477212)
+1760 train 3.590554 (lr=2.5840e-04) (hash(x)=19884708)
+1770 train 3.603837 (lr=2.5383e-04) (hash(x)=20256662)
+1780 train 3.572342 (lr=2.4928e-04) (hash(x)=18622247)
+1790 train 3.535561 (lr=2.4476e-04) (hash(x)=21400648)
+1800 val loss 3.5610
+1800 val perplexity 35.2001
+1800 train 3.362972 (lr=2.4026e-04) (hash(x)=23156122)
+1810 train 3.604962 (lr=2.3580e-04) (hash(x)=13466715)
+1820 train 3.534439 (lr=2.3136e-04) (hash(x)=18810586)
+1830 train 3.533193 (lr=2.2695e-04) (hash(x)=20834847)
+1840 train 3.355483 (lr=2.2257e-04) (hash(x)=14702111)
+1850 train 3.590870 (lr=2.1823e-04) (hash(x)=18470456)
+1860 train 3.582299 (lr=2.1392e-04) (hash(x)=20701741)
+1870 train 3.515582 (lr=2.0965e-04) (hash(x)=19669016)
+1880 train 3.496598 (lr=2.0542e-04) (hash(x)=20381930)
+1890 train 3.273951 (lr=2.0122e-04) (hash(x)=21303953)
+1900 val loss 3.5348
+1900 val perplexity 34.2885
+1900 train 3.557092 (lr=1.9706e-04) (hash(x)=22119135)
+1910 train 3.610139 (lr=1.9295e-04) (hash(x)=19343834)
+1920 train 3.508832 (lr=1.8887e-04) (hash(x)=20131130)
+1930 train 3.357190 (lr=1.8485e-04) (hash(x)=21254128)
+1940 train 3.516062 (lr=1.8086e-04) (hash(x)=20816939)
+1950 train 3.559673 (lr=1.7692e-04) (hash(x)=19820308)
+1960 train 3.556221 (lr=1.7303e-04) (hash(x)=28009259)
+1970 train 3.459218 (lr=1.6919e-04) (hash(x)=19370262)
+1980 train 3.281868 (lr=1.6540e-04) (hash(x)=19401165)
+1990 train 3.559883 (lr=1.6166e-04) (hash(x)=18628676)
+2000 val loss 3.5102
+2000 val perplexity 33.4536
+2000 train 3.556651 (lr=1.5797e-04) (hash(x)=20151188)
+2010 train 3.560676 (lr=1.5433e-04) (hash(x)=18053856)
+2020 train 3.555864 (lr=1.5075e-04) (hash(x)=20481771)
+2030 train 3.581336 (lr=1.4723e-04) (hash(x)=18607449)
+2040 train 3.465315 (lr=1.4376e-04) (hash(x)=21897936)
+2050 train 3.493999 (lr=1.4035e-04) (hash(x)=19648528)
+2060 train 3.518195 (lr=1.3699e-04) (hash(x)=21541784)
+2070 train 3.538033 (lr=1.3370e-04) (hash(x)=19466104)
+2080 train 3.507005 (lr=1.3047e-04) (hash(x)=19540614)
+2090 train 3.487651 (lr=1.2730e-04) (hash(x)=18783653)
+2100 val loss 3.4960
+2100 val perplexity 32.9817
+2100 train 3.423492 (lr=1.2419e-04) (hash(x)=30861019)
+2110 train 3.449903 (lr=1.2115e-04) (hash(x)=20187147)
+2120 train 3.550241 (lr=1.1817e-04) (hash(x)=17350218)
+2130 train 3.589391 (lr=1.1525e-04) (hash(x)=21668918)
+2140 train 3.452075 (lr=1.1241e-04) (hash(x)=16953068)
+2150 train 3.563969 (lr=1.0963e-04) (hash(x)=22435234)
+2160 train 3.457865 (lr=1.0692e-04) (hash(x)=14277862)
+2170 train 3.532546 (lr=1.0427e-04) (hash(x)=21051545)
+2180 train 3.505602 (lr=1.0170e-04) (hash(x)=16728289)
+2190 train 3.509662 (lr=9.9199e-05) (hash(x)=18519101)
+2200 val loss 3.4758
+2200 val perplexity 32.3226
+2200 train 3.510487 (lr=9.6769e-05) (hash(x)=19727249)
+2210 train 3.360144 (lr=9.4412e-05) (hash(x)=18890853)
+2220 train 3.518687 (lr=9.2127e-05) (hash(x)=20660564)
+2230 train 3.533267 (lr=8.9916e-05) (hash(x)=22774110)
+2240 train 3.499394 (lr=8.7779e-05) (hash(x)=19389926)
+2250 train 3.532896 (lr=8.5717e-05) (hash(x)=17041691)
+2260 train 3.413473 (lr=8.3731e-05) (hash(x)=20807441)
+2270 train 3.540822 (lr=8.1821e-05) (hash(x)=22538437)
+2280 train 3.563267 (lr=7.9988e-05) (hash(x)=21551546)
+2290 train 3.437593 (lr=7.8232e-05) (hash(x)=24167699)
+2300 val loss 3.4634
+2300 val perplexity 31.9268
+2300 train 3.348315 (lr=7.6555e-05) (hash(x)=16362157)
+2310 train 3.321218 (lr=7.4956e-05) (hash(x)=23722796)
+2320 train 3.548417 (lr=7.3436e-05) (hash(x)=18346470)
+2330 train 3.499416 (lr=7.1995e-05) (hash(x)=17096354)
+2340 train 3.500489 (lr=7.0635e-05) (hash(x)=23151918)
+2350 train 3.315239 (lr=6.9354e-05) (hash(x)=17604704)
+2360 train 3.455883 (lr=6.8155e-05) (hash(x)=20390213)
+2370 train 3.502772 (lr=6.7036e-05) (hash(x)=24875449)
+2380 train 3.524223 (lr=6.5999e-05) (hash(x)=21185640)
+2390 train 3.458211 (lr=6.5044e-05) (hash(x)=14839939)
+2400 val loss 3.4516
+2400 val perplexity 31.5520
+2400 train 3.525647 (lr=6.4171e-05) (hash(x)=17498688)
+2410 train 3.393266 (lr=6.3380e-05) (hash(x)=20544025)
+2420 train 3.402240 (lr=6.2672e-05) (hash(x)=21093275)
+2430 train 3.387079 (lr=6.2046e-05) (hash(x)=19270991)
+2440 train 3.512996 (lr=6.1504e-05) (hash(x)=17536544)
+2450 train 3.443870 (lr=6.1045e-05) (hash(x)=16978919)
+2460 train 3.446483 (lr=6.0669e-05) (hash(x)=18893211)
+2470 train 3.424699 (lr=6.0376e-05) (hash(x)=17072230)
+2480 train 3.343539 (lr=6.0167e-05) (hash(x)=21601992)
+2490 train 3.478189 (lr=6.0042e-05) (hash(x)=23385916)
+2499 val loss 3.4453
+2499 val perplexity 31.3542

allowing_more_selection_patterns/one_mask_per_head_1_latent_vector_degenerate_seed_1338/model_02499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a56a8848993993ef75b1d1baceafcdd3a8bb0ff4a5dbb75ffb43f54bf93b450d
+size 555244826

allowing_more_selection_patterns/one_mask_per_head_1_latent_vector_degenerate_seed_1338/optimizer_02499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:301236b9c726bc3b495839bca87caee7ae5a5a2d1d6925f60cc8b0c83ea8d731
+size 1009837470