Upload folder using huggingface_hub

one_mask_per_head/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "allowing_more_selection_patterns/one_mask_per_head", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 12, "n_embd": 768, "head_dim": 64, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": null, "warmup_steps": null, "group": "allowing_more_selection_patterns", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1337, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "one_mask_per_head", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": null, "batch_size": 4, "total_batch_size": null, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": false, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": null, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": null}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "allowing_more_selection_patterns/one_mask_per_head", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 12, "n_embd": 768, "head_dim": 64, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": null, "warmup_steps": null, "group": "allowing_more_selection_patterns", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "one_mask_per_head", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": null, "batch_size": 4, "total_batch_size": null, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": false, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": null, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": null}

one_mask_per_head/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 2500
-0 val loss 10.9988
-0 val perplexity 59800.5273
-0 train 10.997852 (lr=8.3916e-07) (hash(x)=23347629)
-10 train 10.213652 (lr=9.2308e-06) (hash(x)=19636655)
-20 train 9.692251 (lr=1.7622e-05) (hash(x)=24490248)
-30 train 9.494464 (lr=2.6014e-05) (hash(x)=21733558)
-40 train 9.236492 (lr=3.4406e-05) (hash(x)=31750616)
-50 train 8.924097 (lr=4.2797e-05) (hash(x)=20041895)
-60 train 8.558031 (lr=5.1189e-05) (hash(x)=21416280)
-70 train 8.223051 (lr=5.9580e-05) (hash(x)=18066780)
-80 train 7.902947 (lr=6.7972e-05) (hash(x)=21809427)
-90 train 7.679000 (lr=7.6364e-05) (hash(x)=22061595)
-100 val loss 7.4737
-100 val perplexity 1761.1528
-100 train 7.422636 (lr=8.4755e-05) (hash(x)=17468914)
-110 train 7.239632 (lr=9.3147e-05) (hash(x)=20421520)
-120 train 7.248019 (lr=1.0154e-04) (hash(x)=23140133)
-130 train 7.109382 (lr=1.0993e-04) (hash(x)=20006835)
-140 train 6.874236 (lr=1.1832e-04) (hash(x)=18628134)
-150 train 6.840735 (lr=1.2671e-04) (hash(x)=21913447)
-160 train 6.771353 (lr=1.3510e-04) (hash(x)=22997297)
-170 train 6.570288 (lr=1.4350e-04) (hash(x)=17904821)
-180 train 6.434371 (lr=1.5189e-04) (hash(x)=18826982)
-190 train 6.515481 (lr=1.6028e-04) (hash(x)=21387716)
-200 val loss 6.3950
-200 val perplexity 598.8182
-200 train 6.315248 (lr=1.6867e-04) (hash(x)=16236395)
-210 train 6.192067 (lr=1.7706e-04) (hash(x)=21428681)
-220 train 6.333644 (lr=1.8545e-04) (hash(x)=20461738)
-230 train 6.156755 (lr=1.9385e-04) (hash(x)=20394865)
-240 train 6.050922 (lr=2.0224e-04) (hash(x)=18028671)
-250 train 5.910237 (lr=2.1063e-04) (hash(x)=18122073)
-260 train 5.831408 (lr=2.1902e-04) (hash(x)=22994143)
-270 train 6.015007 (lr=2.2741e-04) (hash(x)=20646260)
-280 train 5.864094 (lr=2.3580e-04) (hash(x)=23204708)
-290 train 5.771457 (lr=2.4420e-04) (hash(x)=17900252)
-300 val loss 5.8071
-300 val perplexity 332.6530
-300 train 5.649515 (lr=2.5259e-04) (hash(x)=19073819)
-310 train 5.807569 (lr=2.6098e-04) (hash(x)=22378232)
-320 train 5.728095 (lr=2.6937e-04) (hash(x)=19910430)
-330 train 5.645929 (lr=2.7776e-04) (hash(x)=25168753)
-340 train 5.526587 (lr=2.8615e-04) (hash(x)=20665256)
-350 train 5.392931 (lr=2.9455e-04) (hash(x)=19951160)
-360 train 5.613574 (lr=3.0294e-04) (hash(x)=21178202)
-370 train 5.452641 (lr=3.1133e-04) (hash(x)=20435886)
-380 train 5.391888 (lr=3.1972e-04) (hash(x)=21255097)
-390 train 5.407190 (lr=3.2811e-04) (hash(x)=19596330)
-400 val loss 5.3424
-400 val perplexity 209.0189
-400 train 5.368718 (lr=3.3650e-04) (hash(x)=21789426)
-410 train 5.270738 (lr=3.4490e-04) (hash(x)=18366401)
-420 train 5.175039 (lr=3.5329e-04) (hash(x)=18614660)
-430 train 5.290539 (lr=3.6168e-04) (hash(x)=21649063)
-440 train 5.177164 (lr=3.7007e-04) (hash(x)=18505097)
-450 train 5.171529 (lr=3.7846e-04) (hash(x)=23434312)
-460 train 5.090730 (lr=3.8685e-04) (hash(x)=16584569)
-470 train 4.938862 (lr=3.9524e-04) (hash(x)=13447111)
-480 train 5.157804 (lr=4.0364e-04) (hash(x)=22645130)
-490 train 5.005649 (lr=4.1203e-04) (hash(x)=19362777)
-500 val loss 4.9958
-500 val perplexity 147.7923
-500 train 4.929473 (lr=4.2042e-04) (hash(x)=20921874)
-510 train 4.819194 (lr=4.2881e-04) (hash(x)=22123048)
-520 train 4.844348 (lr=4.3720e-04) (hash(x)=21215368)
-530 train 5.024788 (lr=4.4559e-04) (hash(x)=20717936)
-540 train 4.859024 (lr=4.5399e-04) (hash(x)=20118007)
-550 train 4.720725 (lr=4.6238e-04) (hash(x)=21402522)
-560 train 4.654364 (lr=4.7077e-04) (hash(x)=19057180)
-570 train 4.847821 (lr=4.7916e-04) (hash(x)=19360553)
-580 train 4.737807 (lr=4.8755e-04) (hash(x)=18194693)
-590 train 4.643238 (lr=4.9594e-04) (hash(x)=19863826)
-600 val loss 4.5869
-600 val perplexity 98.1939
-600 train 4.541944 (lr=5.0434e-04) (hash(x)=21794551)
-610 train 4.515571 (lr=5.1273e-04) (hash(x)=18685622)
-620 train 4.413346 (lr=5.2112e-04) (hash(x)=23473104)
-630 train 4.504895 (lr=5.2951e-04) (hash(x)=18671672)
-640 train 4.514694 (lr=5.3790e-04) (hash(x)=19234240)
-650 train 4.398890 (lr=5.4629e-04) (hash(x)=23183903)
-660 train 4.275322 (lr=5.5469e-04) (hash(x)=23665543)
-670 train 4.399535 (lr=5.6308e-04) (hash(x)=22055313)
-680 train 4.381485 (lr=5.7147e-04) (hash(x)=22321668)
-690 train 4.487019 (lr=5.7986e-04) (hash(x)=21568926)
-700 val loss 4.3270
-700 val perplexity 75.7199
-700 train 4.331699 (lr=5.8825e-04) (hash(x)=19783143)
-710 train 4.273065 (lr=5.9664e-04) (hash(x)=21565258)
-720 train 4.351468 (lr=5.9999e-04) (hash(x)=16127212)
-730 train 4.270386 (lr=5.9991e-04) (hash(x)=19658174)
-740 train 4.252528 (lr=5.9974e-04) (hash(x)=16454319)
-750 train 4.199822 (lr=5.9949e-04) (hash(x)=17993154)
-760 train 4.084884 (lr=5.9915e-04) (hash(x)=19256989)
-770 train 4.087481 (lr=5.9874e-04) (hash(x)=17351653)
-780 train 4.102091 (lr=5.9824e-04) (hash(x)=21626120)
-790 train 4.231406 (lr=5.9765e-04) (hash(x)=16367412)
-800 val loss 4.1594
-800 val perplexity 64.0332
-800 train 4.112117 (lr=5.9698e-04) (hash(x)=22321504)
-810 train 3.993076 (lr=5.9623e-04) (hash(x)=19869755)
-820 train 4.175332 (lr=5.9540e-04) (hash(x)=24183665)
-830 train 4.135139 (lr=5.9449e-04) (hash(x)=19194113)
-840 train 3.987583 (lr=5.9349e-04) (hash(x)=17710503)
-850 train 3.998139 (lr=5.9241e-04) (hash(x)=17231851)
-860 train 4.144989 (lr=5.9126e-04) (hash(x)=20584272)
-870 train 4.034049 (lr=5.9002e-04) (hash(x)=17468790)
-880 train 3.961010 (lr=5.8869e-04) (hash(x)=22304409)
-890 train 4.061734 (lr=5.8729e-04) (hash(x)=18871567)
-900 val loss 4.0301
-900 val perplexity 56.2668
-900 train 4.005404 (lr=5.8581e-04) (hash(x)=20924720)
-910 train 3.914894 (lr=5.8425e-04) (hash(x)=21163464)
-920 train 3.858732 (lr=5.8262e-04) (hash(x)=20120862)
-930 train 4.035140 (lr=5.8090e-04) (hash(x)=19565198)
-940 train 3.952987 (lr=5.7911e-04) (hash(x)=18517540)
-950 train 3.920708 (lr=5.7723e-04) (hash(x)=18300759)
-960 train 3.957448 (lr=5.7529e-04) (hash(x)=21443001)
-970 train 3.898010 (lr=5.7326e-04) (hash(x)=15210971)
-980 train 4.078487 (lr=5.7116e-04) (hash(x)=20874579)
-990 train 3.991467 (lr=5.6899e-04) (hash(x)=20690802)
-1000 val loss 3.9387
-1000 val perplexity 51.3542
-1000 train 3.858361 (lr=5.6674e-04) (hash(x)=18635877)
-1010 train 3.967668 (lr=5.6442e-04) (hash(x)=21141462)
-1020 train 3.998059 (lr=5.6202e-04) (hash(x)=19483400)
-1030 train 3.898628 (lr=5.5956e-04) (hash(x)=24792825)
-1040 train 3.870465 (lr=5.5702e-04) (hash(x)=16743928)
-1050 train 3.942699 (lr=5.5441e-04) (hash(x)=22540052)
-1060 train 3.958479 (lr=5.5174e-04) (hash(x)=20546637)
-1070 train 3.843789 (lr=5.4899e-04) (hash(x)=13739753)
-1080 train 3.885016 (lr=5.4618e-04) (hash(x)=20071304)
-1090 train 3.857270 (lr=5.4330e-04) (hash(x)=19339656)
-1100 val loss 3.8710
-1100 val perplexity 47.9882
-1100 train 3.837739 (lr=5.4035e-04) (hash(x)=20039628)
-1110 train 3.897439 (lr=5.3734e-04) (hash(x)=19249691)
-1120 train 3.965502 (lr=5.3426e-04) (hash(x)=19048809)
-1130 train 3.866039 (lr=5.3113e-04) (hash(x)=19309000)
-1140 train 3.810650 (lr=5.2792e-04) (hash(x)=21787927)
-1150 train 3.719844 (lr=5.2466e-04) (hash(x)=19998536)
-1160 train 3.874866 (lr=5.2134e-04) (hash(x)=23724694)
-1170 train 3.956590 (lr=5.1796e-04) (hash(x)=17148204)
-1180 train 3.850413 (lr=5.1452e-04) (hash(x)=19511852)
-1190 train 3.833079 (lr=5.1102e-04) (hash(x)=19352862)
-1200 val loss 3.8416
-1200 val perplexity 46.6002
-1200 train 3.753381 (lr=5.0746e-04) (hash(x)=19530144)
-1210 train 3.923264 (lr=5.0386e-04) (hash(x)=19897392)
-1220 train 3.850474 (lr=5.0019e-04) (hash(x)=24402963)
-1230 train 3.767090 (lr=4.9648e-04) (hash(x)=18589754)
-1240 train 3.764629 (lr=4.9271e-04) (hash(x)=21951750)
-1250 train 3.827978 (lr=4.8889e-04) (hash(x)=18611379)
-1260 train 3.856350 (lr=4.8503e-04) (hash(x)=20338521)
-1270 train 3.790913 (lr=4.8111e-04) (hash(x)=18308405)
-1280 train 3.815897 (lr=4.7715e-04) (hash(x)=20920945)
-1290 train 3.759041 (lr=4.7315e-04) (hash(x)=20094152)
-1300 val loss 3.7916
-1300 val perplexity 44.3268
-1300 train 3.867037 (lr=4.6909e-04) (hash(x)=19003947)
-1310 train 3.944770 (lr=4.6500e-04) (hash(x)=19356356)
-1320 train 3.830324 (lr=4.6086e-04) (hash(x)=19390343)
-1330 train 3.737658 (lr=4.5669e-04) (hash(x)=17535982)
-1340 train 3.755102 (lr=4.5247e-04) (hash(x)=21959104)
-1350 train 3.899371 (lr=4.4822e-04) (hash(x)=21982099)
-1360 train 3.844067 (lr=4.4393e-04) (hash(x)=17176243)
-1370 train 3.776604 (lr=4.3960e-04) (hash(x)=21223627)
-1380 train 3.833573 (lr=4.3524e-04) (hash(x)=22462931)
-1390 train 3.923160 (lr=4.3085e-04) (hash(x)=19153464)
-1400 val loss 3.8221
-1400 val perplexity 45.7013
-1400 train 3.743105 (lr=4.2643e-04) (hash(x)=20146496)
-1410 train 3.780858 (lr=4.2197e-04) (hash(x)=21781327)
-1420 train 3.864768 (lr=4.1749e-04) (hash(x)=19234051)
-1430 train 3.921422 (lr=4.1298e-04) (hash(x)=20769898)
-1440 train 3.745297 (lr=4.0845e-04) (hash(x)=22042430)
-1450 train 3.760057 (lr=4.0389e-04) (hash(x)=20527387)
-1460 train 3.777525 (lr=3.9931e-04) (hash(x)=18925766)
-1470 train 3.801388 (lr=3.9470e-04) (hash(x)=18198851)
-1480 train 3.857671 (lr=3.9008e-04) (hash(x)=21847281)
-1490 train 3.836791 (lr=3.8544e-04) (hash(x)=15645274)
-1500 val loss 3.8190
-1500 val perplexity 45.5603
-1500 train 3.736359 (lr=3.8078e-04) (hash(x)=14693370)
-1510 train 3.705180 (lr=3.7610e-04) (hash(x)=21810857)
-1520 train 3.802008 (lr=3.7142e-04) (hash(x)=18980052)
-1530 train 3.900253 (lr=3.6671e-04) (hash(x)=22868842)
-1540 train 3.773475 (lr=3.6200e-04) (hash(x)=17289166)
-1550 train 3.908131 (lr=3.5728e-04) (hash(x)=18109373)
-1560 train 3.900035 (lr=3.5255e-04) (hash(x)=22377619)
-1570 train 3.691564 (lr=3.4781e-04) (hash(x)=20307182)
-1580 train 3.842846 (lr=3.4306e-04) (hash(x)=20391291)
-1590 train 3.776944 (lr=3.3831e-04) (hash(x)=18690939)
-1600 val loss 3.8696
-1600 val perplexity 47.9254
-1600 train 3.831928 (lr=3.3356e-04) (hash(x)=16719886)
-1610 train 3.924848 (lr=3.2881e-04) (hash(x)=16040840)
-1620 train 3.984766 (lr=3.2406e-04) (hash(x)=25795140)
-1630 train 3.890431 (lr=3.1931e-04) (hash(x)=21936964)
-1640 train 3.799555 (lr=3.1456e-04) (hash(x)=11593027)
-1650 train 3.890815 (lr=3.0982e-04) (hash(x)=23256718)
-1660 train 3.811165 (lr=3.0509e-04) (hash(x)=20289140)
-1670 train 3.760187 (lr=3.0036e-04) (hash(x)=18382509)
-1680 train 3.820527 (lr=2.9564e-04) (hash(x)=26604329)
-1690 train 3.881943 (lr=2.9093e-04) (hash(x)=22122534)
-1700 val loss 3.8225
-1700 val perplexity 45.7183
-1700 train 3.873223 (lr=2.8624e-04) (hash(x)=22884812)
-1710 train 3.779017 (lr=2.8156e-04) (hash(x)=18598496)
-1720 train 3.790547 (lr=2.7689e-04) (hash(x)=22147447)
-1730 train 3.790515 (lr=2.7224e-04) (hash(x)=18912135)
-1740 train 3.621968 (lr=2.6761e-04) (hash(x)=21952974)
-1750 train 3.723291 (lr=2.6299e-04) (hash(x)=18390978)
-1760 train 3.777165 (lr=2.5840e-04) (hash(x)=17979288)
-1770 train 3.767013 (lr=2.5383e-04) (hash(x)=18979099)
-1780 train 3.716678 (lr=2.4928e-04) (hash(x)=20511922)
-1790 train 3.772341 (lr=2.4476e-04) (hash(x)=21814060)
-1800 val loss 3.8093
-1800 val perplexity 45.1177
-1800 train 3.820219 (lr=2.4026e-04) (hash(x)=18076243)
-1810 train 3.792329 (lr=2.3580e-04) (hash(x)=19721209)
-1820 train 3.694084 (lr=2.3136e-04) (hash(x)=18140847)
-1830 train 3.771098 (lr=2.2695e-04) (hash(x)=22065413)
-1840 train 3.652997 (lr=2.2257e-04) (hash(x)=21166111)
-1850 train 3.639060 (lr=2.1823e-04) (hash(x)=21904302)
-1860 train 3.777632 (lr=2.1392e-04) (hash(x)=18283828)
-1870 train 3.794472 (lr=2.0965e-04) (hash(x)=20892773)
-1880 train 3.759484 (lr=2.0542e-04) (hash(x)=17781264)
-1890 train 3.721720 (lr=2.0122e-04) (hash(x)=20612962)
-1900 val loss 3.7486
-1900 val perplexity 42.4608
-1900 train 3.808373 (lr=1.9706e-04) (hash(x)=20421058)
-1910 train 3.795221 (lr=1.9295e-04) (hash(x)=18554808)
-1920 train 3.750236 (lr=1.8887e-04) (hash(x)=18189279)
-1930 train 3.749361 (lr=1.8485e-04) (hash(x)=19923279)
-1940 train 3.728091 (lr=1.8086e-04) (hash(x)=21665511)
-1950 train 3.755924 (lr=1.7692e-04) (hash(x)=22038489)
-1960 train 3.723630 (lr=1.7303e-04) (hash(x)=18896956)
-1970 train 3.750365 (lr=1.6919e-04) (hash(x)=20408383)
-1980 train 3.725852 (lr=1.6540e-04) (hash(x)=23039940)
-1990 train 3.858301 (lr=1.6166e-04) (hash(x)=20631172)
-2000 val loss 3.7238
-2000 val perplexity 41.4209
-2000 train 3.646249 (lr=1.5797e-04) (hash(x)=20206106)
-2010 train 3.768192 (lr=1.5433e-04) (hash(x)=21864020)
-2020 train 3.748118 (lr=1.5075e-04) (hash(x)=21391128)
-2030 train 3.693563 (lr=1.4723e-04) (hash(x)=18676636)
-2040 train 3.765742 (lr=1.4376e-04) (hash(x)=15987134)
-2050 train 3.772542 (lr=1.4035e-04) (hash(x)=23940769)
-2060 train 3.738853 (lr=1.3699e-04) (hash(x)=17152840)
-2070 train 3.687091 (lr=1.3370e-04) (hash(x)=22958445)
-2080 train 3.780243 (lr=1.3047e-04) (hash(x)=23616724)
-2090 train 3.720351 (lr=1.2730e-04) (hash(x)=16571283)
-2100 val loss 3.7288
-2100 val perplexity 41.6310
-2100 train 3.726877 (lr=1.2419e-04) (hash(x)=19070327)
-2110 train 3.804023 (lr=1.2115e-04) (hash(x)=20758626)
-2120 train 3.721495 (lr=1.1817e-04) (hash(x)=18397810)
-2130 train 3.687106 (lr=1.1525e-04) (hash(x)=18045234)
-2140 train 3.807865 (lr=1.1241e-04) (hash(x)=18969565)
-2150 train 3.743411 (lr=1.0963e-04) (hash(x)=17149701)
-2160 train 3.655609 (lr=1.0692e-04) (hash(x)=23075443)
-2170 train 3.705993 (lr=1.0427e-04) (hash(x)=17977337)
-2180 train 3.772941 (lr=1.0170e-04) (hash(x)=20657407)
-2190 train 3.749608 (lr=9.9199e-05) (hash(x)=19985933)
-2200 val loss 3.6977
-2200 val perplexity 40.3531
-2200 train 3.678438 (lr=9.6769e-05) (hash(x)=19025387)
-2210 train 3.711391 (lr=9.4412e-05) (hash(x)=23318724)
-2220 train 3.781911 (lr=9.2127e-05) (hash(x)=19308039)
-2230 train 3.610281 (lr=8.9916e-05) (hash(x)=20733682)
-2240 train 3.728448 (lr=8.7779e-05) (hash(x)=18519560)
-2250 train 3.743205 (lr=8.5717e-05) (hash(x)=20299444)
-2260 train 3.706665 (lr=8.3731e-05) (hash(x)=18819615)
-2270 train 3.717638 (lr=8.1821e-05) (hash(x)=18743705)
-2280 train 3.726284 (lr=7.9988e-05) (hash(x)=17190793)
-2290 train 3.756134 (lr=7.8232e-05) (hash(x)=23126906)
-2300 val loss 3.7040
-2300 val perplexity 40.6098
-2300 train 3.677019 (lr=7.6555e-05) (hash(x)=24812267)
-2310 train 3.677190 (lr=7.4956e-05) (hash(x)=20147105)
-2320 train 3.800451 (lr=7.3436e-05) (hash(x)=24619936)
-2330 train 3.638439 (lr=7.1995e-05) (hash(x)=17730658)
-2340 train 3.604113 (lr=7.0635e-05) (hash(x)=19345731)
-2350 train 3.690310 (lr=6.9354e-05) (hash(x)=26415058)
-2360 train 3.693834 (lr=6.8155e-05) (hash(x)=18622999)
-2370 train 3.613158 (lr=6.7036e-05) (hash(x)=18848567)
-2380 train 3.747236 (lr=6.5999e-05) (hash(x)=21903360)
-2390 train 3.755122 (lr=6.5044e-05) (hash(x)=23479216)
-2400 val loss 3.6984
-2400 val perplexity 40.3838
-2400 train 3.718506 (lr=6.4171e-05) (hash(x)=19762013)
-2410 train 3.613304 (lr=6.3380e-05) (hash(x)=20622912)
-2420 train 3.774021 (lr=6.2672e-05) (hash(x)=17669226)
-2430 train 3.670013 (lr=6.2046e-05) (hash(x)=19536719)
-2440 train 3.604573 (lr=6.1504e-05) (hash(x)=19913725)
-2450 train 3.682737 (lr=6.1045e-05) (hash(x)=20782072)
-2460 train 3.642077 (lr=6.0669e-05) (hash(x)=19795002)
-2470 train 3.688946 (lr=6.0376e-05) (hash(x)=17003051)
-2480 train 3.635295 (lr=6.0167e-05) (hash(x)=21414190)
-2490 train 3.708986 (lr=6.0042e-05) (hash(x)=20943620)
-2499 val loss 3.6888
-2499 val perplexity 39.9976
+0 val loss 10.9681
+0 val perplexity 57992.1758
+0 train 10.968928 (lr=8.3916e-07) (hash(x)=17183924)
+10 train 10.165615 (lr=9.2308e-06) (hash(x)=19047484)
+20 train 9.648811 (lr=1.7622e-05) (hash(x)=20001614)
+30 train 9.416683 (lr=2.6014e-05) (hash(x)=21278543)
+40 train 9.170694 (lr=3.4406e-05) (hash(x)=29208111)
+50 train 8.862000 (lr=4.2797e-05) (hash(x)=20801292)
+60 train 8.523182 (lr=5.1189e-05) (hash(x)=27442073)
+70 train 8.159751 (lr=5.9580e-05) (hash(x)=19918406)
+80 train 7.802775 (lr=6.7972e-05) (hash(x)=20569806)
+90 train 7.686491 (lr=7.6364e-05) (hash(x)=26013441)
+100 val loss 7.4656
+100 val perplexity 1746.9067
+100 train 7.441105 (lr=8.4755e-05) (hash(x)=29039793)
+110 train 7.268086 (lr=9.3147e-05) (hash(x)=16550842)
+120 train 7.100458 (lr=1.0154e-04) (hash(x)=17250570)
+130 train 7.124823 (lr=1.0993e-04) (hash(x)=17154654)
+140 train 6.982364 (lr=1.1832e-04) (hash(x)=22437415)
+150 train 6.916542 (lr=1.2671e-04) (hash(x)=18645582)
+160 train 6.645574 (lr=1.3510e-04) (hash(x)=19460261)
+170 train 6.489112 (lr=1.4350e-04) (hash(x)=20953331)
+180 train 6.576027 (lr=1.5189e-04) (hash(x)=20883978)
+190 train 6.445021 (lr=1.6028e-04) (hash(x)=25571428)
+200 val loss 6.3931
+200 val perplexity 597.7213
+200 train 6.315374 (lr=1.6867e-04) (hash(x)=16049634)
+210 train 6.290833 (lr=1.7706e-04) (hash(x)=15886413)
+220 train 6.333457 (lr=1.8545e-04) (hash(x)=21731617)
+230 train 6.125866 (lr=1.9385e-04) (hash(x)=22509449)
+240 train 6.068533 (lr=2.0224e-04) (hash(x)=17396544)
+250 train 6.102439 (lr=2.1063e-04) (hash(x)=17748336)
+260 train 5.988758 (lr=2.1902e-04) (hash(x)=21263532)
+270 train 5.927268 (lr=2.2741e-04) (hash(x)=19623735)
+280 train 5.914373 (lr=2.3580e-04) (hash(x)=18337637)
+290 train 5.867254 (lr=2.4420e-04) (hash(x)=20005903)
+300 val loss 5.7703
+300 val perplexity 320.6483
+300 train 5.787907 (lr=2.5259e-04) (hash(x)=20938811)
+310 train 5.689990 (lr=2.6098e-04) (hash(x)=20688368)
+320 train 5.700755 (lr=2.6937e-04) (hash(x)=19776598)
+330 train 5.619760 (lr=2.7776e-04) (hash(x)=19721837)
+340 train 5.552475 (lr=2.8615e-04) (hash(x)=16578291)
+350 train 5.468309 (lr=2.9455e-04) (hash(x)=19998371)
+360 train 5.573801 (lr=3.0294e-04) (hash(x)=21565819)
+370 train 5.469988 (lr=3.1133e-04) (hash(x)=19484104)
+380 train 5.338028 (lr=3.1972e-04) (hash(x)=20659288)
+390 train 5.304610 (lr=3.2811e-04) (hash(x)=22061179)
+400 val loss 5.3242
+400 val perplexity 205.2523
+400 train 5.395622 (lr=3.3650e-04) (hash(x)=18557926)
+410 train 5.277606 (lr=3.4490e-04) (hash(x)=24034052)
+420 train 5.146585 (lr=3.5329e-04) (hash(x)=19768493)
+430 train 5.253931 (lr=3.6168e-04) (hash(x)=21678884)
+440 train 5.166972 (lr=3.7007e-04) (hash(x)=19589702)
+450 train 5.188886 (lr=3.7846e-04) (hash(x)=19902911)
+460 train 4.981719 (lr=3.8685e-04) (hash(x)=18881029)
+470 train 5.095175 (lr=3.9524e-04) (hash(x)=24329185)
+480 train 4.986784 (lr=4.0364e-04) (hash(x)=18709451)
+490 train 4.952910 (lr=4.1203e-04) (hash(x)=18834705)
+500 val loss 4.9471
+500 val perplexity 140.7662
+500 train 4.983137 (lr=4.2042e-04) (hash(x)=24221098)
+510 train 4.940765 (lr=4.2881e-04) (hash(x)=20010758)
+520 train 4.901899 (lr=4.3720e-04) (hash(x)=18547084)
+530 train 4.709805 (lr=4.4559e-04) (hash(x)=18282458)
+540 train 4.880599 (lr=4.5399e-04) (hash(x)=20671087)
+550 train 4.685594 (lr=4.6238e-04) (hash(x)=18634421)
+560 train 4.621721 (lr=4.7077e-04) (hash(x)=17067920)
+570 train 4.790645 (lr=4.7916e-04) (hash(x)=22276881)
+580 train 4.636317 (lr=4.8755e-04) (hash(x)=22973651)
+590 train 4.639351 (lr=4.9594e-04) (hash(x)=18096900)
+600 val loss 4.5544
+600 val perplexity 95.0525
+600 train 4.501267 (lr=5.0434e-04) (hash(x)=20617414)
+610 train 4.357404 (lr=5.1273e-04) (hash(x)=20722525)
+620 train 4.568745 (lr=5.2112e-04) (hash(x)=20785452)
+630 train 4.524608 (lr=5.2951e-04) (hash(x)=16647858)
+640 train 4.442771 (lr=5.3790e-04) (hash(x)=20798993)
+650 train 4.383072 (lr=5.4629e-04) (hash(x)=20912393)
+660 train 4.219800 (lr=5.5469e-04) (hash(x)=21505604)
+670 train 4.438077 (lr=5.6308e-04) (hash(x)=20263120)
+680 train 4.365936 (lr=5.7147e-04) (hash(x)=20779495)
+690 train 4.279251 (lr=5.7986e-04) (hash(x)=16555678)
+700 val loss 4.3291
+700 val perplexity 75.8747
+700 train 4.151181 (lr=5.8825e-04) (hash(x)=20156261)
+710 train 4.116940 (lr=5.9664e-04) (hash(x)=17064702)
+720 train 4.314353 (lr=5.9999e-04) (hash(x)=19321113)
+730 train 4.284961 (lr=5.9991e-04) (hash(x)=19738613)
+740 train 4.206601 (lr=5.9974e-04) (hash(x)=18585508)
+750 train 4.094906 (lr=5.9949e-04) (hash(x)=18605696)
+760 train 4.260360 (lr=5.9915e-04) (hash(x)=22711193)
+770 train 4.203087 (lr=5.9874e-04) (hash(x)=17163545)
+780 train 4.099117 (lr=5.9824e-04) (hash(x)=22239165)
+790 train 4.215289 (lr=5.9765e-04) (hash(x)=19202751)
+800 val loss 4.1404
+800 val perplexity 62.8257
+800 train 4.200274 (lr=5.9698e-04) (hash(x)=22502753)
+810 train 4.177048 (lr=5.9623e-04) (hash(x)=20197889)
+820 train 4.108479 (lr=5.9540e-04) (hash(x)=19376072)
+830 train 3.963338 (lr=5.9449e-04) (hash(x)=18293711)
+840 train 4.127880 (lr=5.9349e-04) (hash(x)=19061676)
+850 train 4.136122 (lr=5.9241e-04) (hash(x)=20557626)
+860 train 4.068823 (lr=5.9126e-04) (hash(x)=18642266)
+870 train 3.933425 (lr=5.9002e-04) (hash(x)=20957453)
+880 train 3.972992 (lr=5.8869e-04) (hash(x)=21781033)
+890 train 4.083718 (lr=5.8729e-04) (hash(x)=21094646)
+900 val loss 4.0139
+900 val perplexity 55.3599
+900 train 4.072251 (lr=5.8581e-04) (hash(x)=21499150)
+910 train 4.055771 (lr=5.8425e-04) (hash(x)=19423438)
+920 train 3.863294 (lr=5.8262e-04) (hash(x)=20543615)
+930 train 4.026150 (lr=5.8090e-04) (hash(x)=19245236)
+940 train 4.047243 (lr=5.7911e-04) (hash(x)=24035334)
+950 train 3.974447 (lr=5.7723e-04) (hash(x)=16703311)
+960 train 3.949943 (lr=5.7529e-04) (hash(x)=23101805)
+970 train 3.903610 (lr=5.7326e-04) (hash(x)=20320872)
+980 train 3.964460 (lr=5.7116e-04) (hash(x)=20990946)
+990 train 3.961750 (lr=5.6899e-04) (hash(x)=18628956)
+1000 val loss 3.9211
+1000 val perplexity 50.4556
+1000 train 3.898526 (lr=5.6674e-04) (hash(x)=22968748)
+1010 train 3.879448 (lr=5.6442e-04) (hash(x)=20541414)
+1020 train 3.944512 (lr=5.6202e-04) (hash(x)=19274167)
+1030 train 3.892857 (lr=5.5956e-04) (hash(x)=22645317)
+1040 train 3.871137 (lr=5.5702e-04) (hash(x)=17884034)
+1050 train 3.901952 (lr=5.5441e-04) (hash(x)=19299722)
+1060 train 3.902329 (lr=5.5174e-04) (hash(x)=18409931)
+1070 train 3.867874 (lr=5.4899e-04) (hash(x)=22455477)
+1080 train 3.845795 (lr=5.4618e-04) (hash(x)=20085073)
+1090 train 3.870390 (lr=5.4330e-04) (hash(x)=20371024)
+1100 val loss 3.8475
+1100 val perplexity 46.8739
+1100 train 3.825941 (lr=5.4035e-04) (hash(x)=17534387)
+1110 train 3.828694 (lr=5.3734e-04) (hash(x)=18107517)
+1120 train 3.860199 (lr=5.3426e-04) (hash(x)=21862096)
+1130 train 3.876420 (lr=5.3113e-04) (hash(x)=24296067)
+1140 train 3.768576 (lr=5.2792e-04) (hash(x)=25489932)
+1150 train 3.869433 (lr=5.2466e-04) (hash(x)=19256895)
+1160 train 3.774481 (lr=5.2134e-04) (hash(x)=20915643)
+1170 train 3.849969 (lr=5.1796e-04) (hash(x)=22053731)
+1180 train 3.792248 (lr=5.1452e-04) (hash(x)=20012827)
+1190 train 3.787998 (lr=5.1102e-04) (hash(x)=21694459)
+1200 val loss 3.7888
+1200 val perplexity 44.2048
+1200 train 3.829067 (lr=5.0746e-04) (hash(x)=21755992)
+1210 train 3.819084 (lr=5.0386e-04) (hash(x)=21880996)
+1220 train 3.782514 (lr=5.0019e-04) (hash(x)=20520945)
+1230 train 3.764915 (lr=4.9648e-04) (hash(x)=19770784)
+1240 train 3.776699 (lr=4.9271e-04) (hash(x)=18571983)
+1250 train 3.777975 (lr=4.8889e-04) (hash(x)=20735694)
+1260 train 3.740625 (lr=4.8503e-04) (hash(x)=21156163)
+1270 train 3.826717 (lr=4.8111e-04) (hash(x)=21271664)
+1280 train 3.794382 (lr=4.7715e-04) (hash(x)=19315272)
+1290 train 3.813817 (lr=4.7315e-04) (hash(x)=21142191)
+1300 val loss 3.7944
+1300 val perplexity 44.4533
+1300 train 3.829432 (lr=4.6909e-04) (hash(x)=17145293)
+1310 train 4.007178 (lr=4.6500e-04) (hash(x)=17713742)
+1320 train 3.790285 (lr=4.6086e-04) (hash(x)=33348747)
+1330 train 3.732153 (lr=4.5669e-04) (hash(x)=20702111)
+1340 train 3.799348 (lr=4.5247e-04) (hash(x)=17898284)
+1350 train 3.720208 (lr=4.4822e-04) (hash(x)=17904915)
+1360 train 3.730905 (lr=4.4393e-04) (hash(x)=23325893)
+1370 train 3.658915 (lr=4.3960e-04) (hash(x)=20717754)
+1380 train 3.760874 (lr=4.3524e-04) (hash(x)=16859207)
+1390 train 3.720601 (lr=4.3085e-04) (hash(x)=21029500)
+1400 val loss 3.7120
+1400 val perplexity 40.9372
+1400 train 3.653262 (lr=4.2643e-04) (hash(x)=17789655)
+1410 train 3.768733 (lr=4.2197e-04) (hash(x)=19965237)
+1420 train 3.770688 (lr=4.1749e-04) (hash(x)=19262560)
+1430 train 3.671060 (lr=4.1298e-04) (hash(x)=18593169)
+1440 train 3.661405 (lr=4.0845e-04) (hash(x)=20607672)
+1450 train 3.707865 (lr=4.0389e-04) (hash(x)=20682942)
+1460 train 3.652582 (lr=3.9931e-04) (hash(x)=20220347)
+1470 train 3.631529 (lr=3.9470e-04) (hash(x)=15833389)
+1480 train 3.760796 (lr=3.9008e-04) (hash(x)=19495027)
+1490 train 3.673120 (lr=3.8544e-04) (hash(x)=22138106)
+1500 val loss 3.6755
+1500 val perplexity 39.4675
+1500 train 3.633246 (lr=3.8078e-04) (hash(x)=24123427)
+1510 train 3.594836 (lr=3.7610e-04) (hash(x)=19024825)
+1520 train 3.733757 (lr=3.7142e-04) (hash(x)=19780005)
+1530 train 3.742644 (lr=3.6671e-04) (hash(x)=19134410)
+1540 train 3.650724 (lr=3.6200e-04) (hash(x)=19855253)
+1550 train 3.605511 (lr=3.5728e-04) (hash(x)=20259196)
+1560 train 3.607465 (lr=3.5255e-04) (hash(x)=18870222)
+1570 train 3.669847 (lr=3.4781e-04) (hash(x)=16067782)
+1580 train 3.675256 (lr=3.4306e-04) (hash(x)=21604205)
+1590 train 3.646662 (lr=3.3831e-04) (hash(x)=19076298)
+1600 val loss 3.6794
+1600 val perplexity 39.6213
+1600 train 3.682337 (lr=3.3356e-04) (hash(x)=19064055)
+1610 train 3.727690 (lr=3.2881e-04) (hash(x)=19196041)
+1620 train 3.719979 (lr=3.2406e-04) (hash(x)=25625303)
+1630 train 3.640513 (lr=3.1931e-04) (hash(x)=20618762)
+1640 train 3.663329 (lr=3.1456e-04) (hash(x)=21102615)
+1650 train 3.603742 (lr=3.0982e-04) (hash(x)=21088682)
+1660 train 3.583385 (lr=3.0509e-04) (hash(x)=15297157)
+1670 train 3.648555 (lr=3.0036e-04) (hash(x)=19690570)
+1680 train 3.685071 (lr=2.9564e-04) (hash(x)=21445411)
+1690 train 3.652365 (lr=2.9093e-04) (hash(x)=21683876)
+1700 val loss 3.6337
+1700 val perplexity 37.8539
+1700 train 3.648287 (lr=2.8624e-04) (hash(x)=17449466)
+1710 train 3.655129 (lr=2.8156e-04) (hash(x)=16754900)
+1720 train 3.643159 (lr=2.7689e-04) (hash(x)=20736439)
+1730 train 3.688151 (lr=2.7224e-04) (hash(x)=21475325)
+1740 train 3.609847 (lr=2.6761e-04) (hash(x)=31466218)
+1750 train 3.686771 (lr=2.6299e-04) (hash(x)=17740223)
+1760 train 3.616422 (lr=2.5840e-04) (hash(x)=17626006)
+1770 train 3.509679 (lr=2.5383e-04) (hash(x)=22276216)
+1780 train 3.605086 (lr=2.4928e-04) (hash(x)=14277004)
+1790 train 3.657889 (lr=2.4476e-04) (hash(x)=20600488)
+1800 val loss 3.6838
+1800 val perplexity 39.7980
+1800 train 3.686995 (lr=2.4026e-04) (hash(x)=17508978)
+1810 train 3.583800 (lr=2.3580e-04) (hash(x)=22374263)
+1820 train 3.726002 (lr=2.3136e-04) (hash(x)=18959095)
+1830 train 3.658959 (lr=2.2695e-04) (hash(x)=19485836)
+1840 train 3.650826 (lr=2.2257e-04) (hash(x)=24276692)
+1850 train 3.657848 (lr=2.1823e-04) (hash(x)=20554224)
+1860 train 3.638371 (lr=2.1392e-04) (hash(x)=18068030)
+1870 train 3.644769 (lr=2.0965e-04) (hash(x)=18349056)
+1880 train 3.573873 (lr=2.0542e-04) (hash(x)=20640059)
+1890 train 3.586307 (lr=2.0122e-04) (hash(x)=19667347)
+1900 val loss 3.6294
+1900 val perplexity 37.6914
+1900 train 3.665259 (lr=1.9706e-04) (hash(x)=19559109)
+1910 train 3.690634 (lr=1.9295e-04) (hash(x)=22490711)
+1920 train 3.614237 (lr=1.8887e-04) (hash(x)=18268544)
+1930 train 3.580101 (lr=1.8485e-04) (hash(x)=20585037)
+1940 train 3.667114 (lr=1.8086e-04) (hash(x)=30293578)
+1950 train 3.587557 (lr=1.7692e-04) (hash(x)=21836208)
+1960 train 3.508948 (lr=1.7303e-04) (hash(x)=23417084)
+1970 train 3.605651 (lr=1.6919e-04) (hash(x)=19390086)
+1980 train 3.531624 (lr=1.6540e-04) (hash(x)=20129488)
+1990 train 3.567268 (lr=1.6166e-04) (hash(x)=20177413)
+2000 val loss 3.5751
+2000 val perplexity 35.6977
+2000 train 3.498045 (lr=1.5797e-04) (hash(x)=15921967)
+2010 train 3.578220 (lr=1.5433e-04) (hash(x)=20819557)
+2020 train 3.592425 (lr=1.5075e-04) (hash(x)=18729616)
+2030 train 3.532372 (lr=1.4723e-04) (hash(x)=15730659)
+2040 train 3.583822 (lr=1.4376e-04) (hash(x)=18679863)
+2050 train 3.634854 (lr=1.4035e-04) (hash(x)=17110863)
+2060 train 3.539146 (lr=1.3699e-04) (hash(x)=18291952)
+2070 train 3.566314 (lr=1.3370e-04) (hash(x)=18249172)
+2080 train 3.614030 (lr=1.3047e-04) (hash(x)=22686659)
+2090 train 3.554323 (lr=1.2730e-04) (hash(x)=18354325)
+2100 val loss 3.5748
+2100 val perplexity 35.6867
+2100 train 3.544341 (lr=1.2419e-04) (hash(x)=18204536)
+2110 train 3.619862 (lr=1.2115e-04) (hash(x)=22394614)
+2120 train 3.498032 (lr=1.1817e-04) (hash(x)=18121192)
+2130 train 3.533305 (lr=1.1525e-04) (hash(x)=24393898)
+2140 train 3.554592 (lr=1.1241e-04) (hash(x)=27349908)
+2150 train 3.563727 (lr=1.0963e-04) (hash(x)=18262196)
+2160 train 3.496200 (lr=1.0692e-04) (hash(x)=20081235)
+2170 train 3.534118 (lr=1.0427e-04) (hash(x)=16984900)
+2180 train 3.618517 (lr=1.0170e-04) (hash(x)=24294588)
+2190 train 3.564797 (lr=9.9199e-05) (hash(x)=17042450)
+2200 val loss 3.5540
+2200 val perplexity 34.9521
+2200 train 3.448763 (lr=9.6769e-05) (hash(x)=23876593)
+2210 train 3.603014 (lr=9.4412e-05) (hash(x)=19173831)
+2220 train 3.501960 (lr=9.2127e-05) (hash(x)=21211298)
+2230 train 3.504155 (lr=8.9916e-05) (hash(x)=18721700)
+2240 train 3.510086 (lr=8.7779e-05) (hash(x)=19825319)
+2250 train 3.552556 (lr=8.5717e-05) (hash(x)=18378838)
+2260 train 3.535671 (lr=8.3731e-05) (hash(x)=13199654)
+2270 train 3.446128 (lr=8.1821e-05) (hash(x)=18597807)
+2280 train 3.555393 (lr=7.9988e-05) (hash(x)=22012917)
+2290 train 3.467770 (lr=7.8232e-05) (hash(x)=22305542)
+2300 val loss 3.5383
+2300 val perplexity 34.4068
+2300 train 3.566592 (lr=7.6555e-05) (hash(x)=19782726)
+2310 train 3.534157 (lr=7.4956e-05) (hash(x)=22042291)
+2320 train 3.538757 (lr=7.3436e-05) (hash(x)=17370832)
+2330 train 3.581960 (lr=7.1995e-05) (hash(x)=19612132)
+2340 train 3.555169 (lr=7.0635e-05) (hash(x)=18803286)
+2350 train 3.536891 (lr=6.9354e-05) (hash(x)=20196070)
+2360 train 3.496360 (lr=6.8155e-05) (hash(x)=19997454)
+2370 train 3.516991 (lr=6.7036e-05) (hash(x)=16704483)
+2380 train 3.533637 (lr=6.5999e-05) (hash(x)=19131833)
+2390 train 3.476454 (lr=6.5044e-05) (hash(x)=20507035)
+2400 val loss 3.5357
+2400 val perplexity 34.3178
+2400 train 3.596515 (lr=6.4171e-05) (hash(x)=20254090)
+2410 train 3.535757 (lr=6.3380e-05) (hash(x)=17926964)
+2420 train 3.567580 (lr=6.2672e-05) (hash(x)=20607860)
+2430 train 3.495036 (lr=6.2046e-05) (hash(x)=11288168)
+2440 train 3.622654 (lr=6.1504e-05) (hash(x)=21486139)
+2450 train 3.554534 (lr=6.1045e-05) (hash(x)=18994546)
+2460 train 3.485203 (lr=6.0669e-05) (hash(x)=21218220)
+2470 train 3.588367 (lr=6.0376e-05) (hash(x)=20504341)
+2480 train 3.535358 (lr=6.0167e-05) (hash(x)=17937386)
+2490 train 3.445849 (lr=6.0042e-05) (hash(x)=21870434)
+2499 val loss 3.5314
+2499 val perplexity 34.1710

one_mask_per_head/model_02499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03e9292f95d1d885cc37c198a5f339236e191be3d17185f84cc9bbd075b6c323
+oid sha256:eeaf02304527960bbb6f1f6b53165d74e147cb97a8ce5b10a6b03e49a085daf5
 size 633197954

one_mask_per_head/optimizer_02499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a1c48ef84ee6da0616967117c231539bafe177e126efc21aa78269c3a719338a
+oid sha256:a6512b36a95176756ffd714f01262ff75a232fe109c5d169dc88fcbb50659883
 size 1165743366