Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_10240_2_1338", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.0001, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "10e-5_10240_2_1338", "n_embd": 128}

attention_kindselective_n_heads2_seed1338/log2.txt

@@ -1,606 +1,582 @@
 max_steps: 10000
-max_steps: 10000
-0 val loss 11.7640
-0 val perplexity 128545.8516
+100 val loss 9.3442
+100 val perplexity 11431.8779
+100 train 9.297195 (lr=7.5750e-05) (hash(x)=52740221)
+200 val loss 7.7377
+200 val perplexity 2293.1240
+200 train 7.742150 (lr=1.5000e-04) (hash(x)=49034180)
 0 val loss 11.7640
 0 val perplexity 128545.8516
-0 train 11.762399 (lr=2.5000e-07) (hash(x)=50671684)
-0 train 11.762399 (lr=3.5000e-07) (hash(x)=50671684)
-100 val loss 10.2612
-100 val perplexity 28599.8301
-100 val loss 10.0855
-100 val perplexity 23992.4160
-100 train 10.199640 (lr=2.5250e-05) (hash(x)=52740221)
-100 train 10.027347 (lr=3.5350e-05) (hash(x)=52740221)
-200 val loss 9.0641
-200 val perplexity 8639.7002
-200 train 9.082980 (lr=5.0000e-05) (hash(x)=49034180)
-200 val loss 8.3482
-200 val perplexity 4222.5566
-200 train 8.351851 (lr=7.0000e-05) (hash(x)=49034180)
-300 val loss 8.1940
-300 val perplexity 3619.2537
-300 train 8.292101 (lr=4.9988e-05) (hash(x)=63180688)
-300 val loss 7.7296
-300 val perplexity 2274.6714
-300 train 7.949653 (lr=6.9984e-05) (hash(x)=63180688)
-400 val loss 7.8544
-400 val perplexity 2577.1091
-400 train 7.816474 (lr=4.9954e-05) (hash(x)=50373500)
-400 val loss 7.6567
-400 val perplexity 2114.7312
-400 train 7.666688 (lr=6.9935e-05) (hash(x)=50373500)
-500 val loss 7.6951
-500 val perplexity 2197.5679
-500 train 7.606697 (lr=4.9896e-05) (hash(x)=44547422)
-500 val loss 7.6381
-500 val perplexity 2075.7231
-500 train 7.539029 (lr=6.9854e-05) (hash(x)=44547422)
-600 val loss 7.6593
-600 val perplexity 2120.3745
-600 train 7.554819 (lr=4.9815e-05) (hash(x)=47184699)
-600 val loss 7.6413
-600 val perplexity 2082.3872
-600 train 7.530231 (lr=6.9741e-05) (hash(x)=47184699)
-700 val loss 7.6521
-700 val perplexity 2104.9644
-700 train 7.565280 (lr=4.9712e-05) (hash(x)=51374582)
-700 val loss 7.6389
-700 val perplexity 2077.5461
-700 train 7.553903 (lr=6.9596e-05) (hash(x)=51374582)
-800 val loss 7.6369
-800 val perplexity 2073.2136
-800 train 7.378779 (lr=4.9585e-05) (hash(x)=46264805)
-800 val loss 7.6290
-800 val perplexity 2057.0747
-800 train 7.379150 (lr=6.9419e-05) (hash(x)=46264805)
-900 val loss 7.6547
-900 val perplexity 2110.6108
-900 train 7.937618 (lr=4.9436e-05) (hash(x)=61178712)
-900 val loss 7.6323
-900 val perplexity 2063.8235
-900 train 7.921933 (lr=6.9210e-05) (hash(x)=61178712)
-1000 val loss 7.6362
-1000 val perplexity 2071.7578
-1000 train 7.647681 (lr=4.9264e-05) (hash(x)=50886520)
-1000 val loss 7.6263
-1000 val perplexity 2051.3552
-1000 train 7.646755 (lr=6.8970e-05) (hash(x)=50886520)
-1100 val loss 7.6266
-1100 val perplexity 2052.0225
-1100 train 7.411226 (lr=4.9070e-05) (hash(x)=48600099)
-1100 val loss 7.6231
-1100 val perplexity 2044.9672
-1100 train 7.445827 (lr=6.8698e-05) (hash(x)=48600099)
-1200 val loss 7.6176
-1200 val perplexity 2033.6812
-1200 train 7.354834 (lr=4.8854e-05) (hash(x)=50146792)
-1200 val loss 7.6272
-1200 val perplexity 2053.3645
-1200 train 7.383700 (lr=6.8395e-05) (hash(x)=50146792)
-1300 val loss 7.6195
-1300 val perplexity 2037.5619
-1300 train 7.491829 (lr=4.8616e-05) (hash(x)=52617313)
-1300 val loss 7.6346
-1300 val perplexity 2068.6394
-1300 train 7.519730 (lr=6.8062e-05) (hash(x)=52617313)
-1400 val loss 7.6085
-1400 val perplexity 2015.1982
-1400 train 7.284451 (lr=4.8356e-05) (hash(x)=49794446)
-1400 val loss 7.6309
-1400 val perplexity 2060.8892
-1400 train 7.316226 (lr=6.7698e-05) (hash(x)=49794446)
-1500 val loss 7.6101
-1500 val perplexity 2018.5585
-1500 train 7.262372 (lr=4.8074e-05) (hash(x)=50766317)
-1500 val loss 7.6201
-1500 val perplexity 2038.8419
-1500 train 7.261213 (lr=6.7304e-05) (hash(x)=50766317)
-1600 val loss 7.6007
-1600 val perplexity 1999.5605
-1600 train 7.493066 (lr=4.7772e-05) (hash(x)=55551175)
-1600 val loss 7.5985
-1600 val perplexity 1995.1804
-1600 train 7.499768 (lr=6.6881e-05) (hash(x)=55551175)
-1700 val loss 7.5795
-1700 val perplexity 1957.7035
-1700 train 7.733157 (lr=4.7448e-05) (hash(x)=56717172)
-1700 val loss 7.5674
-1700 val perplexity 1934.0377
-1700 train 7.691600 (lr=6.6428e-05) (hash(x)=56717172)
-1800 val loss 7.5356
-1800 val perplexity 1873.6436
-1800 train 7.837944 (lr=4.7105e-05) (hash(x)=55376447)
-1800 val loss 7.5258
-1800 val perplexity 1855.3097
-1800 train 7.796223 (lr=6.5947e-05) (hash(x)=55376447)
-1900 val loss 7.5072
-1900 val perplexity 1821.0981
-1900 train 7.223547 (lr=4.6741e-05) (hash(x)=43810837)
-1900 val loss 7.5078
-1900 val perplexity 1822.2770
-1900 train 7.237650 (lr=6.5437e-05) (hash(x)=43810837)
-2000 val loss 7.4870
-2000 val perplexity 1784.7079
-2000 train 7.566131 (lr=4.6357e-05) (hash(x)=50881655)
-2000 val loss 7.4835
-2000 val perplexity 1778.4316
-2000 train 7.579944 (lr=6.4900e-05) (hash(x)=50881655)
-2100 val loss 7.4819
-2100 val perplexity 1775.6853
-2100 train 7.358100 (lr=4.5954e-05) (hash(x)=49386015)
-2100 val loss 7.4574
-2100 val perplexity 1732.5850
-2100 train 7.333408 (lr=6.4335e-05) (hash(x)=49386015)
-2200 val loss 7.4734
-2200 val perplexity 1760.6473
-2200 train 7.392198 (lr=4.5532e-05) (hash(x)=48572079)
-2200 val loss 7.4651
-2200 val perplexity 1745.9648
-2200 train 7.405998 (lr=6.3745e-05) (hash(x)=48572079)
-2300 val loss 7.4649
-2300 val perplexity 1745.6702
-2300 train 7.528904 (lr=4.5091e-05) (hash(x)=54950719)
-2300 val loss 7.4181
-2300 val perplexity 1665.8230
-2300 train 7.485063 (lr=6.3128e-05) (hash(x)=54950719)
-2400 val loss 7.4481
-2400 val perplexity 1716.5986
-2400 train 7.104608 (lr=4.4633e-05) (hash(x)=42190240)
-2400 val loss 7.3888
-2400 val perplexity 1617.7377
-2400 train 7.051908 (lr=6.2486e-05) (hash(x)=42190240)
-2500 val loss 7.4384
-2500 val perplexity 1700.0844
-2500 train 7.503536 (lr=4.4156e-05) (hash(x)=45223539)
-2500 val loss 7.3688
-2500 val perplexity 1585.7074
-2500 train 7.440190 (lr=6.1819e-05) (hash(x)=45223539)
-2600 val loss 7.4214
-2600 val perplexity 1671.3839
-2600 train 7.425591 (lr=4.3663e-05) (hash(x)=54037353)
-2600 val loss 7.3434
-2600 val perplexity 1546.0243
-2600 train 7.361808 (lr=6.1128e-05) (hash(x)=54037353)
-2700 val loss 7.3990
-2700 val perplexity 1634.3760
-2700 train 7.943274 (lr=4.3153e-05) (hash(x)=59131616)
-2700 val loss 7.3256
-2700 val perplexity 1518.7589
-2700 train 7.771473 (lr=6.0414e-05) (hash(x)=59131616)
-2800 val loss 7.3895
-2800 val perplexity 1618.9755
-2800 train 7.214322 (lr=4.2627e-05) (hash(x)=45882743)
-2800 val loss 7.2809
-2800 val perplexity 1452.2246
-2800 train 7.131896 (lr=5.9677e-05) (hash(x)=45882743)
-2900 val loss 7.3756
-2900 val perplexity 1596.5599
-2900 train 6.986534 (lr=4.2085e-05) (hash(x)=43758910)
-3000 val loss 7.4025
-3000 val perplexity 1640.0563
-3000 train 7.337453 (lr=4.1529e-05) (hash(x)=47965974)
-2900 val loss 7.2484
-2900 val perplexity 1405.8976
-2900 train 6.918225 (lr=5.8919e-05) (hash(x)=43758910)
-3100 val loss 7.3828
-3100 val perplexity 1608.0157
-3100 train 7.282964 (lr=4.0957e-05) (hash(x)=48205243)
-3000 val loss 7.2421
-3000 val perplexity 1397.0543
-3000 train 7.161748 (lr=5.8140e-05) (hash(x)=47965974)
-3200 val loss 7.3542
-3200 val perplexity 1562.7103
-3200 train 7.431534 (lr=4.0373e-05) (hash(x)=54511383)
-3100 val loss 7.2156
-3100 val perplexity 1360.4982
-3100 train 7.109374 (lr=5.7340e-05) (hash(x)=48205243)
-3300 val loss 7.3452
-3300 val perplexity 1548.7839
-3300 train 7.324124 (lr=3.9775e-05) (hash(x)=54428388)
-3200 val loss 7.2021
-3200 val perplexity 1342.2188
-3200 train 7.273560 (lr=5.6522e-05) (hash(x)=54511383)
-3400 val loss 7.3255
-3400 val perplexity 1518.5460
-3400 train 7.373497 (lr=3.9164e-05) (hash(x)=48115990)
-3300 val loss 7.1854
-3300 val perplexity 1320.0015
-3300 train 7.137845 (lr=5.5684e-05) (hash(x)=54428388)
-3500 val loss 7.3067
-3500 val perplexity 1490.2999
-3500 train 6.951129 (lr=3.8541e-05) (hash(x)=41137345)
-3400 val loss 7.1642
-3400 val perplexity 1292.2944
-3400 train 7.198205 (lr=5.4829e-05) (hash(x)=48115990)
-3600 val loss 7.3099
-3600 val perplexity 1495.0908
-3600 train 7.210977 (lr=3.7907e-05) (hash(x)=55186224)
-3500 val loss 7.1480
-3500 val perplexity 1271.5587
-3500 train 6.770042 (lr=5.3958e-05) (hash(x)=41137345)
-3700 val loss 7.2953
-3700 val perplexity 1473.4081
-3700 train 7.162465 (lr=3.7262e-05) (hash(x)=54990049)
-3600 val loss 7.1600
-3600 val perplexity 1286.9684
-3600 train 7.056310 (lr=5.3070e-05) (hash(x)=55186224)
-3800 val loss 7.2900
-3800 val perplexity 1465.5238
-3800 train 7.036756 (lr=3.6608e-05) (hash(x)=46288812)
-3700 val loss 7.1365
-3700 val perplexity 1257.0605
-3700 train 7.001248 (lr=5.2167e-05) (hash(x)=54990049)
-3900 val loss 7.2680
-3900 val perplexity 1433.7078
-3900 train 6.930634 (lr=3.5944e-05) (hash(x)=45829773)
-3800 val loss 7.1138
-3800 val perplexity 1228.7883
-3800 train 6.855778 (lr=5.1251e-05) (hash(x)=46288812)
-4000 val loss 7.2640
-4000 val perplexity 1427.9847
-4000 train 7.045815 (lr=3.5271e-05) (hash(x)=52499943)
-3900 val loss 7.1105
-3900 val perplexity 1224.7800
-3900 train 6.770267 (lr=5.0321e-05) (hash(x)=45829773)
-4100 val loss 7.2214
-4100 val perplexity 1368.4528
-4100 train 7.119732 (lr=3.4590e-05) (hash(x)=48563796)
-4000 val loss 7.0887
-4000 val perplexity 1198.3544
-4000 train 6.878484 (lr=4.9379e-05) (hash(x)=52499943)
-4200 val loss 7.1987
-4200 val perplexity 1337.7422
-4200 train 7.161106 (lr=3.3902e-05) (hash(x)=49165143)
-4100 val loss 7.0505
-4100 val perplexity 1153.4874
-4100 train 6.955088 (lr=4.8426e-05) (hash(x)=48563796)
-4300 val loss 7.1848
-4300 val perplexity 1319.2406
-4300 train 7.263635 (lr=3.3207e-05) (hash(x)=50973176)
-4200 val loss 6.9988
-4200 val perplexity 1095.3329
-4200 train 6.969129 (lr=4.7463e-05) (hash(x)=49165143)
-4400 val loss 7.1592
-4400 val perplexity 1285.8815
-4400 train 7.206085 (lr=3.2507e-05) (hash(x)=55275124)
-4300 val loss 6.9748
-4300 val perplexity 1069.3761
-4300 train 7.032444 (lr=4.6490e-05) (hash(x)=50973176)
-4500 val loss 7.1423
-4500 val perplexity 1264.3616
-4500 train 7.480568 (lr=3.1801e-05) (hash(x)=58646505)
-4400 val loss 6.9587
-4400 val perplexity 1052.2388
-4400 train 7.023558 (lr=4.5509e-05) (hash(x)=55275124)
-4600 val loss 7.1201
-4600 val perplexity 1236.5511
-4600 train 6.934882 (lr=3.1091e-05) (hash(x)=42554666)
-4500 val loss 6.9399
-4500 val perplexity 1032.6294
-4500 train 7.294427 (lr=4.4521e-05) (hash(x)=58646505)
-4700 val loss 7.1123
-4700 val perplexity 1226.9720
-4700 train 7.021177 (lr=3.0377e-05) (hash(x)=47846764)
-4600 val loss 6.9231
-4600 val perplexity 1015.4349
-4600 train 6.728717 (lr=4.3527e-05) (hash(x)=42554666)
-4800 val loss 7.1072
-4800 val perplexity 1220.7145
-4800 train 7.640239 (lr=2.9661e-05) (hash(x)=58239019)
-4700 val loss 6.9024
-4700 val perplexity 994.6642
-4700 train 6.794252 (lr=4.2528e-05) (hash(x)=47846764)
-4900 val loss 7.0912
-4900 val perplexity 1201.3014
-4900 train 7.122570 (lr=2.8942e-05) (hash(x)=50711220)
-4800 val loss 6.8864
-4800 val perplexity 978.8246
-4800 train 7.385535 (lr=4.1525e-05) (hash(x)=58239019)
-5000 val loss 7.0806
-5000 val perplexity 1188.6388
-5000 train 7.011305 (lr=2.8221e-05) (hash(x)=45994194)
-4900 val loss 6.8732
-4900 val perplexity 966.0390
-4900 train 6.911656 (lr=4.0518e-05) (hash(x)=50711220)
-5100 val loss 7.0847
-5100 val perplexity 1193.5157
-5100 train 6.948140 (lr=2.7500e-05) (hash(x)=48659050)
-5000 val loss 6.8794
-5000 val perplexity 972.0590
-5000 train 6.808244 (lr=3.9510e-05) (hash(x)=45994194)
-5200 val loss 7.0560
-5200 val perplexity 1159.7809
-5200 train 6.998636 (lr=2.6779e-05) (hash(x)=49369682)
-5100 val loss 6.8575
-5100 val perplexity 950.9432
-5100 train 6.727626 (lr=3.8500e-05) (hash(x)=48659050)
-5300 val loss 7.0524
-5300 val perplexity 1155.6340
-5300 train 7.432145 (lr=2.6058e-05) (hash(x)=57787700)
-5200 val loss 6.8367
-5200 val perplexity 931.4388
-5200 train 6.751755 (lr=3.7490e-05) (hash(x)=49369682)
-5400 val loss 7.0336
-5400 val perplexity 1134.0551
-5400 train 6.964849 (lr=2.5339e-05) (hash(x)=49365400)
-5300 val loss 6.8303
-5300 val perplexity 925.4502
-5300 train 7.234496 (lr=3.6482e-05) (hash(x)=57787700)
-5500 val loss 7.0214
-5500 val perplexity 1120.3920
-5500 train 6.973766 (lr=2.4623e-05) (hash(x)=48720412)
-5400 val loss 6.8203
-5400 val perplexity 916.2604
-5400 train 6.747429 (lr=3.5475e-05) (hash(x)=49365400)
-5600 val loss 7.0085
-5600 val perplexity 1106.0455
-5600 train 7.324727 (lr=2.3909e-05) (hash(x)=55784800)
-5500 val loss 6.8173
-5500 val perplexity 913.5103
-5500 train 6.778089 (lr=3.4472e-05) (hash(x)=48720412)
-5700 val loss 7.0040
-5700 val perplexity 1101.0533
-5700 train 6.863188 (lr=2.3199e-05) (hash(x)=50073634)
-5600 val loss 6.8012
-5600 val perplexity 898.9412
-5600 train 7.131571 (lr=3.3473e-05) (hash(x)=55784800)
-5800 val loss 6.9817
-5800 val perplexity 1076.7632
-5800 train 6.810428 (lr=2.2493e-05) (hash(x)=50170324)
-5700 val loss 6.7964
-5700 val perplexity 894.6154
-5700 train 6.646752 (lr=3.2479e-05) (hash(x)=50073634)
-5900 val loss 6.9721
-5900 val perplexity 1066.4613
-5900 train 6.721549 (lr=2.1793e-05) (hash(x)=48410268)
-5800 val loss 6.7835
-5800 val perplexity 883.1710
-5800 train 6.606258 (lr=3.1491e-05) (hash(x)=50170324)
-6000 val loss 6.9624
-6000 val perplexity 1056.1648
-6000 train 6.858176 (lr=2.1098e-05) (hash(x)=49527342)
-5900 val loss 6.7863
-5900 val perplexity 885.6093
-5900 train 6.532552 (lr=3.0510e-05) (hash(x)=48410268)
-6100 val loss 6.9515
-6100 val perplexity 1044.7220
-6100 train 6.763265 (lr=2.0410e-05) (hash(x)=49550294)
-6000 val loss 6.7822
-6000 val perplexity 881.9771
-6000 train 6.668611 (lr=2.9537e-05) (hash(x)=49527342)
-6200 val loss 6.9387
-6200 val perplexity 1031.4410
-6200 train 6.494091 (lr=1.9729e-05) (hash(x)=42126106)
-6100 val loss 6.7750
-6100 val perplexity 875.6949
-6100 train 6.587618 (lr=2.8574e-05) (hash(x)=49550294)
-6300 val loss 6.9254
-6300 val perplexity 1017.7720
-6300 train 6.696783 (lr=1.9056e-05) (hash(x)=49608772)
-6200 val loss 6.7762
-6200 val perplexity 876.6934
-6200 train 6.334079 (lr=2.7621e-05) (hash(x)=42126106)
-6400 val loss 6.9128
-6400 val perplexity 1005.0568
-6400 train 6.485902 (lr=1.8392e-05) (hash(x)=52324417)
-6300 val loss 6.7694
-6300 val perplexity 870.7931
-6300 train 6.520870 (lr=2.6679e-05) (hash(x)=49608772)
-6500 val loss 6.8815
-6500 val perplexity 974.0597
-6500 train 6.923303 (lr=1.7738e-05) (hash(x)=46207215)
-6400 val loss 6.7631
-6400 val perplexity 865.2852
-6400 train 6.338704 (lr=2.5749e-05) (hash(x)=52324417)
-6600 val loss 6.8604
-6600 val perplexity 953.7169
-6600 train 6.797771 (lr=1.7093e-05) (hash(x)=49027014)
-6500 val loss 6.7358
-6500 val perplexity 841.9837
-6500 train 6.789601 (lr=2.4833e-05) (hash(x)=46207215)
-6700 val loss 6.8459
-6700 val perplexity 939.9939
-6700 train 6.819520 (lr=1.6459e-05) (hash(x)=46232513)
-6800 val loss 6.8301
-6800 val perplexity 925.2843
-6800 train 6.766226 (lr=1.5836e-05) (hash(x)=47348403)
-6600 val loss 6.7171
-6600 val perplexity 826.3766
-6600 train 6.647032 (lr=2.3930e-05) (hash(x)=49027014)
-6900 val loss 6.8254
-6900 val perplexity 920.9736
-6900 train 6.799843 (lr=1.5225e-05) (hash(x)=49806647)
-6700 val loss 6.7033
-6700 val perplexity 815.0651
-6700 train 6.672481 (lr=2.3042e-05) (hash(x)=46232513)
-7000 val loss 6.8056
-7000 val perplexity 902.8452
-7000 train 6.843875 (lr=1.4627e-05) (hash(x)=50893018)
-6800 val loss 6.7007
-6800 val perplexity 812.9648
-6800 train 6.624757 (lr=2.2171e-05) (hash(x)=47348403)
-7100 val loss 6.7984
-7100 val perplexity 896.4370
-7100 train 6.822917 (lr=1.4043e-05) (hash(x)=49157639)
-6900 val loss 6.7069
-6900 val perplexity 817.9907
-6900 train 6.675504 (lr=2.1316e-05) (hash(x)=49806647)
-7200 val loss 6.7897
-7200 val perplexity 888.6813
-7200 train 6.761259 (lr=1.3471e-05) (hash(x)=47014759)
-7000 val loss 6.6812
-7000 val perplexity 797.3110
-7000 train 6.741058 (lr=2.0478e-05) (hash(x)=50893018)
-7300 val loss 6.7779
-7300 val perplexity 878.1854
-7300 train 6.779035 (lr=1.2915e-05) (hash(x)=47325591)
-7100 val loss 6.6752
-7100 val perplexity 792.4863
-7100 train 6.693850 (lr=1.9660e-05) (hash(x)=49157639)
-7400 val loss 6.7720
-7400 val perplexity 873.0865
-7400 train 6.654572 (lr=1.2373e-05) (hash(x)=49184604)
-7200 val loss 6.6680
-7200 val perplexity 786.8536
-7200 train 6.651248 (lr=1.8860e-05) (hash(x)=47014759)
-7500 val loss 6.7629
-7500 val perplexity 865.1734
-7500 train 7.052269 (lr=1.1847e-05) (hash(x)=55053584)
-7300 val loss 6.6610
-7300 val perplexity 781.3257
-7300 train 6.657533 (lr=1.8081e-05) (hash(x)=47325591)
-7600 val loss 6.7585
-7600 val perplexity 861.3661
-7600 train 6.715505 (lr=1.1337e-05) (hash(x)=48693923)
-7400 val loss 6.6557
-7400 val perplexity 777.1942
-7400 train 6.538068 (lr=1.7323e-05) (hash(x)=49184604)
-7700 val loss 6.7544
-7700 val perplexity 857.8153
-7700 train 6.298505 (lr=1.0844e-05) (hash(x)=40952882)
-7500 val loss 6.6487
-7500 val perplexity 771.8184
-7500 train 6.927235 (lr=1.6586e-05) (hash(x)=55053584)
-7800 val loss 6.7480
-7800 val perplexity 852.3216
-7800 train 6.836084 (lr=1.0367e-05) (hash(x)=52487845)
-7600 val loss 6.6504
-7600 val perplexity 773.0807
-7600 train 6.592214 (lr=1.5872e-05) (hash(x)=48693923)
-7900 val loss 6.7444
-7900 val perplexity 849.3254
-7900 train 6.821626 (lr=9.9088e-06) (hash(x)=50221547)
-7700 val loss 6.6482
-7700 val perplexity 771.4020
-7700 train 6.223924 (lr=1.5181e-05) (hash(x)=40952882)
-8000 val loss 6.7372
-8000 val perplexity 843.2168
-8000 train 7.036058 (lr=9.4682e-06) (hash(x)=62294204)
-7800 val loss 6.6412
-7800 val perplexity 765.9936
-7800 train 6.724183 (lr=1.4514e-05) (hash(x)=52487845)
-8100 val loss 6.7324
-8100 val perplexity 839.1255
-8100 train 6.425050 (lr=9.0461e-06) (hash(x)=44401967)
-7900 val loss 6.6360
-7900 val perplexity 762.0717
-7900 train 6.738249 (lr=1.3872e-05) (hash(x)=50221547)
-8200 val loss 6.7270
-8200 val perplexity 834.6295
-8200 train 6.687708 (lr=8.6430e-06) (hash(x)=52769095)
-8000 val loss 6.6358
-8000 val perplexity 761.8933
-8000 train 6.951126 (lr=1.3255e-05) (hash(x)=62294204)
-8300 val loss 6.7317
-8300 val perplexity 838.6075
-8300 train 6.673268 (lr=8.2593e-06) (hash(x)=56829883)
-8100 val loss 6.6309
-8100 val perplexity 758.1688
-8100 train 6.340541 (lr=1.2665e-05) (hash(x)=44401967)
-8400 val loss 6.7297
-8400 val perplexity 836.9218
-8400 train 6.691971 (lr=7.8953e-06) (hash(x)=52147375)
-8200 val loss 6.6290
-8200 val perplexity 756.7592
-8200 train 6.576173 (lr=1.2100e-05) (hash(x)=52769095)
-8500 val loss 6.7269
-8500 val perplexity 834.5331
-8500 train 6.963628 (lr=7.5515e-06) (hash(x)=60197820)
-8300 val loss 6.6358
-8300 val perplexity 761.8755
-8300 train 6.583430 (lr=1.1563e-05) (hash(x)=56829883)
-8600 val loss 6.7228
-8600 val perplexity 831.1801
-8600 train 6.488226 (lr=7.2282e-06) (hash(x)=49377068)
-8400 val loss 6.6368
-8400 val perplexity 762.6450
-8400 train 6.574768 (lr=1.1053e-05) (hash(x)=52147375)
-8700 val loss 6.7243
-8700 val perplexity 832.3748
-8700 train 6.683083 (lr=6.9257e-06) (hash(x)=51092724)
-8500 val loss 6.6341
-8500 val perplexity 760.6315
-8500 train 6.861284 (lr=1.0572e-05) (hash(x)=60197820)
-8800 val loss 6.7192
-8800 val perplexity 828.1299
-8800 train 6.728143 (lr=6.6444e-06) (hash(x)=48642928)
-8600 val loss 6.6271
-8600 val perplexity 755.2836
-8600 train 6.383428 (lr=1.0119e-05) (hash(x)=49377068)
-8900 val loss 6.7147
-8900 val perplexity 824.3984
-8900 train 6.926672 (lr=6.3845e-06) (hash(x)=55342246)
-8700 val loss 6.6293
-8700 val perplexity 756.9796
-8700 train 6.602247 (lr=9.6960e-06) (hash(x)=51092724)
-9000 val loss 6.6997
-9000 val perplexity 812.1678
-9000 train 6.671376 (lr=6.1462e-06) (hash(x)=48093368)
-8800 val loss 6.6263
-8800 val perplexity 754.6838
-8800 train 6.621269 (lr=9.3021e-06) (hash(x)=48642928)
-9100 val loss 6.6943
-9100 val perplexity 807.8050
-9100 train 6.789246 (lr=5.9300e-06) (hash(x)=48578183)
-8900 val loss 6.6234
-8900 val perplexity 752.5152
-8900 train 6.830116 (lr=8.9382e-06) (hash(x)=55342246)
-9200 val loss 6.6896
-9200 val perplexity 803.9971
-9200 train 6.888481 (lr=5.7359e-06) (hash(x)=50794720)
-9000 val loss 6.6079
-9000 val perplexity 740.9231
-9000 train 6.601912 (lr=8.6047e-06) (hash(x)=48093368)
-9300 val loss 6.6857
-9300 val perplexity 800.8321
-9300 train 6.454147 (lr=5.5641e-06) (hash(x)=46513190)
-9100 val loss 6.6025
-9100 val perplexity 736.9377
-9100 train 6.679001 (lr=8.3020e-06) (hash(x)=48578183)
-9400 val loss 6.6830
-9400 val perplexity 798.7258
-9400 train 6.323279 (lr=5.4149e-06) (hash(x)=43808238)
-9200 val loss 6.5976
-9200 val perplexity 733.3257
-9200 train 6.786611 (lr=8.0302e-06) (hash(x)=50794720)
-9500 val loss 6.6799
-9500 val perplexity 796.2480
-9500 train 6.484011 (lr=5.2884e-06) (hash(x)=45021888)
-9300 val loss 6.5947
-9300 val perplexity 731.1756
-9300 train 6.369024 (lr=7.7898e-06) (hash(x)=46513190)
-9600 val loss 6.6766
-9600 val perplexity 793.6359
-9600 train 6.773294 (lr=5.1847e-06) (hash(x)=56525570)
-9400 val loss 6.5938
-9400 val perplexity 730.5848
-9400 train 6.250168 (lr=7.5809e-06) (hash(x)=43808238)
-9700 val loss 6.6736
-9700 val perplexity 791.2244
-9700 train 6.871308 (lr=5.1040e-06) (hash(x)=52585913)
-9500 val loss 6.5891
-9500 val perplexity 727.1143
-9500 train 6.406715 (lr=7.4038e-06) (hash(x)=45021888)
-9800 val loss 6.6725
-9800 val perplexity 790.3431
-9800 train 6.852726 (lr=5.0462e-06) (hash(x)=52344698)
-9600 val loss 6.5886
-9600 val perplexity 726.7500
-9600 train 6.674443 (lr=7.2586e-06) (hash(x)=56525570)
-9900 val loss 6.6691
-9900 val perplexity 787.7227
-9900 train 6.682109 (lr=5.0116e-06) (hash(x)=51740945)
-9700 val loss 6.5857
-9700 val perplexity 724.6240
-9700 train 6.775486 (lr=7.1456e-06) (hash(x)=52585913)
-9999 val loss 6.6652
-9999 val perplexity 784.6176
-9800 val loss 6.5830
-9800 val perplexity 722.7222
-9800 train 6.780724 (lr=7.0647e-06) (hash(x)=52344698)
-9900 val loss 6.5810
-9900 val perplexity 721.2598
-9900 train 6.589230 (lr=7.0162e-06) (hash(x)=51740945)
-9999 val loss 6.5788
-9999 val perplexity 719.6898
+300 val loss 7.6958
+300 val perplexity 2199.1255
+300 train 7.961709 (lr=1.4997e-04) (hash(x)=63180688)
+400 val loss 7.6637
+400 val perplexity 2129.5801
+400 train 7.680656 (lr=1.4986e-04) (hash(x)=50373500)
+0 train 11.762399 (lr=5.0000e-07) (hash(x)=50671684)
+500 val loss 7.6519
+500 val perplexity 2104.6372
+500 train 7.527692 (lr=1.4969e-04) (hash(x)=44547422)
+100 val loss 9.8118
+100 val perplexity 18248.5859
+600 val loss 7.7028
+600 val perplexity 2214.5046
+100 train 9.757762 (lr=5.0500e-05) (hash(x)=52740221)
+600 train 7.582813 (lr=1.4945e-04) (hash(x)=47184699)
+700 val loss 7.6679
+700 val perplexity 2138.6357
+700 train 7.571599 (lr=1.4913e-04) (hash(x)=51374582)
+200 val loss 8.1949
+200 val perplexity 3622.3582
+200 train 8.195848 (lr=1.0000e-04) (hash(x)=49034180)
+300 val loss 7.7078
+300 val perplexity 2225.6382
+800 val loss 7.6399
+800 val perplexity 2079.5740
+300 train 7.954569 (lr=9.9977e-05) (hash(x)=63180688)
+800 train 7.360402 (lr=1.4876e-04) (hash(x)=46264805)
+900 val loss 7.6276
+900 val perplexity 2054.1938
+900 train 7.878948 (lr=1.4831e-04) (hash(x)=61178712)
+400 val loss 7.6837
+400 val perplexity 2172.6035
+400 train 7.705904 (lr=9.9908e-05) (hash(x)=50373500)
+1000 val loss 7.6159
+1000 val perplexity 2030.1835
+1000 train 7.582415 (lr=1.4779e-04) (hash(x)=50886520)
+500 val loss 7.6574
+500 val perplexity 2116.2241
+500 train 7.541328 (lr=9.9792e-05) (hash(x)=44547422)
+1100 val loss 7.5475
+1100 val perplexity 1895.9528
+1100 train 7.305693 (lr=1.4721e-04) (hash(x)=48600099)
+600 val loss 7.6596
+600 val perplexity 2120.9651
+600 train 7.551372 (lr=9.9631e-05) (hash(x)=47184699)
+1200 val loss 7.5112
+1200 val perplexity 1828.3958
+1200 train 7.210708 (lr=1.4656e-04) (hash(x)=50146792)
+700 val loss 7.6287
+700 val perplexity 2056.4128
+700 train 7.550183 (lr=9.9423e-05) (hash(x)=51374582)
+1300 val loss 7.4972
+1300 val perplexity 1802.9724
+1300 train 7.355508 (lr=1.4585e-04) (hash(x)=52617313)
+800 val loss 7.6292
+800 val perplexity 2057.5054
+800 train 7.363524 (lr=9.9170e-05) (hash(x)=46264805)
+1400 val loss 7.4674
+1400 val perplexity 1749.9814
+1400 train 7.136637 (lr=1.4507e-04) (hash(x)=49794446)
+900 val loss 7.6407
+900 val perplexity 2081.2048
+900 train 7.929552 (lr=9.8872e-05) (hash(x)=61178712)
+1500 val loss 7.4519
+1500 val perplexity 1723.1036
+1500 train 7.113922 (lr=1.4422e-04) (hash(x)=50766317)
+1000 val loss 7.6259
+1000 val perplexity 2050.5642
+1000 train 7.627928 (lr=9.8528e-05) (hash(x)=50886520)
+1600 val loss 7.4349
+1600 val perplexity 1694.0500
+1600 train 7.307274 (lr=1.4332e-04) (hash(x)=55551175)
+1100 val loss 7.6154
+1100 val perplexity 2029.2988
+1100 train 7.433861 (lr=9.8140e-05) (hash(x)=48600099)
+1700 val loss 7.3709
+1700 val perplexity 1589.0658
+1700 train 7.524827 (lr=1.4235e-04) (hash(x)=56717172)
+1200 val loss 7.6086
+1200 val perplexity 2015.4730
+1200 train 7.319845 (lr=9.7708e-05) (hash(x)=50146792)
+1800 val loss 7.3132
+1800 val perplexity 1499.9122
+1800 train 7.543727 (lr=1.4131e-04) (hash(x)=55376447)
+1300 val loss 7.6147
+1300 val perplexity 2027.8499
+1300 train 7.483940 (lr=9.7231e-05) (hash(x)=52617313)
+1900 val loss 7.2845
+1900 val perplexity 1457.4705
+1900 train 7.016429 (lr=1.4022e-04) (hash(x)=43810837)
+1400 val loss 7.6027
+1400 val perplexity 2003.5959
+1400 train 7.282799 (lr=9.6711e-05) (hash(x)=49794446)
+2000 val loss 7.2760
+2000 val perplexity 1445.2384
+2000 train 7.324104 (lr=1.3907e-04) (hash(x)=50881655)
+1500 val loss 7.5973
+1500 val perplexity 1992.9032
+1500 train 7.251457 (lr=9.6149e-05) (hash(x)=50766317)
+2100 val loss 7.2904
+2100 val perplexity 1466.1522
+2100 train 7.182340 (lr=1.3786e-04) (hash(x)=49386015)
+2200 val loss 7.2854
+2200 val perplexity 1458.8618
+2200 train 7.207294 (lr=1.3660e-04) (hash(x)=48572079)
+1600 val loss 7.5756
+1600 val perplexity 1949.9363
+1600 train 7.480996 (lr=9.5544e-05) (hash(x)=55551175)
+2300 val loss 7.2263
+2300 val perplexity 1375.1674
+2300 train 7.262286 (lr=1.3527e-04) (hash(x)=54950719)
+1700 val loss 7.5376
+1700 val perplexity 1877.3765
+1700 train 7.684578 (lr=9.4897e-05) (hash(x)=56717172)
+2400 val loss 7.2174
+2400 val perplexity 1362.8987
+2400 train 6.879970 (lr=1.3390e-04) (hash(x)=42190240)
+1800 val loss 7.4586
+1800 val perplexity 1734.6367
+1800 train 7.710993 (lr=9.4209e-05) (hash(x)=55376447)
+2500 val loss 7.1974
+2500 val perplexity 1335.8898
+2500 train 7.284463 (lr=1.3247e-04) (hash(x)=45223539)
+1900 val loss 7.4584
+1900 val perplexity 1734.4408
+1900 train 7.186811 (lr=9.3481e-05) (hash(x)=43810837)
+2600 val loss 7.1936
+2600 val perplexity 1330.8302
+2600 train 7.223955 (lr=1.3099e-04) (hash(x)=54037353)
+2000 val loss 7.4416
+2000 val perplexity 1705.4586
+2000 train 7.528937 (lr=9.2714e-05) (hash(x)=50881655)
+2700 val loss 7.2011
+2700 val perplexity 1340.8951
+2700 train 7.754804 (lr=1.2946e-04) (hash(x)=59131616)
+2100 val loss 7.4330
+2100 val perplexity 1690.8485
+2100 train 7.312185 (lr=9.1908e-05) (hash(x)=49386015)
+2800 val loss 7.1920
+2800 val perplexity 1328.6990
+2800 train 7.032724 (lr=1.2788e-04) (hash(x)=45882743)
+2200 val loss 7.3995
+2200 val perplexity 1635.2444
+2200 train 7.314746 (lr=9.1064e-05) (hash(x)=48572079)
+2900 val loss 7.1831
+2900 val perplexity 1316.9196
+2900 train 6.815891 (lr=1.2626e-04) (hash(x)=43758910)
+2300 val loss 7.3938
+2300 val perplexity 1625.8793
+2300 train 7.450395 (lr=9.0182e-05) (hash(x)=54950719)
+3000 val loss 7.1706
+3000 val perplexity 1300.5831
+3000 train 7.098728 (lr=1.2459e-04) (hash(x)=47965974)
+2400 val loss 7.3578
+2400 val perplexity 1568.3666
+2400 train 7.028030 (lr=8.9265e-05) (hash(x)=42190240)
+3100 val loss 7.1539
+3100 val perplexity 1279.1355
+3100 train 7.044363 (lr=1.2287e-04) (hash(x)=48205243)
+2500 val loss 7.3452
+2500 val perplexity 1548.7137
+2500 train 7.422577 (lr=8.8313e-05) (hash(x)=45223539)
+3200 val loss 7.1652
+3200 val perplexity 1293.6329
+3200 train 7.250436 (lr=1.2112e-04) (hash(x)=54511383)
+2600 val loss 7.3222
+2600 val perplexity 1513.4851
+2600 train 7.332448 (lr=8.7326e-05) (hash(x)=54037353)
+3300 val loss 7.1444
+3300 val perplexity 1266.9409
+3300 train 7.101440 (lr=1.1932e-04) (hash(x)=54428388)
+2700 val loss 7.3374
+2700 val perplexity 1536.7114
+2700 train 7.783456 (lr=8.6306e-05) (hash(x)=59131616)
+3400 val loss 7.1267
+3400 val perplexity 1244.7963
+3400 train 7.159663 (lr=1.1749e-04) (hash(x)=48115990)
+2800 val loss 7.3442
+2800 val perplexity 1547.1792
+2800 train 7.180212 (lr=8.5254e-05) (hash(x)=45882743)
+3500 val loss 7.1118
+3500 val perplexity 1226.2948
+3500 train 6.734808 (lr=1.1562e-04) (hash(x)=41137345)
+2900 val loss 7.3278
+2900 val perplexity 1522.0278
+2900 train 6.986289 (lr=8.4170e-05) (hash(x)=43758910)
+3600 val loss 7.1108
+3600 val perplexity 1225.1632
+3600 train 7.007893 (lr=1.1372e-04) (hash(x)=55186224)
+3000 val loss 7.3364
+3000 val perplexity 1535.1770
+3000 train 7.256237 (lr=8.3057e-05) (hash(x)=47965974)
+3700 val loss 7.1188
+3700 val perplexity 1235.0267
+3700 train 6.970138 (lr=1.1179e-04) (hash(x)=54990049)
+3100 val loss 7.2971
+3100 val perplexity 1476.0162
+3100 train 7.206101 (lr=8.1915e-05) (hash(x)=48205243)
+3800 val loss 7.0919
+3800 val perplexity 1202.2446
+3800 train 6.816278 (lr=1.0982e-04) (hash(x)=46288812)
+3200 val loss 7.2821
+3200 val perplexity 1454.0240
+3200 train 7.345427 (lr=8.0745e-05) (hash(x)=54511383)
+3900 val loss 7.0675
+3900 val perplexity 1173.2322
+3900 train 6.737766 (lr=1.0783e-04) (hash(x)=45829773)
+3300 val loss 7.2780
+3300 val perplexity 1448.0736
+3300 train 7.247321 (lr=7.9549e-05) (hash(x)=54428388)
+4000 val loss 7.0704
+4000 val perplexity 1176.6615
+4000 train 6.859371 (lr=1.0581e-04) (hash(x)=52499943)
+3400 val loss 7.2686
+3400 val perplexity 1434.5167
+3400 train 7.313385 (lr=7.8328e-05) (hash(x)=48115990)
+4100 val loss 7.0087
+4100 val perplexity 1106.1769
+4100 train 6.920537 (lr=1.0377e-04) (hash(x)=48563796)
+3500 val loss 7.2712
+3500 val perplexity 1438.2900
+3500 train 6.913240 (lr=7.7082e-05) (hash(x)=41137345)
+4200 val loss 6.9935
+4200 val perplexity 1089.5476
+4200 train 6.965348 (lr=1.0171e-04) (hash(x)=49165143)
+3600 val loss 7.2623
+3600 val perplexity 1425.5751
+3600 train 7.169919 (lr=7.5814e-05) (hash(x)=55186224)
+4300 val loss 6.9627
+4300 val perplexity 1056.4453
+4300 train 7.010408 (lr=9.9622e-05) (hash(x)=50973176)
+3700 val loss 7.2633
+3700 val perplexity 1426.9270
+3700 train 7.101009 (lr=7.4525e-05) (hash(x)=54990049)
+4400 val loss 6.9507
+4400 val perplexity 1043.8556
+4400 train 7.006516 (lr=9.7520e-05) (hash(x)=55275124)
+4500 val loss 6.9582
+4500 val perplexity 1051.7567
+4500 train 7.286173 (lr=9.5403e-05) (hash(x)=58646505)
+3800 val loss 7.2887
+3800 val perplexity 1463.6641
+3800 train 7.029315 (lr=7.3215e-05) (hash(x)=46288812)
+4600 val loss 6.9107
+4600 val perplexity 1002.9967
+4600 train 6.718947 (lr=9.3273e-05) (hash(x)=42554666)
+3900 val loss 7.2681
+3900 val perplexity 1433.8232
+3900 train 6.927856 (lr=7.1887e-05) (hash(x)=45829773)
+4700 val loss 6.9037
+4700 val perplexity 995.9628
+4700 train 6.815775 (lr=9.1132e-05) (hash(x)=47846764)
+4000 val loss 7.2776
+4000 val perplexity 1447.4598
+4000 train 7.064430 (lr=7.0541e-05) (hash(x)=52499943)
+4800 val loss 6.8718
+4800 val perplexity 964.6489
+4800 train 7.392079 (lr=8.8982e-05) (hash(x)=58239019)
+4100 val loss 7.2279
+4100 val perplexity 1377.2673
+4100 train 7.127439 (lr=6.9180e-05) (hash(x)=48563796)
+4900 val loss 6.8548
+4900 val perplexity 948.4195
+4900 train 6.890259 (lr=8.6825e-05) (hash(x)=50711220)
+4200 val loss 7.2032
+4200 val perplexity 1343.6794
+4200 train 7.175355 (lr=6.7804e-05) (hash(x)=49165143)
+5000 val loss 6.8666
+5000 val perplexity 959.6856
+5000 train 6.797673 (lr=8.4663e-05) (hash(x)=45994194)
+4300 val loss 7.1964
+4300 val perplexity 1334.5923
+4300 train 7.246315 (lr=6.6414e-05) (hash(x)=50973176)
+5100 val loss 6.8380
+5100 val perplexity 932.5778
+5100 train 6.689332 (lr=8.2500e-05) (hash(x)=48659050)
+4400 val loss 7.1871
+4400 val perplexity 1322.2667
+4400 train 7.213579 (lr=6.5013e-05) (hash(x)=55275124)
+5200 val loss 6.8171
+5200 val perplexity 913.3361
+5200 train 6.733484 (lr=8.0337e-05) (hash(x)=49369682)
+4500 val loss 7.1887
+4500 val perplexity 1324.3516
+4500 train 7.501039 (lr=6.3602e-05) (hash(x)=58646505)
+5300 val loss 6.8009
+5300 val perplexity 898.6852
+5300 train 7.195892 (lr=7.8175e-05) (hash(x)=57787700)
+4600 val loss 7.1673
+4600 val perplexity 1296.2893
+4600 train 6.993137 (lr=6.2182e-05) (hash(x)=42554666)
+5400 val loss 6.8112
+5400 val perplexity 907.9326
+5400 train 6.721237 (lr=7.6018e-05) (hash(x)=49365400)
+4700 val loss 7.1780
+4700 val perplexity 1310.3413
+4700 train 7.083921 (lr=6.0754e-05) (hash(x)=47846764)
+5500 val loss 6.7819
+5500 val perplexity 881.7007
+5500 train 6.746484 (lr=7.3868e-05) (hash(x)=48720412)
+4800 val loss 7.1616
+4800 val perplexity 1289.0020
+4800 train 7.689332 (lr=5.9321e-05) (hash(x)=58239019)
+5600 val loss 6.7828
+5600 val perplexity 882.5264
+5600 train 7.133316 (lr=7.1727e-05) (hash(x)=55784800)
+4900 val loss 7.1571
+4900 val perplexity 1283.2035
+4900 train 7.220632 (lr=5.7883e-05) (hash(x)=50711220)
+5700 val loss 6.7763
+5700 val perplexity 876.8293
+5700 train 6.594879 (lr=6.9597e-05) (hash(x)=50073634)
+5000 val loss 7.1550
+5000 val perplexity 1280.5098
+5000 train 7.076353 (lr=5.6442e-05) (hash(x)=45994194)
+5800 val loss 6.7572
+5800 val perplexity 860.2529
+5800 train 6.577003 (lr=6.7480e-05) (hash(x)=50170324)
+5100 val loss 7.1429
+5100 val perplexity 1265.0339
+5100 train 7.000565 (lr=5.5000e-05) (hash(x)=48659050)
+5900 val loss 6.7658
+5900 val perplexity 867.6870
+5900 train 6.507083 (lr=6.5378e-05) (hash(x)=48410268)
+5200 val loss 7.1410
+5200 val perplexity 1262.6818
+5200 train 7.094159 (lr=5.3558e-05) (hash(x)=49369682)
+6000 val loss 6.7626
+6000 val perplexity 864.9288
+6000 train 6.629282 (lr=6.3294e-05) (hash(x)=49527342)
+5300 val loss 7.1292
+5300 val perplexity 1247.8777
+5300 train 7.497270 (lr=5.2117e-05) (hash(x)=57787700)
+6100 val loss 6.7591
+6100 val perplexity 861.8850
+6100 train 6.519784 (lr=6.1230e-05) (hash(x)=49550294)
+5400 val loss 7.1447
+5400 val perplexity 1267.4213
+5400 train 7.050255 (lr=5.0679e-05) (hash(x)=49365400)
+6200 val loss 6.7512
+6200 val perplexity 855.1146
+6200 train 6.288379 (lr=5.9188e-05) (hash(x)=42126106)
+5500 val loss 7.1224
+5500 val perplexity 1239.3687
+5500 train 7.068456 (lr=4.9246e-05) (hash(x)=48720412)
+6300 val loss 6.7449
+6300 val perplexity 849.7552
+6300 train 6.486180 (lr=5.7169e-05) (hash(x)=49608772)
+5600 val loss 7.1197
+5600 val perplexity 1236.0317
+5600 train 7.443053 (lr=4.7818e-05) (hash(x)=55784800)
+6400 val loss 6.7319
+6400 val perplexity 838.7435
+6400 train 6.273153 (lr=5.5177e-05) (hash(x)=52324417)
+5700 val loss 7.1170
+5700 val perplexity 1232.7433
+5700 train 6.958042 (lr=4.6398e-05) (hash(x)=50073634)
+6500 val loss 6.6949
+6500 val perplexity 808.3016
+6500 train 6.726796 (lr=5.3213e-05) (hash(x)=46207215)
+5800 val loss 7.1130
+5800 val perplexity 1227.8770
+5800 train 6.936908 (lr=4.4987e-05) (hash(x)=50170324)
+6600 val loss 6.6733
+6600 val perplexity 791.0082
+6600 train 6.626161 (lr=5.1279e-05) (hash(x)=49027014)
+5900 val loss 7.1241
+5900 val perplexity 1241.5690
+5900 train 6.890422 (lr=4.3586e-05) (hash(x)=48410268)
+6700 val loss 6.6657
+6700 val perplexity 784.9918
+6700 train 6.616896 (lr=4.9377e-05) (hash(x)=46232513)
+6000 val loss 7.1221
+6000 val perplexity 1239.0070
+6000 train 7.008251 (lr=4.2196e-05) (hash(x)=49527342)
+6800 val loss 6.6663
+6800 val perplexity 785.5156
+6800 train 6.590093 (lr=4.7509e-05) (hash(x)=47348403)
+6100 val loss 7.1235
+6100 val perplexity 1240.8168
+6100 train 6.934540 (lr=4.0820e-05) (hash(x)=49550294)
+6900 val loss 6.6790
+6900 val perplexity 795.5314
+6900 train 6.656285 (lr=4.5676e-05) (hash(x)=49806647)
+6200 val loss 7.1133
+6200 val perplexity 1228.1627
+6200 train 6.688022 (lr=3.9459e-05) (hash(x)=42126106)
+7000 val loss 6.6343
+7000 val perplexity 760.7117
+7000 train 6.703082 (lr=4.3882e-05) (hash(x)=50893018)
+6300 val loss 7.1152
+6300 val perplexity 1230.5397
+6300 train 6.896497 (lr=3.8113e-05) (hash(x)=49608772)
+7100 val loss 6.6442
+7100 val perplexity 768.3418
+7100 train 6.674629 (lr=4.2128e-05) (hash(x)=49157639)
+6400 val loss 7.1097
+6400 val perplexity 1223.7263
+6400 train 6.698913 (lr=3.6785e-05) (hash(x)=52324417)
+7200 val loss 6.6269
+7200 val perplexity 755.1032
+7200 train 6.619589 (lr=4.0414e-05) (hash(x)=47014759)
+7300 val loss 6.6245
+7300 val perplexity 753.3093
+7300 train 6.609889 (lr=3.8745e-05) (hash(x)=47325591)
+6500 val loss 7.0836
+6500 val perplexity 1192.3104
+6500 train 7.098672 (lr=3.5475e-05) (hash(x)=46207215)
+7400 val loss 6.6178
+7400 val perplexity 748.2969
+7400 train 6.509352 (lr=3.7120e-05) (hash(x)=49184604)
+6600 val loss 7.0634
+6600 val perplexity 1168.4180
+6600 train 6.992548 (lr=3.4186e-05) (hash(x)=49027014)
+7500 val loss 6.6078
+7500 val perplexity 740.8351
+7500 train 6.835536 (lr=3.5541e-05) (hash(x)=55053584)
+6700 val loss 7.0569
+6700 val perplexity 1160.8925
+6700 train 6.979507 (lr=3.2918e-05) (hash(x)=46232513)
+7600 val loss 6.6067
+7600 val perplexity 740.0150
+7600 train 6.554074 (lr=3.4011e-05) (hash(x)=48693923)
+6800 val loss 7.0579
+6800 val perplexity 1162.0583
+6800 train 6.994317 (lr=3.1672e-05) (hash(x)=47348403)
+7700 val loss 6.6059
+7700 val perplexity 739.4333
+7700 train 6.199418 (lr=3.2531e-05) (hash(x)=40952882)
+6900 val loss 7.0640
+6900 val perplexity 1169.1509
+6900 train 7.008790 (lr=3.0451e-05) (hash(x)=49806647)
+7800 val loss 6.6001
+7800 val perplexity 735.1403
+7800 train 6.685646 (lr=3.1102e-05) (hash(x)=52487845)
+7000 val loss 7.0359
+7000 val perplexity 1136.6608
+7000 train 7.066860 (lr=2.9255e-05) (hash(x)=50893018)
+7900 val loss 6.5936
+7900 val perplexity 730.3727
+7900 train 6.667073 (lr=2.9726e-05) (hash(x)=50221547)
+7100 val loss 7.0304
+7100 val perplexity 1130.4474
+7100 train 7.072404 (lr=2.8085e-05) (hash(x)=49157639)
+8000 val loss 6.5985
+8000 val perplexity 733.9662
+8000 train 6.816906 (lr=2.8405e-05) (hash(x)=62294204)
+7200 val loss 7.0286
+7200 val perplexity 1128.4714
+7200 train 6.968489 (lr=2.6943e-05) (hash(x)=47014759)
+8100 val loss 6.5919
+8100 val perplexity 729.1611
+8100 train 6.290696 (lr=2.7138e-05) (hash(x)=44401967)
+7300 val loss 7.0301
+7300 val perplexity 1130.1548
+7300 train 7.023178 (lr=2.5830e-05) (hash(x)=47325591)
+8200 val loss 6.5940
+8200 val perplexity 730.6852
+8200 train 6.534984 (lr=2.5929e-05) (hash(x)=52769095)
+7400 val loss 7.0179
+7400 val perplexity 1116.3906
+7400 train 6.888594 (lr=2.4746e-05) (hash(x)=49184604)
+8300 val loss 6.6070
+8300 val perplexity 740.2457
+8300 train 6.528926 (lr=2.4778e-05) (hash(x)=56829883)
+7500 val loss 7.0154
+7500 val perplexity 1113.6748
+7500 train 7.263641 (lr=2.3694e-05) (hash(x)=55053584)
+8400 val loss 6.6083
+8400 val perplexity 741.1849
+8400 train 6.546304 (lr=2.3686e-05) (hash(x)=52147375)
+7600 val loss 7.0119
+7600 val perplexity 1109.7356
+7600 train 6.966333 (lr=2.2674e-05) (hash(x)=48693923)
+8500 val loss 6.6043
+8500 val perplexity 738.2313
+8500 train 6.838852 (lr=2.2655e-05) (hash(x)=60197820)
+7700 val loss 7.0081
+7700 val perplexity 1105.5388
+7700 train 6.596260 (lr=2.1687e-05) (hash(x)=40952882)
+8600 val loss 6.5924
+8600 val perplexity 729.5060
+8600 train 6.325205 (lr=2.1685e-05) (hash(x)=49377068)
+7800 val loss 7.0164
+7800 val perplexity 1114.7831
+7800 train 7.092906 (lr=2.0735e-05) (hash(x)=52487845)
+8700 val loss 6.6001
+8700 val perplexity 735.1347
+8700 train 6.556189 (lr=2.0777e-05) (hash(x)=51092724)
+7900 val loss 7.0032
+7900 val perplexity 1100.1108
+7900 train 7.059808 (lr=1.9818e-05) (hash(x)=50221547)
+8800 val loss 6.5955
+8800 val perplexity 731.8264
+8800 train 6.565369 (lr=1.9933e-05) (hash(x)=48642928)
+8000 val loss 7.0017
+8000 val perplexity 1098.5424
+8000 train 7.273840 (lr=1.8936e-05) (hash(x)=62294204)
+8900 val loss 6.5768
+8900 val perplexity 718.2245
+8900 train 6.758434 (lr=1.9153e-05) (hash(x)=55342246)
+8100 val loss 6.9990
+8100 val perplexity 1095.5623
+8100 train 6.701060 (lr=1.8092e-05) (hash(x)=44401967)
+9000 val loss 6.5617
+9000 val perplexity 707.4598
+9000 train 6.568685 (lr=1.8439e-05) (hash(x)=48093368)
+8200 val loss 7.0037
+8200 val perplexity 1100.6927
+8200 train 6.968071 (lr=1.7286e-05) (hash(x)=52769095)
+9100 val loss 6.5556
+9100 val perplexity 703.1831
+9100 train 6.602783 (lr=1.7790e-05) (hash(x)=48578183)
+8300 val loss 7.0071
+8300 val perplexity 1104.4213
+8300 train 6.974947 (lr=1.6519e-05) (hash(x)=56829883)
+9200 val loss 6.5544
+9200 val perplexity 702.3309
+9200 train 6.698106 (lr=1.7208e-05) (hash(x)=50794720)
+8400 val loss 7.0149
+8400 val perplexity 1113.1216
+8400 train 6.988824 (lr=1.5791e-05) (hash(x)=52147375)
+9300 val loss 6.5496
+9300 val perplexity 698.9913
+9300 train 6.316019 (lr=1.6692e-05) (hash(x)=46513190)
+8500 val loss 7.0100
+8500 val perplexity 1107.6707
+8500 train 7.226442 (lr=1.5103e-05) (hash(x)=60197820)
+9400 val loss 6.5440
+9400 val perplexity 695.0316
+9400 train 6.202390 (lr=1.6245e-05) (hash(x)=43808238)
+8600 val loss 7.0079
+8600 val perplexity 1105.2942
+8600 train 6.780334 (lr=1.4456e-05) (hash(x)=49377068)
+9500 val loss 6.5441
+9500 val perplexity 695.0981
+9500 train 6.356673 (lr=1.5865e-05) (hash(x)=45021888)
+8700 val loss 7.0063
+8700 val perplexity 1103.5695
+8700 train 6.962779 (lr=1.3851e-05) (hash(x)=51092724)
+9600 val loss 6.5407
+9600 val perplexity 692.7766
+9600 train 6.637218 (lr=1.5554e-05) (hash(x)=56525570)
+9700 val loss 6.5368
+9700 val perplexity 690.1021
+9700 train 6.719944 (lr=1.5312e-05) (hash(x)=52585913)
+8800 val loss 7.0045
+8800 val perplexity 1101.6248
+8800 train 7.003560 (lr=1.3289e-05) (hash(x)=48642928)
+9800 val loss 6.5355
+9800 val perplexity 689.1559
+9800 train 6.706218 (lr=1.5139e-05) (hash(x)=52344698)
+8900 val loss 6.9983
+8900 val perplexity 1094.7372
+8900 train 7.272505 (lr=1.2769e-05) (hash(x)=55342246)
+9900 val loss 6.5324
+9900 val perplexity 687.0436
+9900 train 6.540698 (lr=1.5035e-05) (hash(x)=51740945)
+9000 val loss 6.9807
+9000 val perplexity 1075.6921
+9000 train 6.939584 (lr=1.2292e-05) (hash(x)=48093368)
+9999 val loss 6.5308
+9999 val perplexity 685.9659
+9100 val loss 6.9772
+9100 val perplexity 1071.9318
+9100 train 7.052067 (lr=1.1860e-05) (hash(x)=48578183)
+9200 val loss 6.9744
+9200 val perplexity 1068.9447
+9200 train 7.153374 (lr=1.1472e-05) (hash(x)=50794720)
+9300 val loss 6.9714
+9300 val perplexity 1065.6759
+9300 train 6.762959 (lr=1.1128e-05) (hash(x)=46513190)

attention_kindselective_n_heads2_seed1338/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88a13d00e75e1cf66932d118b2f84bb24f00babd0ee75e7f5801725ef5c13418
+oid sha256:b9157bcc1377e68b286966c0a0341a5e69ef9fc2e367bcc48aba49860b5ba4d4
 size 38587970

attention_kindselective_n_heads2_seed1338/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:32d5afc7324a3d22f4c1218ec09c6c54149343ba625fcba9940acb396b796c11
+oid sha256:9f41aca9adf817c5b42c183649b46a97fcd1baba88fea3644adb57163bd39ec2
 size 38587970

attention_kindselective_n_heads2_seed1338/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05a121807594d5ecb5feed74e34dfa1e69b69860960bb12751f372377bb006a6
+oid sha256:f15805825428b7a60e7c78f2ff49c625a1b71822b49facc1d5e458f2e62df939
 size 38587970

attention_kindselective_n_heads2_seed1338/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:40e0b112f277f6cbba421303357ec83f7697b417831c9e4ff43b473f4f3e1114
+oid sha256:d811c4612e563d7ca45c8d8885b79dc2a5745e774a70903db459198e33ac0f4a
 size 38587970

attention_kindselective_n_heads2_seed1338/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d833216fc7bfe1147572ad064c3757e6d11dab6d14b15ed9dbc193d3fe0c652
+oid sha256:47848b329fdc5de70e6f1d2c743be1b06f2fc0459c1128539106cc34eb7bd6cb
 size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42125ee96db8d16feb270e483c050b0f0ad3baa06ce04fb49898faaa1eb18ca6
+oid sha256:23d2ca9117d48314c932bb7aa5c430b045f768930d81a6a6a25c137857f81aad
 size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4c2b49578a7dc4e2283cd26e5e389f35e17e5b918115b0f054f8c487f39e3d0
+oid sha256:b8615fd9af3900fecade21ff7a416b6897eb44dab958718ebeb0d5baae84031b
 size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42529f3655fc336117c939c1c8ec0fb125634901be3c730bed8da0466e4741ef
+oid sha256:9d832d53521de3812a1554dab9c41a6afb51235fdd9d9e0b4f8e9dfe2e3ccd20
 size 70895430