Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 7e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "7e-5_10240_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "15e-5_10240_4_1339", "n_embd": 256}

attention_kindselective_n_heads4_seed1339/log2.txt

@@ -1,606 +1,598 @@
+0 train 11.297240 (lr=5.0000e-07) (hash(x)=55241167)
 max_steps: 10000
-max_steps: 10000
-0 val loss 11.2766
-0 val perplexity 78951.9062
+100 val loss 9.4113
+100 val perplexity 12226.2285
+100 train 9.392327 (lr=5.0500e-05) (hash(x)=46387919)
 0 val loss 11.2766
 0 val perplexity 78951.9062
-0 train 11.297240 (lr=2.5000e-07) (hash(x)=55241167)
-0 train 11.297240 (lr=3.5000e-07) (hash(x)=55241167)
-100 val loss 9.8269
-100 val perplexity 18526.1348
-100 train 9.819736 (lr=2.5250e-05) (hash(x)=46387919)
-100 val loss 9.6928
-100 val perplexity 16200.6123
-100 train 9.682398 (lr=3.5350e-05) (hash(x)=46387919)
-200 val loss 8.7461
-200 val perplexity 6286.3667
-200 train 8.832648 (lr=5.0000e-05) (hash(x)=51276369)
-200 val loss 8.1592
-200 val perplexity 3495.5115
-200 train 8.252799 (lr=7.0000e-05) (hash(x)=51276369)
-300 val loss 8.0583
-300 val perplexity 3160.0583
-300 train 7.995393 (lr=4.9988e-05) (hash(x)=48572728)
-300 val loss 7.6698
-300 val perplexity 2142.7515
-300 train 7.577228 (lr=6.9984e-05) (hash(x)=48572728)
-400 val loss 7.7693
-400 val perplexity 2366.7744
-400 train 7.522111 (lr=4.9954e-05) (hash(x)=44943400)
-400 val loss 7.6067
-400 val perplexity 2011.6699
-400 train 7.332860 (lr=6.9935e-05) (hash(x)=44943400)
-500 val loss 7.6528
-500 val perplexity 2106.5469
-500 train 7.579549 (lr=4.9896e-05) (hash(x)=48434590)
-500 val loss 7.5459
-500 val perplexity 1892.9581
-500 train 7.433667 (lr=6.9854e-05) (hash(x)=48434590)
-600 val loss 7.6072
-600 val perplexity 2012.7042
-600 train 7.477727 (lr=4.9815e-05) (hash(x)=47442266)
-600 val loss 7.4994
-600 val perplexity 1806.9426
-600 train 7.327074 (lr=6.9741e-05) (hash(x)=47442266)
-700 val loss 7.5548
-700 val perplexity 1909.8828
-700 train 7.611083 (lr=4.9712e-05) (hash(x)=56495712)
-700 val loss 7.4660
-700 val perplexity 1747.5291
-700 train 7.532721 (lr=6.9596e-05) (hash(x)=56495712)
-800 val loss 7.5249
-800 val perplexity 1853.5676
-800 train 7.523900 (lr=4.9585e-05) (hash(x)=50093738)
-800 val loss 7.4516
-800 val perplexity 1722.6797
-800 train 7.450620 (lr=6.9419e-05) (hash(x)=50093738)
-900 val loss 7.5121
-900 val perplexity 1830.0836
-900 train 7.484754 (lr=4.9436e-05) (hash(x)=49436061)
-900 val loss 7.4213
-900 val perplexity 1671.1631
-900 train 7.381460 (lr=6.9210e-05) (hash(x)=49436061)
-1000 val loss 7.5026
-1000 val perplexity 1812.8086
-1000 train 7.425063 (lr=4.9264e-05) (hash(x)=48818282)
-1000 val loss 7.3975
-1000 val perplexity 1631.8879
-1000 train 7.324411 (lr=6.8970e-05) (hash(x)=48818282)
-1100 val loss 7.4779
-1100 val perplexity 1768.5020
-1100 train 7.553754 (lr=4.9070e-05) (hash(x)=55387425)
-1100 val loss 7.3392
-1100 val perplexity 1539.5417
-1100 train 7.412958 (lr=6.8698e-05) (hash(x)=55387425)
-1200 val loss 7.4630
-1200 val perplexity 1742.3455
-1200 train 7.370140 (lr=4.8854e-05) (hash(x)=48328378)
-1200 val loss 7.3034
-1200 val perplexity 1485.2927
-1200 train 7.183446 (lr=6.8395e-05) (hash(x)=48328378)
-1300 val loss 7.4402
-1300 val perplexity 1703.0768
-1300 train 7.354329 (lr=4.8616e-05) (hash(x)=48956717)
-1300 val loss 7.2614
-1300 val perplexity 1424.2318
-1300 train 7.164160 (lr=6.8062e-05) (hash(x)=48956717)
-1400 val loss 7.4390
-1400 val perplexity 1701.0858
-1400 train 7.325079 (lr=4.8356e-05) (hash(x)=49782571)
-1400 val loss 7.2354
-1400 val perplexity 1387.6724
-1400 train 7.128095 (lr=6.7698e-05) (hash(x)=49782571)
-1500 val loss 7.4168
-1500 val perplexity 1663.6846
-1500 train 7.217878 (lr=4.8074e-05) (hash(x)=51081994)
-1500 val loss 7.2017
-1500 val perplexity 1341.7664
-1500 train 6.971539 (lr=6.7304e-05) (hash(x)=51081994)
-1600 val loss 7.4079
-1600 val perplexity 1649.0146
-1600 train 7.481792 (lr=4.7772e-05) (hash(x)=48791204)
-1600 val loss 7.1565
-1600 val perplexity 1282.4695
-1600 train 7.212631 (lr=6.6881e-05) (hash(x)=48791204)
-1700 val loss 7.3899
-1700 val perplexity 1619.6024
-1700 train 7.112850 (lr=4.7448e-05) (hash(x)=46425011)
-1700 val loss 7.1197
-1700 val perplexity 1236.1090
-1700 train 6.791309 (lr=6.6428e-05) (hash(x)=46425011)
-1800 val loss 7.0838
-1800 val perplexity 1192.4474
-1800 train 6.699710 (lr=6.5947e-05) (hash(x)=44919141)
-1800 val loss 7.3832
-1800 val perplexity 1608.7535
-1800 train 7.020766 (lr=4.7105e-05) (hash(x)=44919141)
-1900 val loss 7.0346
-1900 val perplexity 1135.2908
-1900 train 7.144853 (lr=6.5437e-05) (hash(x)=48299675)
-1900 val loss 7.3649
-1900 val perplexity 1579.5735
-1900 train 7.448961 (lr=4.6741e-05) (hash(x)=48299675)
-2000 val loss 6.9733
-2000 val perplexity 1067.7863
-2000 train 7.800293 (lr=6.4900e-05) (hash(x)=61804797)
-2000 val loss 7.3407
-2000 val perplexity 1541.7869
-2000 train 8.138843 (lr=4.6357e-05) (hash(x)=61804797)
-2100 val loss 6.9329
-2100 val perplexity 1025.4562
-2100 train 7.034851 (lr=6.4335e-05) (hash(x)=53638030)
-2100 val loss 7.3219
-2100 val perplexity 1513.0824
-2100 train 7.405197 (lr=4.5954e-05) (hash(x)=53638030)
-2200 val loss 6.9051
-2200 val perplexity 997.3262
-2200 train 7.128452 (lr=6.3745e-05) (hash(x)=56364593)
-2200 val loss 7.3057
-2200 val perplexity 1488.7239
-2200 train 7.551863 (lr=4.5532e-05) (hash(x)=56364593)
-2300 val loss 6.8697
-2300 val perplexity 962.6473
-2300 train 6.986196 (lr=6.3128e-05) (hash(x)=52945580)
-2300 val loss 7.3142
-2300 val perplexity 1501.4471
-2300 train 7.455043 (lr=4.5091e-05) (hash(x)=52945580)
-2400 val loss 6.8283
-2400 val perplexity 923.5925
-2400 train 6.899657 (lr=6.2486e-05) (hash(x)=54595211)
-2400 val loss 7.2843
-2400 val perplexity 1457.2745
-2400 train 7.338539 (lr=4.4633e-05) (hash(x)=54595211)
-2500 val loss 6.8054
-2500 val perplexity 902.7001
-2500 train 6.809216 (lr=6.1819e-05) (hash(x)=52323363)
-2500 val loss 7.2856
-2500 val perplexity 1459.1227
-2500 train 7.312573 (lr=4.4156e-05) (hash(x)=52323363)
-2600 val loss 6.7748
-2600 val perplexity 875.4703
-2600 train 6.542181 (lr=6.1128e-05) (hash(x)=42652707)
-2600 val loss 7.2808
-2600 val perplexity 1452.1907
-2600 train 7.076187 (lr=4.3663e-05) (hash(x)=42652707)
-2700 val loss 6.7567
-2700 val perplexity 859.7908
-2700 train 6.453963 (lr=6.0414e-05) (hash(x)=45687855)
-2700 val loss 7.2677
-2700 val perplexity 1433.2026
-2700 train 6.959708 (lr=4.3153e-05) (hash(x)=45687855)
-2800 val loss 6.7146
-2800 val perplexity 824.3206
-2800 train 6.724705 (lr=5.9677e-05) (hash(x)=49962830)
-2800 val loss 7.2714
-2800 val perplexity 1438.6337
-2800 train 7.254436 (lr=4.2627e-05) (hash(x)=49962830)
-2900 val loss 6.6880
-2900 val perplexity 802.7047
-2900 train 6.716569 (lr=5.8919e-05) (hash(x)=52958451)
-2900 val loss 7.2474
-2900 val perplexity 1404.4911
-2900 train 7.309949 (lr=4.2085e-05) (hash(x)=52958451)
-3000 val loss 6.6736
-3000 val perplexity 791.2670
-3000 train 6.612247 (lr=5.8140e-05) (hash(x)=48896990)
-3000 val loss 7.2392
-3000 val perplexity 1393.0072
-3000 train 7.165527 (lr=4.1529e-05) (hash(x)=48896990)
-3100 val loss 6.6638
-3100 val perplexity 783.4915
-3100 train 6.662642 (lr=5.7340e-05) (hash(x)=47064930)
-3100 val loss 7.2448
-3100 val perplexity 1400.7638
-3100 train 7.213631 (lr=4.0957e-05) (hash(x)=47064930)
-3200 val loss 6.6385
-3200 val perplexity 763.9189
-3200 train 6.342895 (lr=5.6522e-05) (hash(x)=45145921)
-3200 val loss 7.2305
-3200 val perplexity 1380.9763
-3200 train 6.985227 (lr=4.0373e-05) (hash(x)=45145921)
-3300 val loss 6.6241
-3300 val perplexity 753.0123
-3300 train 6.447469 (lr=5.5684e-05) (hash(x)=48085018)
-3300 val loss 7.2399
-3300 val perplexity 1393.9142
-3300 train 7.104490 (lr=3.9775e-05) (hash(x)=48085018)
-3400 val loss 6.6132
-3400 val perplexity 744.8303
-3400 train 6.614527 (lr=5.4829e-05) (hash(x)=49596053)
-3400 val loss 7.2185
-3400 val perplexity 1364.4150
-3400 train 7.253217 (lr=3.9164e-05) (hash(x)=49596053)
-3500 val loss 6.6042
-3500 val perplexity 738.2034
-3500 train 6.221120 (lr=5.3958e-05) (hash(x)=41135634)
-3500 val loss 7.2577
-3500 val perplexity 1418.9370
-3500 train 6.842973 (lr=3.8541e-05) (hash(x)=41135634)
-3600 val loss 6.5929
-3600 val perplexity 729.9281
-3600 train 6.570555 (lr=5.3070e-05) (hash(x)=54286330)
-3600 val loss 7.2149
-3600 val perplexity 1359.6013
-3600 train 7.226971 (lr=3.7907e-05) (hash(x)=54286330)
-3700 val loss 6.5851
-3700 val perplexity 724.2557
-3700 train 6.428621 (lr=5.2167e-05) (hash(x)=57061993)
-3700 val loss 7.2117
-3700 val perplexity 1355.2239
-3700 train 7.127274 (lr=3.7262e-05) (hash(x)=57061993)
-3800 val loss 6.5726
-3800 val perplexity 715.2515
-3800 train 6.137836 (lr=5.1251e-05) (hash(x)=46544884)
-3800 val loss 7.2026
-3800 val perplexity 1342.9222
-3800 train 6.810471 (lr=3.6608e-05) (hash(x)=46544884)
-3900 val loss 6.5798
-3900 val perplexity 720.3782
-3900 train 6.413338 (lr=5.0321e-05) (hash(x)=48937435)
-3900 val loss 7.2019
-3900 val perplexity 1341.9429
-3900 train 7.051498 (lr=3.5944e-05) (hash(x)=48937435)
-4000 val loss 6.5694
-4000 val perplexity 712.9241
-4000 train 6.600487 (lr=4.9379e-05) (hash(x)=54466186)
-4000 val loss 7.1906
-4000 val perplexity 1326.9261
-4000 train 7.239226 (lr=3.5271e-05) (hash(x)=54466186)
-4100 val loss 6.5491
-4100 val perplexity 698.6298
-4100 train 6.357440 (lr=4.8426e-05) (hash(x)=51079995)
-4100 val loss 7.1948
-4100 val perplexity 1332.4652
-4100 train 7.051948 (lr=3.4590e-05) (hash(x)=51079995)
-4200 val loss 6.5377
-4200 val perplexity 690.7164
-4200 train 6.665975 (lr=4.7463e-05) (hash(x)=56495581)
-4200 val loss 7.1759
-4200 val perplexity 1307.5887
-4200 train 7.325971 (lr=3.3902e-05) (hash(x)=56495581)
-4300 val loss 6.5119
-4300 val perplexity 673.0989
-4300 train 6.342508 (lr=4.6490e-05) (hash(x)=45095478)
-4300 val loss 7.1620
-4300 val perplexity 1289.4537
-4300 train 7.022624 (lr=3.3207e-05) (hash(x)=45095478)
-4400 val loss 6.4969
-4400 val perplexity 663.0798
-4400 train 6.514081 (lr=4.5509e-05) (hash(x)=49954028)
-4400 val loss 7.1656
-4400 val perplexity 1294.1129
-4400 train 7.113908 (lr=3.2507e-05) (hash(x)=49954028)
-4500 val loss 6.4922
-4500 val perplexity 659.9440
-4500 train 6.422389 (lr=4.4521e-05) (hash(x)=48975821)
-4500 val loss 7.1499
-4500 val perplexity 1274.0404
-4500 train 7.073869 (lr=3.1801e-05) (hash(x)=48975821)
-4600 val loss 6.4620
-4600 val perplexity 640.3523
-4600 train 6.557837 (lr=4.3527e-05) (hash(x)=49293028)
-4600 val loss 7.1385
-4600 val perplexity 1259.4857
-4600 train 7.196972 (lr=3.1091e-05) (hash(x)=49293028)
-4700 val loss 6.4616
-4700 val perplexity 640.1035
-4700 train 6.356697 (lr=4.2528e-05) (hash(x)=48077753)
-4700 val loss 7.1326
-4700 val perplexity 1252.1656
-4700 train 7.069026 (lr=3.0377e-05) (hash(x)=48077753)
-4800 val loss 6.4458
-4800 val perplexity 630.0588
-4800 train 6.352581 (lr=4.1525e-05) (hash(x)=45037879)
-4800 val loss 7.1220
-4800 val perplexity 1238.9526
-4800 train 6.939941 (lr=2.9661e-05) (hash(x)=45037879)
-4900 val loss 6.4401
-4900 val perplexity 626.4606
-4900 train 6.359092 (lr=4.0518e-05) (hash(x)=50320154)
-4900 val loss 7.1202
-4900 val perplexity 1236.6737
-4900 train 7.031347 (lr=2.8942e-05) (hash(x)=50320154)
-5000 val loss 6.4270
-5000 val perplexity 618.3121
-5000 train 6.702835 (lr=3.9510e-05) (hash(x)=55024523)
-5000 val loss 7.1210
-5000 val perplexity 1237.6714
-5000 train 7.321143 (lr=2.8221e-05) (hash(x)=55024523)
-5100 val loss 6.4187
-5100 val perplexity 613.1933
-5100 train 6.337059 (lr=3.8500e-05) (hash(x)=47570607)
-5100 val loss 7.0966
-5100 val perplexity 1207.8408
-5100 train 7.021636 (lr=2.7500e-05) (hash(x)=47570607)
-5200 val loss 6.4092
-5200 val perplexity 607.4013
-5200 train 6.531659 (lr=3.7490e-05) (hash(x)=54311349)
-5200 val loss 7.0982
-5200 val perplexity 1209.7371
-5200 train 7.204744 (lr=2.6779e-05) (hash(x)=54311349)
-5300 val loss 6.4037
-5300 val perplexity 604.0920
-5300 train 6.300325 (lr=3.6482e-05) (hash(x)=47178524)
-5300 val loss 7.0937
-5300 val perplexity 1204.3757
-5300 train 7.004245 (lr=2.6058e-05) (hash(x)=47178524)
-5400 val loss 6.4010
-5400 val perplexity 602.4624
-5400 train 6.421272 (lr=3.5475e-05) (hash(x)=49425088)
-5400 val loss 7.0886
-5400 val perplexity 1198.2756
-5400 train 7.084049 (lr=2.5339e-05) (hash(x)=49425088)
-5500 val loss 6.3854
-5500 val perplexity 593.1009
-5500 train 6.211343 (lr=3.4472e-05) (hash(x)=46383189)
-5500 val loss 7.0866
-5500 val perplexity 1195.8286
-5500 train 6.919466 (lr=2.4623e-05) (hash(x)=46383189)
-5600 val loss 6.3941
-5600 val perplexity 598.3093
-5600 train 6.562718 (lr=3.3473e-05) (hash(x)=54084990)
-5600 val loss 7.0990
-5600 val perplexity 1210.7424
-5600 train 7.309231 (lr=2.3909e-05) (hash(x)=54084990)
-5700 val loss 6.3745
-5700 val perplexity 586.6678
-5700 train 6.056097 (lr=3.2479e-05) (hash(x)=47384182)
-5700 val loss 7.0692
-5700 val perplexity 1175.1588
-5700 train 6.801169 (lr=2.3199e-05) (hash(x)=47384182)
-5800 val loss 6.3737
-5800 val perplexity 586.2322
-5800 train 6.243022 (lr=3.1491e-05) (hash(x)=51683744)
-5800 val loss 7.0651
-5800 val perplexity 1170.3546
-5800 train 6.965415 (lr=2.2493e-05) (hash(x)=51683744)
-5900 val loss 6.3628
-5900 val perplexity 579.8835
-5900 train 6.376868 (lr=3.0510e-05) (hash(x)=52798836)
-5900 val loss 7.0753
-5900 val perplexity 1182.4492
-5900 train 7.069267 (lr=2.1793e-05) (hash(x)=52798836)
-6000 val loss 6.3711
-6000 val perplexity 584.6808
-6000 train 6.247975 (lr=2.9537e-05) (hash(x)=50945000)
-6000 val loss 7.0655
-6000 val perplexity 1170.8513
-6000 train 6.961160 (lr=2.1098e-05) (hash(x)=50945000)
-6100 val loss 6.3612
-6100 val perplexity 578.9307
-6100 train 6.045362 (lr=2.8574e-05) (hash(x)=48964427)
-6100 val loss 7.0515
-6100 val perplexity 1154.6139
-6100 train 6.805200 (lr=2.0410e-05) (hash(x)=48964427)
-6200 val loss 6.3579
-6200 val perplexity 577.0609
-6200 train 6.127436 (lr=2.7621e-05) (hash(x)=47148610)
-6200 val loss 7.0483
-6200 val perplexity 1150.8965
-6200 train 6.847069 (lr=1.9729e-05) (hash(x)=47148610)
-6300 val loss 6.3510
-6300 val perplexity 573.0493
-6300 train 6.284134 (lr=2.6679e-05) (hash(x)=50860553)
-6300 val loss 7.0491
-6300 val perplexity 1151.8496
-6300 train 6.979808 (lr=1.9056e-05) (hash(x)=50860553)
-6400 val loss 6.3487
-6400 val perplexity 571.7348
-6400 train 6.138104 (lr=2.5749e-05) (hash(x)=50553992)
-6400 val loss 7.0396
-6400 val perplexity 1140.9142
-6400 train 6.835309 (lr=1.8392e-05) (hash(x)=50553992)
-6500 val loss 6.3525
-6500 val perplexity 573.9421
-6500 train 6.159073 (lr=2.4833e-05) (hash(x)=52521332)
-6500 val loss 7.0408
-6500 val perplexity 1142.3137
-6500 train 6.891948 (lr=1.7738e-05) (hash(x)=52521332)
-6600 val loss 6.3276
-6600 val perplexity 559.8296
-6600 train 6.666481 (lr=2.3930e-05) (hash(x)=52609843)
-6600 val loss 7.0210
-6600 val perplexity 1119.9614
-6600 train 7.358459 (lr=1.7093e-05) (hash(x)=52609843)
-6700 val loss 6.3134
-6700 val perplexity 551.8937
-6700 train 6.365655 (lr=2.3042e-05) (hash(x)=53632957)
-6700 val loss 7.0149
-6700 val perplexity 1113.1040
-6700 train 7.083940 (lr=1.6459e-05) (hash(x)=53632957)
-6800 val loss 6.3112
-6800 val perplexity 550.6937
-6800 train 6.342908 (lr=2.2171e-05) (hash(x)=46927608)
-6900 val loss 6.2981
-6900 val perplexity 543.5120
-6900 train 6.319361 (lr=2.1316e-05) (hash(x)=54483057)
-6800 val loss 7.0048
-6800 val perplexity 1101.9363
-6800 train 7.046721 (lr=1.5836e-05) (hash(x)=46927608)
-7000 val loss 6.2918
-7000 val perplexity 540.1509
-7000 train 6.423561 (lr=2.0478e-05) (hash(x)=51493884)
-6900 val loss 7.0009
-6900 val perplexity 1097.6104
-6900 train 7.066904 (lr=1.5225e-05) (hash(x)=54483057)
-7100 val loss 6.2839
-7100 val perplexity 535.8787
-7100 train 6.388433 (lr=1.9660e-05) (hash(x)=53303341)
-7000 val loss 6.9954
-7000 val perplexity 1091.6002
-7000 train 7.167415 (lr=1.4627e-05) (hash(x)=51493884)
-7200 val loss 6.2875
-7200 val perplexity 537.8173
-7200 train 5.993762 (lr=1.8860e-05) (hash(x)=45272178)
-7100 val loss 6.9877
-7100 val perplexity 1083.2349
-7100 train 7.118485 (lr=1.4043e-05) (hash(x)=53303341)
-7300 val loss 6.2765
-7300 val perplexity 531.9218
-7300 train 6.216062 (lr=1.8081e-05) (hash(x)=50389872)
-7200 val loss 6.9902
-7200 val perplexity 1085.9910
-7200 train 6.663062 (lr=1.3471e-05) (hash(x)=45272178)
-7400 val loss 6.2720
-7400 val perplexity 529.5265
-7400 train 5.910453 (lr=1.7323e-05) (hash(x)=43796301)
-7300 val loss 6.9799
-7300 val perplexity 1074.8020
-7300 train 6.914957 (lr=1.2915e-05) (hash(x)=50389872)
-7500 val loss 6.2674
-7500 val perplexity 527.1163
-7500 train 6.223414 (lr=1.6586e-05) (hash(x)=47808686)
-7400 val loss 6.9701
-7400 val perplexity 1064.3251
-7400 train 6.588149 (lr=1.2373e-05) (hash(x)=43796301)
-7600 val loss 6.2662
-7600 val perplexity 526.4735
-7600 train 6.001786 (lr=1.5872e-05) (hash(x)=41936898)
-7500 val loss 6.9646
-7500 val perplexity 1058.4805
-7500 train 6.888280 (lr=1.1847e-05) (hash(x)=47808686)
-7700 val loss 6.2684
-7700 val perplexity 527.6361
-7700 train 6.526704 (lr=1.5181e-05) (hash(x)=57550318)
-7600 val loss 6.9621
-7600 val perplexity 1055.8838
-7600 train 6.685042 (lr=1.1337e-05) (hash(x)=41936898)
-7800 val loss 6.2607
-7800 val perplexity 523.5745
-7800 train 6.117768 (lr=1.4514e-05) (hash(x)=47485210)
-7700 val loss 6.9577
-7700 val perplexity 1051.2378
-7700 train 7.145366 (lr=1.0844e-05) (hash(x)=57550318)
-7900 val loss 6.2583
-7900 val perplexity 522.3495
-7900 train 6.632966 (lr=1.3872e-05) (hash(x)=53228688)
-7800 val loss 6.9528
-7800 val perplexity 1046.0869
-7800 train 6.833837 (lr=1.0367e-05) (hash(x)=47485210)
-8000 val loss 6.2552
-8000 val perplexity 520.6916
-8000 train 6.333637 (lr=1.3255e-05) (hash(x)=52018673)
-7900 val loss 6.9490
-7900 val perplexity 1042.1179
-7900 train 7.275203 (lr=9.9088e-06) (hash(x)=53228688)
-8100 val loss 6.2521
-8100 val perplexity 519.1161
-8100 train 6.090673 (lr=1.2665e-05) (hash(x)=47079349)
-8000 val loss 6.9462
-8000 val perplexity 1039.2284
-8000 train 6.982090 (lr=9.4682e-06) (hash(x)=52018673)
-8200 val loss 6.2518
-8200 val perplexity 518.9553
-8200 train 6.302469 (lr=1.2100e-05) (hash(x)=57921563)
-8100 val loss 6.9412
-8100 val perplexity 1033.9647
-8100 train 6.762253 (lr=9.0461e-06) (hash(x)=47079349)
-8300 val loss 6.2460
-8300 val perplexity 515.9507
-8300 train 5.783485 (lr=1.1563e-05) (hash(x)=45038933)
-8200 val loss 6.9394
-8200 val perplexity 1032.1396
-8200 train 6.955695 (lr=8.6430e-06) (hash(x)=57921563)
-8400 val loss 6.2587
-8400 val perplexity 522.5202
-8400 train 6.118326 (lr=1.1053e-05) (hash(x)=47763246)
-8300 val loss 6.9366
-8300 val perplexity 1029.2189
-8300 train 6.503049 (lr=8.2593e-06) (hash(x)=45038933)
-8500 val loss 6.2517
-8500 val perplexity 518.8879
-8500 train 6.352527 (lr=1.0572e-05) (hash(x)=56176595)
-8400 val loss 6.9368
-8400 val perplexity 1029.5016
-8400 train 6.823294 (lr=7.8953e-06) (hash(x)=47763246)
-8600 val loss 6.2449
-8600 val perplexity 515.3643
-8600 train 6.332339 (lr=1.0119e-05) (hash(x)=55184249)
-8500 val loss 6.9325
-8500 val perplexity 1025.0548
-8500 train 6.934601 (lr=7.5515e-06) (hash(x)=56176595)
-8700 val loss 6.2427
-8700 val perplexity 514.2684
-8700 train 6.192267 (lr=9.6960e-06) (hash(x)=46471646)
-8600 val loss 6.9311
-8600 val perplexity 1023.5968
-8600 train 6.992166 (lr=7.2282e-06) (hash(x)=55184249)
-8800 val loss 6.2414
-8800 val perplexity 513.5671
-8800 train 5.959139 (lr=9.3021e-06) (hash(x)=46233162)
-8700 val loss 6.9270
-8700 val perplexity 1019.4482
-8700 train 6.903887 (lr=6.9257e-06) (hash(x)=46471646)
-8800 val loss 6.9308
-8800 val perplexity 1023.3308
-8800 train 6.686684 (lr=6.6444e-06) (hash(x)=46233162)
-8900 val loss 6.2401
-8900 val perplexity 512.9296
-8900 train 6.172048 (lr=8.9382e-06) (hash(x)=47233684)
-8900 val loss 6.9445
-8900 val perplexity 1037.4430
-8900 train 6.853402 (lr=6.3845e-06) (hash(x)=47233684)
-9000 val loss 6.2341
-9000 val perplexity 509.8349
-9000 train 6.103183 (lr=8.6047e-06) (hash(x)=48374529)
-9000 val loss 6.9224
-9000 val perplexity 1014.7220
-9000 train 6.834804 (lr=6.1462e-06) (hash(x)=48374529)
-9100 val loss 6.2260
-9100 val perplexity 505.7538
-9100 train 6.227113 (lr=8.3020e-06) (hash(x)=48065371)
-9100 val loss 6.9177
-9100 val perplexity 1010.0148
-9100 train 6.851860 (lr=5.9300e-06) (hash(x)=48065371)
-9200 val loss 6.2213
-9200 val perplexity 503.3779
-9200 train 6.224229 (lr=8.0302e-06) (hash(x)=47408078)
-9200 val loss 6.9111
-9200 val perplexity 1003.3564
-9200 train 6.844304 (lr=5.7359e-06) (hash(x)=47408078)
-9300 val loss 6.2190
-9300 val perplexity 502.2050
-9300 train 6.201622 (lr=7.7898e-06) (hash(x)=50749781)
-9300 val loss 6.9071
-9300 val perplexity 999.3813
-9300 train 6.925474 (lr=5.5641e-06) (hash(x)=50749781)
-9400 val loss 6.2138
-9400 val perplexity 499.5949
-9400 train 6.430583 (lr=7.5809e-06) (hash(x)=48560169)
-9400 val loss 6.9031
-9400 val perplexity 995.3218
-9400 train 7.070158 (lr=5.4149e-06) (hash(x)=48560169)
-9500 val loss 6.2137
-9500 val perplexity 499.5220
-9500 train 6.352744 (lr=7.4038e-06) (hash(x)=50936392)
-9500 val loss 6.9017
-9500 val perplexity 993.9194
-9500 train 6.979595 (lr=5.2884e-06) (hash(x)=50936392)
-9600 val loss 6.2106
-9600 val perplexity 497.9799
-9600 train 6.226837 (lr=7.2586e-06) (hash(x)=50651714)
-9600 val loss 6.8968
-9600 val perplexity 989.1498
-9600 train 6.905745 (lr=5.1847e-06) (hash(x)=50651714)
-9700 val loss 6.2115
-9700 val perplexity 498.4665
-9700 train 6.296004 (lr=7.1456e-06) (hash(x)=47311384)
-9700 val loss 6.8969
-9700 val perplexity 989.1899
-9700 train 6.926633 (lr=5.1040e-06) (hash(x)=47311384)
-9800 val loss 6.2085
-9800 val perplexity 496.9543
-9800 train 6.317750 (lr=7.0647e-06) (hash(x)=50921139)
-9800 val loss 6.8953
-9800 val perplexity 987.6577
-9800 train 6.967777 (lr=5.0462e-06) (hash(x)=50921139)
-9900 val loss 6.2224
-9900 val perplexity 503.9033
-9900 train 6.376268 (lr=7.0162e-06) (hash(x)=48142455)
-9900 val loss 6.8939
-9900 val perplexity 986.2411
-9900 train 6.922864 (lr=5.0116e-06) (hash(x)=48142455)
-9999 val loss 6.2055
-9999 val perplexity 495.4679
-9999 val loss 6.8894
-9999 val perplexity 981.8267
+200 val loss 7.8296
+200 val perplexity 2513.9221
+200 train 7.923450 (lr=1.0000e-04) (hash(x)=51276369)
+300 val loss 7.6256
+300 val perplexity 2049.9619
+300 train 7.519519 (lr=9.9977e-05) (hash(x)=48572728)
+0 train 11.297240 (lr=7.5000e-07) (hash(x)=55241167)
+400 val loss 7.5602
+400 val perplexity 1920.2125
+400 train 7.274498 (lr=9.9908e-05) (hash(x)=44943400)
+100 val loss 9.0840
+100 val perplexity 8813.1025
+100 train 9.057368 (lr=7.5750e-05) (hash(x)=46387919)
+500 val loss 7.5149
+500 val perplexity 1835.2683
+500 train 7.412261 (lr=9.9792e-05) (hash(x)=48434590)
+200 val loss 7.7009
+200 val perplexity 2210.4219
+200 train 7.782194 (lr=1.5000e-04) (hash(x)=51276369)
+600 val loss 7.4697
+600 val perplexity 1754.1002
+600 train 7.309482 (lr=9.9631e-05) (hash(x)=47442266)
+300 val loss 7.6257
+300 val perplexity 2050.2629
+300 train 7.521665 (lr=1.4997e-04) (hash(x)=48572728)
+700 val loss 7.4248
+700 val perplexity 1676.9968
+700 train 7.487512 (lr=9.9423e-05) (hash(x)=56495712)
+400 val loss 7.6210
+400 val perplexity 2040.5312
+400 train 7.349378 (lr=1.4986e-04) (hash(x)=44943400)
+800 val loss 7.4002
+800 val perplexity 1636.3661
+800 train 7.382157 (lr=9.9170e-05) (hash(x)=50093738)
+500 val loss 7.5935
+500 val perplexity 1985.2406
+500 train 7.501320 (lr=1.4969e-04) (hash(x)=48434590)
+900 val loss 7.3485
+900 val perplexity 1553.9221
+900 train 7.314708 (lr=9.8872e-05) (hash(x)=49436061)
+600 val loss 7.5432
+600 val perplexity 1887.9182
+600 train 7.397351 (lr=1.4945e-04) (hash(x)=47442266)
+1000 val loss 7.3260
+1000 val perplexity 1519.2688
+1000 train 7.252484 (lr=9.8528e-05) (hash(x)=48818282)
+700 val loss 7.5011
+700 val perplexity 1809.9755
+700 train 7.557298 (lr=1.4913e-04) (hash(x)=56495712)
+1100 val loss 7.2749
+1100 val perplexity 1443.6171
+1100 train 7.333434 (lr=9.8140e-05) (hash(x)=55387425)
+800 val loss 7.4475
+800 val perplexity 1715.6282
+800 train 7.430103 (lr=1.4876e-04) (hash(x)=50093738)
+1200 val loss 7.2674
+1200 val perplexity 1432.7633
+1200 train 7.136762 (lr=9.7708e-05) (hash(x)=48328378)
+900 val loss 7.3857
+900 val perplexity 1612.8206
+900 train 7.335268 (lr=1.4831e-04) (hash(x)=49436061)
+1300 val loss 7.2723
+1300 val perplexity 1439.9219
+1300 train 7.192955 (lr=9.7231e-05) (hash(x)=48956717)
+1000 val loss 7.3329
+1000 val perplexity 1529.8805
+1000 train 7.257126 (lr=1.4779e-04) (hash(x)=48818282)
+1400 val loss 7.2745
+1400 val perplexity 1443.0217
+1400 train 7.159895 (lr=9.6711e-05) (hash(x)=49782571)
+1100 val loss 7.2824
+1100 val perplexity 1454.4734
+1100 train 7.329366 (lr=1.4721e-04) (hash(x)=55387425)
+1500 val loss 7.2582
+1500 val perplexity 1419.7051
+1500 train 7.043027 (lr=9.6149e-05) (hash(x)=51081994)
+1200 val loss 7.2663
+1200 val perplexity 1431.2822
+1200 train 7.143242 (lr=1.4656e-04) (hash(x)=48328378)
+1600 val loss 7.2452
+1600 val perplexity 1401.4271
+1600 train 7.309634 (lr=9.5544e-05) (hash(x)=48791204)
+1300 val loss 7.2275
+1300 val perplexity 1376.7683
+1300 train 7.136340 (lr=1.4585e-04) (hash(x)=48956717)
+1700 val loss 7.2259
+1700 val perplexity 1374.5807
+1700 train 6.931890 (lr=9.4897e-05) (hash(x)=46425011)
+1400 val loss 7.2388
+1400 val perplexity 1392.4169
+1400 train 7.132669 (lr=1.4507e-04) (hash(x)=49782571)
+1800 val loss 7.2145
+1800 val perplexity 1358.9602
+1800 train 6.850976 (lr=9.4209e-05) (hash(x)=44919141)
+1500 val loss 7.2698
+1500 val perplexity 1436.2498
+1500 train 7.063611 (lr=1.4422e-04) (hash(x)=51081994)
+1900 val loss 7.1925
+1900 val perplexity 1329.3683
+1900 train 7.279314 (lr=9.3481e-05) (hash(x)=48299675)
+1600 val loss 7.2511
+1600 val perplexity 1409.6339
+1600 train 7.338916 (lr=1.4332e-04) (hash(x)=48791204)
+2000 val loss 7.1303
+2000 val perplexity 1249.2107
+2000 train 7.887177 (lr=9.2714e-05) (hash(x)=61804797)
+1700 val loss 7.2541
+1700 val perplexity 1413.9268
+1700 train 6.979441 (lr=1.4235e-04) (hash(x)=46425011)
+2100 val loss 7.0845
+2100 val perplexity 1193.3490
+2100 train 7.169927 (lr=9.1908e-05) (hash(x)=53638030)
+1800 val loss 7.2169
+1800 val perplexity 1362.2542
+1800 train 6.886115 (lr=1.4131e-04) (hash(x)=44919141)
+2200 val loss 7.0669
+2200 val perplexity 1172.4698
+2200 train 7.293911 (lr=9.1064e-05) (hash(x)=56364593)
+1900 val loss 7.1736
+1900 val perplexity 1304.4712
+1900 train 7.252931 (lr=1.4022e-04) (hash(x)=48299675)
+2300 val loss 7.0647
+2300 val perplexity 1169.9177
+2300 train 7.168211 (lr=9.0182e-05) (hash(x)=52945580)
+2000 val loss 7.1572
+2000 val perplexity 1283.3210
+2000 train 7.978416 (lr=1.3907e-04) (hash(x)=61804797)
+2400 val loss 7.0197
+2400 val perplexity 1118.4805
+2400 train 7.089632 (lr=8.9265e-05) (hash(x)=54595211)
+2100 val loss 7.0765
+2100 val perplexity 1183.8523
+2500 val loss 7.0181
+2500 val perplexity 1116.6877
+2100 train 7.178714 (lr=1.3786e-04) (hash(x)=53638030)
+2500 train 7.027974 (lr=8.8313e-05) (hash(x)=52323363)
+2200 val loss 7.0702
+2200 val perplexity 1176.4192
+2200 train 7.273838 (lr=1.3660e-04) (hash(x)=56364593)
+2600 val loss 6.9974
+2600 val perplexity 1093.8115
+2600 train 6.790699 (lr=8.7326e-05) (hash(x)=42652707)
+2700 val loss 6.9823
+2700 val perplexity 1077.4113
+2300 val loss 7.0534
+2300 val perplexity 1156.7692
+2700 train 6.681511 (lr=8.6306e-05) (hash(x)=45687855)
+2300 train 7.148480 (lr=1.3527e-04) (hash(x)=52945580)
+2800 val loss 6.9617
+2800 val perplexity 1055.4443
+2800 train 6.951207 (lr=8.5254e-05) (hash(x)=49962830)
+2400 val loss 7.0180
+2400 val perplexity 1116.6062
+2400 train 7.111838 (lr=1.3390e-04) (hash(x)=54595211)
+2900 val loss 6.9430
+2900 val perplexity 1035.8958
+2900 train 6.955583 (lr=8.4170e-05) (hash(x)=52958451)
+2500 val loss 6.9926
+2500 val perplexity 1088.5376
+2500 train 6.990477 (lr=1.3247e-04) (hash(x)=52323363)
+3000 val loss 6.9176
+3000 val perplexity 1009.8487
+3000 train 6.864679 (lr=8.3057e-05) (hash(x)=48896990)
+2600 val loss 7.0060
+2600 val perplexity 1103.2196
+2600 train 6.810057 (lr=1.3099e-04) (hash(x)=42652707)
+3100 val loss 6.9101
+3100 val perplexity 1002.2977
+3100 train 6.856146 (lr=8.1915e-05) (hash(x)=47064930)
+2700 val loss 6.9722
+2700 val perplexity 1066.5864
+2700 train 6.679486 (lr=1.2946e-04) (hash(x)=45687855)
+3200 val loss 6.8815
+3200 val perplexity 974.0536
+3200 train 6.596547 (lr=8.0745e-05) (hash(x)=45145921)
+2800 val loss 6.9433
+2800 val perplexity 1036.2302
+2800 train 6.932369 (lr=1.2788e-04) (hash(x)=49962830)
+3300 val loss 6.8839
+3300 val perplexity 976.3783
+3300 train 6.722473 (lr=7.9549e-05) (hash(x)=48085018)
+2900 val loss 6.9177
+2900 val perplexity 1009.9710
+2900 train 6.917989 (lr=1.2626e-04) (hash(x)=52958451)
+3400 val loss 6.8586
+3400 val perplexity 951.9899
+3400 train 6.852911 (lr=7.8328e-05) (hash(x)=49596053)
+3000 val loss 6.8656
+3000 val perplexity 958.6799
+3000 train 6.826707 (lr=1.2459e-04) (hash(x)=48896990)
+3500 val loss 6.8550
+3500 val perplexity 948.5732
+3500 train 6.454469 (lr=7.7082e-05) (hash(x)=41135634)
+3100 val loss 6.9046
+3100 val perplexity 996.8251
+3100 train 6.876366 (lr=1.2287e-04) (hash(x)=47064930)
+3600 val loss 6.8194
+3600 val perplexity 915.4774
+3600 train 6.813796 (lr=7.5814e-05) (hash(x)=54286330)
+3200 val loss 6.8627
+3200 val perplexity 955.9679
+3200 train 6.592710 (lr=1.2112e-04) (hash(x)=45145921)
+3700 val loss 6.8307
+3700 val perplexity 925.8421
+3700 train 6.721279 (lr=7.4525e-05) (hash(x)=57061993)
+3300 val loss 6.8442
+3300 val perplexity 938.4250
+3300 train 6.669895 (lr=1.1932e-04) (hash(x)=48085018)
+3800 val loss 6.8211
+3800 val perplexity 916.9943
+3800 train 6.377489 (lr=7.3215e-05) (hash(x)=46544884)
+3400 val loss 6.8391
+3400 val perplexity 933.6216
+3400 train 6.844838 (lr=1.1749e-04) (hash(x)=49596053)
+3900 val loss 6.7986
+3900 val perplexity 896.6097
+3900 train 6.638959 (lr=7.1887e-05) (hash(x)=48937435)
+3500 val loss 6.8285
+3500 val perplexity 923.8395
+3500 train 6.439719 (lr=1.1562e-04) (hash(x)=41135634)
+4000 val loss 6.7805
+4000 val perplexity 880.4681
+4000 train 6.811349 (lr=7.0541e-05) (hash(x)=54466186)
+3600 val loss 6.8305
+3600 val perplexity 925.6660
+3600 train 6.827404 (lr=1.1372e-04) (hash(x)=54286330)
+4100 val loss 6.7587
+4100 val perplexity 861.4922
+4100 train 6.585770 (lr=6.9180e-05) (hash(x)=51079995)
+3700 val loss 6.8390
+3700 val perplexity 933.5700
+3700 train 6.702039 (lr=1.1179e-04) (hash(x)=57061993)
+4200 val loss 6.7599
+4200 val perplexity 862.5894
+4200 train 6.885074 (lr=6.7804e-05) (hash(x)=56495581)
+3800 val loss 6.8435
+3800 val perplexity 937.7438
+3800 train 6.419742 (lr=1.0982e-04) (hash(x)=46544884)
+4300 val loss 6.7067
+4300 val perplexity 817.8332
+4300 train 6.550481 (lr=6.6414e-05) (hash(x)=45095478)
+3900 val loss 6.8543
+3900 val perplexity 947.9782
+3900 train 6.697006 (lr=1.0783e-04) (hash(x)=48937435)
+4400 val loss 6.6989
+4400 val perplexity 811.4722
+4400 train 6.710745 (lr=6.5013e-05) (hash(x)=49954028)
+4000 val loss 6.8472
+4000 val perplexity 941.2394
+4000 train 6.863346 (lr=1.0581e-04) (hash(x)=54466186)
+4500 val loss 6.6732
+4500 val perplexity 790.9513
+4500 train 6.604428 (lr=6.3602e-05) (hash(x)=48975821)
+4100 val loss 6.8166
+4100 val perplexity 912.8441
+4100 train 6.650379 (lr=1.0377e-04) (hash(x)=51079995)
+4600 val loss 6.6435
+4600 val perplexity 767.7492
+4600 train 6.736774 (lr=6.2182e-05) (hash(x)=49293028)
+4200 val loss 6.8169
+4200 val perplexity 913.1793
+4200 train 6.960119 (lr=1.0171e-04) (hash(x)=56495581)
+4700 val loss 6.6407
+4700 val perplexity 765.6613
+4700 train 6.549062 (lr=6.0754e-05) (hash(x)=48077753)
+4300 val loss 6.7746
+4300 val perplexity 875.3004
+4300 train 6.632710 (lr=9.9622e-05) (hash(x)=45095478)
+4800 val loss 6.6097
+4800 val perplexity 742.2697
+4800 train 6.485708 (lr=5.9321e-05) (hash(x)=45037879)
+4400 val loss 6.7593
+4400 val perplexity 862.0560
+4400 train 6.739607 (lr=9.7520e-05) (hash(x)=49954028)
+4900 val loss 6.5948
+4900 val perplexity 731.2781
+4900 train 6.503368 (lr=5.7883e-05) (hash(x)=50320154)
+4500 val loss 6.7230
+4500 val perplexity 831.2733
+4500 train 6.650680 (lr=9.5403e-05) (hash(x)=48975821)
+5000 val loss 6.5871
+5000 val perplexity 725.6609
+5000 train 6.830953 (lr=5.6442e-05) (hash(x)=55024523)
+4600 val loss 6.7008
+4600 val perplexity 813.0734
+4600 train 6.782051 (lr=9.3273e-05) (hash(x)=49293028)
+5100 val loss 6.5751
+5100 val perplexity 717.0258
+5100 train 6.487334 (lr=5.5000e-05) (hash(x)=47570607)
+4700 val loss 6.7082
+4700 val perplexity 819.0656
+4700 train 6.629680 (lr=9.1132e-05) (hash(x)=48077753)
+5200 val loss 6.5637
+5200 val perplexity 708.9213
+5200 train 6.681831 (lr=5.3558e-05) (hash(x)=54311349)
+4800 val loss 6.7095
+4800 val perplexity 820.1912
+4800 train 6.573892 (lr=8.8982e-05) (hash(x)=45037879)
+5300 val loss 6.5509
+5300 val perplexity 699.8708
+5300 train 6.444979 (lr=5.2117e-05) (hash(x)=47178524)
+4900 val loss 6.6770
+4900 val perplexity 793.9644
+4900 train 6.600446 (lr=8.6825e-05) (hash(x)=50320154)
+5400 val loss 6.5382
+5400 val perplexity 691.0194
+5400 train 6.540273 (lr=5.0679e-05) (hash(x)=49425088)
+5000 val loss 6.6830
+5000 val perplexity 798.7117
+5000 train 6.917428 (lr=8.4663e-05) (hash(x)=55024523)
+5500 val loss 6.5273
+5500 val perplexity 683.5435
+5500 train 6.369995 (lr=4.9246e-05) (hash(x)=46383189)
+5100 val loss 6.6579
+5100 val perplexity 778.9476
+5100 train 6.595668 (lr=8.2500e-05) (hash(x)=47570607)
+5600 val loss 6.5284
+5600 val perplexity 684.2851
+5600 train 6.712617 (lr=4.7818e-05) (hash(x)=54084990)
+5200 val loss 6.6476
+5200 val perplexity 770.9180
+5200 train 6.782622 (lr=8.0337e-05) (hash(x)=54311349)
+5700 val loss 6.5284
+5700 val perplexity 684.2838
+5700 train 6.222934 (lr=4.6398e-05) (hash(x)=47384182)
+5800 val loss 6.5110
+5800 val perplexity 672.5214
+5800 train 6.396578 (lr=4.4987e-05) (hash(x)=51683744)
+5300 val loss 6.6252
+5300 val perplexity 753.8685
+5300 train 6.532175 (lr=7.8175e-05) (hash(x)=47178524)
+5900 val loss 6.5014
+5900 val perplexity 666.0516
+5900 train 6.510395 (lr=4.3586e-05) (hash(x)=52798836)
+5400 val loss 6.6181
+5400 val perplexity 748.5306
+5400 train 6.616325 (lr=7.6018e-05) (hash(x)=49425088)
+6000 val loss 6.5040
+6000 val perplexity 667.8166
+6000 train 6.393989 (lr=4.2196e-05) (hash(x)=50945000)
+6100 val loss 6.5062
+6100 val perplexity 669.2753
+6100 train 6.196708 (lr=4.0820e-05) (hash(x)=48964427)
+5500 val loss 6.6147
+5500 val perplexity 745.9890
+5500 train 6.448332 (lr=7.3868e-05) (hash(x)=46383189)
+6200 val loss 6.5098
+6200 val perplexity 671.7039
+6200 train 6.268740 (lr=3.9459e-05) (hash(x)=47148610)
+5600 val loss 6.6037
+5600 val perplexity 737.8403
+5600 train 6.795812 (lr=7.1727e-05) (hash(x)=54084990)
+6300 val loss 6.5004
+6300 val perplexity 665.4230
+6300 train 6.412795 (lr=3.8113e-05) (hash(x)=50860553)
+6400 val loss 6.4993
+6400 val perplexity 664.6778
+6400 train 6.270996 (lr=3.6785e-05) (hash(x)=50553992)
+5700 val loss 6.5924
+5700 val perplexity 729.5043
+5700 train 6.275771 (lr=6.9597e-05) (hash(x)=47384182)
+6500 val loss 6.4847
+6500 val perplexity 655.0747
+6500 train 6.289717 (lr=3.5475e-05) (hash(x)=52521332)
+5800 val loss 6.5847
+5800 val perplexity 723.9128
+5800 train 6.476488 (lr=6.7480e-05) (hash(x)=51683744)
+6600 val loss 6.4606
+6600 val perplexity 639.4189
+6600 train 6.806929 (lr=3.4186e-05) (hash(x)=52609843)
+6700 val loss 6.4414
+6700 val perplexity 627.2814
+6700 train 6.480598 (lr=3.2918e-05) (hash(x)=53632957)
+5900 val loss 6.5876
+5900 val perplexity 726.0012
+5900 train 6.579369 (lr=6.5378e-05) (hash(x)=52798836)
+6800 val loss 6.4358
+6800 val perplexity 623.7556
+6800 train 6.441747 (lr=3.1672e-05) (hash(x)=46927608)
+6000 val loss 6.5899
+6000 val perplexity 727.6727
+6000 train 6.478214 (lr=6.3294e-05) (hash(x)=50945000)
+6900 val loss 6.4211
+6900 val perplexity 614.6884
+6900 train 6.448431 (lr=3.0451e-05) (hash(x)=54483057)
+7000 val loss 6.4171
+7000 val perplexity 612.2233
+6100 val loss 6.5737
+6100 val perplexity 716.0176
+7000 train 6.564780 (lr=2.9255e-05) (hash(x)=51493884)
+6100 train 6.276618 (lr=6.1230e-05) (hash(x)=48964427)
+7100 val loss 6.4078
+7100 val perplexity 606.5637
+7100 train 6.538711 (lr=2.8085e-05) (hash(x)=53303341)
+6200 val loss 6.5912
+6200 val perplexity 728.6831
+6200 train 6.360708 (lr=5.9188e-05) (hash(x)=47148610)
+7200 val loss 6.4123
+7200 val perplexity 609.2900
+7200 train 6.105514 (lr=2.6943e-05) (hash(x)=45272178)
+6300 val loss 6.5876
+6300 val perplexity 726.0035
+6300 train 6.502324 (lr=5.7169e-05) (hash(x)=50860553)
+7300 val loss 6.4032
+7300 val perplexity 603.7695
+7300 train 6.337297 (lr=2.5830e-05) (hash(x)=50389872)
+7400 val loss 6.3916
+7400 val perplexity 596.8148
+7400 train 6.031796 (lr=2.4746e-05) (hash(x)=43796301)
+6400 val loss 6.5650
+6400 val perplexity 709.8069
+6400 train 6.330476 (lr=5.5177e-05) (hash(x)=50553992)
+7500 val loss 6.3875
+7500 val perplexity 594.3956
+7500 train 6.322572 (lr=2.3694e-05) (hash(x)=47808686)
+6500 val loss 6.5650
+6500 val perplexity 709.7889
+6500 train 6.371252 (lr=5.3213e-05) (hash(x)=52521332)
+7600 val loss 6.3867
+7600 val perplexity 593.9199
+7600 train 6.118526 (lr=2.2674e-05) (hash(x)=41936898)
+7700 val loss 6.3863
+7700 val perplexity 593.6702
+7700 train 6.602107 (lr=2.1687e-05) (hash(x)=57550318)
+6600 val loss 6.5361
+6600 val perplexity 689.5613
+6600 train 6.896532 (lr=5.1279e-05) (hash(x)=52609843)
+7800 val loss 6.3803
+7800 val perplexity 590.1259
+7800 train 6.245681 (lr=2.0735e-05) (hash(x)=47485210)
+6700 val loss 6.5179
+6700 val perplexity 677.1226
+6700 train 6.555040 (lr=4.9377e-05) (hash(x)=53632957)
+7900 val loss 6.3767
+7900 val perplexity 587.9981
+7900 train 6.719403 (lr=1.9818e-05) (hash(x)=53228688)
+8000 val loss 6.3760
+8000 val perplexity 587.5701
+8000 train 6.432549 (lr=1.8936e-05) (hash(x)=52018673)
+6800 val loss 6.5208
+6800 val perplexity 679.0950
+6800 train 6.524570 (lr=4.7509e-05) (hash(x)=46927608)
+8100 val loss 6.3687
+8100 val perplexity 583.2793
+8100 train 6.201787 (lr=1.8092e-05) (hash(x)=47079349)
+6900 val loss 6.5172
+6900 val perplexity 676.6768
+6900 train 6.538620 (lr=4.5676e-05) (hash(x)=54483057)
+8200 val loss 6.3645
+8200 val perplexity 580.8560
+8200 train 6.434308 (lr=1.7286e-05) (hash(x)=57921563)
+8300 val loss 6.3590
+8300 val perplexity 577.6404
+8300 train 5.898799 (lr=1.6519e-05) (hash(x)=45038933)
+7000 val loss 6.5124
+7000 val perplexity 673.4177
+7000 train 6.681723 (lr=4.3882e-05) (hash(x)=51493884)
+8400 val loss 6.3775
+8400 val perplexity 588.4750
+8400 train 6.225904 (lr=1.5791e-05) (hash(x)=47763246)
+7100 val loss 6.4924
+7100 val perplexity 660.0879
+7100 train 6.627834 (lr=4.2128e-05) (hash(x)=53303341)
+8500 val loss 6.3612
+8500 val perplexity 578.9673
+8500 train 6.429732 (lr=1.5103e-05) (hash(x)=56176595)
+8600 val loss 6.3597
+8600 val perplexity 578.0970
+8600 train 6.434188 (lr=1.4456e-05) (hash(x)=55184249)
+7200 val loss 6.5003
+7200 val perplexity 665.3522
+7200 train 6.209903 (lr=4.0414e-05) (hash(x)=45272178)
+8700 val loss 6.3549
+8700 val perplexity 575.2811
+8700 train 6.294187 (lr=1.3851e-05) (hash(x)=46471646)
+8800 val loss 6.3584
+8800 val perplexity 577.3042
+8800 train 6.057069 (lr=1.3289e-05) (hash(x)=46233162)
+7300 val loss 6.4876
+7300 val perplexity 656.9751
+7300 train 6.401985 (lr=3.8745e-05) (hash(x)=50389872)
+8900 val loss 6.3599
+8900 val perplexity 578.1836
+8900 train 6.283579 (lr=1.2769e-05) (hash(x)=47233684)
+7400 val loss 6.4828
+7400 val perplexity 653.8246
+7400 train 6.129344 (lr=3.7120e-05) (hash(x)=43796301)
+9000 val loss 6.3502
+9000 val perplexity 572.6089
+9000 train 6.235326 (lr=1.2292e-05) (hash(x)=48374529)
+9100 val loss 6.3393
+9100 val perplexity 566.3860
+9100 train 6.325945 (lr=1.1860e-05) (hash(x)=48065371)
+7500 val loss 6.4820
+7500 val perplexity 653.2842
+7500 train 6.422829 (lr=3.5541e-05) (hash(x)=47808686)
+9200 val loss 6.3315
+9200 val perplexity 561.9741
+9200 train 6.326896 (lr=1.1472e-05) (hash(x)=47408078)
+7600 val loss 6.4791
+7600 val perplexity 651.3569
+7600 train 6.215595 (lr=3.4011e-05) (hash(x)=41936898)
+9300 val loss 6.3276
+9300 val perplexity 559.8051
+9300 train 6.309064 (lr=1.1128e-05) (hash(x)=50749781)
+9400 val loss 6.3237
+9400 val perplexity 557.6545
+9400 train 6.525542 (lr=1.0830e-05) (hash(x)=48560169)
+7700 val loss 6.4761
+7700 val perplexity 649.4601
+7700 train 6.682161 (lr=3.2531e-05) (hash(x)=57550318)
+9500 val loss 6.3229
+9500 val perplexity 557.1821
+9500 train 6.435276 (lr=1.0577e-05) (hash(x)=50936392)
+7800 val loss 6.4727
+7800 val perplexity 647.2555
+7800 train 6.342023 (lr=3.1102e-05) (hash(x)=47485210)
+9600 val loss 6.3189
+9600 val perplexity 554.9551
+9600 train 6.341009 (lr=1.0369e-05) (hash(x)=50651714)
+9700 val loss 6.3178
+9700 val perplexity 554.3513
+9700 train 6.391021 (lr=1.0208e-05) (hash(x)=47311384)
+7900 val loss 6.4673
+7900 val perplexity 643.7279
+7900 train 6.802589 (lr=2.9726e-05) (hash(x)=53228688)
+8000 val loss 6.4706
+8000 val perplexity 645.8663
+8000 train 6.519435 (lr=2.8405e-05) (hash(x)=52018673)
+9800 val loss 6.3160
+9800 val perplexity 553.3610
+9800 train 6.418117 (lr=1.0092e-05) (hash(x)=50921139)
+8100 val loss 6.4637
+8100 val perplexity 641.4133
+8100 train 6.298305 (lr=2.7138e-05) (hash(x)=47079349)
+9900 val loss 6.3219
+9900 val perplexity 556.6038
+9900 train 6.467185 (lr=1.0023e-05) (hash(x)=48142455)
+8200 val loss 6.4578
+8200 val perplexity 637.6469
+8200 train 6.539501 (lr=2.5929e-05) (hash(x)=57921563)
+9999 val loss 6.3137
+9999 val perplexity 552.0605
+8300 val loss 6.4564
+8300 val perplexity 636.7405
+8300 train 6.004556 (lr=2.4778e-05) (hash(x)=45038933)
+8400 val loss 6.4678
+8400 val perplexity 644.0859
+8400 train 6.313585 (lr=2.3686e-05) (hash(x)=47763246)
+8500 val loss 6.4619
+8500 val perplexity 640.2839
+8500 train 6.524305 (lr=2.2655e-05) (hash(x)=56176595)
+8600 val loss 6.4634
+8600 val perplexity 641.2393
+8600 train 6.531406 (lr=2.1685e-05) (hash(x)=55184249)
+8700 val loss 6.4605
+8700 val perplexity 639.3777
+8700 train 6.405812 (lr=2.0777e-05) (hash(x)=46471646)
+8800 val loss 6.4620
+8800 val perplexity 640.3541
+8800 train 6.169169 (lr=1.9933e-05) (hash(x)=46233162)
+8900 val loss 6.4575
+8900 val perplexity 637.4733
+8900 train 6.387800 (lr=1.9153e-05) (hash(x)=47233684)
+9000 val loss 6.4577
+9000 val perplexity 637.6104
+9000 train 6.348251 (lr=1.8439e-05) (hash(x)=48374529)
+9100 val loss 6.4384
+9100 val perplexity 625.4025
+9100 train 6.387834 (lr=1.7790e-05) (hash(x)=48065371)
+9200 val loss 6.4323
+9200 val perplexity 621.5872
+9200 train 6.400970 (lr=1.7208e-05) (hash(x)=47408078)
+9300 val loss 6.4267
+9300 val perplexity 618.1290
+9300 train 6.410742 (lr=1.6692e-05) (hash(x)=50749781)
+9400 val loss 6.4225
+9400 val perplexity 615.5146
+9400 train 6.621217 (lr=1.6245e-05) (hash(x)=48560169)
+9500 val loss 6.4212
+9500 val perplexity 614.7256
+9500 train 6.548310 (lr=1.5865e-05) (hash(x)=50936392)
+9600 val loss 6.4178
+9600 val perplexity 612.6835
+9600 train 6.430169 (lr=1.5554e-05) (hash(x)=50651714)
+9700 val loss 6.4189
+9700 val perplexity 613.3140
+9700 train 6.469123 (lr=1.5312e-05) (hash(x)=47311384)
+9800 val loss 6.4171
+9800 val perplexity 612.1956
+9800 train 6.518755 (lr=1.5139e-05) (hash(x)=50921139)

attention_kindselective_n_heads4_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf077b4c7d20c894d8cf19edf6e71253f1c6da1d6da3ddf7a45dfdc81c7786f4
+oid sha256:ad93158bdfd32071e25dc456532ed3fd36c61df4cc86f865449960dcdf6fdfcb
 size 92843394

attention_kindselective_n_heads4_seed1339/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bcc6cba1e8aba35cfffd617b9865dd991d56d86cd65fc851959ab0cc5158369f
+oid sha256:6aa78e6516bbc5459cdc3154c9b778836f34c2c1dbac08076e7043de9bd5fe9c
 size 92843394

attention_kindselective_n_heads4_seed1339/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc8a63ddb0b4127e1cd6e71226f2e5d191a1223a81af5752764c496f77640ec9
+oid sha256:20de18d92ff73af4207c8758d2c668cb193ef56f2e7857b8391859c03bfce23d
 size 92843394

attention_kindselective_n_heads4_seed1339/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b38f151b223e117fda0c4bea4e7fdb720f87797da0b5cf680c831c8939dc8977
+oid sha256:71ce7845c806c8b831eef2b6d89205bf1670c932d983ebd9107e92f8a5065a3f
 size 92843394

attention_kindselective_n_heads4_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4be917af745db25f4dbcccc3b0396860aa3f5a668f97463af1fdf11b774e3b27
+oid sha256:edeb87f6d21d339a0559282c4a2db7e96080f40d421400d4df3816f05159bf21
 size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f2a4ed44be047138dcaf6fa604ef8834bd84327b45720eeceeb37c7f6ec0749f
+oid sha256:3ef24c1b7a4309998b53d8f8c3106922fe90cd2c76ffb6673a28994960c96400
 size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c2105389bd849202e620f1820ffe080b1744d6a2dbede4772ef73b165d137c6
+oid sha256:e1171237d3a16e4a7ecf8a131c6ba1d82e5ffafb1c91d7d035a9b67b6ae1398e
 size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:205196b199d1ff99aa2cf028ddd550a3d532b318089f6fee42998cf85313aafa
+oid sha256:7b24fc0c71518371e90fd12967f37464df151eac9cef39a338c5e0c173d3bd09
 size 179406214