Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_8/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_8", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.95e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.95e-5_30720_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1341", "n_embd": 256}

attention_kindselective_n_heads4_seed1341/log2.txt

@@ -1,1055 +1,528 @@
 max_steps: 17500
 0 val loss 11.3107
 0 val perplexity 81693.5234
-0 val loss 11.3107
-0 val perplexity 81693.5234
-0 train 11.308768 (lr=1.5250e-07) (hash(x)=145079536)
-0 train 11.308768 (lr=1.4750e-07) (hash(x)=145079536)
-100 val loss 9.9800
-100 val perplexity 21591.2500
-100 train 9.920788 (lr=1.5403e-05) (hash(x)=154745873)
-100 val loss 9.9853
-100 val perplexity 21704.6113
-100 train 9.925657 (lr=1.4897e-05) (hash(x)=154745873)
-200 val loss 9.1254
-200 val perplexity 9185.4824
-200 train 9.159418 (lr=3.0500e-05) (hash(x)=155800595)
-200 val loss 9.1438
-200 val perplexity 9356.1064
-200 train 9.176240 (lr=2.9500e-05) (hash(x)=155800595)
-300 val loss 8.1862
-300 val perplexity 3590.9377
-300 train 8.152798 (lr=3.0498e-05) (hash(x)=148595389)
-300 val loss 8.1041
-300 val perplexity 3308.0859
-300 train 8.071339 (lr=2.9498e-05) (hash(x)=148595389)
-400 val loss 7.8583
-400 val perplexity 2587.0869
-400 train 7.791428 (lr=3.0491e-05) (hash(x)=145606733)
-400 val loss 7.8285
-400 val perplexity 2511.0779
-400 train 7.755090 (lr=2.9491e-05) (hash(x)=145606733)
-500 val loss 7.6792
-500 val perplexity 2162.7808
-500 train 7.535973 (lr=3.0480e-05) (hash(x)=138221231)
-500 val loss 7.6787
-500 val perplexity 2161.8848
-500 train 7.532548 (lr=2.9480e-05) (hash(x)=138221231)
-600 val loss 7.5889
-600 val perplexity 1976.1528
-600 train 7.599494 (lr=3.0464e-05) (hash(x)=150367139)
-600 val loss 7.6032
-600 val perplexity 2004.5544
-600 train 7.616951 (lr=2.9465e-05) (hash(x)=150367139)
-700 val loss 7.5359
-700 val perplexity 1874.1886
-700 train 7.685524 (lr=3.0443e-05) (hash(x)=155579314)
-700 val loss 7.5488
-700 val perplexity 1898.4008
-700 train 7.698351 (lr=2.9445e-05) (hash(x)=155579314)
-800 val loss 7.4973
-800 val perplexity 1803.1744
-800 train 7.575051 (lr=3.0419e-05) (hash(x)=155747374)
-800 val loss 7.5007
-800 val perplexity 1809.3680
-800 train 7.579003 (lr=2.9421e-05) (hash(x)=155747374)
-900 val loss 7.4606
-900 val perplexity 1738.2352
-900 train 7.657259 (lr=3.0389e-05) (hash(x)=159334575)
-900 val loss 7.4695
-900 val perplexity 1753.7239
-900 train 7.666729 (lr=2.9393e-05) (hash(x)=159334575)
-1000 val loss 7.4266
-1000 val perplexity 1680.0031
-1000 train 7.309896 (lr=3.0355e-05) (hash(x)=140604760)
-1000 val loss 7.4435
-1000 val perplexity 1708.7472
-1000 train 7.325466 (lr=2.9360e-05) (hash(x)=140604760)
-1100 val loss 7.4025
-1100 val perplexity 1640.0477
-1100 train 7.249547 (lr=3.0317e-05) (hash(x)=146208052)
-1100 val loss 7.4298
-1100 val perplexity 1685.4086
-1100 train 7.283123 (lr=2.9323e-05) (hash(x)=146208052)
-1200 val loss 7.3819
-1200 val perplexity 1606.5811
-1200 train 7.299123 (lr=3.0274e-05) (hash(x)=148404734)
-1200 val loss 7.4091
-1200 val perplexity 1650.9817
-1200 train 7.330080 (lr=2.9282e-05) (hash(x)=148404734)
-1300 val loss 7.3645
-1300 val perplexity 1578.9740
-1300 train 7.307587 (lr=3.0227e-05) (hash(x)=155681970)
-1300 val loss 7.3868
-1300 val perplexity 1614.5410
-1300 train 7.330466 (lr=2.9236e-05) (hash(x)=155681970)
-1400 val loss 7.3493
-1400 val perplexity 1555.0526
-1400 train 7.300859 (lr=3.0175e-05) (hash(x)=148115934)
-1400 val loss 7.3697
-1400 val perplexity 1587.1855
-1400 train 7.325119 (lr=2.9186e-05) (hash(x)=148115934)
-1500 val loss 7.3141
-1500 val perplexity 1501.3477
-1500 train 7.450751 (lr=3.0119e-05) (hash(x)=157074034)
-1500 val loss 7.3486
-1500 val perplexity 1553.9933
-1500 train 7.484334 (lr=2.9132e-05) (hash(x)=157074034)
-1600 val loss 7.2987
-1600 val perplexity 1478.3230
-1600 train 7.071062 (lr=3.0059e-05) (hash(x)=137464699)
-1600 val loss 7.3283
-1600 val perplexity 1522.7560
-1600 train 7.105087 (lr=2.9073e-05) (hash(x)=137464699)
-1700 val loss 7.2904
-1700 val perplexity 1466.2109
-1700 train 7.479377 (lr=2.9994e-05) (hash(x)=166955614)
-1700 val loss 7.2977
-1700 val perplexity 1476.8329
-1700 train 7.492035 (lr=2.9011e-05) (hash(x)=166955614)
-1800 val loss 7.2785
-1800 val perplexity 1448.7428
-1800 train 7.117696 (lr=2.9925e-05) (hash(x)=143886042)
-1800 val loss 7.2734
-1800 val perplexity 1441.3727
-1800 train 7.104486 (lr=2.8944e-05) (hash(x)=143886042)
-1900 val loss 7.2777
-1900 val perplexity 1447.7152
-1900 train 7.967541 (lr=2.9851e-05) (hash(x)=193610391)
-1900 val loss 7.2521
-1900 val perplexity 1411.1040
-1900 train 7.926654 (lr=2.8872e-05) (hash(x)=193610391)
-2000 val loss 7.2664
-2000 val perplexity 1431.3634
-2000 train 7.432469 (lr=2.9773e-05) (hash(x)=163799796)
-2000 val loss 7.2228
-2000 val perplexity 1370.2562
-2000 train 7.387167 (lr=2.8797e-05) (hash(x)=163799796)
-2100 val loss 7.2482
-2100 val perplexity 1405.5966
-2100 train 7.221461 (lr=2.9691e-05) (hash(x)=154107339)
-2100 val loss 7.1847
-2100 val perplexity 1319.0815
-2100 train 7.162979 (lr=2.8718e-05) (hash(x)=154107339)
-2200 val loss 7.2320
-2200 val perplexity 1382.9604
-2200 train 7.214847 (lr=2.9605e-05) (hash(x)=144592844)
-2200 val loss 7.1630
-2200 val perplexity 1290.7468
-2200 train 7.154574 (lr=2.8634e-05) (hash(x)=144592844)
-2300 val loss 7.2169
-2300 val perplexity 1362.2659
-2300 train 7.298916 (lr=2.9514e-05) (hash(x)=154535861)
-2300 val loss 7.1451
-2300 val perplexity 1267.8710
-2300 train 7.221553 (lr=2.8546e-05) (hash(x)=154535861)
-2400 val loss 7.2040
-2400 val perplexity 1344.7562
-2400 train 7.959690 (lr=2.9419e-05) (hash(x)=204706354)
-2400 val loss 7.1208
-2400 val perplexity 1237.4305
-2400 train 7.882274 (lr=2.8455e-05) (hash(x)=204706354)
-2500 val loss 7.1907
-2500 val perplexity 1327.0750
-2500 train 7.010283 (lr=2.9320e-05) (hash(x)=143783202)
-2500 val loss 7.1009
-2500 val perplexity 1213.1066
-2500 train 6.906930 (lr=2.8359e-05) (hash(x)=143783202)
-2600 val loss 7.1676
-2600 val perplexity 1296.7295
-2600 train 7.062464 (lr=2.9217e-05) (hash(x)=150862210)
-2600 val loss 7.0761
-2600 val perplexity 1183.3657
-2600 train 6.971740 (lr=2.8259e-05) (hash(x)=150862210)
-2700 val loss 7.1471
-2700 val perplexity 1270.4170
-2700 train 7.220184 (lr=2.9110e-05) (hash(x)=150753426)
-2700 val loss 7.0589
-2700 val perplexity 1163.1388
-2700 train 7.120690 (lr=2.8155e-05) (hash(x)=150753426)
-2800 val loss 7.1354
-2800 val perplexity 1255.5940
-2800 train 7.114671 (lr=2.8998e-05) (hash(x)=147766811)
-2800 val loss 7.0249
-2800 val perplexity 1124.2938
-2800 train 7.008540 (lr=2.8048e-05) (hash(x)=147766811)
-2900 val loss 7.1137
-2900 val perplexity 1228.6904
-2900 train 7.042453 (lr=2.8883e-05) (hash(x)=151177814)
-2900 val loss 7.0106
-2900 val perplexity 1108.3077
-2900 train 6.945816 (lr=2.7936e-05) (hash(x)=151177814)
-3000 val loss 7.0988
-3000 val perplexity 1210.5618
-3000 train 6.933269 (lr=2.8764e-05) (hash(x)=135925327)
-3000 val loss 6.9929
-3000 val perplexity 1088.9036
-3000 train 6.828811 (lr=2.7821e-05) (hash(x)=135925327)
-3100 val loss 7.0889
-3100 val perplexity 1198.5499
-3100 train 6.771436 (lr=2.8640e-05) (hash(x)=124842353)
-3100 val loss 6.9704
-3100 val perplexity 1064.6707
-3100 train 6.648157 (lr=2.7701e-05) (hash(x)=124842353)
-3200 val loss 7.0705
-3200 val perplexity 1176.7177
-3200 train 7.175928 (lr=2.8513e-05) (hash(x)=160440642)
-3200 val loss 6.9601
-3200 val perplexity 1053.7632
-3200 train 7.071601 (lr=2.7578e-05) (hash(x)=160440642)
-3300 val loss 7.0496
-3300 val perplexity 1152.4396
-3300 train 6.947692 (lr=2.8382e-05) (hash(x)=143017131)
-3300 val loss 6.9228
-3300 val perplexity 1015.1130
-3300 train 6.826661 (lr=2.7452e-05) (hash(x)=143017131)
-3400 val loss 7.0343
-3400 val perplexity 1134.9563
-3400 train 7.279748 (lr=2.8247e-05) (hash(x)=148143474)
-3400 val loss 6.9025
-3400 val perplexity 994.8009
-3400 train 7.148989 (lr=2.7321e-05) (hash(x)=148143474)
-3500 val loss 7.0187
-3500 val perplexity 1117.3562
-3500 train 6.838757 (lr=2.8108e-05) (hash(x)=148732639)
-3500 val loss 6.8836
-3500 val perplexity 976.1451
-3500 train 6.701279 (lr=2.7187e-05) (hash(x)=148732639)
-3600 val loss 7.0037
-3600 val perplexity 1100.6555
-3600 train 6.996209 (lr=2.7966e-05) (hash(x)=146333803)
-3600 val loss 6.8588
-3600 val perplexity 952.2255
-3600 train 6.849241 (lr=2.7049e-05) (hash(x)=146333803)
-3700 val loss 6.9855
-3700 val perplexity 1080.8713
-3700 train 6.787265 (lr=2.7820e-05) (hash(x)=143611445)
-3700 val loss 6.8377
-3700 val perplexity 932.3213
-3700 train 6.636805 (lr=2.6908e-05) (hash(x)=143611445)
-3800 val loss 6.9796
-3800 val perplexity 1074.5413
-3800 train 6.781515 (lr=2.7670e-05) (hash(x)=141272115)
-3800 val loss 6.8190
-3800 val perplexity 915.0916
-3800 train 6.616798 (lr=2.6763e-05) (hash(x)=141272115)
-3900 val loss 6.9563
-3900 val perplexity 1049.7366
-3900 train 6.883613 (lr=2.7517e-05) (hash(x)=149161848)
-3900 val loss 6.7907
-3900 val perplexity 889.5716
-3900 train 6.713684 (lr=2.6615e-05) (hash(x)=149161848)
-4000 val loss 6.9283
-4000 val perplexity 1020.7776
-4000 train 6.997078 (lr=2.7360e-05) (hash(x)=152036955)
-4000 val loss 6.7578
-4000 val perplexity 860.7343
-4000 train 6.830627 (lr=2.6463e-05) (hash(x)=152036955)
-4100 val loss 6.9118
-4100 val perplexity 1004.0648
-4100 train 6.975347 (lr=2.7199e-05) (hash(x)=153260968)
-4100 val loss 6.7335
-4100 val perplexity 840.0572
-4100 train 6.788578 (lr=2.6308e-05) (hash(x)=153260968)
-4200 val loss 6.8963
-4200 val perplexity 988.5967
-4200 train 6.767730 (lr=2.7036e-05) (hash(x)=147629478)
-4200 val loss 6.7033
-4200 val perplexity 815.0834
-4300 val loss 6.8811
-4300 val perplexity 973.6808
-4200 train 6.577802 (lr=2.6149e-05) (hash(x)=147629478)
-4300 train 6.894908 (lr=2.6868e-05) (hash(x)=161701080)
-4400 val loss 6.8704
-4400 val perplexity 963.2938
-4400 train 6.921198 (lr=2.6698e-05) (hash(x)=155290524)
-4300 val loss 6.6825
-4300 val perplexity 798.2952
-4300 train 6.701600 (lr=2.5987e-05) (hash(x)=161701080)
-4500 val loss 6.8619
-4500 val perplexity 955.1368
-4500 train 6.661626 (lr=2.6524e-05) (hash(x)=144726593)
-4400 val loss 6.6601
-4400 val perplexity 780.6301
-4400 train 6.702115 (lr=2.5822e-05) (hash(x)=155290524)
-4600 val loss 6.8494
-4600 val perplexity 943.2910
-4600 train 6.800690 (lr=2.6347e-05) (hash(x)=149517202)
-4500 val loss 6.6362
-4500 val perplexity 762.1920
-4500 train 6.437363 (lr=2.5654e-05) (hash(x)=144726593)
-4700 val loss 6.8320
-4700 val perplexity 927.0809
-4700 train 6.839327 (lr=2.6167e-05) (hash(x)=139403008)
-4600 val loss 6.6158
-4600 val perplexity 746.7887
-4600 train 6.554698 (lr=2.5483e-05) (hash(x)=149517202)
-4800 val loss 6.8108
-4800 val perplexity 907.6088
-4800 train 6.878293 (lr=2.5983e-05) (hash(x)=153387238)
-4700 val loss 6.5846
-4700 val perplexity 723.8365
-4700 train 6.584648 (lr=2.5309e-05) (hash(x)=139403008)
-4900 val loss 6.8000
-4900 val perplexity 897.8363
-4900 train 6.551505 (lr=2.5797e-05) (hash(x)=139881569)
-4800 val loss 6.5628
-4800 val perplexity 708.2746
-4800 train 6.628347 (lr=2.5132e-05) (hash(x)=153387238)
-5000 val loss 6.7815
-5000 val perplexity 881.3514
-5000 train 6.750756 (lr=2.5608e-05) (hash(x)=152602580)
-4900 val loss 6.5436
-4900 val perplexity 694.7764
-4900 train 6.295164 (lr=2.4951e-05) (hash(x)=139881569)
-5100 val loss 6.7641
-5100 val perplexity 866.1529
-5100 train 6.826677 (lr=2.5416e-05) (hash(x)=150598460)
-5000 val loss 6.5202
-5000 val perplexity 678.7405
-5000 train 6.504192 (lr=2.4768e-05) (hash(x)=152602580)
-5200 val loss 6.7500
-5200 val perplexity 854.0352
-5200 train 6.656862 (lr=2.5221e-05) (hash(x)=147660496)
-5100 val loss 6.5029
-5100 val perplexity 667.0970
-5100 train 6.558856 (lr=2.4582e-05) (hash(x)=150598460)
-5300 val loss 6.7398
-5300 val perplexity 845.4194
-5300 train 6.481124 (lr=2.5023e-05) (hash(x)=147693848)
-5200 val loss 6.4858
-5200 val perplexity 655.7319
-5200 train 6.382066 (lr=2.4394e-05) (hash(x)=147660496)
-5400 val loss 6.7286
-5400 val perplexity 835.9406
-5400 train 6.478855 (lr=2.4822e-05) (hash(x)=136048145)
-5300 val loss 6.4653
-5300 val perplexity 642.4832
-5300 train 6.206392 (lr=2.4202e-05) (hash(x)=147693848)
-5500 val loss 6.7010
-5500 val perplexity 813.2169
-5500 train 6.665031 (lr=2.4619e-05) (hash(x)=140313123)
-5400 val loss 6.4551
-5400 val perplexity 635.9492
-5400 train 6.197298 (lr=2.4008e-05) (hash(x)=136048145)
-5600 val loss 6.6817
-5600 val perplexity 797.6700
-5600 train 6.708956 (lr=2.4413e-05) (hash(x)=156997993)
-5500 val loss 6.4288
-5500 val perplexity 619.3992
-5500 train 6.407856 (lr=2.3812e-05) (hash(x)=140313123)
-5700 val loss 6.6645
-5700 val perplexity 784.0786
-5700 train 6.655910 (lr=2.4205e-05) (hash(x)=143513880)
-5600 val loss 6.4102
-5600 val perplexity 608.0330
-5600 train 6.432701 (lr=2.3613e-05) (hash(x)=156997993)
-5800 val loss 6.6478
-5800 val perplexity 771.1063
-5800 train 6.795434 (lr=2.3994e-05) (hash(x)=153736628)
-5700 val loss 6.3935
-5700 val perplexity 597.9434
-5700 train 6.396849 (lr=2.3411e-05) (hash(x)=143513880)
-5900 val loss 6.6253
-5900 val perplexity 753.9224
-5900 train 6.822607 (lr=2.3781e-05) (hash(x)=166385417)
-5800 val loss 6.3797
-5800 val perplexity 589.7309
-5800 train 6.521539 (lr=2.3207e-05) (hash(x)=153736628)
-6000 val loss 6.6068
-6000 val perplexity 740.0859
-6000 train 6.815539 (lr=2.3565e-05) (hash(x)=153216517)
-5900 val loss 6.3682
-5900 val perplexity 583.0204
-5900 train 6.552328 (lr=2.3001e-05) (hash(x)=166385417)
-6100 val loss 6.6040
-6100 val perplexity 738.0760
-6100 train 6.474722 (lr=2.3348e-05) (hash(x)=145387545)
-6000 val loss 6.3527
-6000 val perplexity 574.0220
-6000 train 6.577559 (lr=2.2793e-05) (hash(x)=153216517)
-6200 val loss 6.5862
-6200 val perplexity 725.0466
-6200 train 6.693987 (lr=2.3128e-05) (hash(x)=156306460)
-6100 val loss 6.3463
-6100 val perplexity 570.3692
-6100 train 6.220591 (lr=2.2582e-05) (hash(x)=145387545)
-6300 val loss 6.5661
-6300 val perplexity 710.5793
-6300 train 6.470700 (lr=2.2906e-05) (hash(x)=148610447)
-6200 val loss 6.3381
-6200 val perplexity 565.7350
-6200 train 6.445853 (lr=2.2370e-05) (hash(x)=156306460)
-6400 val loss 6.5424
-6400 val perplexity 693.9788
-6400 train 6.756941 (lr=2.2682e-05) (hash(x)=158343535)
-6300 val loss 6.3124
-6300 val perplexity 551.3668
-6300 train 6.230397 (lr=2.2155e-05) (hash(x)=148610447)
-6500 val loss 6.5271
-6500 val perplexity 683.4457
-6500 train 6.676983 (lr=2.2456e-05) (hash(x)=157742557)
-6400 val loss 6.2979
-6400 val perplexity 543.4504
-6400 train 6.518247 (lr=2.1938e-05) (hash(x)=158343535)
-6600 val loss 6.5155
-6600 val perplexity 675.5356
-6600 train 6.439126 (lr=2.2228e-05) (hash(x)=150457308)
-6500 val loss 6.2847
-6500 val perplexity 536.3230
-6500 train 6.441398 (lr=2.1720e-05) (hash(x)=157742557)
-6700 val loss 6.5050
-6700 val perplexity 668.4550
-6700 train 6.388369 (lr=2.1999e-05) (hash(x)=142426205)
-6600 val loss 6.2757
-6600 val perplexity 531.5118
-6600 train 6.189404 (lr=2.1499e-05) (hash(x)=150457308)
-6800 val loss 6.4901
-6800 val perplexity 658.6183
-6800 train 6.303302 (lr=2.1767e-05) (hash(x)=151522525)
-6700 val loss 6.2676
-6700 val perplexity 527.1980
-6700 train 6.134061 (lr=2.1277e-05) (hash(x)=142426205)
-6900 val loss 6.4792
-6900 val perplexity 651.4750
-6900 train 6.403502 (lr=2.1534e-05) (hash(x)=152029630)
-6800 val loss 6.2599
-6800 val perplexity 523.1764
-6800 train 6.053487 (lr=2.1054e-05) (hash(x)=151522525)
-7000 val loss 6.4564
-7000 val perplexity 636.7639
-7000 train 6.314115 (lr=2.1300e-05) (hash(x)=142978954)
-6900 val loss 6.2425
-6900 val perplexity 514.1436
-6900 train 6.178641 (lr=2.0828e-05) (hash(x)=152029630)
-7100 val loss 6.4525
-7100 val perplexity 634.2632
-7100 train 6.332360 (lr=2.1064e-05) (hash(x)=148064359)
-7000 val loss 6.2267
-7000 val perplexity 506.0734
-7000 train 6.081713 (lr=2.0601e-05) (hash(x)=142978954)
-7200 val loss 6.4379
-7200 val perplexity 625.0662
-7200 train 6.348963 (lr=2.0826e-05) (hash(x)=155771845)
-7100 val loss 6.2153
-7100 val perplexity 500.3705
-7100 train 6.090960 (lr=2.0373e-05) (hash(x)=148064359)
-7300 val loss 6.4266
-7300 val perplexity 618.0421
-7300 train 6.214900 (lr=2.0587e-05) (hash(x)=141639844)
-7200 val loss 6.2075
-7200 val perplexity 496.4699
-7200 train 6.113291 (lr=2.0143e-05) (hash(x)=155771845)
-7400 val loss 6.4262
-7400 val perplexity 617.8482
-7400 train 6.478831 (lr=2.0347e-05) (hash(x)=153466021)
-7300 val loss 6.1980
-7300 val perplexity 491.7664
-7300 train 5.992127 (lr=1.9912e-05) (hash(x)=141639844)
-7500 val loss 6.4100
-7500 val perplexity 607.8655
-7500 train 6.410825 (lr=2.0106e-05) (hash(x)=148843587)
-7400 val loss 6.1947
-7400 val perplexity 490.1655
-7400 train 6.239578 (lr=1.9680e-05) (hash(x)=153466021)
-7600 val loss 6.3920
-7600 val perplexity 597.0245
-7600 train 6.529988 (lr=1.9864e-05) (hash(x)=152053933)
-7500 val loss 6.1802
-7500 val perplexity 483.0656
-7500 train 6.184921 (lr=1.9447e-05) (hash(x)=148843587)
-7700 val loss 6.3836
-7700 val perplexity 592.0662
-7700 train 6.192592 (lr=1.9620e-05) (hash(x)=141564003)
-7600 val loss 6.1675
-7600 val perplexity 477.0140
-7600 train 6.304901 (lr=1.9213e-05) (hash(x)=152053933)
-7800 val loss 6.3741
-7800 val perplexity 586.4751
-7800 train 6.245174 (lr=1.9376e-05) (hash(x)=148617843)
-7700 val loss 6.1576
-7700 val perplexity 472.2927
-7700 train 5.997015 (lr=1.8977e-05) (hash(x)=141564003)
-7900 val loss 6.3675
-7900 val perplexity 582.5914
-7900 train 6.316991 (lr=1.9131e-05) (hash(x)=146417668)
-7800 val loss 6.1480
-7800 val perplexity 467.8008
-7800 train 6.016080 (lr=1.8741e-05) (hash(x)=148617843)
-8000 val loss 6.3591
-8000 val perplexity 577.7357
-8000 train 6.161639 (lr=1.8885e-05) (hash(x)=150165488)
-7900 val loss 6.1427
-7900 val perplexity 465.3265
-8100 val loss 6.3476
-8100 val perplexity 571.1010
-7900 train 6.106478 (lr=1.8504e-05) (hash(x)=146417668)
-8100 train 6.322338 (lr=1.8639e-05) (hash(x)=144818855)
-8200 val loss 6.3364
-8200 val perplexity 564.7528
-8200 train 6.395426 (lr=1.8391e-05) (hash(x)=143074954)
-8000 val loss 6.1353
-8000 val perplexity 461.8995
-8000 train 5.937020 (lr=1.8266e-05) (hash(x)=150165488)
-8300 val loss 6.3261
-8300 val perplexity 558.9454
-8300 train 6.218925 (lr=1.8144e-05) (hash(x)=145172494)
-8100 val loss 6.1243
-8100 val perplexity 456.8080
-8100 train 6.108549 (lr=1.8027e-05) (hash(x)=144818855)
-8400 val loss 6.3216
-8400 val perplexity 556.4594
-8400 train 6.342124 (lr=1.7895e-05) (hash(x)=152284393)
-8200 val loss 6.1182
-8200 val perplexity 454.0617
-8200 train 6.190479 (lr=1.7788e-05) (hash(x)=143074954)
-8500 val loss 6.3107
-8500 val perplexity 550.4196
-8500 train 6.166171 (lr=1.7647e-05) (hash(x)=147044705)
-8300 val loss 6.1079
-8300 val perplexity 449.3750
-8300 train 5.992473 (lr=1.7549e-05) (hash(x)=145172494)
-8600 val loss 6.3036
-8600 val perplexity 546.5322
-8600 train 6.193993 (lr=1.7398e-05) (hash(x)=147762608)
-8400 val loss 6.1020
-8400 val perplexity 446.7611
-8400 train 6.125968 (lr=1.7309e-05) (hash(x)=152284393)
-8700 val loss 6.2951
-8700 val perplexity 541.9163
-8700 train 6.282552 (lr=1.7149e-05) (hash(x)=141994567)
-8500 val loss 6.0933
-8500 val perplexity 442.8783
-8500 train 5.954424 (lr=1.7068e-05) (hash(x)=147044705)
-8800 val loss 6.2850
-8800 val perplexity 536.4824
-8800 train 6.344272 (lr=1.6900e-05) (hash(x)=152552896)
-8600 val loss 6.0892
-8600 val perplexity 441.0811
-8600 train 5.970642 (lr=1.6827e-05) (hash(x)=147762608)
-8900 val loss 6.2767
-8900 val perplexity 532.0255
-8900 train 6.108348 (lr=1.6650e-05) (hash(x)=139714454)
-8700 val loss 6.0820
-8700 val perplexity 437.9130
-8700 train 6.084789 (lr=1.6587e-05) (hash(x)=141994567)
-9000 val loss 6.2687
-9000 val perplexity 527.8010
-9000 train 6.293616 (lr=1.6401e-05) (hash(x)=152274419)
-8800 val loss 6.0748
-8800 val perplexity 434.7520
-8800 train 6.151562 (lr=1.6346e-05) (hash(x)=152552896)
-9100 val loss 6.2626
-9100 val perplexity 524.5593
-9100 train 6.346933 (lr=1.6152e-05) (hash(x)=160536885)
-8900 val loss 6.0645
-8900 val perplexity 430.2912
-8900 train 5.895317 (lr=1.6104e-05) (hash(x)=139714454)
-9200 val loss 6.2542
-9200 val perplexity 520.1846
-9200 train 6.066760 (lr=1.5903e-05) (hash(x)=138386899)
-9000 val loss 6.0572
-9000 val perplexity 427.1852
-9000 train 6.077891 (lr=1.5863e-05) (hash(x)=152274419)
-9300 val loss 6.2511
-9300 val perplexity 518.5825
-9300 train 6.284897 (lr=1.5655e-05) (hash(x)=149957457)
-9100 val loss 6.0503
-9100 val perplexity 424.2314
-9100 train 6.136716 (lr=1.5623e-05) (hash(x)=160536885)
-9400 val loss 6.2363
-9400 val perplexity 510.9432
-9400 train 6.314912 (lr=1.5406e-05) (hash(x)=161671548)
-9200 val loss 6.0456
-9200 val perplexity 422.2567
-9200 train 5.846539 (lr=1.5382e-05) (hash(x)=138386899)
-9500 val loss 6.2328
-9500 val perplexity 509.2006
-9500 train 6.380620 (lr=1.5159e-05) (hash(x)=171857400)
-9300 val loss 6.0379
-9300 val perplexity 419.0162
-9300 train 6.086740 (lr=1.5141e-05) (hash(x)=149957457)
-9600 val loss 6.2242
-9600 val perplexity 504.8264
-9600 train 5.942055 (lr=1.4911e-05) (hash(x)=135378471)
-9400 val loss 6.0291
-9400 val perplexity 415.3321
-9400 train 6.068547 (lr=1.4901e-05) (hash(x)=161671548)
-9700 val loss 6.2177
-9700 val perplexity 501.5732
-9700 train 6.138317 (lr=1.4665e-05) (hash(x)=149514591)
-9500 val loss 6.0249
-9500 val perplexity 413.6149
-9500 train 6.157710 (lr=1.4662e-05) (hash(x)=171857400)
-9800 val loss 6.2116
-9800 val perplexity 498.5100
-9800 train 6.183671 (lr=1.4419e-05) (hash(x)=152679072)
-9600 val loss 6.0172
-9600 val perplexity 410.4119
-9600 train 5.762091 (lr=1.4423e-05) (hash(x)=135378471)
-9900 val loss 6.2032
-9900 val perplexity 494.3351
-9900 train 6.124578 (lr=1.4174e-05) (hash(x)=150415193)
-9700 val loss 6.0131
-9700 val perplexity 408.7643
-9700 train 5.938748 (lr=1.4184e-05) (hash(x)=149514591)
-10000 val loss 6.1984
-10000 val perplexity 491.9660
-10000 train 6.032781 (lr=1.3930e-05) (hash(x)=146910087)
-9800 val loss 6.0077
-9800 val perplexity 406.5629
-9800 train 5.983809 (lr=1.3946e-05) (hash(x)=152679072)
-10100 val loss 6.1919
-10100 val perplexity 488.7980
-10100 train 6.112926 (lr=1.3686e-05) (hash(x)=152091050)
-9900 val loss 5.9993
-9900 val perplexity 403.1390
-9900 train 5.938206 (lr=1.3709e-05) (hash(x)=150415193)
-10200 val loss 6.1917
-10200 val perplexity 488.6531
-10200 train 6.033022 (lr=1.3444e-05) (hash(x)=143101306)
-10000 val loss 5.9934
-10000 val perplexity 400.7757
-10000 train 5.828973 (lr=1.3473e-05) (hash(x)=146910087)
-10300 val loss 6.1800
-10300 val perplexity 483.0066
-10300 train 6.190086 (lr=1.3203e-05) (hash(x)=150803985)
-10100 val loss 5.9921
-10100 val perplexity 400.2507
-10100 train 5.906441 (lr=1.3237e-05) (hash(x)=152091050)
-10400 val loss 6.1763
-10400 val perplexity 481.1846
-10400 train 6.182276 (lr=1.2963e-05) (hash(x)=148671575)
-10200 val loss 5.9909
-10200 val perplexity 399.7775
-10200 train 5.834992 (lr=1.3003e-05) (hash(x)=143101306)
-10500 val loss 6.1702
-10500 val perplexity 478.2988
-10500 train 5.976685 (lr=1.2724e-05) (hash(x)=146312794)
-10300 val loss 5.9805
-10300 val perplexity 395.6414
-10300 train 5.992455 (lr=1.2770e-05) (hash(x)=150803985)
-10600 val loss 6.1637
-10600 val perplexity 475.1910
-10600 train 5.988032 (lr=1.2486e-05) (hash(x)=137878985)
-10400 val loss 5.9768
-10400 val perplexity 394.1781
-10400 train 5.956397 (lr=1.2538e-05) (hash(x)=148671575)
-10700 val loss 6.1588
-10700 val perplexity 472.8748
-10700 train 5.953914 (lr=1.2250e-05) (hash(x)=148897486)
-10500 val loss 5.9726
-10500 val perplexity 392.5354
-10500 train 5.775092 (lr=1.2307e-05) (hash(x)=146312794)
-10800 val loss 6.1537
-10800 val perplexity 470.4418
-10800 train 6.206979 (lr=1.2016e-05) (hash(x)=147219580)
-10600 val loss 5.9665
-10600 val perplexity 390.1403
-10600 train 5.794254 (lr=1.2077e-05) (hash(x)=137878985)
-10900 val loss 6.1531
-10900 val perplexity 470.1722
-10900 train 6.186432 (lr=1.1783e-05) (hash(x)=152048929)
-10700 val loss 5.9625
-10700 val perplexity 388.5760
-10700 train 5.760836 (lr=1.1849e-05) (hash(x)=148897486)
-11000 val loss 6.1376
-11000 val perplexity 462.9409
-11000 train 6.026800 (lr=1.1551e-05) (hash(x)=146417066)
-10800 val loss 5.9570
-10800 val perplexity 386.4495
-10800 train 6.040039 (lr=1.1622e-05) (hash(x)=147219580)
-11100 val loss 6.1354
-11100 val perplexity 461.9145
-11100 train 5.998727 (lr=1.1322e-05) (hash(x)=148784436)
-10900 val loss 5.9588
-10900 val perplexity 387.1628
-10900 train 5.989281 (lr=1.1396e-05) (hash(x)=152048929)
-11200 val loss 6.1309
-11200 val perplexity 459.8421
-11200 train 6.069459 (lr=1.1094e-05) (hash(x)=144721870)
-11000 val loss 5.9443
-11000 val perplexity 381.5715
-11000 train 5.836518 (lr=1.1173e-05) (hash(x)=146417066)
-11300 val loss 6.1266
-11300 val perplexity 457.8686
-11300 train 6.036793 (lr=1.0868e-05) (hash(x)=149442981)
-11100 val loss 5.9397
-11100 val perplexity 379.8165
-11100 train 5.808575 (lr=1.0951e-05) (hash(x)=148784436)
-11400 val loss 6.1223
-11400 val perplexity 455.9191
-11400 train 6.050245 (lr=1.0644e-05) (hash(x)=154481228)
-11200 val loss 5.9378
-11200 val perplexity 379.0861
-11200 train 5.890636 (lr=1.0730e-05) (hash(x)=144721870)
-11500 val loss 6.1169
-11500 val perplexity 453.4414
-11500 train 5.782500 (lr=1.0422e-05) (hash(x)=115164581)
-11300 val loss 5.9333
-11300 val perplexity 377.4155
-11300 train 5.847861 (lr=1.0512e-05) (hash(x)=149442981)
-11600 val loss 6.1099
-11600 val perplexity 450.3044
-11600 train 6.324108 (lr=1.0202e-05) (hash(x)=155048082)
-11400 val loss 5.9310
-11400 val perplexity 376.5144
-11400 train 5.834333 (lr=1.0295e-05) (hash(x)=154481228)
-11700 val loss 6.1086
-11700 val perplexity 449.6895
-11700 train 6.160451 (lr=9.9846e-06) (hash(x)=152123770)
-11500 val loss 5.9257
-11500 val perplexity 374.5461
-11500 train 5.605324 (lr=1.0080e-05) (hash(x)=115164581)
-11800 val loss 6.1027
-11800 val perplexity 447.0625
-11800 train 6.045491 (lr=9.7691e-06) (hash(x)=152705857)
-11600 val loss 5.9187
-11600 val perplexity 371.9386
-11600 train 6.139134 (lr=9.8678e-06) (hash(x)=155048082)
-11900 val loss 6.1009
-11900 val perplexity 446.2455
-11900 train 5.810202 (lr=9.5560e-06) (hash(x)=137392362)
-11700 val loss 5.9158
-11700 val perplexity 370.8369
-11700 train 5.972539 (lr=9.6572e-06) (hash(x)=152123770)
-12000 val loss 6.0993
-12000 val perplexity 445.5258
-12000 train 5.903187 (lr=9.3452e-06) (hash(x)=143492562)
-11800 val loss 5.9112
-11800 val perplexity 369.1621
-11800 train 5.853813 (lr=9.4488e-06) (hash(x)=152705857)
-12100 val loss 6.0940
-12100 val perplexity 443.1847
-12100 train 6.051395 (lr=9.1369e-06) (hash(x)=141825431)
-11900 val loss 5.9083
-11900 val perplexity 368.0734
-11900 train 5.622472 (lr=9.2426e-06) (hash(x)=137392362)
-12200 val loss 6.0870
-12200 val perplexity 440.1027
-12200 train 5.972810 (lr=8.9310e-06) (hash(x)=141774370)
-12300 val loss 6.0813
-12300 val perplexity 437.6091
-12300 train 6.086730 (lr=8.7278e-06) (hash(x)=152995173)
-12000 val loss 5.9098
-12000 val perplexity 368.6481
-12000 train 5.715115 (lr=9.0388e-06) (hash(x)=143492562)
-12400 val loss 6.0800
-12400 val perplexity 437.0469
-12400 train 5.888199 (lr=8.5273e-06) (hash(x)=144703471)
-12100 val loss 5.9022
-12100 val perplexity 365.8474
-12100 train 5.871357 (lr=8.8373e-06) (hash(x)=141825431)
-12500 val loss 6.0775
-12500 val perplexity 435.9203
-12500 train 5.953711 (lr=8.3294e-06) (hash(x)=144866179)
-12200 val loss 5.8956
-12200 val perplexity 363.4287
-12200 train 5.795891 (lr=8.6382e-06) (hash(x)=141774370)
-12600 val loss 6.0750
-12600 val perplexity 434.8515
-12600 train 6.058091 (lr=8.1343e-06) (hash(x)=157019663)
-12300 val loss 5.8935
-12300 val perplexity 362.6677
-12300 train 5.909986 (lr=8.4417e-06) (hash(x)=152995173)
-12700 val loss 6.0725
-12700 val perplexity 433.7653
-12700 train 6.036210 (lr=7.9421e-06) (hash(x)=152872020)
-12400 val loss 5.8904
-12400 val perplexity 361.5423
-12400 train 5.708963 (lr=8.2477e-06) (hash(x)=144703471)
-12800 val loss 6.0654
-12800 val perplexity 430.7030
-12800 train 6.112625 (lr=7.7528e-06) (hash(x)=158975636)
-12500 val loss 5.8886
-12500 val perplexity 360.9105
-12500 train 5.769452 (lr=8.0563e-06) (hash(x)=144866179)
-12900 val loss 6.0606
-12900 val perplexity 428.6136
-12900 train 6.078387 (lr=7.5665e-06) (hash(x)=143495701)
-12600 val loss 5.8865
-12600 val perplexity 360.1528
-12600 train 5.884514 (lr=7.8676e-06) (hash(x)=157019663)
-13000 val loss 6.0577
-13000 val perplexity 427.3926
-13000 train 5.965330 (lr=7.3832e-06) (hash(x)=151940005)
-12700 val loss 5.8840
-12700 val perplexity 359.2356
-12700 train 5.846691 (lr=7.6817e-06) (hash(x)=152872020)
-13100 val loss 6.0548
-13100 val perplexity 426.1748
-13100 train 6.317828 (lr=7.2030e-06) (hash(x)=155920713)
-12800 val loss 5.8786
-12800 val perplexity 357.3050
-12800 train 5.921139 (lr=7.4986e-06) (hash(x)=158975636)
-13200 val loss 6.0524
-13200 val perplexity 425.1237
-13200 train 6.207733 (lr=7.0260e-06) (hash(x)=152145878)
-12900 val loss 5.8721
-12900 val perplexity 355.0043
-12900 train 5.912667 (lr=7.3184e-06) (hash(x)=143495701)
-13300 val loss 6.0501
-13300 val perplexity 424.1669
-13300 train 5.828300 (lr=6.8522e-06) (hash(x)=134141709)
-13000 val loss 5.8699
-13000 val perplexity 354.2000
-13000 train 5.782695 (lr=7.1412e-06) (hash(x)=151940005)
-13400 val loss 6.0490
-13400 val perplexity 423.6709
-13400 train 5.953373 (lr=6.6816e-06) (hash(x)=146096299)
-13100 val loss 5.8689
-13100 val perplexity 353.8669
-13100 train 6.138092 (lr=6.9669e-06) (hash(x)=155920713)
-13500 val loss 6.0483
-13500 val perplexity 423.3883
-13500 train 5.795309 (lr=6.5144e-06) (hash(x)=144011431)
-13200 val loss 5.8661
-13200 val perplexity 352.8610
-13200 train 6.022047 (lr=6.7956e-06) (hash(x)=152145878)
-13600 val loss 6.0406
-13600 val perplexity 420.1281
-13600 train 6.291119 (lr=6.3506e-06) (hash(x)=163950932)
-13300 val loss 5.8630
-13300 val perplexity 351.7691
-13300 train 5.652020 (lr=6.6275e-06) (hash(x)=134141709)
-13700 val loss 6.0373
-13700 val perplexity 418.7611
-13700 train 6.179534 (lr=6.1902e-06) (hash(x)=156652406)
-13400 val loss 5.8659
-13400 val perplexity 352.8086
-13400 train 5.783862 (lr=6.4626e-06) (hash(x)=146096299)
-13800 val loss 6.0312
-13800 val perplexity 416.2185
-13800 train 5.939419 (lr=6.0333e-06) (hash(x)=145753166)
-13500 val loss 5.8624
-13500 val perplexity 351.5657
-13500 train 5.618773 (lr=6.3008e-06) (hash(x)=144011431)
-13900 val loss 6.0299
-13900 val perplexity 415.6666
-13900 train 5.908447 (lr=5.8799e-06) (hash(x)=147065551)
-13600 val loss 5.8548
-13600 val perplexity 348.9127
-13600 train 6.106495 (lr=6.1424e-06) (hash(x)=163950932)
-14000 val loss 6.0276
-14000 val perplexity 414.7269
-14000 train 6.481697 (lr=5.7301e-06) (hash(x)=179077542)
-13700 val loss 5.8517
-13700 val perplexity 347.8210
-13700 train 5.992522 (lr=5.9872e-06) (hash(x)=156652406)
-14100 val loss 6.0266
-14100 val perplexity 414.3128
-14100 train 6.123082 (lr=5.5840e-06) (hash(x)=171962802)
-13800 val loss 5.8466
-13800 val perplexity 346.0474
-13800 train 5.750800 (lr=5.8355e-06) (hash(x)=145753166)
-14200 val loss 6.0240
-14200 val perplexity 413.2231
-14200 train 6.072022 (lr=5.4416e-06) (hash(x)=153299913)
-13900 val loss 5.8445
-13900 val perplexity 345.3404
-13900 train 5.712620 (lr=5.6871e-06) (hash(x)=147065551)
-14300 val loss 6.0244
-14300 val perplexity 413.4088
-14300 train 5.955797 (lr=5.3029e-06) (hash(x)=154750565)
-14000 val loss 5.8421
-14000 val perplexity 344.4977
-14000 train 6.301637 (lr=5.5423e-06) (hash(x)=179077542)
-14400 val loss 6.0193
-14400 val perplexity 411.3018
-14400 train 6.303262 (lr=5.1679e-06) (hash(x)=154536694)
-14100 val loss 5.8423
-14100 val perplexity 344.5786
-14100 train 5.920660 (lr=5.4009e-06) (hash(x)=171962802)
-14500 val loss 6.0151
-14500 val perplexity 409.5637
-14500 train 6.049838 (lr=5.0368e-06) (hash(x)=151236383)
-14200 val loss 5.8405
-14200 val perplexity 343.9358
-14200 train 5.903329 (lr=5.2632e-06) (hash(x)=153299913)
-14600 val loss 6.0127
-14600 val perplexity 408.6041
-14600 train 6.025821 (lr=4.9096e-06) (hash(x)=152584272)
-14300 val loss 5.8401
-14300 val perplexity 343.8032
-14300 train 5.777838 (lr=5.1290e-06) (hash(x)=154750565)
-14700 val loss 6.0104
-14700 val perplexity 407.6416
-14700 train 6.067557 (lr=4.7863e-06) (hash(x)=175301826)
-14400 val loss 5.8383
-14400 val perplexity 343.2094
-14400 train 6.123693 (lr=4.9985e-06) (hash(x)=154536694)
-14800 val loss 6.0085
-14800 val perplexity 406.8910
-14800 train 5.925559 (lr=4.6670e-06) (hash(x)=148729408)
-14500 val loss 5.8329
-14500 val perplexity 341.3530
-14500 train 5.879202 (lr=4.8717e-06) (hash(x)=151236383)
-14900 val loss 6.0071
-14900 val perplexity 406.2877
-14900 train 6.155755 (lr=4.5516e-06) (hash(x)=150978283)
-14600 val loss 5.8299
-14600 val perplexity 340.3333
-14600 train 5.838913 (lr=4.7487e-06) (hash(x)=152584272)
-15000 val loss 6.0072
-15000 val perplexity 406.3619
-15000 train 5.936687 (lr=4.4403e-06) (hash(x)=153621389)
-14700 val loss 5.8278
-14700 val perplexity 339.6272
-14700 train 5.837060 (lr=4.6294e-06) (hash(x)=175301826)
-15100 val loss 6.0067
-15100 val perplexity 406.1379
-15100 train 5.920137 (lr=4.3330e-06) (hash(x)=150653793)
-14800 val loss 5.8278
-14800 val perplexity 339.6144
-14800 train 5.750486 (lr=4.5139e-06) (hash(x)=148729408)
-15200 val loss 6.0038
-15200 val perplexity 404.9572
-15200 train 6.143285 (lr=4.2298e-06) (hash(x)=140802934)
-14900 val loss 5.8259
-14900 val perplexity 338.9724
-14900 train 5.992436 (lr=4.4024e-06) (hash(x)=150978283)
-15300 val loss 5.9980
-15300 val perplexity 402.6403
-15300 train 5.946833 (lr=4.1308e-06) (hash(x)=152007556)
-15000 val loss 5.8256
-15000 val perplexity 338.8798
-15000 train 5.767491 (lr=4.2947e-06) (hash(x)=153621389)
-15400 val loss 5.9971
-15400 val perplexity 402.2480
-15400 train 5.947176 (lr=4.0360e-06) (hash(x)=149057446)
-15100 val loss 5.8249
-15100 val perplexity 338.6296
-15100 train 5.742505 (lr=4.1909e-06) (hash(x)=150653793)
-15500 val loss 5.9953
-15500 val perplexity 401.5520
-15500 train 5.912923 (lr=3.9453e-06) (hash(x)=147521774)
-15200 val loss 5.8225
-15200 val perplexity 337.8013
-15200 train 5.995377 (lr=4.0912e-06) (hash(x)=140802934)
-15600 val loss 5.9934
-15600 val perplexity 400.7822
-15600 train 5.960157 (lr=3.8589e-06) (hash(x)=148944405)
-15300 val loss 5.8174
-15300 val perplexity 336.0993
-15300 train 5.777167 (lr=3.9954e-06) (hash(x)=152007556)
-15700 val loss 5.9923
-15700 val perplexity 400.3158
-15700 train 5.936503 (lr=3.7767e-06) (hash(x)=148472532)
-15400 val loss 5.8164
-15400 val perplexity 335.7530
-15400 train 5.763052 (lr=3.9036e-06) (hash(x)=149057446)
-15800 val loss 5.9913
-15800 val perplexity 399.9413
-15800 train 5.785662 (lr=3.6988e-06) (hash(x)=149048780)
-15500 val loss 5.8142
-15500 val perplexity 335.0070
-15500 train 5.744727 (lr=3.8160e-06) (hash(x)=147521774)
-15900 val loss 5.9919
-15900 val perplexity 400.1645
-15900 train 5.937081 (lr=3.6253e-06) (hash(x)=150175778)
-15600 val loss 5.8137
-15600 val perplexity 334.8482
-15600 train 5.782406 (lr=3.7324e-06) (hash(x)=148944405)
-16000 val loss 5.9920
-16000 val perplexity 400.2128
-16000 train 5.800823 (lr=3.5560e-06) (hash(x)=145943831)
-15700 val loss 5.8122
-15700 val perplexity 334.3681
-15700 train 5.767058 (lr=3.6529e-06) (hash(x)=148472532)
-16100 val loss 5.9856
-16100 val perplexity 397.6656
-16100 train 6.125412 (lr=3.4912e-06) (hash(x)=134314842)
-15800 val loss 5.8110
-15800 val perplexity 333.9406
-15800 train 5.598567 (lr=3.5776e-06) (hash(x)=149048780)
-16200 val loss 5.9824
-16200 val perplexity 396.3834
-16200 train 6.008039 (lr=3.4307e-06) (hash(x)=153755135)
-15900 val loss 5.8127
-15900 val perplexity 334.5194
-15900 train 5.770451 (lr=3.5064e-06) (hash(x)=150175778)
-16300 val loss 5.9822
-16300 val perplexity 396.2938
-16300 train 6.110716 (lr=3.3746e-06) (hash(x)=146916346)
-16000 val loss 5.8132
-16000 val perplexity 334.6827
-16000 train 5.611537 (lr=3.4394e-06) (hash(x)=145943831)
-16400 val loss 5.9822
-16400 val perplexity 396.3155
-16400 train 5.904950 (lr=3.3229e-06) (hash(x)=155023643)
-16100 val loss 5.8063
-16100 val perplexity 332.3815
-16100 train 5.956073 (lr=3.3767e-06) (hash(x)=134314842)
-16500 val loss 5.9803
-16500 val perplexity 395.5616
-16500 train 6.034608 (lr=3.2757e-06) (hash(x)=148910191)
-16200 val loss 5.8038
-16200 val perplexity 331.5721
-16600 val loss 5.9780
-16600 val perplexity 394.6545
-16200 train 5.827353 (lr=3.3182e-06) (hash(x)=153755135)
-16600 train 5.877320 (lr=3.2329e-06) (hash(x)=150107746)
-16700 val loss 5.9756
-16700 val perplexity 393.7152
-16700 train 6.089258 (lr=3.1946e-06) (hash(x)=159943976)
-16300 val loss 5.8046
-16300 val perplexity 331.8285
-16300 train 5.931115 (lr=3.2639e-06) (hash(x)=146916346)
-16800 val loss 5.9737
-16800 val perplexity 392.9377
-16800 train 5.983416 (lr=3.1607e-06) (hash(x)=160255226)
-16400 val loss 5.8037
-16400 val perplexity 331.5388
-16400 train 5.724208 (lr=3.2140e-06) (hash(x)=155023643)
-16900 val loss 5.9729
-16900 val perplexity 392.6606
-16900 train 5.963253 (lr=3.1314e-06) (hash(x)=157121826)
-16500 val loss 5.8007
-16500 val perplexity 330.5226
-16500 train 5.872295 (lr=3.1683e-06) (hash(x)=148910191)
-17000 val loss 5.9715
-17000 val perplexity 392.1104
-17000 train 6.106241 (lr=3.1065e-06) (hash(x)=159298029)
-16600 val loss 5.8002
-16600 val perplexity 330.3697
-16600 train 5.711960 (lr=3.1269e-06) (hash(x)=150107746)
-17100 val loss 5.9710
-17100 val perplexity 391.8808
-17100 train 5.918888 (lr=3.0862e-06) (hash(x)=149355903)
-16700 val loss 5.7966
-16700 val perplexity 329.1867
-16700 train 5.922190 (lr=3.0898e-06) (hash(x)=159943976)
-17200 val loss 5.9695
-17200 val perplexity 391.2908
-17200 train 5.779477 (lr=3.0704e-06) (hash(x)=144467909)
-16800 val loss 5.7955
-16800 val perplexity 328.8318
-16800 train 5.792985 (lr=3.0571e-06) (hash(x)=160255226)
-17300 val loss 5.9684
-17300 val perplexity 390.8734
-17300 train 6.135820 (lr=3.0591e-06) (hash(x)=157150350)
-16900 val loss 5.7936
-16900 val perplexity 328.2051
-16900 train 5.787290 (lr=3.0287e-06) (hash(x)=157121826)
-17400 val loss 5.9656
-17400 val perplexity 389.7708
-17400 train 6.016390 (lr=3.0523e-06) (hash(x)=154825212)
-17000 val loss 5.7933
-17000 val perplexity 328.1010
-17000 train 5.921300 (lr=3.0047e-06) (hash(x)=159298029)
-17499 val loss 5.9656
-17499 val perplexity 389.8025
-17100 val loss 5.7923
-17100 val perplexity 327.7756
-17100 train 5.742039 (lr=2.9850e-06) (hash(x)=149355903)
-17200 val loss 5.7920
-17200 val perplexity 327.6541
-17200 train 5.608214 (lr=2.9697e-06) (hash(x)=144467909)
-17300 val loss 5.7898
-17300 val perplexity 326.9355
-17300 train 5.949254 (lr=2.9588e-06) (hash(x)=157150350)
-17400 val loss 5.7880
-17400 val perplexity 326.3664
-17400 train 5.832754 (lr=2.9522e-06) (hash(x)=154825212)
-17499 val loss 5.7870
-17499 val perplexity 326.0410
+0 train 11.308768 (lr=1.5000e-07) (hash(x)=145079536)
+100 val loss 9.9830
+100 val perplexity 21656.2090
+100 train 9.924451 (lr=1.5150e-05) (hash(x)=154745873)
+200 val loss 9.2262
+200 val perplexity 10159.3574
+200 train 9.254306 (lr=3.0000e-05) (hash(x)=155800595)
+300 val loss 8.0887
+300 val perplexity 3257.6125
+300 train 8.057108 (lr=2.9998e-05) (hash(x)=148595389)
+400 val loss 7.7043
+400 val perplexity 2217.7786
+400 train 7.619694 (lr=2.9991e-05) (hash(x)=145606733)
+500 val loss 7.5578
+500 val perplexity 1915.6061
+500 train 7.391018 (lr=2.9980e-05) (hash(x)=138221231)
+600 val loss 7.4932
+600 val perplexity 1795.7026
+600 train 7.504253 (lr=2.9964e-05) (hash(x)=150367139)
+700 val loss 7.4564
+700 val perplexity 1730.8922
+700 train 7.611702 (lr=2.9944e-05) (hash(x)=155579314)
+800 val loss 7.4182
+800 val perplexity 1665.9652
+800 train 7.493585 (lr=2.9920e-05) (hash(x)=155747374)
+900 val loss 7.3806
+900 val perplexity 1604.6064
+900 train 7.583446 (lr=2.9891e-05) (hash(x)=159334575)
+1000 val loss 7.3464
+1000 val perplexity 1550.5729
+1000 train 7.226415 (lr=2.9858e-05) (hash(x)=140604760)
+1100 val loss 7.3188
+1100 val perplexity 1508.4453
+1100 train 7.163220 (lr=2.9820e-05) (hash(x)=146208052)
+1200 val loss 7.2921
+1200 val perplexity 1468.6635
+1200 train 7.210934 (lr=2.9778e-05) (hash(x)=148404734)
+1300 val loss 7.2604
+1300 val perplexity 1422.8925
+1300 train 7.194634 (lr=2.9732e-05) (hash(x)=155681970)
+1400 val loss 7.2315
+1400 val perplexity 1382.2393
+1400 train 7.185406 (lr=2.9681e-05) (hash(x)=148115934)
+1500 val loss 7.1955
+1500 val perplexity 1333.4130
+1500 train 7.345188 (lr=2.9626e-05) (hash(x)=157074034)
+1600 val loss 7.1643
+1600 val perplexity 1292.4220
+1600 train 6.927887 (lr=2.9566e-05) (hash(x)=137464699)
+1700 val loss 7.1353
+1700 val perplexity 1255.4856
+1700 train 7.315463 (lr=2.9502e-05) (hash(x)=166955614)
+1800 val loss 7.1089
+1800 val perplexity 1222.8240
+1800 train 6.936053 (lr=2.9434e-05) (hash(x)=143886042)
+1900 val loss 7.0820
+1900 val perplexity 1190.2955
+1900 train 7.707833 (lr=2.9362e-05) (hash(x)=193610391)
+2000 val loss 7.0571
+2000 val perplexity 1161.1222
+2000 train 7.232715 (lr=2.9285e-05) (hash(x)=163799796)
+2100 val loss 7.0157
+2100 val perplexity 1113.9966
+2100 train 6.991386 (lr=2.9204e-05) (hash(x)=154107339)
+2200 val loss 6.9856
+2200 val perplexity 1080.9934
+2200 train 6.980697 (lr=2.9119e-05) (hash(x)=144592844)
+2300 val loss 6.9576
+2300 val perplexity 1051.1511
+2300 train 7.036345 (lr=2.9030e-05) (hash(x)=154535861)
+2400 val loss 6.9332
+2400 val perplexity 1025.8033
+2400 train 7.687595 (lr=2.8937e-05) (hash(x)=204706354)
+2500 val loss 6.9026
+2500 val perplexity 994.8754
+2500 train 6.700395 (lr=2.8840e-05) (hash(x)=143783202)
+2600 val loss 6.8699
+2600 val perplexity 962.8300
+2600 train 6.765809 (lr=2.8738e-05) (hash(x)=150862210)
+2700 val loss 6.8375
+2700 val perplexity 932.1981
+2700 train 6.894982 (lr=2.8633e-05) (hash(x)=150753426)
+2800 val loss 6.8110
+2800 val perplexity 907.7455
+2800 train 6.813390 (lr=2.8523e-05) (hash(x)=147766811)
+2900 val loss 6.7701
+2900 val perplexity 871.4382
+2900 train 6.708093 (lr=2.8410e-05) (hash(x)=151177814)
+3000 val loss 6.7345
+3000 val perplexity 840.8976
+3000 train 6.587459 (lr=2.8292e-05) (hash(x)=135925327)
+3100 val loss 6.7059
+3100 val perplexity 817.2025
+3100 train 6.357706 (lr=2.8171e-05) (hash(x)=124842353)
+3200 val loss 6.6830
+3200 val perplexity 798.7388
+3200 train 6.789571 (lr=2.8046e-05) (hash(x)=160440642)
+3300 val loss 6.6456
+3300 val perplexity 769.3922
+3300 train 6.545945 (lr=2.7917e-05) (hash(x)=143017131)
+3400 val loss 6.6180
+3400 val perplexity 748.4457
+3400 train 6.906633 (lr=2.7784e-05) (hash(x)=148143474)
+3500 val loss 6.6012
+3500 val perplexity 735.9523
+3500 train 6.420564 (lr=2.7648e-05) (hash(x)=148732639)
+3600 val loss 6.5736
+3600 val perplexity 715.9206
+3600 train 6.569450 (lr=2.7508e-05) (hash(x)=146333803)
+3700 val loss 6.5550
+3700 val perplexity 702.7313
+3700 train 6.353135 (lr=2.7364e-05) (hash(x)=143611445)
+3800 val loss 6.5419
+3800 val perplexity 693.6258
+3800 train 6.329458 (lr=2.7216e-05) (hash(x)=141272115)
+3900 val loss 6.5180
+3900 val perplexity 677.2001
+3900 train 6.447794 (lr=2.7066e-05) (hash(x)=149161848)
+4000 val loss 6.4837
+4000 val perplexity 654.3748
+4000 train 6.561291 (lr=2.6911e-05) (hash(x)=152036955)
+4100 val loss 6.4627
+4100 val perplexity 640.7753
+4100 train 6.516588 (lr=2.6754e-05) (hash(x)=153260968)
+4200 val loss 6.4425
+4200 val perplexity 627.9695
+4200 train 6.316812 (lr=2.6592e-05) (hash(x)=147629478)
+4300 val loss 6.4259
+4300 val perplexity 617.6288
+4300 train 6.424664 (lr=2.6428e-05) (hash(x)=161701080)
+4400 val loss 6.4103
+4400 val perplexity 608.0597
+4400 train 6.460846 (lr=2.6260e-05) (hash(x)=155290524)
+4500 val loss 6.4012
+4500 val perplexity 602.5555
+4500 train 6.182449 (lr=2.6089e-05) (hash(x)=144726593)
+4600 val loss 6.3826
+4600 val perplexity 591.4775
+4600 train 6.321618 (lr=2.5915e-05) (hash(x)=149517202)
+4700 val loss 6.3610
+4700 val perplexity 578.8070
+4700 train 6.363839 (lr=2.5738e-05) (hash(x)=139403008)
+4800 val loss 6.3395
+4800 val perplexity 566.5151
+4800 train 6.422396 (lr=2.5558e-05) (hash(x)=153387238)
+4900 val loss 6.3396
+4900 val perplexity 566.5673
+4900 train 6.089847 (lr=2.5374e-05) (hash(x)=139881569)
+5000 val loss 6.3176
+5000 val perplexity 554.2572
+5000 train 6.305876 (lr=2.5188e-05) (hash(x)=152602580)
+5100 val loss 6.3054
+5100 val perplexity 547.5158
+5100 train 6.356030 (lr=2.4999e-05) (hash(x)=150598460)
+5200 val loss 6.2945
+5200 val perplexity 541.5656
+5200 train 6.180711 (lr=2.4807e-05) (hash(x)=147660496)
+5300 val loss 6.2825
+5300 val perplexity 535.1163
+5300 train 6.018123 (lr=2.4613e-05) (hash(x)=147693848)
+5400 val loss 6.2812
+5400 val perplexity 534.4260
+5400 train 6.027461 (lr=2.4415e-05) (hash(x)=136048145)
+5500 val loss 6.2582
+5500 val perplexity 522.2893
+5500 train 6.254835 (lr=2.4215e-05) (hash(x)=140313123)
+5600 val loss 6.2472
+5600 val perplexity 516.5477
+5600 train 6.269135 (lr=2.4013e-05) (hash(x)=156997993)
+5700 val loss 6.2285
+5700 val perplexity 506.9784
+5700 train 6.237016 (lr=2.3808e-05) (hash(x)=143513880)
+5800 val loss 6.2168
+5800 val perplexity 501.1166
+5800 train 6.343710 (lr=2.3601e-05) (hash(x)=153736628)
+5900 val loss 6.2070
+5900 val perplexity 496.1949
+5900 train 6.360643 (lr=2.3391e-05) (hash(x)=166385417)
+6000 val loss 6.1948
+6000 val perplexity 490.2095
+6000 train 6.430104 (lr=2.3179e-05) (hash(x)=153216517)
+6100 val loss 6.1878
+6100 val perplexity 486.7515
+6100 train 6.067350 (lr=2.2965e-05) (hash(x)=145387545)
+6200 val loss 6.1868
+6200 val perplexity 486.2852
+6200 train 6.303199 (lr=2.2749e-05) (hash(x)=156306460)
+6300 val loss 6.1642
+6300 val perplexity 475.4185
+6300 train 6.087062 (lr=2.2530e-05) (hash(x)=148610447)
+6400 val loss 6.1500
+6400 val perplexity 468.7014
+6400 train 6.373208 (lr=2.2310e-05) (hash(x)=158343535)
+6500 val loss 6.1425
+6500 val perplexity 465.1938
+6500 train 6.307102 (lr=2.2088e-05) (hash(x)=157742557)
+6600 val loss 6.1336
+6600 val perplexity 461.1154
+6600 train 6.029664 (lr=2.1864e-05) (hash(x)=150457308)
+6700 val loss 6.1283
+6700 val perplexity 458.6432
+6700 train 5.993783 (lr=2.1638e-05) (hash(x)=142426205)
+6800 val loss 6.1207
+6800 val perplexity 455.1693
+6800 train 5.913719 (lr=2.1410e-05) (hash(x)=151522525)
+6900 val loss 6.1069
+6900 val perplexity 448.9227
+6900 train 6.053385 (lr=2.1181e-05) (hash(x)=152029630)
+7000 val loss 6.0944
+7000 val perplexity 443.3823
+7000 train 5.944612 (lr=2.0951e-05) (hash(x)=142978954)
+7100 val loss 6.0889
+7100 val perplexity 440.9485
+7100 train 5.960811 (lr=2.0718e-05) (hash(x)=148064359)
+7200 val loss 6.0795
+7200 val perplexity 436.8317
+7200 train 5.983203 (lr=2.0485e-05) (hash(x)=155771845)
+7300 val loss 6.0744
+7300 val perplexity 434.5692
+7300 train 5.878072 (lr=2.0250e-05) (hash(x)=141639844)
+7400 val loss 6.0725
+7400 val perplexity 433.7572
+7400 train 6.120244 (lr=2.0014e-05) (hash(x)=153466021)
+7500 val loss 6.0606
+7500 val perplexity 428.6192
+7500 train 6.057049 (lr=1.9777e-05) (hash(x)=148843587)
+7600 val loss 6.0469
+7600 val perplexity 422.8022
+7600 train 6.166252 (lr=1.9538e-05) (hash(x)=152053933)
+7700 val loss 6.0409
+7700 val perplexity 420.2540
+7700 train 5.897380 (lr=1.9299e-05) (hash(x)=141564003)
+7800 val loss 6.0309
+7800 val perplexity 416.0959
+7800 train 5.900531 (lr=1.9059e-05) (hash(x)=148617843)
+7900 val loss 6.0284
+7900 val perplexity 415.0530
+7900 train 5.994560 (lr=1.8817e-05) (hash(x)=146417668)
+8000 val loss 6.0244
+8000 val perplexity 413.3800
+8000 train 5.822050 (lr=1.8576e-05) (hash(x)=150165488)
+8100 val loss 6.0155
+8100 val perplexity 409.7361
+8100 train 6.002255 (lr=1.8333e-05) (hash(x)=144818855)
+8200 val loss 6.0087
+8200 val perplexity 406.9713
+8200 train 6.095023 (lr=1.8090e-05) (hash(x)=143074954)
+8300 val loss 5.9975
+8300 val perplexity 402.4019
+8300 train 5.877024 (lr=1.7846e-05) (hash(x)=145172494)
+8400 val loss 5.9932
+8400 val perplexity 400.7092
+8400 train 6.021608 (lr=1.7602e-05) (hash(x)=152284393)
+8500 val loss 5.9849
+8500 val perplexity 397.3773
+8500 train 5.836091 (lr=1.7357e-05) (hash(x)=147044705)
+8600 val loss 5.9824
+8600 val perplexity 396.3830
+8600 train 5.866980 (lr=1.7113e-05) (hash(x)=147762608)
+8700 val loss 5.9753
+8700 val perplexity 393.5761
+8700 train 5.968902 (lr=1.6868e-05) (hash(x)=141994567)
+8800 val loss 5.9670
+8800 val perplexity 390.3279
+8800 train 6.055788 (lr=1.6623e-05) (hash(x)=152552896)
+8900 val loss 5.9606
+8900 val perplexity 387.8401
+8900 train 5.794694 (lr=1.6377e-05) (hash(x)=139714454)
+9000 val loss 5.9507
+9000 val perplexity 384.0324
+9000 train 5.985847 (lr=1.6132e-05) (hash(x)=152274419)
+9100 val loss 5.9520
+9100 val perplexity 384.5033
+9100 train 6.051697 (lr=1.5887e-05) (hash(x)=160536885)
+9200 val loss 5.9469
+9200 val perplexity 382.5706
+9200 train 5.750583 (lr=1.5643e-05) (hash(x)=138386899)
+9300 val loss 5.9390
+9300 val perplexity 379.5516
+9300 train 5.991132 (lr=1.5398e-05) (hash(x)=149957457)
+9400 val loss 5.9296
+9400 val perplexity 375.9923
+9400 train 5.968359 (lr=1.5154e-05) (hash(x)=161671548)
+9500 val loss 5.9283
+9500 val perplexity 375.4995
+9500 train 6.068425 (lr=1.4910e-05) (hash(x)=171857400)
+9600 val loss 5.9188
+9600 val perplexity 371.9698
+9600 train 5.667602 (lr=1.4667e-05) (hash(x)=135378471)
+9700 val loss 5.9142
+9700 val perplexity 370.2644
+9700 train 5.849272 (lr=1.4424e-05) (hash(x)=149514591)
+9800 val loss 5.9104
+9800 val perplexity 368.8677
+9800 train 5.887805 (lr=1.4183e-05) (hash(x)=152679072)
+9900 val loss 5.9027
+9900 val perplexity 366.0232
+9900 train 5.846486 (lr=1.3941e-05) (hash(x)=150415193)
+10000 val loss 5.8980
+10000 val perplexity 364.3136
+10000 train 5.735629 (lr=1.3701e-05) (hash(x)=146910087)
+10100 val loss 5.8937
+10100 val perplexity 362.7524
+10100 train 5.806296 (lr=1.3462e-05) (hash(x)=152091050)
+10200 val loss 5.8943
+10200 val perplexity 362.9474
+10200 train 5.742750 (lr=1.3223e-05) (hash(x)=143101306)
+10300 val loss 5.8847
+10300 val perplexity 359.4808
+10300 train 5.894190 (lr=1.2986e-05) (hash(x)=150803985)
+10400 val loss 5.8856
+10400 val perplexity 359.8033
+10400 train 5.868137 (lr=1.2750e-05) (hash(x)=148671575)
+10500 val loss 5.8745
+10500 val perplexity 355.8593
+10500 train 5.674866 (lr=1.2515e-05) (hash(x)=146312794)
+10600 val loss 5.8748
+10600 val perplexity 355.9517
+10600 train 5.699982 (lr=1.2282e-05) (hash(x)=137878985)
+10700 val loss 5.8703
+10700 val perplexity 354.3533
+10700 train 5.673525 (lr=1.2049e-05) (hash(x)=148897486)
+10800 val loss 5.8628
+10800 val perplexity 351.6937
+10800 train 5.969318 (lr=1.1819e-05) (hash(x)=147219580)
+10900 val loss 5.8686
+10900 val perplexity 353.7402
+10900 train 5.893168 (lr=1.1590e-05) (hash(x)=152048929)
+11000 val loss 5.8514
+11000 val perplexity 347.7291
+11000 train 5.751215 (lr=1.1362e-05) (hash(x)=146417066)
+11100 val loss 5.8479
+11100 val perplexity 346.4909
+11100 train 5.713552 (lr=1.1136e-05) (hash(x)=148784436)
+11200 val loss 5.8447
+11200 val perplexity 345.3947
+11200 train 5.803506 (lr=1.0912e-05) (hash(x)=144721870)
+11300 val loss 5.8406
+11300 val perplexity 343.9958
+11300 train 5.762826 (lr=1.0690e-05) (hash(x)=149442981)
+11400 val loss 5.8382
+11400 val perplexity 343.1647
+11400 train 5.735843 (lr=1.0470e-05) (hash(x)=154481228)
+11500 val loss 5.8336
+11500 val perplexity 341.5893
+11500 train 5.523773 (lr=1.0251e-05) (hash(x)=115164581)
+11600 val loss 5.8279
+11600 val perplexity 339.6416
+11600 train 6.049501 (lr=1.0035e-05) (hash(x)=155048082)
+11700 val loss 5.8249
+11700 val perplexity 338.6386
+11700 train 5.881338 (lr=9.8209e-06) (hash(x)=152123770)
+11800 val loss 5.8224
+11800 val perplexity 337.7885
+11800 train 5.759107 (lr=9.6090e-06) (hash(x)=152705857)
+11900 val loss 5.8188
+11900 val perplexity 336.5702
+11900 train 5.538026 (lr=9.3993e-06) (hash(x)=137392362)
+12000 val loss 5.8193
+12000 val perplexity 336.7239
+12000 train 5.628447 (lr=9.1920e-06) (hash(x)=143492562)
+12100 val loss 5.8126
+12100 val perplexity 334.4966
+12100 train 5.782037 (lr=8.9871e-06) (hash(x)=141825431)
+12200 val loss 5.8071
+12200 val perplexity 332.6457
+12200 train 5.707171 (lr=8.7846e-06) (hash(x)=141774370)
+12300 val loss 5.8049
+12300 val perplexity 331.9221
+12300 train 5.825204 (lr=8.5847e-06) (hash(x)=152995173)
+12400 val loss 5.8022
+12400 val perplexity 331.0165
+12400 train 5.623291 (lr=8.3875e-06) (hash(x)=144703471)
+12500 val loss 5.7999
+12500 val perplexity 330.2648
+12500 train 5.694048 (lr=8.1929e-06) (hash(x)=144866179)
+12600 val loss 5.7973
+12600 val perplexity 329.4210
+12600 train 5.801395 (lr=8.0010e-06) (hash(x)=157019663)
+12700 val loss 5.7957
+12700 val perplexity 328.8947
+12700 train 5.754167 (lr=7.8119e-06) (hash(x)=152872020)
+12800 val loss 5.7910
+12800 val perplexity 327.3514
+12800 train 5.827247 (lr=7.6257e-06) (hash(x)=158975636)
+12900 val loss 5.7854
+12900 val perplexity 325.5142
+12900 train 5.825149 (lr=7.4425e-06) (hash(x)=143495701)
+13000 val loss 5.7849
+13000 val perplexity 325.3500
+13000 train 5.703864 (lr=7.2622e-06) (hash(x)=151940005)
+13100 val loss 5.7801
+13100 val perplexity 323.7766
+13100 train 6.062990 (lr=7.0850e-06) (hash(x)=155920713)
+13200 val loss 5.7798
+13200 val perplexity 323.7076
+13200 train 5.957345 (lr=6.9108e-06) (hash(x)=152145878)
+13300 val loss 5.7770
+13300 val perplexity 322.7914
+13300 train 5.568100 (lr=6.7398e-06) (hash(x)=134141709)
+13400 val loss 5.7791
+13400 val perplexity 323.4539
+13400 train 5.697982 (lr=6.5721e-06) (hash(x)=146096299)
+13500 val loss 5.7754
+13500 val perplexity 322.2620
+13500 train 5.530023 (lr=6.4076e-06) (hash(x)=144011431)
+13600 val loss 5.7691
+13600 val perplexity 320.2527
+13600 train 6.011714 (lr=6.2465e-06) (hash(x)=163950932)
+13700 val loss 5.7677
+13700 val perplexity 319.8127
+13700 train 5.914011 (lr=6.0887e-06) (hash(x)=156652406)
+13800 val loss 5.7617
+13800 val perplexity 317.8942
+13800 train 5.671328 (lr=5.9344e-06) (hash(x)=145753166)
+13900 val loss 5.7614
+13900 val perplexity 317.7904
+13900 train 5.626029 (lr=5.7835e-06) (hash(x)=147065551)
+14000 val loss 5.7574
+14000 val perplexity 316.5275
+14000 train 6.217151 (lr=5.6362e-06) (hash(x)=179077542)
+14100 val loss 5.7576
+14100 val perplexity 316.5848
+14100 train 5.831569 (lr=5.4925e-06) (hash(x)=171962802)
+14200 val loss 5.7569
+14200 val perplexity 316.3727
+14200 train 5.824154 (lr=5.3524e-06) (hash(x)=153299913)
+14300 val loss 5.7570
+14300 val perplexity 316.4114
+14300 train 5.687628 (lr=5.2159e-06) (hash(x)=154750565)
+14400 val loss 5.7550
+14400 val perplexity 315.7674
+14400 train 6.040416 (lr=5.0832e-06) (hash(x)=154536694)
+14500 val loss 5.7488
+14500 val perplexity 313.8270
+14500 train 5.798073 (lr=4.9543e-06) (hash(x)=151236383)
+14600 val loss 5.7461
+14600 val perplexity 312.9566
+14600 train 5.754922 (lr=4.8291e-06) (hash(x)=152584272)
+14700 val loss 5.7443
+14700 val perplexity 312.4029
+14700 train 5.761162 (lr=4.7079e-06) (hash(x)=175301826)
+14800 val loss 5.7434
+14800 val perplexity 312.1136
+14800 train 5.661438 (lr=4.5905e-06) (hash(x)=148729408)
+14900 val loss 5.7431
+14900 val perplexity 312.0179
+14900 train 5.918625 (lr=4.4770e-06) (hash(x)=150978283)
+15000 val loss 5.7427
+15000 val perplexity 311.9147
+15000 train 5.683867 (lr=4.3675e-06) (hash(x)=153621389)
+15100 val loss 5.7424
+15100 val perplexity 311.8216
+15100 train 5.667899 (lr=4.2620e-06) (hash(x)=150653793)
+15200 val loss 5.7405
+15200 val perplexity 311.2079
+15200 train 5.915138 (lr=4.1605e-06) (hash(x)=140802934)
+15300 val loss 5.7337
+15300 val perplexity 309.1233
+15300 train 5.712039 (lr=4.0631e-06) (hash(x)=152007556)
+15400 val loss 5.7335
+15400 val perplexity 309.0579
+15400 train 5.686319 (lr=3.9698e-06) (hash(x)=149057446)
+15500 val loss 5.7319
+15500 val perplexity 308.5640
+15500 train 5.655775 (lr=3.8806e-06) (hash(x)=147521774)
+15600 val loss 5.7299
+15600 val perplexity 307.9506
+15600 train 5.693157 (lr=3.7956e-06) (hash(x)=148944405)
+15700 val loss 5.7300
+15700 val perplexity 307.9562
+15700 train 5.683930 (lr=3.7148e-06) (hash(x)=148472532)
+15800 val loss 5.7297
+15800 val perplexity 307.8897
+15800 train 5.504209 (lr=3.6382e-06) (hash(x)=149048780)
+15900 val loss 5.7301
+15900 val perplexity 308.0150
+15900 train 5.690699 (lr=3.5658e-06) (hash(x)=150175778)
+16000 val loss 5.7306
+16000 val perplexity 308.1541
+16000 train 5.519766 (lr=3.4977e-06) (hash(x)=145943831)
+16100 val loss 5.7238
+16100 val perplexity 306.0658
+16100 train 5.874838 (lr=3.4339e-06) (hash(x)=134314842)
+16200 val loss 5.7219
+16200 val perplexity 305.4991
+16200 train 5.747156 (lr=3.3744e-06) (hash(x)=153755135)
+16300 val loss 5.7224
+16300 val perplexity 305.6330
+16300 train 5.851801 (lr=3.3193e-06) (hash(x)=146916346)
+16400 val loss 5.7225
+16400 val perplexity 305.6758
+16400 train 5.643525 (lr=3.2684e-06) (hash(x)=155023643)
+16500 val loss 5.7201
+16500 val perplexity 304.9290
+16500 train 5.794312 (lr=3.2220e-06) (hash(x)=148910191)
+16600 val loss 5.7190
+16600 val perplexity 304.6099
+16600 train 5.634387 (lr=3.1799e-06) (hash(x)=150107746)
+16700 val loss 5.7152
+16700 val perplexity 303.4561
+16700 train 5.852514 (lr=3.1422e-06) (hash(x)=159943976)
+16800 val loss 5.7141
+16800 val perplexity 303.1232
+16800 train 5.711351 (lr=3.1089e-06) (hash(x)=160255226)
+16900 val loss 5.7129
+16900 val perplexity 302.7485
+16900 train 5.699970 (lr=3.0801e-06) (hash(x)=157121826)
+17000 val loss 5.7126
+17000 val perplexity 302.6575
+17000 train 5.846539 (lr=3.0556e-06) (hash(x)=159298029)
+17100 val loss 5.7119
+17100 val perplexity 302.4565
+17100 train 5.660579 (lr=3.0356e-06) (hash(x)=149355903)
+17200 val loss 5.7121
+17200 val perplexity 302.5085
+17200 train 5.529926 (lr=3.0200e-06) (hash(x)=144467909)
+17300 val loss 5.7097
+17300 val perplexity 301.7834
+17300 train 5.881879 (lr=3.0089e-06) (hash(x)=157150350)
+17400 val loss 5.7080
+17400 val perplexity 301.2650
+17400 train 5.758003 (lr=3.0022e-06) (hash(x)=154825212)
+17499 val loss 5.7072
+17499 val perplexity 301.0173

attention_kindselective_n_heads4_seed1341/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21781f9ec9fc41cd2811b83af43d365d33498e2c128b382097af52ca94799329
+oid sha256:d0a2f2b27fd5eb105bf18c7fb787a33f930c1969ed34839a6a4e3d80c229c920
 size 92843394

attention_kindselective_n_heads4_seed1341/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:edc43771715c6528e6c556e29a3ece52d4ecc9a678712e1ad61a51d7127d8540
+oid sha256:a3c839abef22afa1e6ca2833950a7cb0f4fa92ec28ef715594bf1dda52b98965
 size 179406214