Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_30720_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1341", "n_embd": 256}

attention_kindselective_n_heads4_seed1341/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

attention_kindselective_n_heads4_seed1341/log2.txt

@@ -1,1055 +1,414 @@
-max_steps: 17500
-0 val loss 11.3107
-0 val perplexity 81693.5234
-0 val loss 11.3107
-0 val perplexity 81693.5234
-0 train 11.308768 (lr=1.8750e-07) (hash(x)=145079536)
-0 train 11.308768 (lr=2.0000e-07) (hash(x)=145079536)
-100 val loss 9.9475
-100 val perplexity 20900.9492
-100 val loss 9.9305
-100 val perplexity 20547.3184
-100 train 9.887659 (lr=1.8937e-05) (hash(x)=154745873)
-100 train 9.869424 (lr=2.0200e-05) (hash(x)=154745873)
-200 val loss 8.8930
-200 val perplexity 7280.7363
-200 val loss 8.6350
-200 val perplexity 5625.3657
-200 train 8.938697 (lr=3.7500e-05) (hash(x)=155800595)
-200 train 8.700185 (lr=4.0000e-05) (hash(x)=155800595)
-300 val loss 7.9772
-300 val perplexity 2913.7043
-300 val loss 7.7205
-300 val perplexity 2253.9800
-300 train 7.942238 (lr=3.7497e-05) (hash(x)=148595389)
-300 train 7.677109 (lr=3.9997e-05) (hash(x)=148595389)
-400 val loss 7.7660
-400 val perplexity 2359.0081
-400 train 7.692175 (lr=3.7489e-05) (hash(x)=145606733)
-400 val loss 7.5253
-400 val perplexity 1854.3280
-400 train 7.429271 (lr=3.9988e-05) (hash(x)=145606733)
-500 val loss 7.6252
-500 val perplexity 2049.1321
-500 train 7.473835 (lr=3.7475e-05) (hash(x)=138221231)
-500 val loss 7.4571
-500 val perplexity 1732.1571
-500 train 7.274553 (lr=3.9973e-05) (hash(x)=138221231)
-600 val loss 7.5417
-600 val perplexity 1885.0631
-600 train 7.552938 (lr=3.7456e-05) (hash(x)=150367139)
-600 val loss 7.4091
-600 val perplexity 1651.0148
-600 train 7.414547 (lr=3.9953e-05) (hash(x)=150367139)
-700 val loss 7.4944
-700 val perplexity 1797.9723
-700 train 7.645608 (lr=3.7430e-05) (hash(x)=155579314)
-700 val loss 7.3478
-700 val perplexity 1552.7489
-700 train 7.501362 (lr=3.9926e-05) (hash(x)=155579314)
-800 val loss 7.4339
-800 val perplexity 1692.3730
-800 train 7.515994 (lr=3.7400e-05) (hash(x)=155747374)
-800 val loss 7.2731
-800 val perplexity 1441.0717
-800 train 7.349138 (lr=3.9893e-05) (hash(x)=155747374)
-900 val loss 7.3800
-900 val perplexity 1603.6160
-900 train 7.586455 (lr=3.7364e-05) (hash(x)=159334575)
-900 val loss 7.2117
-900 val perplexity 1355.2490
-900 train 7.430864 (lr=3.9855e-05) (hash(x)=159334575)
-1000 val loss 7.3251
-1000 val perplexity 1517.9445
-1000 train 7.211405 (lr=3.7322e-05) (hash(x)=140604760)
-1000 val loss 7.1531
-1000 val perplexity 1278.1234
-1000 train 7.017516 (lr=3.9810e-05) (hash(x)=140604760)
-1100 val loss 7.2738
-1100 val perplexity 1441.9576
-1100 train 7.113899 (lr=3.7275e-05) (hash(x)=146208052)
-1100 val loss 7.0989
-1100 val perplexity 1210.6454
-1100 train 6.925892 (lr=3.9760e-05) (hash(x)=146208052)
-1200 val loss 7.2267
-1200 val perplexity 1375.7426
-1200 train 7.126267 (lr=3.7223e-05) (hash(x)=148404734)
-1200 val loss 7.0482
-1200 val perplexity 1150.7867
-1200 train 6.938736 (lr=3.9704e-05) (hash(x)=148404734)
-1300 val loss 7.1736
-1300 val perplexity 1304.5807
-1300 train 7.107864 (lr=3.7164e-05) (hash(x)=155681970)
-1300 val loss 7.0146
-1300 val perplexity 1112.7230
-1300 train 6.945279 (lr=3.9642e-05) (hash(x)=155681970)
-1400 val loss 7.1235
-1400 val perplexity 1240.8369
-1400 train 7.073852 (lr=3.7101e-05) (hash(x)=148115934)
-1400 val loss 6.9556
-1400 val perplexity 1049.0596
-1400 train 6.901715 (lr=3.9574e-05) (hash(x)=148115934)
-1500 val loss 7.0624
-1500 val perplexity 1167.1885
-1500 train 7.218080 (lr=3.7032e-05) (hash(x)=157074034)
-1500 val loss 6.8954
-1500 val perplexity 987.7095
-1500 train 7.059405 (lr=3.9501e-05) (hash(x)=157074034)
-1600 val loss 6.9965
-1600 val perplexity 1092.7928
-1600 train 6.758173 (lr=3.6958e-05) (hash(x)=137464699)
-1600 val loss 6.8454
-1600 val perplexity 939.5462
-1600 train 6.603693 (lr=3.9421e-05) (hash(x)=137464699)
-1700 val loss 6.9363
-1700 val perplexity 1028.9852
-1700 train 7.090509 (lr=3.6878e-05) (hash(x)=166955614)
-1700 val loss 6.7900
-1700 val perplexity 888.9067
-1700 train 6.936810 (lr=3.9336e-05) (hash(x)=166955614)
-1800 val loss 6.8773
-1800 val perplexity 970.0369
-1800 train 6.705127 (lr=3.6793e-05) (hash(x)=143886042)
-1800 val loss 6.7448
-1800 val perplexity 849.5964
-1800 train 6.570076 (lr=3.9246e-05) (hash(x)=143886042)
-1900 val loss 6.8304
-1900 val perplexity 925.5649
-1900 train 7.393573 (lr=3.6702e-05) (hash(x)=193610391)
-1900 val loss 6.6976
-1900 val perplexity 810.4745
-1900 train 7.261598 (lr=3.9149e-05) (hash(x)=193610391)
-2000 val loss 6.7689
-2000 val perplexity 870.3190
-2000 train 6.956485 (lr=3.6606e-05) (hash(x)=163799796)
-2000 val loss 6.6535
-2000 val perplexity 775.5157
-2000 train 6.857941 (lr=3.9047e-05) (hash(x)=163799796)
-2100 val loss 6.7018
-2100 val perplexity 813.8666
-2100 train 6.689584 (lr=3.6505e-05) (hash(x)=154107339)
-2100 val loss 6.6103
-2100 val perplexity 742.7250
-2100 train 6.602917 (lr=3.8939e-05) (hash(x)=154107339)
-2200 val loss 6.6605
-2200 val perplexity 780.9216
-2200 train 6.669657 (lr=3.6399e-05) (hash(x)=144592844)
-2200 val loss 6.5823
-2200 val perplexity 722.1735
-2200 train 6.591764 (lr=3.8826e-05) (hash(x)=144592844)
-2300 val loss 6.6079
-2300 val perplexity 740.9298
-2300 train 6.679429 (lr=3.6288e-05) (hash(x)=154535861)
-2300 val loss 6.5377
-2300 val perplexity 690.7051
-2300 train 6.601418 (lr=3.8707e-05) (hash(x)=154535861)
-2400 val loss 6.5631
-2400 val perplexity 708.4941
-2400 train 7.329075 (lr=3.6171e-05) (hash(x)=204706354)
-2400 val loss 6.5092
-2400 val perplexity 671.2576
-2400 train 7.266478 (lr=3.8583e-05) (hash(x)=204706354)
-2500 val loss 6.5323
-2500 val perplexity 687.0010
-2500 train 6.326653 (lr=3.6049e-05) (hash(x)=143783202)
-2500 val loss 6.4879
-2500 val perplexity 657.1377
-2500 train 6.280975 (lr=3.8453e-05) (hash(x)=143783202)
-2600 val loss 6.4941
-2600 val perplexity 661.2271
-2600 train 6.404017 (lr=3.5923e-05) (hash(x)=150862210)
-2600 val loss 6.4599
-2600 val perplexity 638.9831
-2600 train 6.371375 (lr=3.8317e-05) (hash(x)=150862210)
-2700 val loss 6.4670
-2700 val perplexity 643.5612
-2700 train 6.509830 (lr=3.5791e-05) (hash(x)=150753426)
-2700 val loss 6.4312
-2700 val perplexity 620.9432
-2700 train 6.481739 (lr=3.8177e-05) (hash(x)=150753426)
-2800 val loss 6.4464
-2800 val perplexity 630.4233
-2800 train 6.465562 (lr=3.5654e-05) (hash(x)=147766811)
-2800 val loss 6.4168
-2800 val perplexity 612.0283
-2800 train 6.451035 (lr=3.8031e-05) (hash(x)=147766811)
-2900 val loss 6.3979
-2900 val perplexity 600.5582
-2900 train 6.329579 (lr=3.5512e-05) (hash(x)=151177814)
-2900 val loss 6.3885
-2900 val perplexity 594.9806
-2900 train 6.327680 (lr=3.7879e-05) (hash(x)=151177814)
-3000 val loss 6.3747
-3000 val perplexity 586.7963
-3000 train 6.239610 (lr=3.5365e-05) (hash(x)=135925327)
-3000 val loss 6.3632
-3000 val perplexity 580.0998
-3000 train 6.232740 (lr=3.7723e-05) (hash(x)=135925327)
-3100 val loss 6.3508
-3100 val perplexity 572.9315
-3100 train 5.987338 (lr=3.5214e-05) (hash(x)=124842353)
-3100 val loss 6.3534
-3100 val perplexity 574.4473
-3100 train 5.985197 (lr=3.7561e-05) (hash(x)=124842353)
-3200 val loss 6.3339
-3200 val perplexity 563.3701
-3200 train 6.424920 (lr=3.5057e-05) (hash(x)=160440642)
-3200 val loss 6.3508
-3200 val perplexity 572.9645
-3200 train 6.449918 (lr=3.7394e-05) (hash(x)=160440642)
-3300 val loss 6.3022
-3300 val perplexity 545.7853
-3300 train 6.219065 (lr=3.4896e-05) (hash(x)=143017131)
-3300 val loss 6.3124
-3300 val perplexity 551.3576
-3300 train 6.227836 (lr=3.7222e-05) (hash(x)=143017131)
-3400 val loss 6.2772
-3400 val perplexity 532.2734
-3400 train 6.647445 (lr=3.4730e-05) (hash(x)=148143474)
-3400 val loss 6.2929
-3400 val perplexity 540.7349
-3400 train 6.647270 (lr=3.7045e-05) (hash(x)=148143474)
-3500 val loss 6.2591
-3500 val perplexity 522.7372
-3500 train 6.077838 (lr=3.4560e-05) (hash(x)=148732639)
-3500 val loss 6.2809
-3500 val perplexity 534.2861
-3500 train 6.102720 (lr=3.6864e-05) (hash(x)=148732639)
-3600 val loss 6.2419
-3600 val perplexity 513.8419
-3600 train 6.244680 (lr=3.4384e-05) (hash(x)=146333803)
-3600 val loss 6.2661
-3600 val perplexity 526.4097
-3600 train 6.271680 (lr=3.6677e-05) (hash(x)=146333803)
-3700 val loss 6.2261
-3700 val perplexity 505.7911
-3700 train 6.029370 (lr=3.4205e-05) (hash(x)=143611445)
-3700 val loss 6.2551
-3700 val perplexity 520.6425
-3700 train 6.060405 (lr=3.6485e-05) (hash(x)=143611445)
-3800 val loss 6.2239
-3800 val perplexity 504.6608
-3800 train 6.008307 (lr=3.4021e-05) (hash(x)=141272115)
-3800 val loss 6.2480
-3800 val perplexity 516.9984
-3800 train 6.028848 (lr=3.6289e-05) (hash(x)=141272115)
-3900 val loss 6.2016
-3900 val perplexity 493.5485
-3900 train 6.137794 (lr=3.3832e-05) (hash(x)=149161848)
-3900 val loss 6.2293
-3900 val perplexity 507.4062
-3900 train 6.166859 (lr=3.6088e-05) (hash(x)=149161848)
-4000 val loss 6.1724
-4000 val perplexity 479.3447
-4000 train 6.246626 (lr=3.3639e-05) (hash(x)=152036955)
-4000 val loss 6.1951
-4000 val perplexity 490.3334
-4000 train 6.271713 (lr=3.5882e-05) (hash(x)=152036955)
-4100 val loss 6.1561
-4100 val perplexity 471.5683
-4100 train 6.201271 (lr=3.3442e-05) (hash(x)=153260968)
-4100 val loss 6.1798
-4100 val perplexity 482.8908
-4100 train 6.228312 (lr=3.5671e-05) (hash(x)=153260968)
-4200 val loss 6.1402
-4200 val perplexity 464.1679
-4200 train 6.029848 (lr=3.3240e-05) (hash(x)=147629478)
-4200 val loss 6.1666
-4200 val perplexity 476.5579
-4200 train 6.052325 (lr=3.5457e-05) (hash(x)=147629478)
-4300 val loss 6.1300
-4300 val perplexity 459.4489
-4300 train 6.085530 (lr=3.3035e-05) (hash(x)=161701080)
-4300 val loss 6.1532
-4300 val perplexity 470.1998
-4300 train 6.117069 (lr=3.5237e-05) (hash(x)=161701080)
-4400 val loss 6.1198
-4400 val perplexity 454.7519
-4400 train 6.167680 (lr=3.2825e-05) (hash(x)=155290524)
-4400 val loss 6.1466
-4400 val perplexity 467.1368
-4400 train 6.194079 (lr=3.5014e-05) (hash(x)=155290524)
-4500 val loss 6.1049
-4500 val perplexity 448.0491
-4500 train 5.891583 (lr=3.2611e-05) (hash(x)=144726593)
-4500 val loss 6.1316
-4500 val perplexity 460.1760
-4500 train 5.918100 (lr=3.4786e-05) (hash(x)=144726593)
-4600 val loss 6.0951
-4600 val perplexity 443.6972
-4600 train 6.051011 (lr=3.2394e-05) (hash(x)=149517202)
-4600 val loss 6.1236
-4600 val perplexity 456.5213
-4600 train 6.087997 (lr=3.4553e-05) (hash(x)=149517202)
-4700 val loss 6.0744
-4700 val perplexity 434.5847
-4700 train 6.067421 (lr=3.2172e-05) (hash(x)=139403008)
-4700 val loss 6.1027
-4700 val perplexity 447.0701
-4700 train 6.095179 (lr=3.4317e-05) (hash(x)=139403008)
-4800 val loss 6.0584
-4800 val perplexity 427.6721
-4800 train 6.180338 (lr=3.1947e-05) (hash(x)=153387238)
-4800 val loss 6.0834
-4800 val perplexity 438.5213
-4800 train 6.183041 (lr=3.4077e-05) (hash(x)=153387238)
-4900 val loss 6.0543
-4900 val perplexity 425.9605
-4900 train 5.802900 (lr=3.1718e-05) (hash(x)=139881569)
-4900 val loss 6.0811
-4900 val perplexity 437.5304
-4900 train 5.832566 (lr=3.3832e-05) (hash(x)=139881569)
-5000 val loss 6.0322
-5000 val perplexity 416.6431
-5000 train 6.041107 (lr=3.1485e-05) (hash(x)=152602580)
-5000 val loss 6.0648
-5000 val perplexity 430.4250
-5000 train 6.070035 (lr=3.3584e-05) (hash(x)=152602580)
-5100 val loss 6.0235
-5100 val perplexity 413.0090
-5100 train 6.071440 (lr=3.1249e-05) (hash(x)=150598460)
-5100 val loss 6.0549
-5100 val perplexity 426.1752
-5100 train 6.101313 (lr=3.3332e-05) (hash(x)=150598460)
-5200 val loss 6.0116
-5200 val perplexity 408.1312
-5200 train 5.894236 (lr=3.1009e-05) (hash(x)=147660496)
-5200 val loss 6.0430
-5200 val perplexity 421.1597
-5200 train 5.921767 (lr=3.3076e-05) (hash(x)=147660496)
-5300 val loss 6.0084
-5300 val perplexity 406.8307
-5300 train 5.754591 (lr=3.0766e-05) (hash(x)=147693848)
-5300 val loss 6.0379
-5300 val perplexity 419.0294
-5300 train 5.791131 (lr=3.2817e-05) (hash(x)=147693848)
-5400 val loss 6.0008
-5400 val perplexity 403.7434
-5400 train 5.763398 (lr=3.0519e-05) (hash(x)=136048145)
-5400 val loss 6.0402
-5400 val perplexity 419.9767
-5400 train 5.802145 (lr=3.2554e-05) (hash(x)=136048145)
-5500 val loss 5.9803
-5500 val perplexity 395.5671
-5500 train 6.009804 (lr=3.0269e-05) (hash(x)=140313123)
-5500 val loss 6.0176
-5500 val perplexity 410.5718
-5500 train 6.038659 (lr=3.2287e-05) (hash(x)=140313123)
-5600 val loss 5.9709
-5600 val perplexity 391.8647
-5600 train 6.000503 (lr=3.0016e-05) (hash(x)=156997993)
-5600 val loss 6.0068
-5600 val perplexity 406.1997
-5600 train 6.036944 (lr=3.2017e-05) (hash(x)=156997993)
-5700 val loss 5.9577
-5700 val perplexity 386.7048
-5700 train 5.974874 (lr=2.9760e-05) (hash(x)=143513880)
-5700 val loss 5.9976
-5700 val perplexity 402.4466
-5700 train 6.007860 (lr=3.1744e-05) (hash(x)=143513880)
-5800 val loss 5.9488
-5800 val perplexity 383.3037
-5800 train 6.059963 (lr=2.9501e-05) (hash(x)=153736628)
-5800 val loss 5.9862
-5800 val perplexity 397.9112
-5800 train 6.105621 (lr=3.1468e-05) (hash(x)=153736628)
-5900 val loss 5.9390
-5900 val perplexity 379.5628
-5900 train 6.063349 (lr=2.9239e-05) (hash(x)=166385417)
-5900 val loss 5.9854
-5900 val perplexity 397.5974
-5900 train 6.117510 (lr=3.1188e-05) (hash(x)=166385417)
-6000 val loss 5.9255
-6000 val perplexity 374.4518
-6000 train 6.170571 (lr=2.8974e-05) (hash(x)=153216517)
-6000 val loss 5.9711
-6000 val perplexity 391.9264
-6000 train 6.213697 (lr=3.0905e-05) (hash(x)=153216517)
-6100 val loss 5.9246
-6100 val perplexity 374.1172
-6100 train 5.816231 (lr=2.8706e-05) (hash(x)=145387545)
-6100 val loss 5.9652
-6100 val perplexity 389.6248
-6100 train 5.857091 (lr=3.0620e-05) (hash(x)=145387545)
-6200 val loss 5.9235
-6200 val perplexity 373.7118
-6200 train 6.051315 (lr=2.8436e-05) (hash(x)=156306460)
-6200 val loss 5.9607
-6200 val perplexity 387.8851
-6200 train 6.091648 (lr=3.0332e-05) (hash(x)=156306460)
-6300 val loss 5.9015
-6300 val perplexity 365.5843
-6300 train 5.842206 (lr=2.8163e-05) (hash(x)=148610447)
-6300 val loss 5.9386
-6300 val perplexity 379.4168
-6300 train 5.881337 (lr=3.0040e-05) (hash(x)=148610447)
-6400 val loss 5.8869
-6400 val perplexity 360.2831
-6400 train 6.113601 (lr=2.7888e-05) (hash(x)=158343535)
-6400 val loss 5.9284
-6400 val perplexity 375.5528
-6400 train 6.155274 (lr=2.9747e-05) (hash(x)=158343535)
-6500 val loss 5.8778
-6500 val perplexity 357.0185
-6500 train 6.037257 (lr=2.7610e-05) (hash(x)=157742557)
-6500 val loss 5.9186
-6500 val perplexity 371.8734
-6500 train 6.085427 (lr=2.9450e-05) (hash(x)=157742557)
-6600 val loss 5.8700
-6600 val perplexity 354.2516
-6600 train 5.763630 (lr=2.7330e-05) (hash(x)=150457308)
-6600 val loss 5.9109
-6600 val perplexity 369.0442
-6600 train 5.798040 (lr=2.9152e-05) (hash(x)=150457308)
-6700 val loss 5.8640
-6700 val perplexity 352.1355
-6700 train 5.719018 (lr=2.7047e-05) (hash(x)=142426205)
-6700 val loss 5.9085
-6700 val perplexity 368.1502
-6700 train 5.757698 (lr=2.8851e-05) (hash(x)=142426205)
-6800 val loss 5.8662
-6800 val perplexity 352.8958
-6800 train 5.642277 (lr=2.6763e-05) (hash(x)=151522525)
-6800 val loss 5.9020
-6800 val perplexity 365.7698
-6800 train 5.676707 (lr=2.8547e-05) (hash(x)=151522525)
-6900 val loss 5.8505
-6900 val perplexity 347.3940
-6900 train 5.823310 (lr=2.6477e-05) (hash(x)=152029630)
-6900 val loss 5.8897
-6900 val perplexity 361.2955
-6900 train 5.848513 (lr=2.8242e-05) (hash(x)=152029630)
-7000 val loss 5.8346
-7000 val perplexity 341.9149
-7000 train 5.679857 (lr=2.6188e-05) (hash(x)=142978954)
-7000 val loss 5.8743
-7000 val perplexity 355.7711
-7000 train 5.724010 (lr=2.7934e-05) (hash(x)=142978954)
-7100 val loss 5.8294
-7100 val perplexity 340.1654
-7100 train 5.713923 (lr=2.5898e-05) (hash(x)=148064359)
-7100 val loss 5.8667
-7100 val perplexity 353.0793
-7100 train 5.747740 (lr=2.7624e-05) (hash(x)=148064359)
-7200 val loss 5.8227
-7200 val perplexity 337.8991
-7200 train 5.718215 (lr=2.5606e-05) (hash(x)=155771845)
-7200 val loss 5.8588
-7200 val perplexity 350.3069
-7200 train 5.752675 (lr=2.7313e-05) (hash(x)=155771845)
-7300 val loss 5.8232
-7300 val perplexity 338.0643
-7300 train 5.622985 (lr=2.5312e-05) (hash(x)=141639844)
-7300 val loss 5.8545
-7300 val perplexity 348.7894
-7300 train 5.646307 (lr=2.7000e-05) (hash(x)=141639844)
-7400 val loss 5.8208
-7400 val perplexity 337.2388
-7400 train 5.871749 (lr=2.5017e-05) (hash(x)=153466021)
-7400 val loss 5.8575
-7400 val perplexity 349.8495
-7400 train 5.910269 (lr=2.6685e-05) (hash(x)=153466021)
-7500 val loss 5.8062
-7500 val perplexity 332.3530
-7500 train 5.788906 (lr=2.4721e-05) (hash(x)=148843587)
-7500 val loss 5.8403
-7500 val perplexity 343.8840
-7500 train 5.822583 (lr=2.6369e-05) (hash(x)=148843587)
-7600 val loss 5.7932
-7600 val perplexity 328.0590
-7600 train 5.915468 (lr=2.4423e-05) (hash(x)=152053933)
-7600 val loss 5.8307
-7600 val perplexity 340.6115
-7600 train 5.955600 (lr=2.6051e-05) (hash(x)=152053933)
-7700 val loss 5.7872
-7700 val perplexity 326.0890
-7700 train 5.670220 (lr=2.4124e-05) (hash(x)=141564003)
-7700 val loss 5.8225
-7700 val perplexity 337.8119
-7700 train 5.695722 (lr=2.5732e-05) (hash(x)=141564003)
-7800 val loss 5.7776
-7800 val perplexity 322.9746
-7800 train 5.652206 (lr=2.3823e-05) (hash(x)=148617843)
-7800 val loss 5.8148
-7800 val perplexity 335.2256
-7800 train 5.692866 (lr=2.5411e-05) (hash(x)=148617843)
-7900 val loss 5.7755
-7900 val perplexity 322.3173
-7900 train 5.765408 (lr=2.3522e-05) (hash(x)=146417668)
-7900 val loss 5.8111
-7900 val perplexity 333.9737
-7900 train 5.790226 (lr=2.5090e-05) (hash(x)=146417668)
-8000 val loss 5.7700
-8000 val perplexity 320.5481
-8000 train 5.556377 (lr=2.3219e-05) (hash(x)=150165488)
-8000 val loss 5.8064
-8000 val perplexity 332.4193
-8000 train 5.587837 (lr=2.4767e-05) (hash(x)=150165488)
-8100 val loss 5.7600
-8100 val perplexity 317.3409
-8100 train 5.759561 (lr=2.2916e-05) (hash(x)=144818855)
-8100 val loss 5.7962
-8100 val perplexity 329.0320
-8100 train 5.799084 (lr=2.4444e-05) (hash(x)=144818855)
-8200 val loss 5.7594
-8200 val perplexity 317.1577
-8200 train 5.855401 (lr=2.2612e-05) (hash(x)=143074954)
-8200 val loss 5.7938
-8200 val perplexity 328.2616
-8200 train 5.898982 (lr=2.4120e-05) (hash(x)=143074954)
-8300 val loss 5.7483
-8300 val perplexity 313.6542
-8300 train 5.614585 (lr=2.2308e-05) (hash(x)=145172494)
-8300 val loss 5.7821
-8300 val perplexity 324.4247
-8300 train 5.645600 (lr=2.3795e-05) (hash(x)=145172494)
-8400 val loss 5.7403
-8400 val perplexity 311.1508
-8400 train 5.787636 (lr=2.2002e-05) (hash(x)=152284393)
-8400 val loss 5.7760
-8400 val perplexity 322.4603
-8400 train 5.819298 (lr=2.3469e-05) (hash(x)=152284393)
-8500 val loss 5.7384
-8500 val perplexity 310.5595
-8500 train 5.599001 (lr=2.1697e-05) (hash(x)=147044705)
-8500 val loss 5.7701
-8500 val perplexity 320.5808
-8500 train 5.625921 (lr=2.3143e-05) (hash(x)=147044705)
-8600 val loss 5.7350
-8600 val perplexity 309.5278
-8600 train 5.594783 (lr=2.1391e-05) (hash(x)=147762608)
-8600 val loss 5.7662
-8600 val perplexity 319.3280
-8600 train 5.629746 (lr=2.2817e-05) (hash(x)=147762608)
-8700 val loss 5.7243
-8700 val perplexity 306.2231
-8700 train 5.737029 (lr=2.1085e-05) (hash(x)=141994567)
-8700 val loss 5.7582
-8700 val perplexity 316.7644
-8700 train 5.775930 (lr=2.2490e-05) (hash(x)=141994567)
-8800 val loss 5.7175
-8800 val perplexity 304.1554
-8800 train 5.835638 (lr=2.0778e-05) (hash(x)=152552896)
-8800 val loss 5.7513
-8800 val perplexity 314.6034
-8800 train 5.866969 (lr=2.2163e-05) (hash(x)=152552896)
-8900 val loss 5.7123
-8900 val perplexity 302.5586
-8900 train 5.556566 (lr=2.0472e-05) (hash(x)=139714454)
-8900 val loss 5.7443
-8900 val perplexity 312.4159
-8900 train 5.590520 (lr=2.1837e-05) (hash(x)=139714454)
-9000 val loss 5.7048
-9000 val perplexity 300.2996
-9000 train 5.737811 (lr=2.0165e-05) (hash(x)=152274419)
-9000 val loss 5.7367
-9000 val perplexity 310.0361
-9000 train 5.766550 (lr=2.1510e-05) (hash(x)=152274419)
-9100 val loss 5.7037
-9100 val perplexity 299.9624
-9100 train 5.813401 (lr=1.9859e-05) (hash(x)=160536885)
-9100 val loss 5.7347
-9100 val perplexity 309.4271
-9100 train 5.826126 (lr=2.1183e-05) (hash(x)=160536885)
-9200 val loss 5.6971
-9200 val perplexity 298.0017
-9200 train 5.508581 (lr=1.9553e-05) (hash(x)=138386899)
-9200 val loss 5.7295
-9200 val perplexity 307.8032
-9200 train 5.536484 (lr=2.0857e-05) (hash(x)=138386899)
-9300 val loss 5.6941
-9300 val perplexity 297.0971
-9300 train 5.746347 (lr=1.9248e-05) (hash(x)=149957457)
-9300 val loss 5.7248
-9300 val perplexity 306.3625
-9300 train 5.772715 (lr=2.0531e-05) (hash(x)=149957457)
-9400 val loss 5.6848
-9400 val perplexity 294.3666
-9400 train 5.670947 (lr=1.8942e-05) (hash(x)=161671548)
-9400 val loss 5.7138
-9400 val perplexity 303.0352
-9400 train 5.702243 (lr=2.0205e-05) (hash(x)=161671548)
-9500 val loss 5.6870
-9500 val perplexity 295.0154
-9500 train 5.817723 (lr=1.8638e-05) (hash(x)=171857400)
-9500 val loss 5.7160
-9500 val perplexity 303.6863
-9500 train 5.834106 (lr=1.9880e-05) (hash(x)=171857400)
-9600 val loss 5.6747
-9600 val perplexity 291.4071
-9600 train 5.456934 (lr=1.8334e-05) (hash(x)=135378471)
-9600 val loss 5.7046
-9600 val perplexity 300.2393
-9600 train 5.474633 (lr=1.9556e-05) (hash(x)=135378471)
-9700 val loss 5.6693
-9700 val perplexity 289.8326
-9700 train 5.607586 (lr=1.8031e-05) (hash(x)=149514591)
-9700 val loss 5.7020
-9700 val perplexity 299.4644
-9700 train 5.642845 (lr=1.9233e-05) (hash(x)=149514591)
-9800 val loss 5.6663
-9800 val perplexity 288.9694
-9800 train 5.640245 (lr=1.7728e-05) (hash(x)=152679072)
-9800 val loss 5.6972
-9800 val perplexity 298.0387
-9800 train 5.668010 (lr=1.8910e-05) (hash(x)=152679072)
-9900 val loss 5.6595
-9900 val perplexity 287.0002
-9900 train 5.624960 (lr=1.7427e-05) (hash(x)=150415193)
-9900 val loss 5.6929
-9900 val perplexity 296.7478
-9900 train 5.654785 (lr=1.8589e-05) (hash(x)=150415193)
-10000 val loss 5.6569
-10000 val perplexity 286.2522
-10000 val loss 5.6856
-10000 val perplexity 294.5803
-10000 train 5.507225 (lr=1.7126e-05) (hash(x)=146910087)
-10000 train 5.539617 (lr=1.8268e-05) (hash(x)=146910087)
-10100 val loss 5.6824
-10100 val perplexity 293.6487
-10100 val loss 5.6513
-10100 val perplexity 284.6697
-10100 train 5.601520 (lr=1.7949e-05) (hash(x)=152091050)
-10100 train 5.568396 (lr=1.6827e-05) (hash(x)=152091050)
-10200 val loss 5.6810
-10200 val perplexity 293.2429
-10200 val loss 5.6508
-10200 val perplexity 284.5271
-10200 train 5.529841 (lr=1.7631e-05) (hash(x)=143101306)
-10200 train 5.501368 (lr=1.6529e-05) (hash(x)=143101306)
-10300 val loss 5.6724
-10300 val perplexity 290.7270
-10300 train 5.686642 (lr=1.7315e-05) (hash(x)=150803985)
-10300 val loss 5.6468
-10300 val perplexity 283.3883
-10300 train 5.661108 (lr=1.6233e-05) (hash(x)=150803985)
-10400 val loss 5.6745
-10400 val perplexity 291.3510
-10400 train 5.633561 (lr=1.7000e-05) (hash(x)=148671575)
-10400 val loss 5.6431
-10400 val perplexity 282.3257
-10400 train 5.613005 (lr=1.5938e-05) (hash(x)=148671575)
-10500 val loss 5.6646
-10500 val perplexity 288.4691
-10500 train 5.477703 (lr=1.6687e-05) (hash(x)=146312794)
-10500 val loss 5.6361
-10500 val perplexity 280.3604
-10500 train 5.438739 (lr=1.5644e-05) (hash(x)=146312794)
-10600 val loss 5.6643
-10600 val perplexity 288.3757
-10600 train 5.490323 (lr=1.6376e-05) (hash(x)=137878985)
-10600 val loss 5.6384
-10600 val perplexity 281.0232
-10600 train 5.461352 (lr=1.5352e-05) (hash(x)=137878985)
-10700 val loss 5.6600
-10700 val perplexity 287.1475
-10700 train 5.472318 (lr=1.6066e-05) (hash(x)=148897486)
-10700 val loss 5.6302
-10700 val perplexity 278.7291
-10700 train 5.449248 (lr=1.5062e-05) (hash(x)=148897486)
-10800 val loss 5.6559
-10800 val perplexity 285.9793
-10800 train 5.805680 (lr=1.5758e-05) (hash(x)=147219580)
-10800 val loss 5.6267
-10800 val perplexity 277.7428
-10800 train 5.778777 (lr=1.4773e-05) (hash(x)=147219580)
-10900 val loss 5.6570
-10900 val perplexity 286.2779
-10900 train 5.681130 (lr=1.5453e-05) (hash(x)=152048929)
-10900 val loss 5.6266
-10900 val perplexity 277.7201
-10900 train 5.650803 (lr=1.4487e-05) (hash(x)=152048929)
-11000 val loss 5.6419
-11000 val perplexity 282.0003
-11000 train 5.529566 (lr=1.5149e-05) (hash(x)=146417066)
-11000 val loss 5.6135
-11000 val perplexity 274.1082
-11000 train 5.504382 (lr=1.4203e-05) (hash(x)=146417066)
-11100 val loss 5.6401
-11100 val perplexity 281.4987
-11100 train 5.493770 (lr=1.4848e-05) (hash(x)=148784436)
-11100 val loss 5.6129
-11100 val perplexity 273.9335
-11100 train 5.457813 (lr=1.3920e-05) (hash(x)=148784436)
-11200 val loss 5.6382
-11200 val perplexity 280.9495
-11200 train 5.628141 (lr=1.4550e-05) (hash(x)=144721870)
-11200 val loss 5.6098
-11200 val perplexity 273.0796
-11200 train 5.594026 (lr=1.3640e-05) (hash(x)=144721870)
-11300 val loss 5.6353
-11300 val perplexity 280.1469
-11300 train 5.545784 (lr=1.4253e-05) (hash(x)=149442981)
-11300 val loss 5.6067
-11300 val perplexity 272.2455
-11300 train 5.517210 (lr=1.3362e-05) (hash(x)=149442981)
-11400 val loss 5.6329
-11400 val perplexity 279.4653
-11400 train 5.514412 (lr=1.3960e-05) (hash(x)=154481228)
-11400 val loss 5.6055
-11400 val perplexity 271.9057
-11400 train 5.480088 (lr=1.3087e-05) (hash(x)=154481228)
-11500 val loss 5.6285
-11500 val perplexity 278.2540
-11500 train 5.319666 (lr=1.3668e-05) (hash(x)=115164581)
-11500 val loss 5.6020
-11500 val perplexity 270.9755
-11500 train 5.307278 (lr=1.2814e-05) (hash(x)=115164581)
-11600 val loss 5.6203
-11600 val perplexity 275.9729
-11600 train 5.839368 (lr=1.3380e-05) (hash(x)=155048082)
-11600 val loss 5.5953
-11600 val perplexity 269.1717
-11600 train 5.827364 (lr=1.2544e-05) (hash(x)=155048082)
-11700 val loss 5.6196
-11700 val perplexity 275.7854
-11700 train 5.684441 (lr=1.3095e-05) (hash(x)=152123770)
-11700 val loss 5.5941
-11700 val perplexity 268.8488
-11700 train 5.663855 (lr=1.2276e-05) (hash(x)=152123770)
-11800 val loss 5.6168
-11800 val perplexity 275.0091
-11800 train 5.545686 (lr=1.2812e-05) (hash(x)=152705857)
-11800 val loss 5.5908
-11800 val perplexity 267.9501
-11800 train 5.517049 (lr=1.2011e-05) (hash(x)=152705857)
-11900 val loss 5.6153
-11900 val perplexity 274.5948
-11900 train 5.349757 (lr=1.2532e-05) (hash(x)=137392362)
-11900 val loss 5.5877
-11900 val perplexity 267.1139
-11900 train 5.319537 (lr=1.1749e-05) (hash(x)=137392362)
-12000 val loss 5.6149
-12000 val perplexity 274.4831
-12000 train 5.422106 (lr=1.2256e-05) (hash(x)=143492562)
-12000 val loss 5.5900
-12000 val perplexity 267.7459
-12000 train 5.393394 (lr=1.1490e-05) (hash(x)=143492562)
-12100 val loss 5.6086
-12100 val perplexity 272.7590
-12100 train 5.589555 (lr=1.1983e-05) (hash(x)=141825431)
-12100 val loss 5.5840
-12100 val perplexity 266.1208
-12100 train 5.570032 (lr=1.1234e-05) (hash(x)=141825431)
-12200 val loss 5.6035
-12200 val perplexity 271.3845
-12200 train 5.514888 (lr=1.1713e-05) (hash(x)=141774370)
-12200 val loss 5.5780
-12200 val perplexity 264.5513
-12200 train 5.495852 (lr=1.0981e-05) (hash(x)=141774370)
-12300 val loss 5.5996
-12300 val perplexity 270.3284
-12300 train 5.632575 (lr=1.1446e-05) (hash(x)=152995173)
-12300 val loss 5.5736
-12300 val perplexity 263.3873
-12300 train 5.604387 (lr=1.0731e-05) (hash(x)=152995173)
-12400 val loss 5.6003
-12400 val perplexity 270.5031
-12400 train 5.425294 (lr=1.1183e-05) (hash(x)=144703471)
-12400 val loss 5.5743
-12400 val perplexity 263.5755
-12400 train 5.402729 (lr=1.0484e-05) (hash(x)=144703471)
-12500 val loss 5.5969
-12500 val perplexity 269.5911
-12500 train 5.497826 (lr=1.0924e-05) (hash(x)=144866179)
-12500 val loss 5.5720
-12500 val perplexity 262.9684
-12500 train 5.477533 (lr=1.0241e-05) (hash(x)=144866179)
-12600 val loss 5.5966
-12600 val perplexity 269.4970
-12600 train 5.607377 (lr=1.0668e-05) (hash(x)=157019663)
-12600 val loss 5.5686
-12600 val perplexity 262.0779
-12600 train 5.579659 (lr=1.0001e-05) (hash(x)=157019663)
-12700 val loss 5.5925
-12700 val perplexity 268.3951
-12700 train 5.546249 (lr=1.0416e-05) (hash(x)=152872020)
-12700 val loss 5.5681
-12700 val perplexity 261.9382
-12700 train 5.541395 (lr=9.7649e-06) (hash(x)=152872020)
-12800 val loss 5.5872
-12800 val perplexity 266.9830
-12800 train 5.617934 (lr=1.0168e-05) (hash(x)=158975636)
-12800 val loss 5.5627
-12800 val perplexity 260.5228
-12800 train 5.591032 (lr=9.5322e-06) (hash(x)=158975636)
-12900 val loss 5.5833
-12900 val perplexity 265.9386
-12900 train 5.631866 (lr=9.9233e-06) (hash(x)=143495701)
-12900 val loss 5.5572
-12900 val perplexity 259.1091
-12900 train 5.616541 (lr=9.3031e-06) (hash(x)=143495701)
-13000 val loss 5.5830
-13000 val perplexity 265.8771
-13000 train 5.510010 (lr=9.6829e-06) (hash(x)=151940005)
-13000 val loss 5.5570
-13000 val perplexity 259.0425
-13000 train 5.482976 (lr=9.0777e-06) (hash(x)=151940005)
-13100 val loss 5.5801
-13100 val perplexity 265.1006
-13100 train 5.866326 (lr=9.4466e-06) (hash(x)=155920713)
-13100 val loss 5.5544
-13100 val perplexity 258.3700
-13100 train 5.851393 (lr=8.8562e-06) (hash(x)=155920713)
-13200 val loss 5.5774
-13200 val perplexity 264.3724
-13200 train 5.770224 (lr=9.2144e-06) (hash(x)=152145878)
-13200 val loss 5.5530
-13200 val perplexity 258.0057
-13200 train 5.751560 (lr=8.6385e-06) (hash(x)=152145878)
-13300 val loss 5.5752
-13300 val perplexity 263.8147
-13300 train 5.402647 (lr=8.9865e-06) (hash(x)=134141709)
-13300 val loss 5.5511
-13300 val perplexity 257.5152
-13300 train 5.376328 (lr=8.4248e-06) (hash(x)=134141709)
-13400 val loss 5.5769
-13400 val perplexity 264.2384
-13400 train 5.506146 (lr=8.7628e-06) (hash(x)=146096299)
-13400 val loss 5.5543
-13400 val perplexity 258.3470
-13400 train 5.476796 (lr=8.2151e-06) (hash(x)=146096299)
-13500 val loss 5.5751
-13500 val perplexity 263.7630
-13500 train 5.330158 (lr=8.5435e-06) (hash(x)=144011431)
-13500 val loss 5.5502
-13500 val perplexity 257.2935
-13500 train 5.308886 (lr=8.0095e-06) (hash(x)=144011431)
-13600 val loss 5.5676
-13600 val perplexity 261.8164
-13600 train 5.802654 (lr=8.3286e-06) (hash(x)=163950932)
-13600 val loss 5.5438
-13600 val perplexity 255.6573
-13600 train 5.783780 (lr=7.8081e-06) (hash(x)=163950932)
-13700 val loss 5.5651
-13700 val perplexity 261.1624
-13700 train 5.712312 (lr=8.1183e-06) (hash(x)=156652406)
-13700 val loss 5.5410
-13700 val perplexity 254.9451
-13700 train 5.686758 (lr=7.6109e-06) (hash(x)=156652406)
-13800 val loss 5.5605
-13800 val perplexity 259.9570
-13800 train 5.476258 (lr=7.9125e-06) (hash(x)=145753166)
-13800 val loss 5.5367
-13800 val perplexity 253.8321
-13800 train 5.461190 (lr=7.4180e-06) (hash(x)=145753166)
-13900 val loss 5.5595
-13900 val perplexity 259.6913
-13900 train 5.424074 (lr=7.7114e-06) (hash(x)=147065551)
-13900 val loss 5.5352
-13900 val perplexity 253.4479
-13900 train 5.404427 (lr=7.2294e-06) (hash(x)=147065551)
-14000 val loss 5.5569
-14000 val perplexity 259.0065
-14000 train 6.028049 (lr=7.5149e-06) (hash(x)=179077542)
-14000 val loss 5.5337
-14000 val perplexity 253.0728
-14000 train 6.001717 (lr=7.0452e-06) (hash(x)=179077542)
-14100 val loss 5.5554
-14100 val perplexity 258.6318
-14100 train 5.623073 (lr=7.3233e-06) (hash(x)=171962802)
-14100 val loss 5.5331
-14100 val perplexity 252.9184
-14100 train 5.588490 (lr=6.8656e-06) (hash(x)=171962802)
-14200 val loss 5.5564
-14200 val perplexity 258.8788
-14200 train 5.642525 (lr=7.1365e-06) (hash(x)=153299913)
-14200 val loss 5.5331
-14200 val perplexity 252.9215
-14200 train 5.619051 (lr=6.6904e-06) (hash(x)=153299913)
-14300 val loss 5.5553
-14300 val perplexity 258.6111
-14300 train 5.481285 (lr=6.9546e-06) (hash(x)=154750565)
-14300 val loss 5.5332
-14300 val perplexity 252.9503
-14300 train 5.454321 (lr=6.5199e-06) (hash(x)=154750565)
-14400 val loss 5.5537
-14400 val perplexity 258.1823
-14400 train 5.830606 (lr=6.7776e-06) (hash(x)=154536694)
-14400 val loss 5.5308
-14400 val perplexity 252.3419
-14400 train 5.808904 (lr=6.3540e-06) (hash(x)=154536694)
-14500 val loss 5.5497
-14500 val perplexity 257.1502
-14500 train 5.613526 (lr=6.6057e-06) (hash(x)=151236383)
-14500 val loss 5.5264
-14500 val perplexity 251.2405
-14500 train 5.590186 (lr=6.1928e-06) (hash(x)=151236383)
-14600 val loss 5.5451
-14600 val perplexity 255.9854
-14600 train 5.570751 (lr=6.4389e-06) (hash(x)=152584272)
-14600 val loss 5.5217
-14600 val perplexity 250.0630
-14600 train 5.553623 (lr=6.0364e-06) (hash(x)=152584272)
-14700 val loss 5.5435
-14700 val perplexity 255.5691
-14700 train 5.501938 (lr=6.2771e-06) (hash(x)=175301826)
-14700 val loss 5.5211
-14700 val perplexity 249.9111
-14700 train 5.493054 (lr=5.8848e-06) (hash(x)=175301826)
-14800 val loss 5.5438
-14800 val perplexity 255.6586
-14800 train 5.460385 (lr=6.1206e-06) (hash(x)=148729408)
-14800 val loss 5.5212
-14800 val perplexity 249.9372
-14800 train 5.445827 (lr=5.7381e-06) (hash(x)=148729408)
-14900 val loss 5.5426
-14900 val perplexity 255.3284
-14900 train 5.746192 (lr=5.9693e-06) (hash(x)=150978283)
-14900 val loss 5.5200
-14900 val perplexity 249.6369
-14900 train 5.723087 (lr=5.5962e-06) (hash(x)=150978283)
-15000 val loss 5.5426
-15000 val perplexity 255.3446
-15000 train 5.486250 (lr=5.8233e-06) (hash(x)=153621389)
-15000 val loss 5.5219
-15000 val perplexity 250.1049
-15000 train 5.470625 (lr=5.4593e-06) (hash(x)=153621389)
-15100 val loss 5.5417
-15100 val perplexity 255.1073
-15100 train 5.458971 (lr=5.6826e-06) (hash(x)=150653793)
-15100 val loss 5.5205
-15100 val perplexity 249.7702
-15100 train 5.437407 (lr=5.3275e-06) (hash(x)=150653793)
-15200 val loss 5.5392
-15200 val perplexity 254.4813
-15200 train 5.726518 (lr=5.5473e-06) (hash(x)=140802934)
-15200 val loss 5.5179
-15200 val perplexity 249.0995
-15200 train 5.714012 (lr=5.2006e-06) (hash(x)=140802934)
-15300 val loss 5.5347
-15300 val perplexity 253.3340
-15300 train 5.516016 (lr=5.4175e-06) (hash(x)=152007556)
-15300 val loss 5.5123
-15300 val perplexity 247.7215
-15300 train 5.528864 (lr=5.0789e-06) (hash(x)=152007556)
-15400 val loss 5.5336
-15400 val perplexity 253.0472
-15400 train 5.484604 (lr=5.2931e-06) (hash(x)=149057446)
-15400 val loss 5.5114
-15400 val perplexity 247.4870
-15400 train 5.471828 (lr=4.9622e-06) (hash(x)=149057446)
-15500 val loss 5.5319
-15500 val perplexity 252.6117
-15500 train 5.448268 (lr=5.1742e-06) (hash(x)=147521774)
-15500 val loss 5.5098
-15500 val perplexity 247.1077
-15500 train 5.432381 (lr=4.8508e-06) (hash(x)=147521774)
-15600 val loss 5.5318
-15600 val perplexity 252.5913
-15600 train 5.508049 (lr=5.0608e-06) (hash(x)=148944405)
-15600 val loss 5.5085
-15600 val perplexity 246.7899
-15600 train 5.490562 (lr=4.7445e-06) (hash(x)=148944405)
-15700 val loss 5.5308
-15700 val perplexity 252.3476
-15700 train 5.481038 (lr=4.9531e-06) (hash(x)=148472532)
-15700 val loss 5.5086
-15700 val perplexity 246.8106
-15700 train 5.453821 (lr=4.6435e-06) (hash(x)=148472532)
-15800 val loss 5.5300
-15800 val perplexity 252.1432
-15800 train 5.296927 (lr=4.8509e-06) (hash(x)=149048780)
-15800 val loss 5.5086
-15800 val perplexity 246.8076
-15800 train 5.272242 (lr=4.5478e-06) (hash(x)=149048780)
-15900 val loss 5.5318
-15900 val perplexity 252.5956
-15900 train 5.496135 (lr=4.7545e-06) (hash(x)=150175778)
-15900 val loss 5.5102
-15900 val perplexity 247.2009
-15900 train 5.477364 (lr=4.4573e-06) (hash(x)=150175778)
-16000 val loss 5.5324
-16000 val perplexity 252.7528
-16000 train 5.306297 (lr=4.6637e-06) (hash(x)=145943831)
-16000 val loss 5.5102
-16000 val perplexity 247.2036
-16000 train 5.279723 (lr=4.3722e-06) (hash(x)=145943831)
-16100 val loss 5.5258
-16100 val perplexity 251.0751
-16100 train 5.708304 (lr=4.5786e-06) (hash(x)=134314842)
-16100 val loss 5.5040
-16100 val perplexity 245.6816
-16100 train 5.692306 (lr=4.2924e-06) (hash(x)=134314842)
-16200 val loss 5.5230
-16200 val perplexity 250.3804
-16200 train 5.557369 (lr=4.4993e-06) (hash(x)=153755135)
-16200 val loss 5.5016
-16200 val perplexity 245.0840
-16200 train 5.526541 (lr=4.2180e-06) (hash(x)=153755135)
-16300 val loss 5.5230
-16300 val perplexity 250.3745
-16300 train 5.654873 (lr=4.4257e-06) (hash(x)=146916346)
-16300 val loss 5.5019
-16300 val perplexity 245.1468
-16300 train 5.636758 (lr=4.1491e-06) (hash(x)=146916346)
-16400 val loss 5.5236
-16400 val perplexity 250.5255
-16400 train 5.434968 (lr=4.3579e-06) (hash(x)=155023643)
-16400 val loss 5.5035
-16400 val perplexity 245.5604
-16400 train 5.417802 (lr=4.0856e-06) (hash(x)=155023643)
-16500 val loss 5.5213
-16500 val perplexity 249.9600
-16500 train 5.613109 (lr=4.2960e-06) (hash(x)=148910191)
-16500 val loss 5.5012
-16500 val perplexity 244.9902
-16500 train 5.604181 (lr=4.0275e-06) (hash(x)=148910191)
-16600 val loss 5.5208
-16600 val perplexity 249.8469
-16600 train 5.450554 (lr=4.2399e-06) (hash(x)=150107746)
-16600 val loss 5.5008
-16600 val perplexity 244.8920
-16600 train 5.428391 (lr=3.9749e-06) (hash(x)=150107746)
-16700 val loss 5.5178
-16700 val perplexity 249.0921
-16700 train 5.644423 (lr=4.1896e-06) (hash(x)=159943976)
-16700 val loss 5.4975
-16700 val perplexity 244.0741
-16700 train 5.637831 (lr=3.9278e-06) (hash(x)=159943976)
-16800 val loss 5.5155
-16800 val perplexity 248.5153
-16800 train 5.501436 (lr=4.1452e-06) (hash(x)=160255226)
-16800 val loss 5.4954
-16800 val perplexity 243.5715
-16800 train 5.489238 (lr=3.8862e-06) (hash(x)=160255226)
-16900 val loss 5.5147
-16900 val perplexity 248.3036
-16900 train 5.508539 (lr=4.1067e-06) (hash(x)=157121826)
-16900 val loss 5.4945
-16900 val perplexity 243.3470
-16900 train 5.483582 (lr=3.8501e-06) (hash(x)=157121826)
-17000 val loss 5.5137
-17000 val perplexity 248.0701
-17000 train 5.639942 (lr=4.0741e-06) (hash(x)=159298029)
-17000 val loss 5.4932
-17000 val perplexity 243.0335
-17000 train 5.625374 (lr=3.8195e-06) (hash(x)=159298029)
-17100 val loss 5.5142
-17100 val perplexity 248.1973
-17100 train 5.479031 (lr=4.0475e-06) (hash(x)=149355903)
-17100 val loss 5.4936
-17100 val perplexity 243.1309
-17100 train 5.459307 (lr=3.7945e-06) (hash(x)=149355903)
-17200 val loss 5.5136
-17200 val perplexity 248.0547
-17200 train 5.324676 (lr=4.0267e-06) (hash(x)=144467909)
-17200 val loss 5.4942
-17200 val perplexity 243.2700
-17200 train 5.309157 (lr=3.7750e-06) (hash(x)=144467909)
-17300 val loss 5.5118
-17300 val perplexity 247.5893
-17300 train 5.678904 (lr=4.0119e-06) (hash(x)=157150350)
-17300 val loss 5.4913
-17300 val perplexity 242.5806
-17300 train 5.669414 (lr=3.7611e-06) (hash(x)=157150350)
-17400 val loss 5.5102
-17400 val perplexity 247.2052
-17400 train 5.547256 (lr=4.0030e-06) (hash(x)=154825212)
-17400 val loss 5.4896
-17400 val perplexity 242.1596
-17400 train 5.547105 (lr=3.7528e-06) (hash(x)=154825212)
-17499 val loss 5.5081
-17499 val perplexity 246.6936
-17499 val loss 5.4884
-17499 val perplexity 241.8788
+max_steps: 8750
+1900 val loss 6.4169
+1900 val perplexity 612.1060
+1900 train 6.250515 (lr=3.7502e-05) (hash(x)=149434659)
+0 val loss 11.3095
+0 val perplexity 81594.2500
+2000 val loss 6.3543
+2000 val perplexity 574.9713
+2000 train 6.255687 (lr=3.7143e-05) (hash(x)=152285486)
+0 train 11.313040 (lr=1.0000e-07) (hash(x)=145079536)
+2100 val loss 6.3165
+2100 val perplexity 553.6381
+2100 train 6.064629 (lr=3.6761e-05) (hash(x)=144294295)
+100 val loss 10.0523
+100 val perplexity 23209.7812
+100 train 10.045820 (lr=1.0100e-05) (hash(x)=155800595)
+2200 val loss 6.2910
+2200 val perplexity 539.7109
+2200 train 6.295984 (lr=3.6358e-05) (hash(x)=175030215)
+200 val loss 9.5099
+200 val perplexity 13492.4014
+200 train 9.462710 (lr=2.0100e-05) (hash(x)=145606733)
+2300 val loss 6.2671
+2300 val perplexity 526.9221
+2300 train 6.077068 (lr=3.5935e-05) (hash(x)=150831428)
+300 val loss 8.3310
+300 val perplexity 4150.4053
+300 train 8.320391 (lr=3.0100e-05) (hash(x)=150367139)
+2400 val loss 6.2306
+2400 val perplexity 508.0812
+2400 train 6.415508 (lr=3.5491e-05) (hash(x)=140808297)
+400 val loss 7.6708
+400 val perplexity 2144.7644
+400 train 7.970377 (lr=4.0100e-05) (hash(x)=155747374)
+2500 val loss 6.2058
+2500 val perplexity 495.6054
+2500 train 6.186115 (lr=3.5027e-05) (hash(x)=153160275)
+500 val loss 7.5037
+500 val perplexity 1814.6550
+500 train 7.391925 (lr=5.0000e-05) (hash(x)=140604760)
+2600 val loss 6.1802
+2600 val perplexity 483.0939
+2600 train 5.962604 (lr=3.4545e-05) (hash(x)=133990623)
+600 val loss 7.4009
+600 val perplexity 1637.5042
+600 train 7.347689 (lr=4.9984e-05) (hash(x)=148404734)
+2700 val loss 6.1672
+2700 val perplexity 476.8687
+2700 train 6.135607 (lr=3.4044e-05) (hash(x)=142860944)
+700 val loss 7.3296
+700 val perplexity 1524.7316
+700 train 7.283709 (lr=4.9935e-05) (hash(x)=148115934)
+2800 val loss 6.1343
+2800 val perplexity 461.4319
+2800 train 6.149925 (lr=3.3526e-05) (hash(x)=137959511)
+800 val loss 7.2290
+800 val perplexity 1378.7905
+800 train 7.112580 (lr=4.9853e-05) (hash(x)=137464699)
+2900 val loss 6.1097
+2900 val perplexity 450.2144
+2900 train 6.063270 (lr=3.2992e-05) (hash(x)=147009873)
+900 val loss 7.1585
+900 val perplexity 1284.9253
+900 train 6.972766 (lr=4.9739e-05) (hash(x)=143886042)
+3000 val loss 6.0910
+3000 val perplexity 441.8452
+3000 train 5.950695 (lr=3.2441e-05) (hash(x)=158264841)
+1000 val loss 7.0824
+1000 val perplexity 1190.8246
+1000 train 7.084828 (lr=4.9593e-05) (hash(x)=163799796)
+3100 val loss 6.0868
+3100 val perplexity 440.0080
+3100 train 5.932418 (lr=3.1875e-05) (hash(x)=139232251)
+1100 val loss 7.0104
+1100 val perplexity 1108.0588
+1100 train 6.966660 (lr=4.9415e-05) (hash(x)=144592844)
+3200 val loss 6.0559
+3200 val perplexity 426.6234
+3200 train 6.098489 (lr=3.1295e-05) (hash(x)=153436104)
+1200 val loss 6.9273
+1200 val perplexity 1019.7122
+1200 train 7.257290 (lr=4.9205e-05) (hash(x)=204706354)
+3300 val loss 6.0380
+3300 val perplexity 419.0717
+3300 train 5.860431 (lr=3.0702e-05) (hash(x)=149681831)
+1300 val loss 6.8551
+1300 val perplexity 948.7275
+1300 train 6.785756 (lr=4.8964e-05) (hash(x)=150862210)
+3400 val loss 6.0271
+3400 val perplexity 414.5083
+3400 train 5.904068 (lr=3.0095e-05) (hash(x)=168425516)
+1400 val loss 6.7796
+1400 val perplexity 879.7139
+1400 train 6.770991 (lr=4.8691e-05) (hash(x)=147766811)
+3500 val loss 6.0053
+3500 val perplexity 405.5786
+3500 train 6.031878 (lr=2.9477e-05) (hash(x)=163104338)
+1500 val loss 6.6920
+1500 val perplexity 805.9559
+1500 train 6.509122 (lr=4.8388e-05) (hash(x)=135925327)
+3600 val loss 5.9910
+3600 val perplexity 399.8305
+3600 train 5.969259 (lr=2.8849e-05) (hash(x)=165109772)
+1600 val loss 6.6294
+1600 val perplexity 757.0620
+1600 train 6.598938 (lr=4.8055e-05) (hash(x)=160440642)
+3700 val loss 5.9877
+3700 val perplexity 398.4793
+3700 train 5.721743 (lr=2.8210e-05) (hash(x)=153420306)
+1700 val loss 6.5596
+1700 val perplexity 705.9973
+1700 train 6.660162 (lr=4.7691e-05) (hash(x)=151184106)
+3800 val loss 5.9577
+3800 val perplexity 386.7325
+3800 train 5.977221 (lr=2.7562e-05) (hash(x)=160168863)
+1800 val loss 6.5071
+1800 val perplexity 669.8593
+1800 train 6.396586 (lr=4.7299e-05) (hash(x)=148100580)
+3900 val loss 5.9497
+3900 val perplexity 383.6512
+3900 train 5.866309 (lr=2.6907e-05) (hash(x)=153906073)
+1900 val loss 6.4710
+1900 val perplexity 646.0992
+1900 train 6.298414 (lr=4.6878e-05) (hash(x)=149434659)
+4000 val loss 5.9363
+4000 val perplexity 378.5279
+4000 train 5.876613 (lr=2.6244e-05) (hash(x)=151055067)
+2000 val loss 6.4085
+2000 val perplexity 606.9621
+2000 train 6.308722 (lr=4.6428e-05) (hash(x)=152285486)
+4100 val loss 5.9223
+4100 val perplexity 373.2578
+4100 train 5.903517 (lr=2.5575e-05) (hash(x)=149629830)
+2100 val loss 6.3651
+2100 val perplexity 581.1993
+2100 train 6.112900 (lr=4.5951e-05) (hash(x)=144294295)
+4200 val loss 5.9090
+4200 val perplexity 368.3522
+4200 train 5.792145 (lr=2.4900e-05) (hash(x)=143101381)
+2200 val loss 6.3310
+2200 val perplexity 561.7220
+2200 train 6.346328 (lr=4.5448e-05) (hash(x)=175030215)
+4300 val loss 5.9024
+4300 val perplexity 365.9272
+4300 train 5.808477 (lr=2.4222e-05) (hash(x)=149712044)
+2300 val loss 6.3039
+2300 val perplexity 546.6758
+2300 train 6.113389 (lr=4.4918e-05) (hash(x)=150831428)
+4400 val loss 5.8847
+4400 val perplexity 359.4937
+4400 train 5.809499 (lr=2.3540e-05) (hash(x)=153446449)
+2400 val loss 6.2631
+2400 val perplexity 524.8505
+2400 train 6.434492 (lr=4.4363e-05) (hash(x)=140808297)
+4500 val loss 5.8707
+4500 val perplexity 354.4794
+4500 train 5.820339 (lr=2.2856e-05) (hash(x)=146086947)
+2500 val loss 6.2334
+2500 val perplexity 509.4646
+2500 train 6.218014 (lr=4.3784e-05) (hash(x)=153160275)
+4600 val loss 5.8615
+4600 val perplexity 351.2458
+4600 train 5.702044 (lr=2.2171e-05) (hash(x)=153800173)
+2600 val loss 6.2117
+2600 val perplexity 498.5354
+2600 train 5.992179 (lr=4.3181e-05) (hash(x)=133990623)
+4700 val loss 5.8508
+4700 val perplexity 347.5017
+4700 train 5.917328 (lr=2.1486e-05) (hash(x)=155962726)
+2700 val loss 6.1884
+2700 val perplexity 487.0828
+2700 train 6.157396 (lr=4.2555e-05) (hash(x)=142860944)
+4800 val loss 5.8399
+4800 val perplexity 343.7307
+4800 train 6.078052 (lr=2.0801e-05) (hash(x)=142045616)
+2800 val loss 6.1528
+2800 val perplexity 470.0521
+2800 train 6.166156 (lr=4.1908e-05) (hash(x)=137959511)
+4900 val loss 5.8263
+4900 val perplexity 339.1056
+4900 train 5.799638 (lr=2.0118e-05) (hash(x)=143418248)
+2900 val loss 6.1300
+2900 val perplexity 459.4257
+2900 train 6.085161 (lr=4.1240e-05) (hash(x)=147009873)
+5000 val loss 5.8207
+5000 val perplexity 337.2140
+5000 train 5.740273 (lr=1.9438e-05) (hash(x)=145789790)
+3000 val loss 6.1100
+3000 val perplexity 450.3489
+3000 train 5.963087 (lr=4.0551e-05) (hash(x)=158264841)
+5100 val loss 5.8133
+5100 val perplexity 334.7339
+5100 train 5.792392 (lr=1.8762e-05) (hash(x)=137795633)
+3100 val loss 6.0980
+3100 val perplexity 444.9706
+3100 train 5.941044 (lr=3.9844e-05) (hash(x)=139232251)
+5200 val loss 5.8031
+5200 val perplexity 331.3376
+5200 train 5.965540 (lr=1.8090e-05) (hash(x)=148907132)
+3200 val loss 6.0665
+3200 val perplexity 431.1898
+3200 train 6.106855 (lr=3.9119e-05) (hash(x)=153436104)
+5300 val loss 5.7981
+5300 val perplexity 329.6798
+5300 train 5.675566 (lr=1.7424e-05) (hash(x)=152343580)
+3300 val loss 6.0532
+3300 val perplexity 425.4629
+3300 train 5.876822 (lr=3.8377e-05) (hash(x)=149681831)
+5400 val loss 5.7863
+5400 val perplexity 325.8111
+5400 train 5.640113 (lr=1.6765e-05) (hash(x)=148578264)
+3400 val loss 6.0428
+3400 val perplexity 421.0752
+3400 train 5.926132 (lr=3.7619e-05) (hash(x)=168425516)
+5500 val loss 5.7738
+5500 val perplexity 321.7622
+5500 train 5.940225 (lr=1.6113e-05) (hash(x)=145635833)
+3500 val loss 6.0154
+3500 val perplexity 409.6693
+3500 train 6.041699 (lr=3.6847e-05) (hash(x)=163104338)
+5600 val loss 5.7682
+5600 val perplexity 319.9536
+5600 train 5.752505 (lr=1.5469e-05) (hash(x)=156337844)
+3600 val loss 6.0012
+3600 val perplexity 403.8958
+3600 train 5.974601 (lr=3.6061e-05) (hash(x)=165109772)
+5700 val loss 5.7646
+5700 val perplexity 318.8028
+5700 train 5.645874 (lr=1.4836e-05) (hash(x)=147168506)
+3700 val loss 5.9942
+3700 val perplexity 401.1147
+3700 train 5.726319 (lr=3.5263e-05) (hash(x)=153420306)
+5800 val loss 5.7550
+5800 val perplexity 315.7605
+5800 train 5.812964 (lr=1.4212e-05) (hash(x)=159566920)
+3800 val loss 5.9667
+3800 val perplexity 390.2202
+3800 train 5.985947 (lr=3.4453e-05) (hash(x)=160168863)
+5900 val loss 5.7469
+5900 val perplexity 313.2336
+5900 train 5.812284 (lr=1.3600e-05) (hash(x)=158273929)
+3900 val loss 5.9539
+3900 val perplexity 385.2505
+3900 train 5.868129 (lr=3.3633e-05) (hash(x)=153906073)
+6000 val loss 5.7454
+6000 val perplexity 312.7557
+6000 train 5.718481 (lr=1.3000e-05) (hash(x)=156649749)
+4000 val loss 5.9410
+4000 val perplexity 380.3042
+4000 train 5.883380 (lr=3.2805e-05) (hash(x)=151055067)
+6100 val loss 5.7351
+6100 val perplexity 309.5356
+6100 train 5.649320 (lr=1.2413e-05) (hash(x)=146812388)
+4100 val loss 5.9261
+4100 val perplexity 374.7030
+4100 train 5.906900 (lr=3.1968e-05) (hash(x)=149629830)
+6200 val loss 5.7306
+6200 val perplexity 308.1406
+6200 train 5.558667 (lr=1.1840e-05) (hash(x)=143522146)
+4200 val loss 5.9100
+4200 val perplexity 368.6896
+4200 train 5.794271 (lr=3.1126e-05) (hash(x)=143101381)
+6300 val loss 5.7261
+6300 val perplexity 306.7715
+6300 train 5.550650 (lr=1.1282e-05) (hash(x)=150124474)
+4300 val loss 5.9004
+4300 val perplexity 365.2003
+4300 train 5.797774 (lr=3.0277e-05) (hash(x)=149712044)
+6400 val loss 5.7211
+6400 val perplexity 305.2509
+6400 train 5.749485 (lr=1.0739e-05) (hash(x)=141242117)
+4400 val loss 5.8842
+4400 val perplexity 359.3067
+4400 train 5.809937 (lr=2.9425e-05) (hash(x)=153446449)
+6500 val loss 5.7133
+6500 val perplexity 302.8625
+6500 train 5.593424 (lr=1.0213e-05) (hash(x)=143529762)
+4500 val loss 5.8706
+4500 val perplexity 354.4476
+4500 train 5.820799 (lr=2.8571e-05) (hash(x)=146086947)
+6600 val loss 5.7079
+6600 val perplexity 301.2269
+6600 train 5.496843 (lr=9.7032e-06) (hash(x)=136948374)
+4600 val loss 5.8618
+4600 val perplexity 351.3521
+4600 train 5.701613 (lr=2.7714e-05) (hash(x)=153800173)
+6700 val loss 5.7075
+6700 val perplexity 301.1216
+6700 train 5.509098 (lr=9.2116e-06) (hash(x)=146268592)
+4700 val loss 5.8520
+4700 val perplexity 347.9245
+4700 train 5.926853 (lr=2.6857e-05) (hash(x)=155962726)
+6800 val loss 5.6996
+6800 val perplexity 298.7610
+6800 train 5.658401 (lr=8.7387e-06) (hash(x)=152676836)
+4800 val loss 5.8400
+4800 val perplexity 343.7773
+4800 train 6.078588 (lr=2.6002e-05) (hash(x)=142045616)
+6900 val loss 5.6922
+6900 val perplexity 296.5384
+6900 train 5.611579 (lr=8.2849e-06) (hash(x)=134657776)
+4900 val loss 5.8270
+4900 val perplexity 339.3542
+4900 train 5.798401 (lr=2.5148e-05) (hash(x)=143418248)
+7000 val loss 5.6908
+7000 val perplexity 296.1258
+7000 train 5.645500 (lr=7.8510e-06) (hash(x)=166721861)
+5000 val loss 5.8158
+5000 val perplexity 335.5614
+5000 train 5.741745 (lr=2.4298e-05) (hash(x)=145789790)
+7100 val loss 5.6882
+7100 val perplexity 295.3699
+7100 train 5.487141 (lr=7.4377e-06) (hash(x)=135496702)
+5100 val loss 5.8126
+5100 val perplexity 334.4930
+5100 train 5.793215 (lr=2.3452e-05) (hash(x)=137795633)
+7200 val loss 5.6852
+7200 val perplexity 294.4621
+7200 train 5.796127 (lr=7.0455e-06) (hash(x)=155567461)
+5200 val loss 5.7978
+5200 val perplexity 329.5794
+5200 train 5.952859 (lr=2.2613e-05) (hash(x)=148907132)
+7300 val loss 5.6786
+7300 val perplexity 292.5464
+7300 train 5.477703 (lr=6.6749e-06) (hash(x)=142803829)
+5300 val loss 5.7882
+5300 val perplexity 326.4250
+5300 train 5.663875 (lr=2.1780e-05) (hash(x)=152343580)
+7400 val loss 5.6780
+7400 val perplexity 292.3659
+7400 train 5.497990 (lr=6.3266e-06) (hash(x)=145294178)
+5400 val loss 5.7794
+5400 val perplexity 323.5801
+5400 train 5.630389 (lr=2.0956e-05) (hash(x)=148578264)
+7500 val loss 5.6748
+7500 val perplexity 291.4342
+7500 train 5.431237 (lr=6.0010e-06) (hash(x)=150573713)
+5500 val loss 5.7663
+5500 val perplexity 319.3484
+5500 train 5.930579 (lr=2.0141e-05) (hash(x)=145635833)
+7600 val loss 5.6735
+7600 val perplexity 291.0639
+7600 train 5.689526 (lr=5.6986e-06) (hash(x)=142771511)
+5600 val loss 5.7618
+5600 val perplexity 317.9220
+5600 train 5.749690 (lr=1.9337e-05) (hash(x)=156337844)
+7700 val loss 5.6666
+7700 val perplexity 289.0573
+7700 train 5.571441 (lr=5.4198e-06) (hash(x)=143602175)
+5700 val loss 5.7559
+5700 val perplexity 316.0397
+5700 train 5.634243 (lr=1.8545e-05) (hash(x)=147168506)
+7800 val loss 5.6639
+7800 val perplexity 288.2749
+7800 train 5.717690 (lr=5.1650e-06) (hash(x)=152379862)
+5800 val loss 5.7454
+5800 val perplexity 312.7545
+5800 train 5.805884 (lr=1.7765e-05) (hash(x)=159566920)
+7900 val loss 5.6640
+7900 val perplexity 288.2939
+7900 train 5.486330 (lr=4.9347e-06) (hash(x)=146655921)
+5900 val loss 5.7365
+5900 val perplexity 309.9833
+5900 train 5.803931 (lr=1.7000e-05) (hash(x)=158273929)
+8000 val loss 5.6640
+8000 val perplexity 288.3034
+8000 train 5.793463 (lr=4.7291e-06) (hash(x)=148262482)
+6000 val loss 5.7346
+6000 val perplexity 309.3907
+6000 train 5.703155 (lr=1.6250e-05) (hash(x)=156649749)
+8100 val loss 5.6561
+8100 val perplexity 286.0415
+8100 train 5.593901 (lr=4.5486e-06) (hash(x)=147683655)
+6100 val loss 5.7216
+6100 val perplexity 305.3945
+6100 train 5.637345 (lr=1.5516e-05) (hash(x)=146812388)
+8200 val loss 5.6568
+8200 val perplexity 286.2420
+8200 train 5.767795 (lr=4.3933e-06) (hash(x)=157312987)
+6200 val loss 5.7177
+6200 val perplexity 304.2039
+6200 train 5.546375 (lr=1.4800e-05) (hash(x)=143522146)
+8300 val loss 5.6541
+8300 val perplexity 285.4726
+8300 train 5.617525 (lr=4.2636e-06) (hash(x)=141107543)
+6300 val loss 5.7147
+6300 val perplexity 303.2979
+6300 train 5.535329 (lr=1.4102e-05) (hash(x)=150124474)
+8400 val loss 5.6495
+8400 val perplexity 284.1558
+8400 train 5.641396 (lr=4.1596e-06) (hash(x)=141323024)
+6400 val loss 5.7066
+6400 val perplexity 300.8557
+6400 train 5.735716 (lr=1.3424e-05) (hash(x)=141242117)
+8500 val loss 5.6482
+8500 val perplexity 283.7680
+8500 train 5.618681 (lr=4.0815e-06) (hash(x)=150696521)
+6500 val loss 5.6979
+6500 val perplexity 298.2270
+6500 train 5.574231 (lr=1.2766e-05) (hash(x)=143529762)
+8600 val loss 5.6481
+8600 val perplexity 283.7611
+8600 train 5.595167 (lr=4.0294e-06) (hash(x)=162288191)
+6600 val loss 5.6934
+6600 val perplexity 296.9074
+6600 train 5.482853 (lr=1.2129e-05) (hash(x)=136948374)
+8700 val loss 5.6443
+8700 val perplexity 282.6739
+8700 train 5.511504 (lr=4.0033e-06) (hash(x)=152860941)
+8749 val loss 5.6436
+8749 val perplexity 282.4701
+6700 val loss 5.6928
+6700 val perplexity 296.7115
+6700 train 5.493250 (lr=1.1515e-05) (hash(x)=146268592)

attention_kindselective_n_heads4_seed1341/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4eb8ecdad34911d2df3b8385b1e0e53ea3b1187bdb14089047b7248e4e241676
+size 92843394

attention_kindselective_n_heads4_seed1341/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af64c7f2d113763ac94c9f25256a07178eebabde450ce63f95bf18e6262aa7cb
+size 179406214