diff --git a/attention_kindselective_n_heads4_seed1341/args.json b/attention_kindselective_n_heads4_seed1341/args.json
index 7c4876d74e63510973be588a260310884d32ff0a..f087f3f8ca3af0647d57c8d42f54e6cdaf190809 100644
--- a/attention_kindselective_n_heads4_seed1341/args.json
+++ b/attention_kindselective_n_heads4_seed1341/args.json
@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_7/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 50000, "warmup_steps": 200, "group": "wider_is_better_7", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 7e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "7e-5_10240_4_1341", "n_embd": 256}
\ No newline at end of file
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_7/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 50000, "warmup_steps": 200, "group": "wider_is_better_7", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2e-5_10240_4_1341", "n_embd": 256}
\ No newline at end of file
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_10000.pt b/attention_kindselective_n_heads4_seed1341/dataloader_10000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e1452fd777edd0ccb65d6b47710c50208c14b312
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_10000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3858f6c832feea78a674d8c5c384061cc7d4f22cddbd0a2be6de33bc91e2c72
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_12500.pt b/attention_kindselective_n_heads4_seed1341/dataloader_12500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f63806424a0177a7f2d678c2c63138219ed021f3
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_12500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab3779d33c2e0a7873fcd8c39402e44260740665950323ad1445480ec339965a
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_15000.pt b/attention_kindselective_n_heads4_seed1341/dataloader_15000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0ea0f5a1bfab75667c4ebb0ca01b358cdc836a54
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_15000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:450020c7c306c73e5c07c463518ab937102d657515ea5a38da6f2e7291f20324
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_17500.pt b/attention_kindselective_n_heads4_seed1341/dataloader_17500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eb9392348b1209c827e3e376b05eeda80e779aa8
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_17500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0696a655d7c8a9d7d275c7489f74a5a948ee029ac3941b045d6abaf12544a5b1
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_20000.pt b/attention_kindselective_n_heads4_seed1341/dataloader_20000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ba32fca059aec40e4f758de96bdac6df23b9d9f5
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_20000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4eb226bdcee777fd1ca493533704dae226c077ef79c842fc9dc59a534d5381c1
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_22500.pt b/attention_kindselective_n_heads4_seed1341/dataloader_22500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..498dc444f528d893090328a5bd1e2f37da46dc12
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_22500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88833bfd821adac4edd1dd0772083ae007c7b8d33041f66e53a679e1fa8993e0
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_25000.pt b/attention_kindselective_n_heads4_seed1341/dataloader_25000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b657bc134192d0ea956f984c289d0c682979a1f4
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_25000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:554555a425bac43d626c36f1c81c2b0aba51eda3281dab27a9cb56b61f413354
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_27500.pt b/attention_kindselective_n_heads4_seed1341/dataloader_27500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d92d43d89390714f43db4f0782e49af0145b4a90
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_27500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a52940b3b45414e6cdbec0dbaeba848f52d681c2daf78c269027057332d7fbd
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_30000.pt b/attention_kindselective_n_heads4_seed1341/dataloader_30000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..06856f53253b6c8cbefd7d595d9b9b7266b22621
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_30000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:775519ea26122cb70d533c496bcdbbc19f759f3d096e83e98cca1dc10275fe8e
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_32500.pt b/attention_kindselective_n_heads4_seed1341/dataloader_32500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dc0b4b40e57f41aa1046b3bd2697256635160c09
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_32500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab3422c8205fe45210246fed3dd6c317b75df02228cd8b75fba669574ce3b2d9
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_35000.pt b/attention_kindselective_n_heads4_seed1341/dataloader_35000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..62647b540fa7925361626b9f8dfa3959eebb7608
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_35000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:199ed3be67b88981f686112c97a2261729a37e0be3d4b0f4a289985a95d3cdf1
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_37500.pt b/attention_kindselective_n_heads4_seed1341/dataloader_37500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..475d8b538138a8e39b76a4cf04c8eaeac074d295
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_37500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f77178b3be9dd3f8cd03c935236251f73fde6da7948ba9feda0c888fb8912dfe
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_40000.pt b/attention_kindselective_n_heads4_seed1341/dataloader_40000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..50ec00ef6be330bbdb4cdf88e9a1097345da0d4d
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_40000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84f58ba3b4a1f9be7da4e697e06782f0e1ce4d3aca49f1997087fc83aa466dd9
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_42500.pt b/attention_kindselective_n_heads4_seed1341/dataloader_42500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..74e133f73a7293d5f4d6407784703c91f705d6e3
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_42500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf6d24c78d89100d146bce9f26be940db3d71092473d9b55db97d6b35531eac2
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/dataloader_45000.pt b/attention_kindselective_n_heads4_seed1341/dataloader_45000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..718bc149d695b1e9498bdd0693053d7417207818
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/dataloader_45000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22bb90b43d81f3da5454f91a70e1ed29aeb2f470a727ce38390ff8a5c4924889
+size 964
diff --git a/attention_kindselective_n_heads4_seed1341/log2.txt b/attention_kindselective_n_heads4_seed1341/log2.txt
index cec4d91e5c5c79c91b650a74657fb3539efa4904..754344b5af0bc85ea28a9407352851c6761911ff 100644
--- a/attention_kindselective_n_heads4_seed1341/log2.txt
+++ b/attention_kindselective_n_heads4_seed1341/log2.txt
@@ -1,279 +1,3273 @@
 max_steps: 50000
-45100 val loss 5.7904
-45100 val perplexity 327.1403
-45100 train 5.674967 (lr=6.0664e-06) (hash(x)=56386155)
+32200 val loss 5.9773
+32200 val perplexity 394.3710
+32200 train 5.901725 (lr=5.3268e-06) (hash(x)=50998615)
 0 val loss 11.3128
 0 val perplexity 81862.0547
-45200 val loss 5.7941
-45200 val perplexity 328.3590
-45200 train 5.824661 (lr=6.0237e-06) (hash(x)=53389864)
-45300 val loss 5.7925
-45300 val perplexity 327.8424
-45300 train 5.834869 (lr=5.9818e-06) (hash(x)=50817456)
-0 train 11.310279 (lr=3.5000e-07) (hash(x)=47078120)
-45400 val loss 5.7922
-45400 val perplexity 327.7464
-45400 train 5.832580 (lr=5.9407e-06) (hash(x)=50081660)
-100 val loss 9.7348
-100 val perplexity 16895.5098
-100 train 9.668372 (lr=3.5350e-05) (hash(x)=43429388)
-45500 val loss 5.7909
-45500 val perplexity 327.2947
-45500 train 5.550583 (lr=5.9005e-06) (hash(x)=55567121)
-45600 val loss 5.7850
-45600 val perplexity 325.3777
-45600 train 5.868555 (lr=5.8612e-06) (hash(x)=48902854)
-200 val loss 8.2779
-200 val perplexity 3935.9905
-200 train 8.438046 (lr=7.0000e-05) (hash(x)=52929681)
-45700 val loss 5.7834
-45700 val perplexity 324.8476
-45700 train 6.019247 (lr=5.8227e-06) (hash(x)=53803301)
-300 val loss 7.7389
-300 val perplexity 2295.8757
-300 train 7.749171 (lr=6.9999e-05) (hash(x)=49930367)
-45800 val loss 5.7814
-45800 val perplexity 324.2190
-45800 train 5.843807 (lr=5.7851e-06) (hash(x)=47576606)
-400 val loss 7.6187
-400 val perplexity 2035.9438
-400 train 7.430635 (lr=6.9997e-05) (hash(x)=48542946)
-45900 val loss 5.7793
-45900 val perplexity 323.5269
-45900 train 5.661962 (lr=5.7484e-06) (hash(x)=56276981)
-500 val loss 7.5611
-500 val perplexity 1921.9686
-500 train 7.778275 (lr=6.9994e-05) (hash(x)=55286048)
-46000 val loss 5.7781
-46000 val perplexity 323.1599
-46000 train 5.741266 (lr=5.7125e-06) (hash(x)=52174902)
-600 val loss 7.5338
-600 val perplexity 1870.2159
-600 train 7.778021 (lr=6.9990e-05) (hash(x)=51577760)
-46100 val loss 5.7794
-46100 val perplexity 323.5764
-46100 train 5.910207 (lr=5.6775e-06) (hash(x)=52155490)
-700 val loss 7.4794
-700 val perplexity 1771.2211
-700 train 7.640063 (lr=6.9984e-05) (hash(x)=57433471)
-46200 val loss 5.7787
-46200 val perplexity 323.3492
-46200 train 5.635107 (lr=5.6434e-06) (hash(x)=46489984)
-800 val loss 7.4539
-800 val perplexity 1726.5818
-800 train 7.369297 (lr=6.9977e-05) (hash(x)=49799291)
-46300 val loss 5.7793
-46300 val perplexity 323.5242
-46300 train 5.750629 (lr=5.6101e-06) (hash(x)=56603976)
-900 val loss 7.4260
-900 val perplexity 1679.1525
-900 train 7.442145 (lr=6.9969e-05) (hash(x)=49502839)
-46400 val loss 5.7772
-46400 val perplexity 322.8646
-46400 train 5.591374 (lr=5.5777e-06) (hash(x)=51228412)
-46500 val loss 5.7769
-46500 val perplexity 322.7455
-46500 train 5.628456 (lr=5.5462e-06) (hash(x)=48511186)
-1000 val loss 7.3913
-1000 val perplexity 1621.8335
-1000 train 7.669879 (lr=6.9960e-05) (hash(x)=51142904)
-46600 val loss 5.7759
-46600 val perplexity 322.4206
-46600 train 5.813951 (lr=5.5156e-06) (hash(x)=48832039)
-1100 val loss 7.3607
-1100 val perplexity 1572.9337
-1100 train 7.425812 (lr=6.9949e-05) (hash(x)=52751086)
-46700 val loss 5.7767
-46700 val perplexity 322.6856
-46700 train 6.470688 (lr=5.4858e-06) (hash(x)=65821271)
-1200 val loss 7.3503
-1200 val perplexity 1556.7256
-1200 train 7.339065 (lr=6.9937e-05) (hash(x)=51538621)
-46800 val loss 5.7767
-46800 val perplexity 322.6765
-46800 train 5.485295 (lr=5.4569e-06) (hash(x)=48005593)
-1300 val loss 7.3312
-1300 val perplexity 1527.2528
-1300 train 7.411671 (lr=6.9924e-05) (hash(x)=52034040)
-46900 val loss 5.7753
-46900 val perplexity 322.2317
-46900 train 6.050996 (lr=5.4289e-06) (hash(x)=53182453)
-1400 val loss 7.3248
-1400 val perplexity 1517.4226
-1400 train 7.362967 (lr=6.9910e-05) (hash(x)=50640105)
-47000 val loss 5.7761
-47000 val perplexity 322.4943
-47000 train 5.677234 (lr=5.4017e-06) (hash(x)=49385983)
-1500 val loss 7.3135
-1500 val perplexity 1500.4502
-1500 train 7.252579 (lr=6.9894e-05) (hash(x)=49016270)
-47100 val loss 5.7766
-47100 val perplexity 322.6584
-47100 train 5.769415 (lr=5.3755e-06) (hash(x)=48801622)
-1600 val loss 7.2885
-1600 val perplexity 1463.3402
-1600 train 7.062971 (lr=6.9877e-05) (hash(x)=46100488)
-47200 val loss 5.7778
-47200 val perplexity 323.0544
-47200 train 5.511658 (lr=5.3501e-06) (hash(x)=45216690)
-1700 val loss 7.2714
-1700 val perplexity 1438.6268
-1700 train 7.318298 (lr=6.9859e-05) (hash(x)=49185350)
-47300 val loss 5.7776
-47300 val perplexity 322.9699
-47300 train 5.592494 (lr=5.3256e-06) (hash(x)=51185601)
-47400 val loss 5.7751
-47400 val perplexity 322.1754
-47400 train 5.482538 (lr=5.3020e-06) (hash(x)=47743508)
-1800 val loss 7.2397
-1800 val perplexity 1393.7301
-1800 train 7.135669 (lr=6.9840e-05) (hash(x)=48024574)
-47500 val loss 5.7803
-47500 val perplexity 323.8446
-47500 train 5.554578 (lr=5.2792e-06) (hash(x)=49318833)
-1900 val loss 7.2352
-1900 val perplexity 1387.3793
-1900 train 7.070409 (lr=6.9819e-05) (hash(x)=45823189)
-47600 val loss 5.7810
-47600 val perplexity 324.0983
-47600 train 5.741845 (lr=5.2574e-06) (hash(x)=50430826)
-2000 val loss 7.2296
-2000 val perplexity 1379.6533
-2000 train 7.069522 (lr=6.9797e-05) (hash(x)=45703932)
-47700 val loss 5.7792
-47700 val perplexity 323.5000
-47700 train 5.718388 (lr=5.2364e-06) (hash(x)=51966931)
-47800 val loss 5.7772
-47800 val perplexity 322.8683
-47800 train 5.552941 (lr=5.2163e-06) (hash(x)=49517209)
-2100 val loss 7.2192
-2100 val perplexity 1365.4330
-2100 train 7.872229 (lr=6.9774e-05) (hash(x)=58570170)
-47900 val loss 5.7794
-47900 val perplexity 323.5654
-47900 train 5.597253 (lr=5.1972e-06) (hash(x)=49749702)
-48000 val loss 5.7788
-48000 val perplexity 323.3792
-48000 train 5.680276 (lr=5.1788e-06) (hash(x)=52535448)
-2200 val loss 7.2430
-2200 val perplexity 1398.3040
-2200 train 7.315641 (lr=6.9750e-05) (hash(x)=55262880)
-48100 val loss 5.7743
-48100 val perplexity 321.9077
-48100 train 6.455255 (lr=5.1614e-06) (hash(x)=53739181)
-48200 val loss 5.7725
-48200 val perplexity 321.3269
-48200 train 5.751445 (lr=5.1449e-06) (hash(x)=48086710)
-2300 val loss 7.2406
-2300 val perplexity 1394.9767
-2300 train 6.987216 (lr=6.9724e-05) (hash(x)=46415497)
-48300 val loss 5.7727
-48300 val perplexity 321.4185
-48300 train 5.740582 (lr=5.1293e-06) (hash(x)=43991942)
-2400 val loss 7.2117
-2400 val perplexity 1355.2457
-2400 train 7.150833 (lr=6.9697e-05) (hash(x)=49272278)
-48400 val loss 5.7686
-48400 val perplexity 320.0804
-48400 train 5.642438 (lr=5.1145e-06) (hash(x)=50801906)
-48500 val loss 5.7686
-48500 val perplexity 320.1000
-48500 train 5.825756 (lr=5.1007e-06) (hash(x)=48915599)
-2500 val loss 7.2094
-2500 val perplexity 1352.1377
-2500 train 7.084943 (lr=6.9669e-05) (hash(x)=48390803)
-48600 val loss 5.7668
-48600 val perplexity 319.5191
-48600 train 5.929821 (lr=5.0877e-06) (hash(x)=54450172)
-48700 val loss 5.7680
-48700 val perplexity 319.9124
-48700 train 5.660990 (lr=5.0756e-06) (hash(x)=48755509)
-2600 val loss 7.1807
-2600 val perplexity 1313.8004
-2600 train 7.066188 (lr=6.9640e-05) (hash(x)=47450116)
-48800 val loss 5.7696
-48800 val perplexity 320.4213
-48800 train 5.660839 (lr=5.0644e-06) (hash(x)=51380096)
-2700 val loss 7.1767
-2700 val perplexity 1308.5830
-2700 train 7.263812 (lr=6.9609e-05) (hash(x)=52681152)
-48900 val loss 5.7680
-48900 val perplexity 319.9099
-48900 train 5.630540 (lr=5.0542e-06) (hash(x)=43404218)
-2800 val loss 7.1723
-2800 val perplexity 1302.8984
-2800 train 7.220819 (lr=6.9577e-05) (hash(x)=50664094)
-49000 val loss 5.7681
-49000 val perplexity 319.9189
-49000 train 5.758193 (lr=5.0448e-06) (hash(x)=55059739)
-2900 val loss 7.1650
-2900 val perplexity 1293.3875
-2900 train 6.984344 (lr=6.9544e-05) (hash(x)=47067144)
-49100 val loss 5.7706
-49100 val perplexity 320.7290
-49100 train 5.996271 (lr=5.0363e-06) (hash(x)=47889309)
-3000 val loss 7.1467
-3000 val perplexity 1269.8622
-3000 train 6.990779 (lr=6.9510e-05) (hash(x)=45015009)
-49200 val loss 5.7695
-49200 val perplexity 320.3733
-49200 train 5.512230 (lr=5.0286e-06) (hash(x)=46202589)
-3100 val loss 7.1704
-3100 val perplexity 1300.3754
-3100 train 7.004176 (lr=6.9474e-05) (hash(x)=45245896)
-49300 val loss 5.7690
-49300 val perplexity 320.2130
-49300 train 5.721933 (lr=5.0219e-06) (hash(x)=49161813)
-3200 val loss 7.1561
-3200 val perplexity 1281.9510
-3200 train 7.116989 (lr=6.9438e-05) (hash(x)=49995942)
-49400 val loss 5.7681
-49400 val perplexity 319.9419
-49400 train 5.601577 (lr=5.0161e-06) (hash(x)=49505044)
-3300 val loss 7.1272
-3300 val perplexity 1245.4196
-3300 train 7.024605 (lr=6.9400e-05) (hash(x)=52311504)
-49500 val loss 5.7683
-49500 val perplexity 319.9982
-49500 train 5.544023 (lr=5.0112e-06) (hash(x)=46610327)
-3400 val loss 7.1102
-3400 val perplexity 1224.3795
-3400 train 7.109678 (lr=6.9360e-05) (hash(x)=44332917)
-49600 val loss 5.7680
-49600 val perplexity 319.8917
-49600 train 6.107419 (lr=5.0072e-06) (hash(x)=51188240)
-3500 val loss 7.1105
-3500 val perplexity 1224.7303
-3500 train 7.218296 (lr=6.9320e-05) (hash(x)=56517159)
-49700 val loss 5.7667
-49700 val perplexity 319.4920
-49700 train 5.552507 (lr=5.0040e-06) (hash(x)=41492016)
-3600 val loss 7.0860
-3600 val perplexity 1195.0756
-3600 train 6.962664 (lr=6.9278e-05) (hash(x)=50720920)
-49800 val loss 5.7675
-49800 val perplexity 319.7239
-49800 train 5.705134 (lr=5.0018e-06) (hash(x)=53488833)
-3700 val loss 7.0801
-3700 val perplexity 1188.0885
-3700 train 7.562508 (lr=6.9235e-05) (hash(x)=62727701)
-49900 val loss 5.7662
-49900 val perplexity 319.3073
-49900 train 6.118833 (lr=5.0004e-06) (hash(x)=60260935)
-3800 val loss 7.0837
-3800 val perplexity 1192.3132
-3800 train 6.928957 (lr=6.9191e-05) (hash(x)=54772539)
-49999 val loss 5.7655
-49999 val perplexity 319.1097
-3900 val loss 7.0776
-3900 val perplexity 1185.0813
-3900 train 7.083435 (lr=6.9146e-05) (hash(x)=52274485)
-4000 val loss 7.0939
-4000 val perplexity 1204.5883
-4000 train 6.805799 (lr=6.9099e-05) (hash(x)=50118307)
-4100 val loss 7.0658
-4100 val perplexity 1171.2137
-4100 train 6.668285 (lr=6.9051e-05) (hash(x)=42771647)
-4200 val loss 7.0463
-4200 val perplexity 1148.6185
-4200 train 7.126415 (lr=6.9002e-05) (hash(x)=51748836)
+0 train 11.310279 (lr=1.5000e-07) (hash(x)=47078120)
+32300 val loss 5.9761
+32300 val perplexity 393.8897
+32300 train 5.769437 (lr=5.2884e-06) (hash(x)=44030191)
+100 val loss 9.9982
+100 val perplexity 21986.6523
+100 train 9.942831 (lr=1.5150e-05) (hash(x)=43429388)
+32400 val loss 5.9761
+32400 val perplexity 393.8959
+32400 train 5.888457 (lr=5.2502e-06) (hash(x)=47269028)
+0 train 11.310279 (lr=1.0000e-07) (hash(x)=47078120)
+200 val loss 9.4054
+200 val perplexity 12153.4092
+200 train 9.491462 (lr=3.0000e-05) (hash(x)=52929681)
+32500 val loss 5.9739
+32500 val perplexity 393.0167
+32500 train 5.806501 (lr=5.2122e-06) (hash(x)=50128289)
+100 val loss 10.1293
+100 val perplexity 25065.8340
+100 train 10.075195 (lr=1.0100e-05) (hash(x)=43429388)
+300 val loss 8.3246
+300 val perplexity 4124.0371
+300 train 8.336009 (lr=3.0000e-05) (hash(x)=49930367)
+32600 val loss 5.9765
+32600 val perplexity 394.0494
+32600 train 5.789043 (lr=5.1742e-06) (hash(x)=42781351)
+200 val loss 9.7040
+200 val perplexity 16383.7656
+200 train 9.777761 (lr=2.0000e-05) (hash(x)=52929681)
+400 val loss 7.8864
+400 val perplexity 2660.7151
+400 train 7.728951 (lr=2.9999e-05) (hash(x)=48542946)
+32700 val loss 5.9763
+32700 val perplexity 393.9765
+32700 train 6.058278 (lr=5.1363e-06) (hash(x)=51712117)
+300 val loss 9.1506
+300 val perplexity 9419.8252
+300 train 9.147341 (lr=2.0000e-05) (hash(x)=49930367)
+500 val loss 7.7187
+500 val perplexity 2250.0647
+500 train 7.894150 (lr=2.9998e-05) (hash(x)=55286048)
+32800 val loss 5.9671
+32800 val perplexity 390.3791
+32800 train 6.074276 (lr=5.0986e-06) (hash(x)=52478104)
+400 val loss 8.5647
+400 val perplexity 5243.4072
+400 train 8.466437 (lr=1.9999e-05) (hash(x)=48542946)
+600 val loss 7.6330
+600 val perplexity 2065.1831
+600 train 7.851482 (lr=2.9996e-05) (hash(x)=51577760)
+32900 val loss 5.9662
+32900 val perplexity 390.0358
+32900 train 5.934245 (lr=5.0610e-06) (hash(x)=53588694)
+500 val loss 8.1482
+500 val perplexity 3457.2827
+500 train 8.258484 (lr=1.9998e-05) (hash(x)=55286048)
+700 val loss 7.5829
+700 val perplexity 1964.2629
+700 train 7.731767 (lr=2.9993e-05) (hash(x)=57433471)
+600 val loss 7.9240
+600 val perplexity 2762.8931
+600 train 8.044626 (lr=1.9997e-05) (hash(x)=51577760)
+33000 val loss 5.9620
+33000 val perplexity 388.3709
+33000 train 5.849672 (lr=5.0236e-06) (hash(x)=50821436)
+800 val loss 7.5521
+800 val perplexity 1904.7434
+800 train 7.477926 (lr=2.9990e-05) (hash(x)=49799291)
+700 val loss 7.8238
+700 val perplexity 2499.3281
+700 train 7.924374 (lr=1.9996e-05) (hash(x)=57433471)
+33100 val loss 5.9619
+33100 val perplexity 388.3648
+33100 train 5.934168 (lr=4.9862e-06) (hash(x)=50371199)
+900 val loss 7.5272
+900 val perplexity 1857.8461
+900 train 7.535049 (lr=2.9987e-05) (hash(x)=49502839)
+800 val loss 7.7221
+800 val perplexity 2257.6372
+800 train 7.659780 (lr=1.9994e-05) (hash(x)=49799291)
+33200 val loss 5.9622
+33200 val perplexity 388.4455
+33200 train 6.158159 (lr=4.9490e-06) (hash(x)=54544623)
+1000 val loss 7.5077
+1000 val perplexity 1821.9885
+1000 train 7.774044 (lr=2.9983e-05) (hash(x)=51142904)
+900 val loss 7.6434
+900 val perplexity 2086.7976
+900 train 7.654329 (lr=1.9991e-05) (hash(x)=49502839)
+33300 val loss 5.9598
+33300 val perplexity 387.5357
+33300 train 5.831619 (lr=4.9119e-06) (hash(x)=55456921)
+1100 val loss 7.4929
+1100 val perplexity 1795.2112
+1100 train 7.563055 (lr=2.9978e-05) (hash(x)=52751086)
+1000 val loss 7.5907
+1000 val perplexity 1979.6821
+1000 train 7.810211 (lr=1.9989e-05) (hash(x)=51142904)
+33400 val loss 5.9618
+33400 val perplexity 388.2983
+33400 train 5.517099 (lr=4.8750e-06) (hash(x)=37240412)
+1100 val loss 7.5595
+1100 val perplexity 1918.9375
+1100 train 7.621131 (lr=1.9985e-05) (hash(x)=52751086)
+1200 val loss 7.4853
+1200 val perplexity 1781.6069
+1200 train 7.469899 (lr=2.9973e-05) (hash(x)=51538621)
+33500 val loss 5.9606
+33500 val perplexity 387.8316
+33500 train 6.215307 (lr=4.8382e-06) (hash(x)=56551894)
+1200 val loss 7.5372
+1200 val perplexity 1876.6434
+1200 train 7.524499 (lr=1.9982e-05) (hash(x)=51538621)
+1300 val loss 7.4799
+1300 val perplexity 1772.0802
+1300 train 7.563471 (lr=2.9968e-05) (hash(x)=52034040)
+33600 val loss 5.9589
+33600 val perplexity 387.1845
+33600 train 5.856531 (lr=4.8015e-06) (hash(x)=47514225)
+1300 val loss 7.5160
+1300 val perplexity 1837.2858
+1300 train 7.592782 (lr=1.9978e-05) (hash(x)=52034040)
+1400 val loss 7.4644
+1400 val perplexity 1744.7938
+1400 train 7.497466 (lr=2.9961e-05) (hash(x)=50640105)
+33700 val loss 5.9577
+33700 val perplexity 386.7018
+33700 train 5.805673 (lr=4.7650e-06) (hash(x)=51164416)
+1400 val loss 7.4935
+1400 val perplexity 1796.3723
+1400 train 7.517349 (lr=1.9974e-05) (hash(x)=50640105)
+1500 val loss 7.4506
+1500 val perplexity 1720.9236
+1500 train 7.389732 (lr=2.9955e-05) (hash(x)=49016270)
+1500 val loss 7.4750
+1500 val perplexity 1763.4847
+1500 train 7.414174 (lr=1.9970e-05) (hash(x)=49016270)
+33800 val loss 5.9608
+33800 val perplexity 387.9230
+33800 train 5.786061 (lr=4.7286e-06) (hash(x)=50660204)
+1600 val loss 7.4419
+1600 val perplexity 1706.0590
+1600 train 7.218320 (lr=2.9947e-05) (hash(x)=46100488)
+1600 val loss 7.4648
+1600 val perplexity 1745.5178
+1600 train 7.240737 (lr=1.9965e-05) (hash(x)=46100488)
+33900 val loss 5.9567
+33900 val perplexity 386.3478
+33900 train 5.942020 (lr=4.6923e-06) (hash(x)=48511113)
+1700 val loss 7.4340
+1700 val perplexity 1692.6055
+1700 train 7.474668 (lr=2.9940e-05) (hash(x)=49185350)
+1700 val loss 7.4415
+1700 val perplexity 1705.3285
+1700 train 7.473830 (lr=1.9960e-05) (hash(x)=49185350)
+34000 val loss 5.9590
+34000 val perplexity 387.2233
+34000 train 5.564332 (lr=4.6562e-06) (hash(x)=41560072)
+1800 val loss 7.4184
+1800 val perplexity 1666.4269
+1800 train 7.325621 (lr=2.9931e-05) (hash(x)=48024574)
+1800 val loss 7.4335
+1800 val perplexity 1691.6946
+1800 train 7.344293 (lr=1.9954e-05) (hash(x)=48024574)
+34100 val loss 5.9571
+34100 val perplexity 386.4749
+34100 train 5.604887 (lr=4.6202e-06) (hash(x)=50354364)
+1900 val loss 7.4047
+1900 val perplexity 1643.6694
+1900 train 7.235081 (lr=2.9922e-05) (hash(x)=45823189)
+1900 val loss 7.4187
+1900 val perplexity 1666.9403
+1900 train 7.256597 (lr=1.9948e-05) (hash(x)=45823189)
+34200 val loss 5.9568
+34200 val perplexity 386.3579
+34200 train 6.174836 (lr=4.5844e-06) (hash(x)=54291562)
+2000 val loss 7.3980
+2000 val perplexity 1632.7605
+2000 train 7.237600 (lr=2.9913e-05) (hash(x)=45703932)
+2000 val loss 7.4079
+2000 val perplexity 1648.9714
+2000 train 7.251985 (lr=1.9942e-05) (hash(x)=45703932)
+34300 val loss 5.9590
+34300 val perplexity 387.2122
+34300 train 6.328149 (lr=4.5487e-06) (hash(x)=53340739)
+2100 val loss 7.3959
+2100 val perplexity 1629.2693
+2100 train 7.990074 (lr=1.9935e-05) (hash(x)=58570170)
+2100 val loss 7.3932
+2100 val perplexity 1624.8765
+2100 train 7.978726 (lr=2.9903e-05) (hash(x)=58570170)
+34400 val loss 5.9534
+34400 val perplexity 385.0786
+34400 train 5.594629 (lr=4.5132e-06) (hash(x)=44872469)
+2200 val loss 7.3925
+2200 val perplexity 1623.7975
+2200 train 7.418814 (lr=1.9928e-05) (hash(x)=55262880)
+2200 val loss 7.4047
+2200 val perplexity 1643.7148
+2200 train 7.446170 (lr=2.9893e-05) (hash(x)=55262880)
+2300 val loss 7.3624
+2300 val perplexity 1575.5790
+2300 train 7.140794 (lr=1.9921e-05) (hash(x)=46415497)
+34500 val loss 5.9516
+34500 val perplexity 384.3842
+34500 train 5.000134 (lr=4.4778e-06) (hash(x)=23649625)
+2300 val loss 7.3758
+2300 val perplexity 1596.7928
+2300 train 7.149339 (lr=2.9882e-05) (hash(x)=46415497)
+2400 val loss 7.3417
+2400 val perplexity 1543.3057
+2400 train 7.279164 (lr=1.9913e-05) (hash(x)=49272278)
+34600 val loss 5.9546
+34600 val perplexity 385.5230
+34600 train 5.869273 (lr=4.4426e-06) (hash(x)=46230721)
+2400 val loss 7.3582
+2400 val perplexity 1568.9883
+2400 train 7.288417 (lr=2.9870e-05) (hash(x)=49272278)
+2500 val loss 7.3202
+2500 val perplexity 1510.5052
+2500 train 7.184688 (lr=1.9905e-05) (hash(x)=48390803)
+34700 val loss 5.9463
+34700 val perplexity 382.3483
+34700 train 5.704084 (lr=4.4075e-06) (hash(x)=44277516)
+2500 val loss 7.3467
+2500 val perplexity 1550.9937
+2500 train 7.229398 (lr=2.9858e-05) (hash(x)=48390803)
+2600 val loss 7.2998
+2600 val perplexity 1479.9332
+2600 train 7.191774 (lr=1.9897e-05) (hash(x)=47450116)
+34800 val loss 5.9446
+34800 val perplexity 381.6902
+34800 train 6.033243 (lr=4.3725e-06) (hash(x)=50346875)
+2600 val loss 7.3314
+2600 val perplexity 1527.4845
+2600 train 7.211346 (lr=2.9846e-05) (hash(x)=47450116)
+2700 val loss 7.2829
+2700 val perplexity 1455.1796
+2700 train 7.350107 (lr=1.9888e-05) (hash(x)=52681152)
+34900 val loss 5.9443
+34900 val perplexity 381.5750
+34900 train 5.848508 (lr=4.3378e-06) (hash(x)=45110550)
+2700 val loss 7.3294
+2700 val perplexity 1524.5200
+2700 train 7.402822 (lr=2.9832e-05) (hash(x)=52681152)
+2800 val loss 7.2660
+2800 val perplexity 1430.8755
+2800 train 7.349651 (lr=1.9879e-05) (hash(x)=50664094)
+35000 val loss 5.9421
+35000 val perplexity 380.7357
+35000 train 5.919306 (lr=4.3031e-06) (hash(x)=46812915)
+2800 val loss 7.3071
+2800 val perplexity 1490.9104
+2800 train 7.377246 (lr=2.9819e-05) (hash(x)=50664094)
+2900 val loss 7.2523
+2900 val perplexity 1411.2770
+2900 train 7.038266 (lr=1.9870e-05) (hash(x)=47067144)
+35100 val loss 5.9415
+35100 val perplexity 380.5010
+35100 train 6.107468 (lr=4.2687e-06) (hash(x)=54729285)
+2900 val loss 7.3015
+2900 val perplexity 1482.4915
+2900 train 7.119827 (lr=2.9805e-05) (hash(x)=47067144)
+3000 val loss 7.2410
+3000 val perplexity 1395.5176
+3000 train 7.067964 (lr=1.9860e-05) (hash(x)=45015009)
+35200 val loss 5.9440
+35200 val perplexity 381.4518
+35200 train 5.687371 (lr=4.2344e-06) (hash(x)=49651453)
+3100 val loss 7.2261
+3100 val perplexity 1374.8789
+3100 train 7.039052 (lr=1.9850e-05) (hash(x)=45245896)
+3000 val loss 7.2980
+3000 val perplexity 1477.3392
+3000 train 7.109474 (lr=2.9790e-05) (hash(x)=45015009)
+3200 val loss 7.2115
+3200 val perplexity 1354.8691
+3200 train 7.168735 (lr=1.9839e-05) (hash(x)=49995942)
+35300 val loss 5.9428
+35300 val perplexity 381.0115
+35300 train 5.877661 (lr=4.2002e-06) (hash(x)=51398605)
+3100 val loss 7.2809
+3100 val perplexity 1452.2314
+3100 train 7.091062 (lr=2.9775e-05) (hash(x)=45245896)
+3300 val loss 7.1921
+3300 val perplexity 1328.8853
+3300 train 7.084653 (lr=1.9828e-05) (hash(x)=52311504)
+35400 val loss 5.9403
+35400 val perplexity 380.0446
+35400 train 5.755366 (lr=4.1662e-06) (hash(x)=52214092)
+3200 val loss 7.2722
+3200 val perplexity 1439.6993
+3200 train 7.231256 (lr=2.9759e-05) (hash(x)=49995942)
+3400 val loss 7.1770
+3400 val perplexity 1309.0111
+3400 train 7.170058 (lr=1.9817e-05) (hash(x)=44332917)
+35500 val loss 5.9394
+35500 val perplexity 379.7076
+35500 train 6.278767 (lr=4.1324e-06) (hash(x)=60115839)
+3300 val loss 7.2599
+3300 val perplexity 1422.0724
+3300 train 7.151744 (lr=2.9743e-05) (hash(x)=52311504)
+3500 val loss 7.1582
+3500 val perplexity 1284.5908
+3500 train 7.265118 (lr=1.9806e-05) (hash(x)=56517159)
+35600 val loss 5.9559
+35600 val perplexity 386.0056
+35600 train 6.601181 (lr=4.0988e-06) (hash(x)=53051302)
+3400 val loss 7.2501
+3400 val perplexity 1408.3022
+3400 train 7.238114 (lr=2.9726e-05) (hash(x)=44332917)
+3600 val loss 7.1540
+3600 val perplexity 1279.1886
+3600 train 7.036954 (lr=1.9794e-05) (hash(x)=50720920)
+35700 val loss 5.9383
+35700 val perplexity 379.2992
+35700 train 5.654312 (lr=4.0653e-06) (hash(x)=47769916)
+3500 val loss 7.2353
+3500 val perplexity 1387.5149
+3500 train 7.346288 (lr=2.9709e-05) (hash(x)=56517159)
+3700 val loss 7.1343
+3700 val perplexity 1254.2578
+3700 train 7.639817 (lr=1.9782e-05) (hash(x)=62727701)
+35800 val loss 5.9392
+35800 val perplexity 379.6260
+35800 train 5.711969 (lr=4.0319e-06) (hash(x)=48652911)
+3600 val loss 7.2265
+3600 val perplexity 1375.3812
+3600 train 7.123477 (lr=2.9691e-05) (hash(x)=50720920)
+3800 val loss 7.1271
+3800 val perplexity 1245.2527
+3800 train 6.966629 (lr=1.9769e-05) (hash(x)=54772539)
+35900 val loss 5.9384
+35900 val perplexity 379.3417
+35900 train 5.867605 (lr=3.9988e-06) (hash(x)=50452504)
+3700 val loss 7.2204
+3700 val perplexity 1366.9893
+3700 train 7.701910 (lr=2.9672e-05) (hash(x)=62727701)
+3900 val loss 7.1128
+3900 val perplexity 1227.5878
+3900 train 7.136312 (lr=1.9756e-05) (hash(x)=52274485)
+36000 val loss 5.9400
+36000 val perplexity 379.9188
+36000 train 5.920528 (lr=3.9658e-06) (hash(x)=44888966)
+3800 val loss 7.2145
+3800 val perplexity 1359.0348
+3800 train 7.065799 (lr=2.9653e-05) (hash(x)=54772539)
+4000 val loss 7.1080
+4000 val perplexity 1221.6915
+4000 train 6.828626 (lr=1.9743e-05) (hash(x)=50118307)
+36100 val loss 5.9363
+36100 val perplexity 378.5241
+36100 train 5.800508 (lr=3.9330e-06) (hash(x)=48245730)
+4100 val loss 7.0940
+4100 val perplexity 1204.7369
+4100 train 6.690231 (lr=1.9729e-05) (hash(x)=42771647)
+3900 val loss 7.2119
+3900 val perplexity 1355.4009
+3900 train 7.238420 (lr=2.9634e-05) (hash(x)=52274485)
+4200 val loss 7.0801
+4200 val perplexity 1188.1140
+4200 train 7.170380 (lr=1.9715e-05) (hash(x)=51748836)
+36200 val loss 5.9339
+36200 val perplexity 377.6395
+36200 train 5.915541 (lr=3.9003e-06) (hash(x)=51614607)
+4000 val loss 7.2040
+4000 val perplexity 1344.7966
+4000 train 6.956905 (lr=2.9614e-05) (hash(x)=50118307)
+4300 val loss 7.0622
+4300 val perplexity 1166.9775
+4300 train 7.078612 (lr=1.9701e-05) (hash(x)=49021280)
+36300 val loss 5.9336
+36300 val perplexity 377.4983
+36300 train 5.971287 (lr=3.8678e-06) (hash(x)=43069600)
+4100 val loss 7.2090
+4100 val perplexity 1351.5975
+4100 train 6.819117 (lr=2.9593e-05) (hash(x)=42771647)
+4400 val loss 7.0564
+4400 val perplexity 1160.2090
+4400 train 7.157178 (lr=1.9686e-05) (hash(x)=55200309)
+36400 val loss 5.9317
+36400 val perplexity 376.7777
+36400 train 6.059913 (lr=3.8355e-06) (hash(x)=52756265)
+4200 val loss 7.1980
+4200 val perplexity 1336.7557
+4200 train 7.286159 (lr=2.9572e-05) (hash(x)=51748836)
+4500 val loss 7.0390
+4500 val perplexity 1140.3002
+4500 train 7.098520 (lr=1.9671e-05) (hash(x)=52085049)
+36500 val loss 5.9302
+36500 val perplexity 376.2151
+36500 train 5.725226 (lr=3.8034e-06) (hash(x)=47894210)
+4300 val loss 7.1784
+4300 val perplexity 1310.7963
+4300 train 7.191980 (lr=2.9551e-05) (hash(x)=49021280)
+4600 val loss 7.0309
+4600 val perplexity 1131.0530
+4600 train 7.008595 (lr=1.9656e-05) (hash(x)=48935595)
+36600 val loss 5.9310
+36600 val perplexity 376.5466
+36600 train 5.932123 (lr=3.7715e-06) (hash(x)=46906979)
+4400 val loss 7.1732
+4400 val perplexity 1303.9991
+4400 train 7.281921 (lr=2.9529e-05) (hash(x)=55200309)
+4700 val loss 7.0183
+4700 val perplexity 1116.8960
+4700 train 7.190259 (lr=1.9640e-05) (hash(x)=49182380)
+36700 val loss 5.9310
+36700 val perplexity 376.5403
+36700 train 6.633295 (lr=3.7397e-06) (hash(x)=45349010)
+4500 val loss 7.1557
+4500 val perplexity 1281.3746
+4500 train 7.212247 (lr=2.9506e-05) (hash(x)=52085049)
+4800 val loss 7.0047
+4800 val perplexity 1101.8081
+4800 train 6.796061 (lr=1.9624e-05) (hash(x)=43941929)
+36800 val loss 5.9261
+36800 val perplexity 374.6765
+36800 train 5.664440 (lr=3.7081e-06) (hash(x)=40761454)
+4900 val loss 6.9920
+4900 val perplexity 1087.8698
+4600 val loss 7.1444
+4600 val perplexity 1267.0206
+4900 train 7.161346 (lr=1.9607e-05) (hash(x)=51852773)
+4600 train 7.109821 (lr=2.9483e-05) (hash(x)=48935595)
+5000 val loss 6.9906
+5000 val perplexity 1086.3706
+36900 val loss 5.9267
+36900 val perplexity 374.9044
+36900 train 6.048511 (lr=3.6767e-06) (hash(x)=52150265)
+5000 train 6.617741 (lr=1.9591e-05) (hash(x)=40509616)
+4700 val loss 7.1271
+4700 val perplexity 1245.2545
+4700 train 7.270721 (lr=2.9460e-05) (hash(x)=49182380)
+5100 val loss 6.9702
+5100 val perplexity 1064.4276
+5100 train 7.274549 (lr=1.9573e-05) (hash(x)=57585369)
+37000 val loss 5.9270
+37000 val perplexity 375.0217
+37000 train 6.001470 (lr=3.6455e-06) (hash(x)=48982242)
+4800 val loss 7.1151
+4800 val perplexity 1230.3760
+4800 train 6.902147 (lr=2.9436e-05) (hash(x)=43941929)
+5200 val loss 6.9613
+5200 val perplexity 1054.9905
+5200 train 6.977490 (lr=1.9556e-05) (hash(x)=51042313)
+37100 val loss 5.9263
+37100 val perplexity 374.7555
+37100 train 5.883381 (lr=3.6144e-06) (hash(x)=52100108)
+4900 val loss 7.1128
+4900 val perplexity 1227.5350
+4900 train 7.275159 (lr=2.9411e-05) (hash(x)=51852773)
+5300 val loss 6.9514
+5300 val perplexity 1044.6289
+5300 train 7.090740 (lr=1.9538e-05) (hash(x)=52001684)
+37200 val loss 5.9268
+37200 val perplexity 374.9473
+37200 train 5.734054 (lr=3.5836e-06) (hash(x)=47721621)
+5000 val loss 7.1022
+5000 val perplexity 1214.6694
+5000 train 6.736402 (lr=2.9386e-05) (hash(x)=40509616)
+5400 val loss 6.9411
+5400 val perplexity 1033.9420
+5400 train 6.853202 (lr=1.9520e-05) (hash(x)=48831647)
+37300 val loss 5.9253
+37300 val perplexity 374.3751
+37300 train 5.719491 (lr=3.5529e-06) (hash(x)=49312812)
+5100 val loss 7.0904
+5100 val perplexity 1200.3314
+5100 train 7.399484 (lr=2.9360e-05) (hash(x)=57585369)
+5500 val loss 6.9378
+5500 val perplexity 1030.5201
+5500 train 7.299558 (lr=1.9502e-05) (hash(x)=50192069)
+37400 val loss 5.9246
+37400 val perplexity 374.1443
+37400 train 5.764271 (lr=3.5224e-06) (hash(x)=48536877)
+5200 val loss 7.0803
+5200 val perplexity 1188.3090
+5200 train 7.101250 (lr=2.9334e-05) (hash(x)=51042313)
+5600 val loss 6.9244
+5600 val perplexity 1016.7373
+5600 train 6.732193 (lr=1.9483e-05) (hash(x)=47208852)
+37500 val loss 5.9269
+37500 val perplexity 375.0031
+37500 train 5.883107 (lr=3.4921e-06) (hash(x)=50612814)
+5300 val loss 7.0734
+5300 val perplexity 1180.1713
+5300 train 7.199037 (lr=2.9307e-05) (hash(x)=52001684)
+5700 val loss 6.9149
+5700 val perplexity 1007.1322
+5700 train 6.515814 (lr=1.9464e-05) (hash(x)=44061694)
+37600 val loss 5.9249
+37600 val perplexity 374.2451
+37600 train 6.419948 (lr=3.4620e-06) (hash(x)=60685704)
+5800 val loss 6.9008
+5800 val perplexity 993.0355
+5800 train 7.280470 (lr=1.9444e-05) (hash(x)=56513279)
+5400 val loss 7.0761
+5400 val perplexity 1183.3308
+5400 train 6.991091 (lr=2.9280e-05) (hash(x)=48831647)
+37700 val loss 5.9241
+37700 val perplexity 373.9492
+37700 train 6.034066 (lr=3.4321e-06) (hash(x)=56543272)
+5900 val loss 6.8903
+5900 val perplexity 982.6515
+5900 train 7.034351 (lr=1.9424e-05) (hash(x)=50412818)
+5500 val loss 7.0684
+5500 val perplexity 1174.3253
+5500 train 7.414994 (lr=2.9252e-05) (hash(x)=50192069)
+6000 val loss 6.8809
+6000 val perplexity 973.5192
+6000 train 6.691369 (lr=1.9404e-05) (hash(x)=47159634)
+37800 val loss 5.9212
+37800 val perplexity 372.8663
+37800 train 6.050722 (lr=3.4024e-06) (hash(x)=56659623)
+5600 val loss 7.0588
+5600 val perplexity 1162.9951
+5600 train 6.879931 (lr=2.9224e-05) (hash(x)=47208852)
+6100 val loss 6.8656
+6100 val perplexity 958.7585
+6100 train 6.968782 (lr=1.9384e-05) (hash(x)=54312795)
+37900 val loss 5.9225
+37900 val perplexity 373.3545
+37900 train 5.603712 (lr=3.3728e-06) (hash(x)=48577952)
+5700 val loss 7.0565
+5700 val perplexity 1160.4220
+5700 train 6.664793 (lr=2.9196e-05) (hash(x)=44061694)
+6200 val loss 6.8593
+6200 val perplexity 952.6883
+6200 train 7.022208 (lr=1.9363e-05) (hash(x)=54187587)
+38000 val loss 5.9220
+38000 val perplexity 373.1468
+38000 train 5.557166 (lr=3.3435e-06) (hash(x)=46719420)
+5800 val loss 7.0460
+5800 val perplexity 1148.2582
+5800 train 7.409586 (lr=2.9166e-05) (hash(x)=56513279)
+6300 val loss 6.8302
+6300 val perplexity 925.3813
+6300 train 6.914322 (lr=1.9342e-05) (hash(x)=53620387)
+38100 val loss 5.9184
+38100 val perplexity 371.8072
+38100 train 5.884927 (lr=3.3143e-06) (hash(x)=56290069)
+5900 val loss 7.0384
+5900 val perplexity 1139.5918
+5900 train 7.187278 (lr=2.9137e-05) (hash(x)=50412818)
+6400 val loss 6.8179
+6400 val perplexity 914.0562
+6400 train 6.765425 (lr=1.9320e-05) (hash(x)=48761774)
+38200 val loss 5.9155
+38200 val perplexity 370.7310
+38200 train 5.872331 (lr=3.2854e-06) (hash(x)=50056974)
+6000 val loss 7.0273
+6000 val perplexity 1126.9749
+6000 train 6.841527 (lr=2.9106e-05) (hash(x)=47159634)
+6500 val loss 6.8085
+6500 val perplexity 905.5442
+6500 train 7.052934 (lr=1.9299e-05) (hash(x)=56690281)
+38300 val loss 5.9167
+38300 val perplexity 371.1848
+38300 train 5.910731 (lr=3.2566e-06) (hash(x)=48797079)
+6100 val loss 7.0230
+6100 val perplexity 1122.1376
+6100 train 7.122001 (lr=2.9076e-05) (hash(x)=54312795)
+6600 val loss 6.8007
+6600 val perplexity 898.4749
+6600 train 6.619725 (lr=1.9276e-05) (hash(x)=42985269)
+38400 val loss 5.9150
+38400 val perplexity 370.5637
+38400 train 5.799443 (lr=3.2281e-06) (hash(x)=49702043)
+6200 val loss 7.0252
+6200 val perplexity 1124.5979
+6200 train 7.187134 (lr=2.9044e-05) (hash(x)=54187587)
+6700 val loss 6.8142
+6700 val perplexity 910.7267
+6700 train 6.840393 (lr=1.9254e-05) (hash(x)=53315447)
+38500 val loss 5.9171
+38500 val perplexity 371.3172
+38500 train 5.938179 (lr=3.1997e-06) (hash(x)=53375877)
+6300 val loss 7.0149
+6300 val perplexity 1113.1089
+6300 train 7.100558 (lr=2.9013e-05) (hash(x)=53620387)
+6800 val loss 6.7661
+6800 val perplexity 867.9592
+6800 train 7.249754 (lr=1.9231e-05) (hash(x)=61577166)
+6900 val loss 6.7494
+6900 val perplexity 853.5458
+6900 train 6.877337 (lr=1.9208e-05) (hash(x)=54641005)
+38600 val loss 5.9114
+38600 val perplexity 369.2133
+38600 train 5.941380 (lr=3.1716e-06) (hash(x)=52757407)
+6400 val loss 7.0043
+6400 val perplexity 1101.3832
+6400 train 6.946133 (lr=2.8981e-05) (hash(x)=48761774)
+7000 val loss 6.7354
+7000 val perplexity 841.6754
+7000 train 7.286756 (lr=1.9185e-05) (hash(x)=60579512)
+38700 val loss 5.9110
+38700 val perplexity 369.0934
+38700 train 5.992656 (lr=3.1436e-06) (hash(x)=51335882)
+6500 val loss 7.0017
+6500 val perplexity 1098.4722
+6500 train 7.231021 (lr=2.8948e-05) (hash(x)=56690281)
+7100 val loss 6.7341
+7100 val perplexity 840.6198
+7100 train 6.622653 (lr=1.9161e-05) (hash(x)=53151549)
+38800 val loss 5.9109
+38800 val perplexity 369.0406
+38800 train 5.846481 (lr=3.1159e-06) (hash(x)=46048181)
+6600 val loss 6.9955
+6600 val perplexity 1091.7407
+6600 train 6.802545 (lr=2.8915e-05) (hash(x)=42985269)
+7200 val loss 6.7145
+7200 val perplexity 824.2852
+7200 train 7.689163 (lr=1.9137e-05) (hash(x)=71842455)
+6700 val loss 7.0164
+6700 val perplexity 1114.8214
+6700 train 7.019029 (lr=2.8881e-05) (hash(x)=53315447)
+38900 val loss 5.9109
+38900 val perplexity 369.0463
+38900 train 5.914674 (lr=3.0883e-06) (hash(x)=50536450)
+7300 val loss 6.7039
+7300 val perplexity 815.5853
+7300 train 6.474155 (lr=1.9112e-05) (hash(x)=44516452)
+6800 val loss 6.9934
+6800 val perplexity 1089.3829
+6800 train 7.470355 (lr=2.8847e-05) (hash(x)=61577166)
+39000 val loss 5.9088
+39000 val perplexity 368.2807
+39000 train 5.739921 (lr=3.0610e-06) (hash(x)=49201278)
+7400 val loss 6.6920
+7400 val perplexity 805.8956
+7400 train 6.337657 (lr=1.9087e-05) (hash(x)=42667710)
+6900 val loss 6.9748
+6900 val perplexity 1069.3408
+6900 train 7.076023 (lr=2.8812e-05) (hash(x)=54641005)
+39100 val loss 5.9099
+39100 val perplexity 368.6553
+39100 train 5.993381 (lr=3.0339e-06) (hash(x)=52669740)
+7500 val loss 6.6810
+7500 val perplexity 797.1133
+7500 train 6.422154 (lr=1.9062e-05) (hash(x)=47050797)
+7000 val loss 6.9658
+7000 val perplexity 1059.8142
+7000 train 7.505472 (lr=2.8777e-05) (hash(x)=60579512)
+39200 val loss 5.9072
+39200 val perplexity 367.6590
+39200 train 5.910360 (lr=3.0069e-06) (hash(x)=45566341)
+7600 val loss 6.6720
+7600 val perplexity 789.9886
+7600 train 6.583601 (lr=1.9037e-05) (hash(x)=49785056)
+7100 val loss 6.9640
+7100 val perplexity 1057.8220
+7100 train 6.860265 (lr=2.8741e-05) (hash(x)=53151549)
+39300 val loss 5.9074
+39300 val perplexity 367.7390
+39300 train 6.496314 (lr=2.9802e-06) (hash(x)=50582342)
+7700 val loss 6.6532
+7700 val perplexity 775.2761
+7700 train 6.482975 (lr=1.9011e-05) (hash(x)=53232030)
+7200 val loss 6.9515
+7200 val perplexity 1044.7245
+7200 train 7.894550 (lr=2.8705e-05) (hash(x)=71842455)
+7800 val loss 6.6476
+7800 val perplexity 770.9294
+7800 train 6.474964 (lr=1.8985e-05) (hash(x)=48049749)
+39400 val loss 5.9058
+39400 val perplexity 367.1459
+39400 train 5.655417 (lr=2.9537e-06) (hash(x)=43591296)
+7900 val loss 6.6291
+7900 val perplexity 756.7866
+7900 train 6.427439 (lr=1.8959e-05) (hash(x)=44768513)
+7300 val loss 6.9471
+7300 val perplexity 1040.1227
+7300 train 6.709592 (lr=2.8668e-05) (hash(x)=44516452)
+39500 val loss 5.9073
+39500 val perplexity 367.6959
+39500 train 5.697701 (lr=2.9274e-06) (hash(x)=49119520)
+8000 val loss 6.6115
+8000 val perplexity 743.5921
+8000 train 6.469481 (lr=1.8932e-05) (hash(x)=46228039)
+7400 val loss 6.9422
+7400 val perplexity 1035.0262
+7400 train 6.617076 (lr=2.8631e-05) (hash(x)=42667710)
+39600 val loss 5.9060
+39600 val perplexity 367.2266
+39600 train 6.130869 (lr=2.9014e-06) (hash(x)=50278799)
+8100 val loss 6.6183
+8100 val perplexity 748.6688
+8100 train 7.019002 (lr=1.8905e-05) (hash(x)=60017091)
+7500 val loss 6.9354
+7500 val perplexity 1027.9828
+7500 train 6.684983 (lr=2.8594e-05) (hash(x)=47050797)
+39700 val loss 5.9073
+39700 val perplexity 367.7008
+39700 train 5.837137 (lr=2.8755e-06) (hash(x)=54799713)
+8200 val loss 6.5925
+8200 val perplexity 729.5909
+8200 train 6.596733 (lr=1.8878e-05) (hash(x)=49910198)
+7600 val loss 6.9313
+7600 val perplexity 1023.7872
+7600 train 6.834815 (lr=2.8556e-05) (hash(x)=49785056)
+39800 val loss 5.9071
+39800 val perplexity 367.6208
+39800 train 5.982021 (lr=2.8498e-06) (hash(x)=46290010)
+8300 val loss 6.5824
+8300 val perplexity 722.2355
+8300 train 6.964666 (lr=1.8850e-05) (hash(x)=57919055)
+7700 val loss 6.9228
+7700 val perplexity 1015.1658
+7700 train 6.772649 (lr=2.8517e-05) (hash(x)=53232030)
+39900 val loss 5.9058
+39900 val perplexity 367.1671
+39900 train 5.549470 (lr=2.8244e-06) (hash(x)=41503413)
+8400 val loss 6.5706
+8400 val perplexity 713.8079
+8400 train 6.681305 (lr=1.8822e-05) (hash(x)=49694964)
+7800 val loss 6.9070
+7800 val perplexity 999.2861
+7800 train 6.741716 (lr=2.8478e-05) (hash(x)=48049749)
+40000 val loss 5.9043
+40000 val perplexity 366.6274
+40000 train 5.911654 (lr=2.7992e-06) (hash(x)=44949087)
+8500 val loss 6.5613
+8500 val perplexity 707.2082
+8500 train 6.581522 (lr=1.8794e-05) (hash(x)=53762585)
+7900 val loss 6.8940
+7900 val perplexity 986.2900
+7900 train 6.681769 (lr=2.8438e-05) (hash(x)=44768513)
+40100 val loss 5.9048
+40100 val perplexity 366.7914
+40100 train 5.495280 (lr=2.7742e-06) (hash(x)=45005676)
+8600 val loss 6.5529
+8600 val perplexity 701.2430
+8600 train 6.587273 (lr=1.8766e-05) (hash(x)=51166973)
+8000 val loss 6.8759
+8000 val perplexity 968.6295
+8000 train 6.749121 (lr=2.8398e-05) (hash(x)=46228039)
+8700 val loss 6.5433
+8700 val perplexity 694.5820
+8700 train 6.630268 (lr=1.8737e-05) (hash(x)=53968049)
+40200 val loss 5.9051
+40200 val perplexity 366.8949
+40200 train 6.075080 (lr=2.7494e-06) (hash(x)=51341494)
+8100 val loss 6.8648
+8100 val perplexity 957.9236
+8100 train 7.291884 (lr=2.8358e-05) (hash(x)=60017091)
+8800 val loss 6.5369
+8800 val perplexity 690.1122
+8800 train 6.622470 (lr=1.8708e-05) (hash(x)=59231056)
+40300 val loss 5.9057
+40300 val perplexity 367.1190
+40300 train 5.644422 (lr=2.7248e-06) (hash(x)=47686820)
+8200 val loss 6.8442
+8200 val perplexity 938.4005
+8200 train 6.815755 (lr=2.8317e-05) (hash(x)=49910198)
+8900 val loss 6.5361
+8900 val perplexity 689.5646
+8900 train 6.453028 (lr=1.8678e-05) (hash(x)=50488048)
+40400 val loss 5.9046
+40400 val perplexity 366.7326
+40400 train 5.795218 (lr=2.7004e-06) (hash(x)=47142962)
+9000 val loss 6.5168
+9000 val perplexity 676.3849
+9000 train 6.215597 (lr=1.8648e-05) (hash(x)=44492956)
+8300 val loss 6.8339
+8300 val perplexity 928.8366
+8300 train 7.189191 (lr=2.8276e-05) (hash(x)=57919055)
+40500 val loss 5.9047
+40500 val perplexity 366.7492
+40500 train 5.665764 (lr=2.6763e-06) (hash(x)=47585616)
+9100 val loss 6.5286
+9100 val perplexity 684.4349
+9100 train 6.554313 (lr=1.8618e-05) (hash(x)=51134989)
+8400 val loss 6.8249
+8400 val perplexity 920.4854
+8400 train 6.927425 (lr=2.8234e-05) (hash(x)=49694964)
+40600 val loss 5.9061
+40600 val perplexity 367.2821
+40600 train 5.740398 (lr=2.6524e-06) (hash(x)=50075832)
+9200 val loss 6.5059
+9200 val perplexity 669.0823
+9200 train 6.309696 (lr=1.8588e-05) (hash(x)=48636056)
+8500 val loss 6.8170
+8500 val perplexity 913.2006
+8500 train 6.822862 (lr=2.8191e-05) (hash(x)=53762585)
+40700 val loss 5.8990
+40700 val perplexity 364.6791
+40700 train 5.960289 (lr=2.6287e-06) (hash(x)=52764040)
+9300 val loss 6.5000
+9300 val perplexity 665.1661
+9300 train 6.431509 (lr=1.8557e-05) (hash(x)=50200551)
+8600 val loss 6.8183
+8600 val perplexity 914.4538
+8600 train 6.863628 (lr=2.8149e-05) (hash(x)=51166973)
+40800 val loss 5.8982
+40800 val perplexity 364.3813
+40800 train 6.265448 (lr=2.6053e-06) (hash(x)=56727171)
+9400 val loss 6.4952
+9400 val perplexity 661.9431
+9400 train 6.298841 (lr=1.8526e-05) (hash(x)=48057228)
+8700 val loss 6.8010
+8700 val perplexity 898.7273
+8700 train 6.878037 (lr=2.8105e-05) (hash(x)=53968049)
+40900 val loss 5.8983
+40900 val perplexity 364.4335
+40900 train 5.969778 (lr=2.5820e-06) (hash(x)=53111899)
+9500 val loss 6.4800
+9500 val perplexity 651.9874
+9500 train 6.246171 (lr=1.8495e-05) (hash(x)=48125171)
+8800 val loss 6.7914
+8800 val perplexity 890.1478
+8800 train 6.882590 (lr=2.8062e-05) (hash(x)=59231056)
+9600 val loss 6.4949
+9600 val perplexity 661.7307
+9600 train 6.398923 (lr=1.8463e-05) (hash(x)=53375853)
+41000 val loss 5.8948
+41000 val perplexity 363.1500
+41000 train 5.751266 (lr=2.5590e-06) (hash(x)=45951630)
+8900 val loss 6.7842
+8900 val perplexity 883.7616
+8900 train 6.684120 (lr=2.8017e-05) (hash(x)=50488048)
+9700 val loss 6.4699
+9700 val perplexity 645.4162
+9700 train 7.257029 (lr=1.8432e-05) (hash(x)=53924631)
+41100 val loss 5.8965
+41100 val perplexity 363.7515
+41100 train 6.290370 (lr=2.5362e-06) (hash(x)=59664516)
+9000 val loss 6.7676
+9000 val perplexity 869.2527
+9000 train 6.466223 (lr=2.7973e-05) (hash(x)=44492956)
+9800 val loss 6.4543
+9800 val perplexity 635.4154
+9800 train 6.504161 (lr=1.8399e-05) (hash(x)=48895047)
+9100 val loss 6.7695
+9100 val perplexity 870.9086
+9100 train 6.777167 (lr=2.7928e-05) (hash(x)=51134989)
+9900 val loss 6.4522
+9900 val perplexity 634.0721
+9900 train 6.296432 (lr=1.8367e-05) (hash(x)=44269923)
+9200 val loss 6.7513
+9200 val perplexity 855.1868
+9200 train 6.573255 (lr=2.7882e-05) (hash(x)=48636056)
+9300 val loss 6.7498
+9300 val perplexity 853.9117
+9300 train 6.711143 (lr=2.7836e-05) (hash(x)=50200551)
+10000 val loss 6.4551
+10000 val perplexity 635.9598
+10000 train 6.334339 (lr=1.8334e-05) (hash(x)=49666623)
+9400 val loss 6.7390
+9400 val perplexity 844.6905
+9400 train 6.563443 (lr=2.7789e-05) (hash(x)=48057228)
+10100 val loss 6.4409
+10100 val perplexity 626.9396
+10100 train 5.940355 (lr=1.8301e-05) (hash(x)=39732485)
+9500 val loss 6.7267
+9500 val perplexity 834.4134
+9500 train 6.497000 (lr=2.7743e-05) (hash(x)=48125171)
+9600 val loss 6.7241
+9600 val perplexity 832.1874
+9600 train 6.652503 (lr=2.7695e-05) (hash(x)=53375853)
+10200 val loss 6.4302
+10200 val perplexity 620.2892
+10200 train 7.446473 (lr=1.8268e-05) (hash(x)=45176813)
+9700 val loss 6.6984
+9700 val perplexity 811.1329
+9700 train 7.455695 (lr=2.7647e-05) (hash(x)=53924631)
+10300 val loss 6.4171
+10300 val perplexity 612.2554
+10300 train 6.261757 (lr=1.8234e-05) (hash(x)=47779031)
+9800 val loss 6.6904
+9800 val perplexity 804.6119
+9800 train 6.728513 (lr=2.7599e-05) (hash(x)=48895047)
+10400 val loss 6.4175
+10400 val perplexity 612.4774
+10400 train 6.281345 (lr=1.8200e-05) (hash(x)=47802708)
+9900 val loss 6.6833
+9900 val perplexity 798.9330
+9900 train 6.538757 (lr=2.7550e-05) (hash(x)=44269923)
+10000 val loss 6.6736
+10000 val perplexity 791.2455
+10000 train 6.570342 (lr=2.7501e-05) (hash(x)=49666623)
+10500 val loss 6.4047
+10500 val perplexity 604.6678
+10500 train 6.219766 (lr=1.8166e-05) (hash(x)=47245542)
+10100 val loss 6.6558
+10100 val perplexity 777.2894
+10100 train 6.179580 (lr=2.7452e-05) (hash(x)=39732485)
+10600 val loss 6.3986
+10600 val perplexity 601.0154
+10600 train 6.471349 (lr=1.8132e-05) (hash(x)=50902224)
+10200 val loss 6.6427
+10200 val perplexity 767.1436
+10200 train 7.525699 (lr=2.7402e-05) (hash(x)=45176813)
+10700 val loss 6.3968
+10700 val perplexity 599.8931
+10700 train 6.289356 (lr=1.8097e-05) (hash(x)=51581765)
+10300 val loss 6.6382
+10300 val perplexity 763.7065
+10300 train 6.488876 (lr=2.7351e-05) (hash(x)=47779031)
+10400 val loss 6.6277
+10400 val perplexity 755.7404
+10400 train 6.478593 (lr=2.7300e-05) (hash(x)=47802708)
+10800 val loss 6.3922
+10800 val perplexity 597.1857
+10800 train 6.419674 (lr=1.8062e-05) (hash(x)=50137616)
+10500 val loss 6.6167
+10500 val perplexity 747.4396
+10500 train 6.430318 (lr=2.7249e-05) (hash(x)=47245542)
+10900 val loss 6.3893
+10900 val perplexity 595.4694
+10900 train 6.089376 (lr=1.8026e-05) (hash(x)=45446745)
+10600 val loss 6.6043
+10600 val perplexity 738.2361
+10600 train 6.668101 (lr=2.7197e-05) (hash(x)=50902224)
+11000 val loss 6.3815
+11000 val perplexity 590.7982
+11000 train 6.211236 (lr=1.7991e-05) (hash(x)=50403725)
+10700 val loss 6.5934
+10700 val perplexity 730.2362
+10700 train 6.497122 (lr=2.7145e-05) (hash(x)=51581765)
+10800 val loss 6.5874
+10800 val perplexity 725.9222
+10800 train 6.604099 (lr=2.7093e-05) (hash(x)=50137616)
+11100 val loss 6.3782
+11100 val perplexity 588.8416
+11100 train 5.830037 (lr=1.7955e-05) (hash(x)=42015206)
+10900 val loss 6.5874
+10900 val perplexity 725.8852
+10900 train 6.287614 (lr=2.7040e-05) (hash(x)=45446745)
+11200 val loss 6.3727
+11200 val perplexity 585.6443
+11200 train 6.237059 (lr=1.7919e-05) (hash(x)=52280259)
+11000 val loss 6.5813
+11000 val perplexity 721.4620
+11000 train 6.428674 (lr=2.6986e-05) (hash(x)=50403725)
+11300 val loss 6.3679
+11300 val perplexity 582.8486
+11300 train 6.124495 (lr=1.7882e-05) (hash(x)=50264744)
+11100 val loss 6.5698
+11100 val perplexity 713.2064
+11100 train 6.012609 (lr=2.6932e-05) (hash(x)=42015206)
+11200 val loss 6.5617
+11200 val perplexity 707.4396
+11200 train 6.426662 (lr=2.6878e-05) (hash(x)=52280259)
+11400 val loss 6.3668
+11400 val perplexity 582.2203
+11400 train 6.268094 (lr=1.7845e-05) (hash(x)=49817375)
+11300 val loss 6.5636
+11300 val perplexity 708.7875
+11300 train 6.338786 (lr=2.6823e-05) (hash(x)=50264744)
+11500 val loss 6.3553
+11500 val perplexity 575.5437
+11500 train 6.351455 (lr=1.7809e-05) (hash(x)=51662160)
+11400 val loss 6.5569
+11400 val perplexity 704.0655
+11400 train 6.469149 (lr=2.6768e-05) (hash(x)=49817375)
+11600 val loss 6.3556
+11600 val perplexity 575.7087
+11600 train 6.306983 (lr=1.7771e-05) (hash(x)=48891495)
+11500 val loss 6.5505
+11500 val perplexity 699.5775
+11500 train 6.553368 (lr=2.6713e-05) (hash(x)=51662160)
+11700 val loss 6.3338
+11700 val perplexity 563.2967
+11700 train 6.323509 (lr=1.7734e-05) (hash(x)=46306682)
+11600 val loss 6.5542
+11600 val perplexity 702.1796
+11600 train 6.502248 (lr=2.6657e-05) (hash(x)=48891495)
+11800 val loss 6.3276
+11800 val perplexity 559.8133
+11800 train 6.281574 (lr=1.7696e-05) (hash(x)=46605293)
+11700 val loss 6.5216
+11700 val perplexity 679.6616
+11700 train 6.503895 (lr=2.6601e-05) (hash(x)=46306682)
+11900 val loss 6.3177
+11900 val perplexity 554.2987
+11900 train 6.244193 (lr=1.7658e-05) (hash(x)=53746201)
+11800 val loss 6.5176
+11800 val perplexity 676.9421
+11800 train 6.452675 (lr=2.6544e-05) (hash(x)=46605293)
+12000 val loss 6.3148
+12000 val perplexity 552.7012
+12000 train 6.409264 (lr=1.7619e-05) (hash(x)=48587180)
+11900 val loss 6.5102
+11900 val perplexity 671.9752
+11900 train 6.448487 (lr=2.6487e-05) (hash(x)=53746201)
+12100 val loss 6.3118
+12100 val perplexity 551.0380
+12100 train 6.098515 (lr=1.7581e-05) (hash(x)=45786729)
+12000 val loss 6.4949
+12000 val perplexity 661.7803
+12000 train 6.574782 (lr=2.6429e-05) (hash(x)=48587180)
+12200 val loss 6.3044
+12200 val perplexity 546.9933
+12200 train 6.262596 (lr=1.7542e-05) (hash(x)=42858336)
+12100 val loss 6.5034
+12100 val perplexity 667.3970
+12100 train 6.259594 (lr=2.6371e-05) (hash(x)=45786729)
+12300 val loss 6.2982
+12300 val perplexity 543.6165
+12300 train 6.466894 (lr=1.7503e-05) (hash(x)=51944313)
+12200 val loss 6.4847
+12200 val perplexity 655.0447
+12200 train 6.435366 (lr=2.6313e-05) (hash(x)=42858336)
+12400 val loss 6.2888
+12400 val perplexity 538.5133
+12400 train 6.782796 (lr=1.7464e-05) (hash(x)=59357191)
+12300 val loss 6.4811
+12300 val perplexity 652.7029
+12300 train 6.664652 (lr=2.6254e-05) (hash(x)=51944313)
+12500 val loss 6.2804
+12500 val perplexity 533.9772
+12500 train 6.387350 (lr=1.7424e-05) (hash(x)=49770062)
+12400 val loss 6.4830
+12400 val perplexity 653.9244
+12400 train 7.004457 (lr=2.6195e-05) (hash(x)=59357191)
+12600 val loss 6.2823
+12600 val perplexity 534.9946
+12600 train 6.053275 (lr=1.7384e-05) (hash(x)=47235788)
+12500 val loss 6.4612
+12500 val perplexity 639.8331
+12500 train 6.578255 (lr=2.6136e-05) (hash(x)=49770062)
+12700 val loss 6.2731
+12700 val perplexity 530.1288
+12700 train 6.283521 (lr=1.7344e-05) (hash(x)=51700943)
+12600 val loss 6.4535
+12600 val perplexity 634.9377
+12600 train 6.222001 (lr=2.6076e-05) (hash(x)=47235788)
+12800 val loss 6.2684
+12800 val perplexity 527.6228
+12800 train 6.227781 (lr=1.7303e-05) (hash(x)=50532124)
+12700 val loss 6.4535
+12700 val perplexity 634.9272
+12700 train 6.476688 (lr=2.6016e-05) (hash(x)=51700943)
+12900 val loss 6.2658
+12900 val perplexity 526.2704
+12900 train 6.361206 (lr=1.7263e-05) (hash(x)=51948273)
+12800 val loss 6.4446
+12800 val perplexity 629.3238
+12800 train 6.396770 (lr=2.5955e-05) (hash(x)=50532124)
+13000 val loss 6.2694
+13000 val perplexity 528.1793
+13000 train 6.454553 (lr=1.7222e-05) (hash(x)=46732738)
+12900 val loss 6.4409
+12900 val perplexity 626.9483
+12900 train 6.545595 (lr=2.5894e-05) (hash(x)=51948273)
+13100 val loss 6.2587
+13100 val perplexity 522.5568
+13100 train 6.257236 (lr=1.7181e-05) (hash(x)=49837250)
+13000 val loss 6.4394
+13000 val perplexity 626.0610
+13000 train 6.611993 (lr=2.5833e-05) (hash(x)=46732738)
+13200 val loss 6.2575
+13200 val perplexity 521.9365
+13200 train 6.552484 (lr=1.7139e-05) (hash(x)=57646893)
+13100 val loss 6.4218
+13100 val perplexity 615.1276
+13100 train 6.428285 (lr=2.5771e-05) (hash(x)=49837250)
+13300 val loss 6.2591
+13300 val perplexity 522.7662
+13300 train 5.796292 (lr=1.7098e-05) (hash(x)=42055305)
+13200 val loss 6.4274
+13200 val perplexity 618.5604
+13200 train 6.766737 (lr=2.5709e-05) (hash(x)=57646893)
+13400 val loss 6.2540
+13400 val perplexity 520.0995
+13400 train 5.953706 (lr=1.7056e-05) (hash(x)=44684844)
+13300 val loss 6.4196
+13300 val perplexity 613.7654
+13300 train 5.951127 (lr=2.5647e-05) (hash(x)=42055305)
+13500 val loss 6.2497
+13500 val perplexity 517.8377
+13500 train 5.921538 (lr=1.7014e-05) (hash(x)=46971715)
+13400 val loss 6.4179
+13400 val perplexity 612.6996
+13400 train 6.118713 (lr=2.5584e-05) (hash(x)=44684844)
+13600 val loss 6.2417
+13600 val perplexity 513.7351
+13600 train 6.077806 (lr=1.6971e-05) (hash(x)=48728506)
+13500 val loss 6.4154
+13500 val perplexity 611.1861
+13500 train 6.093008 (lr=2.5521e-05) (hash(x)=46971715)
+13700 val loss 6.2473
+13700 val perplexity 516.6006
+13700 train 6.045406 (lr=1.6929e-05) (hash(x)=50739726)
+13600 val loss 6.4053
+13600 val perplexity 605.0511
+13600 train 6.265106 (lr=2.5457e-05) (hash(x)=48728506)
+13800 val loss 6.2404
+13800 val perplexity 513.0594
+13800 train 6.369393 (lr=1.6886e-05) (hash(x)=52174318)
+13700 val loss 6.4141
+13700 val perplexity 610.3922
+13700 train 6.237026 (lr=2.5393e-05) (hash(x)=50739726)
+13900 val loss 6.2337
+13900 val perplexity 509.6327
+13900 train 5.618373 (lr=1.6843e-05) (hash(x)=44141266)
+13800 val loss 6.4075
+13800 val perplexity 606.3760
+13800 train 6.525609 (lr=2.5329e-05) (hash(x)=52174318)
+14000 val loss 6.2259
+14000 val perplexity 505.6831
+14000 train 6.193916 (lr=1.6800e-05) (hash(x)=50880921)
+13900 val loss 6.4080
+13900 val perplexity 606.6499
+13900 train 5.806569 (lr=2.5264e-05) (hash(x)=44141266)
+14100 val loss 6.2212
+14100 val perplexity 503.3112
+14100 train 6.140870 (lr=1.6756e-05) (hash(x)=44179328)
+14000 val loss 6.3950
+14000 val perplexity 598.8296
+14000 train 6.393941 (lr=2.5199e-05) (hash(x)=50880921)
+14200 val loss 6.2077
+14200 val perplexity 496.5558
+14200 train 6.319089 (lr=1.6712e-05) (hash(x)=52989772)
+14100 val loss 6.3790
+14100 val perplexity 589.3093
+14100 train 6.311073 (lr=2.5134e-05) (hash(x)=44179328)
+14300 val loss 6.2026
+14300 val perplexity 494.0191
+14300 train 6.438374 (lr=1.6668e-05) (hash(x)=55406567)
+14200 val loss 6.3757
+14200 val perplexity 587.3710
+14200 train 6.483571 (lr=2.5068e-05) (hash(x)=52989772)
+14400 val loss 6.1976
+14400 val perplexity 491.5441
+14400 train 6.237109 (lr=1.6624e-05) (hash(x)=47274361)
+14300 val loss 6.3646
+14300 val perplexity 580.9161
+14300 train 6.578245 (lr=2.5002e-05) (hash(x)=55406567)
+14500 val loss 6.1941
+14500 val perplexity 489.8303
+14500 train 6.637053 (lr=1.6580e-05) (hash(x)=53333322)
+14400 val loss 6.3545
+14400 val perplexity 575.0757
+14400 train 6.380944 (lr=2.4936e-05) (hash(x)=47274361)
+14600 val loss 6.1889
+14600 val perplexity 487.3140
+14600 train 6.320680 (lr=1.6535e-05) (hash(x)=48578234)
+14500 val loss 6.3581
+14500 val perplexity 577.1357
+14500 train 6.808034 (lr=2.4869e-05) (hash(x)=53333322)
+14700 val loss 6.1928
+14700 val perplexity 489.2320
+14700 train 5.850684 (lr=1.6490e-05) (hash(x)=45694052)
+14600 val loss 6.3516
+14600 val perplexity 573.4229
+14600 train 6.476131 (lr=2.4802e-05) (hash(x)=48578234)
+14800 val loss 6.1793
+14800 val perplexity 482.6633
+14800 train 5.962896 (lr=1.6445e-05) (hash(x)=46245798)
+14700 val loss 6.3706
+14700 val perplexity 584.4319
+14700 train 6.043706 (lr=2.4735e-05) (hash(x)=45694052)
+14900 val loss 6.1838
+14900 val perplexity 484.8154
+14900 train 6.326663 (lr=1.6400e-05) (hash(x)=55970820)
+14800 val loss 6.3362
+14800 val perplexity 564.6424
+14800 train 6.141839 (lr=2.4668e-05) (hash(x)=46245798)
+15000 val loss 6.1824
+15000 val perplexity 484.1367
+15000 train 6.292761 (lr=1.6354e-05) (hash(x)=55635805)
+14900 val loss 6.3500
+14900 val perplexity 572.4984
+14900 train 6.491947 (lr=2.4600e-05) (hash(x)=55970820)
+15100 val loss 6.1630
+15100 val perplexity 474.8546
+15100 train 6.079277 (lr=1.6308e-05) (hash(x)=48884573)
+15000 val loss 6.3397
+15000 val perplexity 566.6356
+15000 train 6.432809 (lr=2.4531e-05) (hash(x)=55635805)
+15200 val loss 6.1633
+15200 val perplexity 475.0038
+15200 train 6.344537 (lr=1.6262e-05) (hash(x)=59283076)
+15100 val loss 6.3262
+15100 val perplexity 559.0528
+15100 train 6.228620 (lr=2.4463e-05) (hash(x)=48884573)
+15300 val loss 6.1599
+15300 val perplexity 473.3729
+15300 train 6.083650 (lr=1.6216e-05) (hash(x)=48258741)
+15200 val loss 6.3199
+15200 val perplexity 555.5445
+15200 train 6.512674 (lr=2.4394e-05) (hash(x)=59283076)
+15400 val loss 6.1581
+15400 val perplexity 472.5211
+15400 train 6.202772 (lr=1.6170e-05) (hash(x)=49433427)
+15300 val loss 6.3152
+15300 val perplexity 552.9125
+15300 train 6.246204 (lr=2.4324e-05) (hash(x)=48258741)
+15500 val loss 6.1571
+15500 val perplexity 472.0676
+15500 train 5.914767 (lr=1.6123e-05) (hash(x)=42504793)
+15400 val loss 6.3197
+15400 val perplexity 555.3824
+15400 train 6.393365 (lr=2.4255e-05) (hash(x)=49433427)
+15600 val loss 6.1523
+15600 val perplexity 469.7962
+15600 train 6.036044 (lr=1.6077e-05) (hash(x)=48860388)
+15500 val loss 6.3186
+15500 val perplexity 554.8186
+15500 train 6.070224 (lr=2.4185e-05) (hash(x)=42504793)
+15700 val loss 6.1422
+15700 val perplexity 465.0862
+15700 train 5.961047 (lr=1.6030e-05) (hash(x)=49229381)
+15600 val loss 6.3235
+15600 val perplexity 557.5032
+15600 train 6.205709 (lr=2.4115e-05) (hash(x)=48860388)
+15800 val loss 6.1415
+15800 val perplexity 464.7309
+15800 train 5.947126 (lr=1.5982e-05) (hash(x)=48471211)
+15700 val loss 6.2982
+15700 val perplexity 543.5794
+15700 train 6.102509 (lr=2.4044e-05) (hash(x)=49229381)
+15900 val loss 6.1466
+15900 val perplexity 467.1286
+15900 train 5.860680 (lr=1.5935e-05) (hash(x)=47555877)
+15800 val loss 6.2967
+15800 val perplexity 542.7612
+15800 train 6.088617 (lr=2.3974e-05) (hash(x)=48471211)
+16000 val loss 6.1421
+16000 val perplexity 465.0377
+16000 train 5.859025 (lr=1.5887e-05) (hash(x)=48397986)
+15900 val loss 6.2989
+15900 val perplexity 543.9979
+15900 train 6.013615 (lr=2.3903e-05) (hash(x)=47555877)
+16100 val loss 6.1424
+16100 val perplexity 465.1572
+16100 train 6.235885 (lr=1.5840e-05) (hash(x)=53299004)
+16000 val loss 6.2968
+16000 val perplexity 542.8060
+16000 train 5.999140 (lr=2.3831e-05) (hash(x)=48397986)
+16100 val loss 6.2971
+16100 val perplexity 542.9978
+16100 train 6.387906 (lr=2.3760e-05) (hash(x)=53299004)
+16200 val loss 6.1543
+16200 val perplexity 470.7380
+16200 train 5.936277 (lr=1.5792e-05) (hash(x)=45763277)
+16200 val loss 6.2963
+16200 val perplexity 542.5806
+16200 train 6.080934 (lr=2.3688e-05) (hash(x)=45763277)
+16300 val loss 6.1405
+16300 val perplexity 464.2948
+16300 train 5.616697 (lr=1.5744e-05) (hash(x)=39675609)
+16300 val loss 6.2885
+16300 val perplexity 538.3557
+16300 train 5.748598 (lr=2.3615e-05) (hash(x)=39675609)
+16400 val loss 6.1388
+16400 val perplexity 463.4907
+16400 train 6.200221 (lr=1.5695e-05) (hash(x)=48973557)
+16400 val loss 6.2856
+16400 val perplexity 536.7828
+16400 train 6.355161 (lr=2.3543e-05) (hash(x)=48973557)
+16500 val loss 6.1257
+16500 val perplexity 457.4858
+16500 train 6.098573 (lr=1.5647e-05) (hash(x)=46121211)
+16500 val loss 6.2677
+16500 val perplexity 527.2767
+16500 train 6.225919 (lr=2.3470e-05) (hash(x)=46121211)
+16600 val loss 6.1169
+16600 val perplexity 453.4765
+16600 train 6.395256 (lr=1.5598e-05) (hash(x)=49181055)
+16600 val loss 6.2721
+16600 val perplexity 529.5921
+16600 train 6.534654 (lr=2.3397e-05) (hash(x)=49181055)
+16700 val loss 6.1165
+16700 val perplexity 453.2925
+16700 train 6.058544 (lr=1.5549e-05) (hash(x)=49361213)
+16700 val loss 6.2686
+16700 val perplexity 527.7282
+16700 train 6.207542 (lr=2.3324e-05) (hash(x)=49361213)
+16800 val loss 6.1165
+16800 val perplexity 453.2815
+16800 train 6.144247 (lr=1.5500e-05) (hash(x)=52739987)
+16800 val loss 6.2708
+16800 val perplexity 528.8873
+16800 train 6.306143 (lr=2.3250e-05) (hash(x)=52739987)
+16900 val loss 6.1025
+16900 val perplexity 446.9725
+16900 train 6.289323 (lr=1.5451e-05) (hash(x)=55994797)
+16900 val loss 6.2500
+16900 val perplexity 518.0148
+16900 train 6.456830 (lr=2.3176e-05) (hash(x)=55994797)
+17000 val loss 6.0981
+17000 val perplexity 445.0062
+17000 train 6.122550 (lr=1.5401e-05) (hash(x)=50109331)
+17000 val loss 6.2437
+17000 val perplexity 514.7530
+17000 train 6.282882 (lr=2.3102e-05) (hash(x)=50109331)
+17100 val loss 6.0956
+17100 val perplexity 443.9139
+17100 train 6.156457 (lr=1.5352e-05) (hash(x)=50292169)
+17100 val loss 6.2404
+17100 val perplexity 513.0597
+17100 train 6.318860 (lr=2.3028e-05) (hash(x)=50292169)
+17200 val loss 6.1043
+17200 val perplexity 447.7934
+17200 train 6.153732 (lr=1.5302e-05) (hash(x)=50722386)
+17200 val loss 6.2444
+17200 val perplexity 515.1353
+17200 train 6.294212 (lr=2.2953e-05) (hash(x)=50722386)
+17300 val loss 6.2357
+17300 val perplexity 510.6502
+17300 train 6.356597 (lr=2.2878e-05) (hash(x)=54010042)
+17300 val loss 6.0899
+17300 val perplexity 441.3775
+17300 train 6.219165 (lr=1.5252e-05) (hash(x)=54010042)
+17400 val loss 6.2329
+17400 val perplexity 509.2309
+17400 train 6.217581 (lr=2.2803e-05) (hash(x)=53970990)
+17400 val loss 6.0874
+17400 val perplexity 440.2878
+17400 train 6.029953 (lr=1.5202e-05) (hash(x)=53970990)
+17500 val loss 6.2212
+17500 val perplexity 503.3260
+17500 train 6.313018 (lr=2.2727e-05) (hash(x)=52544667)
+17500 val loss 6.0829
+17500 val perplexity 438.2926
+17500 train 6.161538 (lr=1.5152e-05) (hash(x)=52544667)
+17600 val loss 6.2229
+17600 val perplexity 504.1432
+17600 train 6.167073 (lr=2.2652e-05) (hash(x)=50735496)
+17600 val loss 6.0835
+17600 val perplexity 438.5805
+17600 train 6.038527 (lr=1.5101e-05) (hash(x)=50735496)
+17700 val loss 6.2219
+17700 val perplexity 503.6670
+17700 train 6.598975 (lr=2.2576e-05) (hash(x)=55662551)
+17700 val loss 6.0825
+17700 val perplexity 438.1438
+17700 train 6.456525 (lr=1.5050e-05) (hash(x)=55662551)
+17800 val loss 6.2167
+17800 val perplexity 501.0320
+17800 train 6.096024 (lr=2.2500e-05) (hash(x)=51137009)
+17800 val loss 6.0759
+17800 val perplexity 435.2604
+17800 train 5.948887 (lr=1.5000e-05) (hash(x)=51137009)
+17900 val loss 6.2160
+17900 val perplexity 500.6946
+17900 train 7.101241 (lr=2.2423e-05) (hash(x)=73004834)
+17900 val loss 6.0814
+17900 val perplexity 437.6381
+17900 train 6.895092 (lr=1.4949e-05) (hash(x)=73004834)
+18000 val loss 6.2047
+18000 val perplexity 495.0716
+18000 train 6.404561 (lr=2.2346e-05) (hash(x)=51900245)
+18000 val loss 6.0733
+18000 val perplexity 434.0978
+18000 train 6.287374 (lr=1.4898e-05) (hash(x)=51900245)
+18100 val loss 6.2000
+18100 val perplexity 492.7433
+18100 train 6.487039 (lr=2.2270e-05) (hash(x)=56278625)
+18100 val loss 6.0685
+18100 val perplexity 432.0235
+18100 train 6.347717 (lr=1.4846e-05) (hash(x)=56278625)
+18200 val loss 6.2046
+18200 val perplexity 495.0003
+18200 train 6.074300 (lr=2.2192e-05) (hash(x)=50478164)
+18200 val loss 6.0777
+18200 val perplexity 436.0458
+18200 train 5.941356 (lr=1.4795e-05) (hash(x)=50478164)
+18300 val loss 6.2009
+18300 val perplexity 493.1711
+18300 train 6.167547 (lr=2.2115e-05) (hash(x)=47837565)
+18300 val loss 6.0701
+18300 val perplexity 432.7140
+18300 train 6.044156 (lr=1.4743e-05) (hash(x)=47837565)
+18400 val loss 6.2039
+18400 val perplexity 494.6976
+18400 train 6.075322 (lr=2.2038e-05) (hash(x)=49622704)
+18400 val loss 6.0693
+18400 val perplexity 432.3846
+18400 train 5.925997 (lr=1.4692e-05) (hash(x)=49622704)
+18500 val loss 6.1913
+18500 val perplexity 488.4863
+18500 train 6.025623 (lr=2.1960e-05) (hash(x)=49741203)
+18600 val loss 6.1956
+18600 val perplexity 490.5717
+18600 train 6.680341 (lr=2.1882e-05) (hash(x)=54690634)
+18500 val loss 6.0615
+18500 val perplexity 429.0167
+18500 train 5.897947 (lr=1.4640e-05) (hash(x)=49741203)
+18700 val loss 6.1929
+18700 val perplexity 489.2413
+18700 train 6.065812 (lr=2.1804e-05) (hash(x)=53778909)
+18600 val loss 6.0689
+18600 val perplexity 432.1851
+18600 train 6.561193 (lr=1.4588e-05) (hash(x)=54690634)
+18800 val loss 6.1855
+18800 val perplexity 485.6750
+18800 train 6.364597 (lr=2.1725e-05) (hash(x)=52196791)
+18700 val loss 6.0647
+18700 val perplexity 430.3821
+18700 train 5.939446 (lr=1.4536e-05) (hash(x)=53778909)
+18900 val loss 6.1666
+18900 val perplexity 476.5452
+18900 train 6.114970 (lr=2.1647e-05) (hash(x)=49452227)
+18800 val loss 6.0599
+18800 val perplexity 428.3205
+18800 train 6.224871 (lr=1.4483e-05) (hash(x)=52196791)
+18900 val loss 6.0421
+18900 val perplexity 420.7874
+18900 train 5.993591 (lr=1.4431e-05) (hash(x)=49452227)
+19000 val loss 6.1674
+19000 val perplexity 476.9519
+19000 train 6.146073 (lr=2.1568e-05) (hash(x)=54220718)
+19100 val loss 6.1718
+19100 val perplexity 479.0271
+19000 val loss 6.0396
+19000 val perplexity 419.7349
+19100 train 6.302427 (lr=2.1489e-05) (hash(x)=54899289)
+19000 train 6.034254 (lr=1.4378e-05) (hash(x)=54220718)
+19200 val loss 6.1553
+19200 val perplexity 471.1907
+19200 train 6.327837 (lr=2.1409e-05) (hash(x)=52676298)
+19100 val loss 6.0604
+19100 val perplexity 428.5570
+19100 train 6.200470 (lr=1.4326e-05) (hash(x)=54899289)
+19300 val loss 6.1495
+19300 val perplexity 468.4897
+19200 val loss 6.0357
+19200 val perplexity 418.0893
+19300 train 6.219587 (lr=2.1330e-05) (hash(x)=54877992)
+19200 train 6.210310 (lr=1.4273e-05) (hash(x)=52676298)
+19400 val loss 6.1558
+19400 val perplexity 471.4400
+19400 train 5.985022 (lr=2.1250e-05) (hash(x)=46432347)
+19300 val loss 6.0373
+19300 val perplexity 418.7497
+19300 train 6.102807 (lr=1.4220e-05) (hash(x)=54877992)
+19500 val loss 6.1388
+19500 val perplexity 463.4892
+19500 train 6.240186 (lr=2.1171e-05) (hash(x)=50677944)
+19400 val loss 6.0436
+19400 val perplexity 421.4245
+19400 train 5.865130 (lr=1.4167e-05) (hash(x)=46432347)
+19500 val loss 6.0270
+19500 val perplexity 414.4816
+19500 train 6.123233 (lr=1.4114e-05) (hash(x)=50677944)
+19600 val loss 6.1432
+19600 val perplexity 465.5424
+19600 train 6.014059 (lr=2.1091e-05) (hash(x)=48855410)
+19700 val loss 6.1597
+19700 val perplexity 473.3036
+19700 train 6.459591 (lr=2.1010e-05) (hash(x)=57942870)
+19600 val loss 6.0272
+19600 val perplexity 414.5563
+19600 train 5.876552 (lr=1.4060e-05) (hash(x)=48855410)
+19800 val loss 6.1332
+19800 val perplexity 460.8864
+19800 train 6.139413 (lr=2.0930e-05) (hash(x)=56663417)
+19700 val loss 6.0392
+19700 val perplexity 419.5678
+19700 train 6.338866 (lr=1.4007e-05) (hash(x)=57942870)
+19900 val loss 6.1406
+19900 val perplexity 464.3346
+19900 train 5.973985 (lr=2.0850e-05) (hash(x)=41160884)
+19800 val loss 6.0228
+19800 val perplexity 412.7224
+19800 train 6.014856 (lr=1.3953e-05) (hash(x)=56663417)
+20000 val loss 6.1285
+20000 val perplexity 458.7250
+20000 train 6.074942 (lr=2.0769e-05) (hash(x)=53380308)
+19900 val loss 6.0185
+19900 val perplexity 410.9730
+19900 train 5.895044 (lr=1.3900e-05) (hash(x)=41160884)
+20100 val loss 6.1282
+20100 val perplexity 458.6126
+20100 train 5.981658 (lr=2.0688e-05) (hash(x)=46345322)
+20000 val loss 6.0209
+20000 val perplexity 411.9336
+20000 train 5.968961 (lr=1.3846e-05) (hash(x)=53380308)
+20200 val loss 6.1313
+20200 val perplexity 460.0213
+20200 train 5.936157 (lr=2.0607e-05) (hash(x)=45030198)
+20100 val loss 6.0158
+20100 val perplexity 409.8407
+20100 train 5.872766 (lr=1.3792e-05) (hash(x)=46345322)
+20300 val loss 6.1320
+20300 val perplexity 460.3364
+20300 train 5.551084 (lr=2.0526e-05) (hash(x)=36613892)
+20200 val loss 6.0228
+20200 val perplexity 412.7157
+20200 train 5.854208 (lr=1.3738e-05) (hash(x)=45030198)
+20400 val loss 6.1316
+20400 val perplexity 460.1597
+20400 train 5.795229 (lr=2.0444e-05) (hash(x)=52987869)
+20300 val loss 6.0162
+20300 val perplexity 410.0242
+20300 train 5.457102 (lr=1.3684e-05) (hash(x)=36613892)
+20500 val loss 6.1249
+20500 val perplexity 457.0994
+20500 train 6.024478 (lr=2.0363e-05) (hash(x)=51868861)
+20400 val loss 6.0158
+20400 val perplexity 409.8696
+20400 train 5.658610 (lr=1.3630e-05) (hash(x)=52987869)
+20600 val loss 6.1229
+20600 val perplexity 456.1870
+20600 train 5.991029 (lr=2.0281e-05) (hash(x)=48914729)
+20500 val loss 6.0140
+20500 val perplexity 409.1068
+20500 train 5.905686 (lr=1.3575e-05) (hash(x)=51868861)
+20700 val loss 6.1153
+20700 val perplexity 452.7321
+20700 train 6.055604 (lr=2.0199e-05) (hash(x)=52260642)
+20600 val loss 6.0073
+20600 val perplexity 406.3778
+20600 train 5.874764 (lr=1.3521e-05) (hash(x)=48914729)
+20800 val loss 6.1053
+20800 val perplexity 448.2181
+20800 train 6.623477 (lr=2.0117e-05) (hash(x)=67146535)
+20700 val loss 6.0024
+20700 val perplexity 404.4169
+20700 train 5.932829 (lr=1.3466e-05) (hash(x)=52260642)
+20900 val loss 6.1063
+20900 val perplexity 448.6815
+20900 train 6.185323 (lr=2.0035e-05) (hash(x)=54553323)
+20800 val loss 5.9952
+20800 val perplexity 401.5001
+20800 train 6.504151 (lr=1.3412e-05) (hash(x)=67146535)
+21000 val loss 6.0960
+21000 val perplexity 444.0867
+21000 train 6.078564 (lr=1.9953e-05) (hash(x)=48875330)
+20900 val loss 5.9920
+20900 val perplexity 400.2261
+20900 train 6.064557 (lr=1.3357e-05) (hash(x)=54553323)
+21100 val loss 6.0972
+21100 val perplexity 444.5899
+21100 train 6.324653 (lr=1.9871e-05) (hash(x)=51341973)
+21000 val loss 5.9885
+21000 val perplexity 398.8137
+21000 train 5.961763 (lr=1.3302e-05) (hash(x)=48875330)
+21200 val loss 6.0943
+21200 val perplexity 443.3028
+21200 train 6.055624 (lr=1.9788e-05) (hash(x)=49271158)
+21100 val loss 5.9863
+21100 val perplexity 397.9486
+21100 train 6.234424 (lr=1.3247e-05) (hash(x)=51341973)
+21300 val loss 6.0891
+21300 val perplexity 441.0062
+21300 train 5.931742 (lr=1.9705e-05) (hash(x)=45591170)
+21200 val loss 5.9889
+21200 val perplexity 398.9937
+21200 train 5.970147 (lr=1.3192e-05) (hash(x)=49271158)
+21400 val loss 6.0869
+21400 val perplexity 440.0573
+21400 train 6.178593 (lr=1.9623e-05) (hash(x)=53118930)
+21300 val loss 5.9850
+21300 val perplexity 397.4040
+21300 train 5.818439 (lr=1.3137e-05) (hash(x)=45591170)
+21500 val loss 6.0879
+21500 val perplexity 440.4896
+21500 train 5.838634 (lr=1.9540e-05) (hash(x)=52415119)
+21400 val loss 5.9841
+21400 val perplexity 397.0761
+21400 train 6.053033 (lr=1.3082e-05) (hash(x)=53118930)
+21600 val loss 6.0805
+21600 val perplexity 437.2612
+21600 train 5.916590 (lr=1.9457e-05) (hash(x)=48585730)
+21500 val loss 5.9847
+21500 val perplexity 397.3201
+21500 train 5.814132 (lr=1.3026e-05) (hash(x)=52415119)
+21700 val loss 6.0772
+21700 val perplexity 435.8020
+21700 train 6.266566 (lr=1.9373e-05) (hash(x)=61249800)
+21600 val loss 5.9767
+21600 val perplexity 394.1384
+21600 train 5.822378 (lr=1.2971e-05) (hash(x)=48585730)
+21800 val loss 6.0729
+21800 val perplexity 433.9529
+21800 train 5.850069 (lr=1.9290e-05) (hash(x)=49264022)
+21700 val loss 5.9758
+21700 val perplexity 393.7847
+21700 train 6.174076 (lr=1.2916e-05) (hash(x)=61249800)
+21900 val loss 6.0742
+21900 val perplexity 434.4892
+21900 train 6.023652 (lr=1.9207e-05) (hash(x)=47761044)
+21800 val loss 5.9772
+21800 val perplexity 394.3221
+21800 train 5.760363 (lr=1.2860e-05) (hash(x)=49264022)
+22000 val loss 6.0693
+22000 val perplexity 432.3831
+22000 train 6.181675 (lr=1.9123e-05) (hash(x)=49243894)
+21900 val loss 5.9768
+21900 val perplexity 394.1804
+21900 train 5.914711 (lr=1.2805e-05) (hash(x)=47761044)
+22100 val loss 6.0784
+22100 val perplexity 436.3510
+22100 train 5.782692 (lr=1.9040e-05) (hash(x)=45613618)
+22000 val loss 5.9747
+22000 val perplexity 393.3482
+22000 train 6.090980 (lr=1.2749e-05) (hash(x)=49243894)
+22200 val loss 6.0763
+22200 val perplexity 435.4298
+22200 train 6.013232 (lr=1.8956e-05) (hash(x)=47671047)
+22100 val loss 5.9798
+22100 val perplexity 395.3712
+22100 train 5.700965 (lr=1.2693e-05) (hash(x)=45613618)
+22300 val loss 6.0677
+22300 val perplexity 431.6951
+22300 train 6.072111 (lr=1.8872e-05) (hash(x)=55327350)
+22200 val loss 5.9802
+22200 val perplexity 395.5373
+22200 train 5.923239 (lr=1.2637e-05) (hash(x)=47671047)
+22400 val loss 6.0637
+22400 val perplexity 429.9716
+22400 train 6.255962 (lr=1.8788e-05) (hash(x)=49864138)
+22300 val loss 5.9717
+22300 val perplexity 392.1625
+22300 train 5.982235 (lr=1.2581e-05) (hash(x)=55327350)
+22500 val loss 6.0564
+22500 val perplexity 426.8266
+22500 train 5.993009 (lr=1.8704e-05) (hash(x)=48102324)
+22400 val loss 5.9642
+22400 val perplexity 389.2545
+22400 train 6.145995 (lr=1.2526e-05) (hash(x)=49864138)
+22600 val loss 6.0508
+22600 val perplexity 424.4692
+22600 train 6.088826 (lr=1.8620e-05) (hash(x)=49662723)
+22500 val loss 5.9562
+22500 val perplexity 386.1327
+22500 train 5.894469 (lr=1.2470e-05) (hash(x)=48102324)
+22700 val loss 6.0515
+22700 val perplexity 424.7344
+22700 train 6.240497 (lr=1.8536e-05) (hash(x)=51372261)
+22600 val loss 5.9555
+22600 val perplexity 385.8689
+22600 train 6.016599 (lr=1.2414e-05) (hash(x)=49662723)
+22800 val loss 6.0419
+22800 val perplexity 420.6796
+22800 train 5.992987 (lr=1.8452e-05) (hash(x)=48811743)
+22700 val loss 5.9502
+22700 val perplexity 383.8325
+22700 train 6.162386 (lr=1.2357e-05) (hash(x)=51372261)
+22900 val loss 6.0419
+22900 val perplexity 420.6764
+22900 train 5.952356 (lr=1.8368e-05) (hash(x)=51623299)
+22800 val loss 5.9491
+22800 val perplexity 383.4189
+22800 train 5.886395 (lr=1.2301e-05) (hash(x)=48811743)
+23000 val loss 6.0455
+23000 val perplexity 422.2226
+23000 train 5.950410 (lr=1.8283e-05) (hash(x)=47452756)
+22900 val loss 5.9464
+22900 val perplexity 382.3817
+22900 train 5.841969 (lr=1.2245e-05) (hash(x)=51623299)
+23100 val loss 6.0542
+23100 val perplexity 425.8809
+23100 train 5.799381 (lr=1.8199e-05) (hash(x)=43735997)
+23000 val loss 5.9531
+23000 val perplexity 384.9440
+23000 train 5.843216 (lr=1.2189e-05) (hash(x)=47452756)
+23200 val loss 6.0346
+23200 val perplexity 417.6111
+23200 train 5.953833 (lr=1.8114e-05) (hash(x)=49451282)
+23100 val loss 5.9490
+23100 val perplexity 383.3681
+23100 train 5.726491 (lr=1.2133e-05) (hash(x)=43735997)
+23300 val loss 6.0348
+23300 val perplexity 417.7053
+23300 train 6.093483 (lr=1.8030e-05) (hash(x)=56682096)
+23200 val loss 5.9451
+23200 val perplexity 381.8769
+23200 train 5.867705 (lr=1.2076e-05) (hash(x)=49451282)
+23400 val loss 6.0348
+23400 val perplexity 417.7113
+23400 train 5.733088 (lr=1.7945e-05) (hash(x)=44941532)
+23300 val loss 5.9419
+23300 val perplexity 380.6739
+23300 train 5.975385 (lr=1.2020e-05) (hash(x)=56682096)
+23500 val loss 6.0255
+23500 val perplexity 413.8569
+23500 train 6.023521 (lr=1.7860e-05) (hash(x)=49488756)
+23400 val loss 5.9385
+23400 val perplexity 379.3596
+23400 train 5.640111 (lr=1.1963e-05) (hash(x)=44941532)
+23600 val loss 6.0257
+23600 val perplexity 413.9416
+23600 train 5.907542 (lr=1.7776e-05) (hash(x)=49425437)
+23500 val loss 5.9336
+23500 val perplexity 377.5096
+23500 train 5.948933 (lr=1.1907e-05) (hash(x)=49488756)
+23700 val loss 6.0281
+23700 val perplexity 414.9061
+23700 train 6.419603 (lr=1.7691e-05) (hash(x)=56257944)
+23600 val loss 5.9353
+23600 val perplexity 378.1577
+23600 train 5.820508 (lr=1.1850e-05) (hash(x)=49425437)
+23800 val loss 6.0250
+23800 val perplexity 413.6513
+23800 train 6.110242 (lr=1.7606e-05) (hash(x)=51424843)
+23700 val loss 5.9328
+23700 val perplexity 377.2095
+23700 train 6.330105 (lr=1.1794e-05) (hash(x)=56257944)
+23900 val loss 6.0205
+23900 val perplexity 411.7786
+23900 train 6.073912 (lr=1.7521e-05) (hash(x)=51833590)
+23800 val loss 5.9345
+23800 val perplexity 377.8497
+23800 train 6.020976 (lr=1.1737e-05) (hash(x)=51424843)
+24000 val loss 6.0183
+24000 val perplexity 410.8883
+24000 train 5.833591 (lr=1.7436e-05) (hash(x)=48412152)
+23900 val loss 5.9306
+23900 val perplexity 376.3884
+23900 train 5.977436 (lr=1.1681e-05) (hash(x)=51833590)
+24100 val loss 6.0191
+24100 val perplexity 411.1947
+24100 train 5.926859 (lr=1.7351e-05) (hash(x)=48804835)
+24000 val loss 5.9287
+24000 val perplexity 375.6551
+24000 train 5.723898 (lr=1.1624e-05) (hash(x)=48412152)
+24200 val loss 6.0170
+24200 val perplexity 410.3301
+24200 train 6.413759 (lr=1.7266e-05) (hash(x)=56459963)
+24100 val loss 5.9349
+24100 val perplexity 377.9877
+24100 train 5.840219 (lr=1.1567e-05) (hash(x)=48804835)
+24300 val loss 6.0107
+24300 val perplexity 407.7548
+24300 train 5.919900 (lr=1.7181e-05) (hash(x)=46716240)
+24200 val loss 5.9247
+24200 val perplexity 374.1827
+24200 train 6.329389 (lr=1.1511e-05) (hash(x)=56459963)
+24400 val loss 6.0051
+24400 val perplexity 405.4839
+24400 train 6.166482 (lr=1.7096e-05) (hash(x)=53022938)
+24300 val loss 5.9170
+24300 val perplexity 371.2991
+24300 train 5.834604 (lr=1.1454e-05) (hash(x)=46716240)
+24500 val loss 6.0080
+24500 val perplexity 406.6794
+24500 train 5.977483 (lr=1.7011e-05) (hash(x)=49422455)
+24400 val loss 5.9168
+24400 val perplexity 371.2111
+24400 train 6.079606 (lr=1.1397e-05) (hash(x)=53022938)
+24600 val loss 6.0010
+24600 val perplexity 403.8124
+24600 train 6.258727 (lr=1.6926e-05) (hash(x)=46095419)
+24500 val loss 5.9227
+24500 val perplexity 373.4107
+24500 train 5.885417 (lr=1.1341e-05) (hash(x)=49422455)
+24700 val loss 6.0071
+24700 val perplexity 406.3061
+24700 train 6.134390 (lr=1.6841e-05) (hash(x)=52504918)
+24600 val loss 5.9136
+24600 val perplexity 370.0352
+24600 train 6.186104 (lr=1.1284e-05) (hash(x)=46095419)
+24800 val loss 5.9951
+24800 val perplexity 401.4607
+24800 train 5.942247 (lr=1.6755e-05) (hash(x)=53353568)
+24700 val loss 5.9183
+24700 val perplexity 371.7796
+24700 train 6.040504 (lr=1.1227e-05) (hash(x)=52504918)
+24900 val loss 5.9935
+24900 val perplexity 400.8075
+24900 train 5.906751 (lr=1.6670e-05) (hash(x)=47247169)
+24800 val loss 5.9111
+24800 val perplexity 369.1079
+24800 train 5.865495 (lr=1.1170e-05) (hash(x)=53353568)
+25000 val loss 5.9965
+25000 val perplexity 402.0107
+25000 train 5.875292 (lr=1.6585e-05) (hash(x)=50655375)
+24900 val loss 5.9054
+24900 val perplexity 366.9996
+24900 train 5.821893 (lr=1.1114e-05) (hash(x)=47247169)
+25100 val loss 5.9916
+25100 val perplexity 400.0506
+25100 train 5.813133 (lr=1.6500e-05) (hash(x)=50341890)
+25000 val loss 5.9096
+25000 val perplexity 368.5458
+25000 train 5.773839 (lr=1.1057e-05) (hash(x)=50655375)
+25200 val loss 5.9896
+25200 val perplexity 399.2626
+25200 train 6.039109 (lr=1.6415e-05) (hash(x)=52368607)
+25100 val loss 5.9037
+25100 val perplexity 366.4084
+25100 train 5.723794 (lr=1.1000e-05) (hash(x)=50341890)
+25300 val loss 5.9837
+25300 val perplexity 396.9006
+25300 train 5.637661 (lr=1.6330e-05) (hash(x)=46898454)
+25200 val loss 5.8992
+25200 val perplexity 364.7466
+25200 train 5.956071 (lr=1.0943e-05) (hash(x)=52368607)
+25400 val loss 5.9818
+25400 val perplexity 396.1496
+25400 train 5.897984 (lr=1.6245e-05) (hash(x)=50423971)
+25300 val loss 5.8970
+25300 val perplexity 363.9403
+25300 train 5.544650 (lr=1.0886e-05) (hash(x)=46898454)
+25500 val loss 5.9802
+25500 val perplexity 395.5035
+25500 train 5.874312 (lr=1.6159e-05) (hash(x)=46878539)
+25400 val loss 5.8985
+25400 val perplexity 364.5074
+25400 train 5.836495 (lr=1.0830e-05) (hash(x)=50423971)
+25600 val loss 5.9762
+25600 val perplexity 393.9413
+25600 train 6.179102 (lr=1.6074e-05) (hash(x)=54616245)
+25500 val loss 5.8937
+25500 val perplexity 362.7491
+25500 train 5.785678 (lr=1.0773e-05) (hash(x)=46878539)
+25700 val loss 5.9726
+25700 val perplexity 392.5346
+25700 train 5.730014 (lr=1.5989e-05) (hash(x)=47739146)
+25600 val loss 5.8920
+25600 val perplexity 362.1275
+25600 train 6.093657 (lr=1.0716e-05) (hash(x)=54616245)
+25800 val loss 5.9754
+25800 val perplexity 393.6360
+25800 train 5.882566 (lr=1.5904e-05) (hash(x)=49016232)
+25700 val loss 5.8965
+25700 val perplexity 363.7462
+25700 train 5.661457 (lr=1.0659e-05) (hash(x)=47739146)
+25900 val loss 5.9811
+25900 val perplexity 395.8807
+25900 train 5.902969 (lr=1.5819e-05) (hash(x)=49512666)
+25800 val loss 5.8946
+25800 val perplexity 363.0795
+25800 train 5.792024 (lr=1.0603e-05) (hash(x)=49016232)
+26000 val loss 5.9724
+26000 val perplexity 392.4349
+26000 train 5.798389 (lr=1.5734e-05) (hash(x)=44586031)
+25900 val loss 5.8983
+25900 val perplexity 364.4059
+25900 train 5.809695 (lr=1.0546e-05) (hash(x)=49512666)
+26100 val loss 5.9630
+26100 val perplexity 388.7891
+26100 train 5.995256 (lr=1.5649e-05) (hash(x)=47151445)
+26000 val loss 5.8921
+26000 val perplexity 362.1809
+26000 train 5.729291 (lr=1.0489e-05) (hash(x)=44586031)
+26200 val loss 5.9602
+26200 val perplexity 387.6819
+26200 train 6.185826 (lr=1.5564e-05) (hash(x)=54800573)
+26100 val loss 5.8836
+26100 val perplexity 359.0883
+26100 train 5.927690 (lr=1.0433e-05) (hash(x)=47151445)
+26300 val loss 5.9618
+26300 val perplexity 388.2972
+26300 train 5.818963 (lr=1.5479e-05) (hash(x)=51810733)
+26200 val loss 5.8833
+26200 val perplexity 358.9922
+26200 train 6.107346 (lr=1.0376e-05) (hash(x)=54800573)
+26400 val loss 5.9569
+26400 val perplexity 386.4206
+26400 train 6.059855 (lr=1.5394e-05) (hash(x)=51482762)
+26300 val loss 5.8823
+26300 val perplexity 358.6396
+26300 train 5.730820 (lr=1.0319e-05) (hash(x)=51810733)
+26500 val loss 5.9646
+26500 val perplexity 389.4006
+26500 train 5.868408 (lr=1.5309e-05) (hash(x)=50129702)
+26400 val loss 5.8768
+26400 val perplexity 356.6590
+26400 train 5.977849 (lr=1.0263e-05) (hash(x)=51482762)
+26600 val loss 5.9509
+26600 val perplexity 384.0854
+26600 train 5.925449 (lr=1.5224e-05) (hash(x)=51104302)
+26500 val loss 5.8870
+26500 val perplexity 360.3133
+26500 train 5.785389 (lr=1.0206e-05) (hash(x)=50129702)
+26700 val loss 5.9505
+26700 val perplexity 383.9572
+26700 train 5.776924 (lr=1.5140e-05) (hash(x)=48490782)
+26600 val loss 5.8739
+26600 val perplexity 355.6246
+26600 train 5.849953 (lr=1.0150e-05) (hash(x)=51104302)
+26800 val loss 5.9535
+26800 val perplexity 385.1151
+26800 train 5.985641 (lr=1.5055e-05) (hash(x)=50602590)
+26700 val loss 5.8760
+26700 val perplexity 356.3904
+26700 train 5.697481 (lr=1.0093e-05) (hash(x)=48490782)
+26900 val loss 5.9496
+26900 val perplexity 383.5881
+26900 train 5.732647 (lr=1.4970e-05) (hash(x)=45624407)
+26800 val loss 5.8754
+26800 val perplexity 356.1647
+26800 train 5.907648 (lr=1.0037e-05) (hash(x)=50602590)
+27000 val loss 5.9431
+27000 val perplexity 381.1289
+27000 train 5.699210 (lr=1.4886e-05) (hash(x)=46448811)
+26900 val loss 5.8684
+26900 val perplexity 353.6670
+26900 train 5.648049 (lr=9.9802e-06) (hash(x)=45624407)
+27100 val loss 5.9421
+27100 val perplexity 380.7151
+27100 train 5.603257 (lr=1.4801e-05) (hash(x)=42263321)
+27000 val loss 5.8699
+27000 val perplexity 354.2099
+27000 train 5.637478 (lr=9.9238e-06) (hash(x)=46448811)
+27200 val loss 5.9381
+27200 val perplexity 379.2099
+27200 train 5.672335 (lr=1.4717e-05) (hash(x)=45998703)
+27100 val loss 5.8690
+27100 val perplexity 353.9018
+27100 train 5.532988 (lr=9.8675e-06) (hash(x)=42263321)
+27300 val loss 5.9416
+27300 val perplexity 380.5411
+27300 train 6.072817 (lr=1.4632e-05) (hash(x)=58008207)
+27200 val loss 5.8648
+27200 val perplexity 352.3957
+27200 train 5.624200 (lr=9.8112e-06) (hash(x)=45998703)
+27400 val loss 5.9471
+27400 val perplexity 382.6534
+27400 train 6.020402 (lr=1.4548e-05) (hash(x)=58914934)
+27300 val loss 5.8762
+27300 val perplexity 356.4473
+27300 train 5.996145 (lr=9.7549e-06) (hash(x)=58008207)
+27500 val loss 5.9475
+27500 val perplexity 382.8053
+27500 train 5.906272 (lr=1.4464e-05) (hash(x)=54961792)
+27400 val loss 5.8715
+27400 val perplexity 354.7728
+27400 train 5.921473 (lr=9.6987e-06) (hash(x)=58914934)
+27600 val loss 5.9343
+27600 val perplexity 377.7920
+27600 train 5.594903 (lr=1.4380e-05) (hash(x)=43436743)
+27500 val loss 5.8646
+27500 val perplexity 352.3293
+27500 train 5.842748 (lr=9.6426e-06) (hash(x)=54961792)
+27700 val loss 5.9319
+27700 val perplexity 376.8519
+27700 train 5.539855 (lr=1.4296e-05) (hash(x)=43954091)
+27600 val loss 5.8642
+27600 val perplexity 352.1909
+27600 train 5.545135 (lr=9.5865e-06) (hash(x)=43436743)
+27800 val loss 5.9291
+27800 val perplexity 375.8271
+27800 train 5.843649 (lr=1.4212e-05) (hash(x)=49556041)
+27700 val loss 5.8651
+27700 val perplexity 352.5293
+27700 train 5.459342 (lr=9.5304e-06) (hash(x)=43954091)
+27900 val loss 5.9294
+27900 val perplexity 375.9375
+27900 train 5.719796 (lr=1.4128e-05) (hash(x)=44542698)
+27800 val loss 5.8633
+27800 val perplexity 351.8735
+27800 train 5.766902 (lr=9.4745e-06) (hash(x)=49556041)
+28000 val loss 5.9222
+28000 val perplexity 373.2163
+28000 train 5.896948 (lr=1.4044e-05) (hash(x)=51737822)
+27900 val loss 5.8566
+27900 val perplexity 349.5311
+27900 train 5.661964 (lr=9.4185e-06) (hash(x)=44542698)
+28100 val loss 5.9187
+28100 val perplexity 371.9289
+28100 train 6.401803 (lr=1.3960e-05) (hash(x)=69830597)
+28000 val loss 5.8522
+28000 val perplexity 348.0018
+28000 train 5.804460 (lr=9.3627e-06) (hash(x)=51737822)
+28200 val loss 5.9156
+28200 val perplexity 370.7883
+28200 train 5.982912 (lr=1.3877e-05) (hash(x)=51670601)
+28100 val loss 5.8542
+28100 val perplexity 348.6792
+28100 train 6.351122 (lr=9.3069e-06) (hash(x)=69830597)
+28300 val loss 5.9109
+28300 val perplexity 369.0479
+28300 train 6.012157 (lr=1.3793e-05) (hash(x)=51628878)
+28200 val loss 5.8462
+28200 val perplexity 345.9212
+28200 train 5.903768 (lr=9.2511e-06) (hash(x)=51670601)
+28400 val loss 5.9146
+28400 val perplexity 370.3912
+28400 train 5.941531 (lr=1.3710e-05) (hash(x)=50973565)
+28300 val loss 5.8440
+28300 val perplexity 345.1626
+28300 train 5.932347 (lr=9.1955e-06) (hash(x)=51628878)
+28500 val loss 5.9128
+28500 val perplexity 369.7237
+28500 train 6.070199 (lr=1.3627e-05) (hash(x)=63914328)
+28400 val loss 5.8446
+28400 val perplexity 345.3715
+28400 train 5.875063 (lr=9.1399e-06) (hash(x)=50973565)
+28600 val loss 5.9062
+28600 val perplexity 367.2940
+28600 train 5.934083 (lr=1.3543e-05) (hash(x)=56517839)
+28500 val loss 5.8446
+28500 val perplexity 345.3555
+28500 train 5.994771 (lr=9.0844e-06) (hash(x)=63914328)
+28700 val loss 5.9038
+28700 val perplexity 366.4173
+28700 train 5.924917 (lr=1.3460e-05) (hash(x)=47631048)
+28600 val loss 5.8425
+28600 val perplexity 344.6243
+28600 train 5.873335 (lr=9.0290e-06) (hash(x)=56517839)
+28800 val loss 5.9046
+28800 val perplexity 366.7172
+28800 train 5.646642 (lr=1.3377e-05) (hash(x)=44376567)
+28700 val loss 5.8381
+28700 val perplexity 343.1184
+28700 train 5.860738 (lr=8.9736e-06) (hash(x)=47631048)
+28900 val loss 5.9024
+28900 val perplexity 365.9141
+28900 train 5.865271 (lr=1.3295e-05) (hash(x)=50934351)
+28800 val loss 5.8395
+28800 val perplexity 343.5935
+28800 train 5.606833 (lr=8.9183e-06) (hash(x)=44376567)
+29000 val loss 5.9020
+29000 val perplexity 365.7646
+29000 train 6.094385 (lr=1.3212e-05) (hash(x)=51712329)
+28900 val loss 5.8396
+28900 val perplexity 343.6252
+28900 train 5.800102 (lr=8.8631e-06) (hash(x)=50934351)
+29100 val loss 5.9031
+29100 val perplexity 366.1618
+29100 train 5.896192 (lr=1.3129e-05) (hash(x)=51282586)
+29000 val loss 5.8349
+29000 val perplexity 342.0140
+29000 train 6.052308 (lr=8.8080e-06) (hash(x)=51712329)
+29200 val loss 5.8938
+29200 val perplexity 362.7666
+29200 train 5.581905 (lr=1.3047e-05) (hash(x)=44457224)
+29100 val loss 5.8387
+29100 val perplexity 343.3264
+29100 train 5.842405 (lr=8.7530e-06) (hash(x)=51282586)
+29300 val loss 5.8956
+29300 val perplexity 363.4376
+29300 train 5.831235 (lr=1.2965e-05) (hash(x)=47669361)
+29200 val loss 5.8363
+29200 val perplexity 342.5129
+29200 train 5.526801 (lr=8.6981e-06) (hash(x)=44457224)
+29400 val loss 5.8919
+29400 val perplexity 362.0883
+29400 train 5.964642 (lr=1.2883e-05) (hash(x)=49442135)
+29300 val loss 5.8349
+29300 val perplexity 342.0238
+29300 train 5.766893 (lr=8.6432e-06) (hash(x)=47669361)
+29500 val loss 5.8988
+29500 val perplexity 364.5858
+29500 train 5.412758 (lr=1.2801e-05) (hash(x)=39350955)
+29400 val loss 5.8335
+29400 val perplexity 341.5370
+29400 train 5.901731 (lr=8.5885e-06) (hash(x)=49442135)
+29600 val loss 5.8890
+29600 val perplexity 361.0511
+29600 train 6.358497 (lr=1.2719e-05) (hash(x)=53114560)
+29500 val loss 5.8393
+29500 val perplexity 343.5519
+29500 train 5.361787 (lr=8.5338e-06) (hash(x)=39350955)
+29700 val loss 5.8883
+29700 val perplexity 360.7818
+29700 train 5.790334 (lr=1.2637e-05) (hash(x)=45302862)
+29600 val loss 5.8351
+29600 val perplexity 342.1006
+29600 train 6.330326 (lr=8.4793e-06) (hash(x)=53114560)
+29800 val loss 5.8877
+29800 val perplexity 360.5828
+29800 train 5.833242 (lr=1.2556e-05) (hash(x)=51679878)
+29700 val loss 5.8276
+29700 val perplexity 339.5544
+29700 train 5.741794 (lr=8.4248e-06) (hash(x)=45302862)
+29900 val loss 5.8874
+29900 val perplexity 360.4805
+29900 train 5.733284 (lr=1.2474e-05) (hash(x)=47344663)
+29800 val loss 5.8253
+29800 val perplexity 338.7754
+29800 train 5.744218 (lr=8.3705e-06) (hash(x)=51679878)
+30000 val loss 5.8840
+30000 val perplexity 359.2462
+30000 train 5.642229 (lr=1.2393e-05) (hash(x)=46530414)
+29900 val loss 5.8278
+29900 val perplexity 339.6131
+29900 train 5.653694 (lr=8.3162e-06) (hash(x)=47344663)
+30100 val loss 5.8857
+30100 val perplexity 359.8524
+30100 train 5.432938 (lr=1.2312e-05) (hash(x)=43262362)
+30000 val loss 5.8257
+30000 val perplexity 338.8935
+30200 val loss 5.8790
+30200 val perplexity 357.4456
+30000 train 5.572353 (lr=8.2621e-06) (hash(x)=46530414)
+30200 train 5.720525 (lr=1.2231e-05) (hash(x)=47879732)
+30300 val loss 5.8788
+30300 val perplexity 357.3746
+30300 train 5.697599 (lr=1.2150e-05) (hash(x)=47335687)
+30100 val loss 5.8249
+30100 val perplexity 338.6239
+30100 train 5.367616 (lr=8.2080e-06) (hash(x)=43262362)
+30400 val loss 5.8784
+30400 val perplexity 357.2348
+30400 train 5.917314 (lr=1.2070e-05) (hash(x)=49940701)
+30200 val loss 5.8242
+30200 val perplexity 338.3829
+30200 train 5.656559 (lr=8.1541e-06) (hash(x)=47879732)
+30500 val loss 5.8763
+30500 val perplexity 356.4738
+30500 train 5.774759 (lr=1.1990e-05) (hash(x)=49860489)
+30300 val loss 5.8215
+30300 val perplexity 337.4719
+30300 train 5.635166 (lr=8.1003e-06) (hash(x)=47335687)
+30600 val loss 5.8784
+30600 val perplexity 357.2401
+30600 train 5.620846 (lr=1.1909e-05) (hash(x)=44643399)
+30400 val loss 5.8198
+30400 val perplexity 336.9133
+30400 train 5.860961 (lr=8.0466e-06) (hash(x)=49940701)
+30700 val loss 5.8760
+30700 val perplexity 356.3649
+30700 train 5.682973 (lr=1.1829e-05) (hash(x)=49388442)
+30500 val loss 5.8184
+30500 val perplexity 336.4339
+30500 train 5.708053 (lr=7.9931e-06) (hash(x)=49860489)
+30800 val loss 5.8741
+30800 val perplexity 355.7114
+30800 train 5.862242 (lr=1.1750e-05) (hash(x)=56160596)
+30600 val loss 5.8239
+30600 val perplexity 338.2841
+30600 train 5.571492 (lr=7.9396e-06) (hash(x)=44643399)
+30900 val loss 5.8735
+30900 val perplexity 355.5057
+30900 train 5.855698 (lr=1.1670e-05) (hash(x)=49005636)
+30700 val loss 5.8195
+30700 val perplexity 336.7962
+30700 train 5.629300 (lr=7.8863e-06) (hash(x)=49388442)
+31000 val loss 5.8733
+31000 val perplexity 355.4042
+31000 train 6.094777 (lr=1.1591e-05) (hash(x)=54646552)
+30800 val loss 5.8169
+30800 val perplexity 335.9203
+30800 train 5.808354 (lr=7.8331e-06) (hash(x)=56160596)
+31100 val loss 5.8636
+31100 val perplexity 352.0015
+31100 train 5.687142 (lr=1.1511e-05) (hash(x)=44816272)
+30900 val loss 5.8155
+30900 val perplexity 335.4492
+30900 train 5.800075 (lr=7.7800e-06) (hash(x)=49005636)
+31200 val loss 5.8656
+31200 val perplexity 352.6889
+31200 train 6.059061 (lr=1.1432e-05) (hash(x)=53040860)
+31000 val loss 5.8192
+31000 val perplexity 336.6967
+31000 train 6.067160 (lr=7.7270e-06) (hash(x)=54646552)
+31300 val loss 5.8607
+31300 val perplexity 350.9629
+31300 train 6.081599 (lr=1.1353e-05) (hash(x)=52051764)
+31100 val loss 5.8099
+31100 val perplexity 333.5960
+31100 train 5.619771 (lr=7.6742e-06) (hash(x)=44816272)
+31400 val loss 5.8591
+31400 val perplexity 350.4079
+31400 train 5.805600 (lr=1.1275e-05) (hash(x)=51564589)
+31200 val loss 5.8143
+31200 val perplexity 335.0458
+31200 train 6.006360 (lr=7.6215e-06) (hash(x)=53040860)
+31500 val loss 5.8560
+31500 val perplexity 349.3149
+31500 train 5.605606 (lr=1.1196e-05) (hash(x)=49676250)
+31300 val loss 5.8097
+31300 val perplexity 333.5237
+31300 train 6.057798 (lr=7.5690e-06) (hash(x)=52051764)
+31600 val loss 5.8569
+31600 val perplexity 349.6333
+31600 train 5.806407 (lr=1.1118e-05) (hash(x)=49244964)
+31400 val loss 5.8073
+31400 val perplexity 332.7039
+31400 train 5.740465 (lr=7.5165e-06) (hash(x)=51564589)
+31700 val loss 5.8532
+31700 val perplexity 348.3606
+31700 train 6.083027 (lr=1.1040e-05) (hash(x)=49527873)
+31500 val loss 5.8061
+31500 val perplexity 332.3088
+31500 train 5.534233 (lr=7.4643e-06) (hash(x)=49676250)
+31800 val loss 5.8658
+31800 val perplexity 352.7789
+31800 train 5.850847 (lr=1.0962e-05) (hash(x)=49566755)
+31600 val loss 5.8038
+31600 val perplexity 331.5609
+31600 train 5.732040 (lr=7.4121e-06) (hash(x)=49244964)
+31900 val loss 5.8512
+31900 val perplexity 347.6664
+31900 train 5.661178 (lr=1.0885e-05) (hash(x)=45456170)
+31700 val loss 5.8009
+31700 val perplexity 330.6069
+31700 train 6.044876 (lr=7.3601e-06) (hash(x)=49527873)
+32000 val loss 5.8480
+32000 val perplexity 346.5530
+32000 train 5.900880 (lr=1.0808e-05) (hash(x)=49986978)
+31800 val loss 5.8172
+31800 val perplexity 336.0214
+31800 train 5.776331 (lr=7.3083e-06) (hash(x)=49566755)
+32100 val loss 5.8568
+32100 val perplexity 349.6205
+32100 train 5.520376 (lr=1.0730e-05) (hash(x)=50539149)
+31900 val loss 5.8032
+31900 val perplexity 331.3619
+31900 train 5.609488 (lr=7.2566e-06) (hash(x)=45456170)
+32200 val loss 5.8512
+32200 val perplexity 347.6600
+32200 train 5.778606 (lr=1.0654e-05) (hash(x)=50998615)
+32000 val loss 5.7998
+32000 val perplexity 330.2473
+32000 train 5.868989 (lr=7.2050e-06) (hash(x)=49986978)
+32300 val loss 5.8477
+32300 val perplexity 346.4478
+32300 train 5.675282 (lr=1.0577e-05) (hash(x)=44030191)
+32100 val loss 5.8008
+32100 val perplexity 330.5757
+32100 train 5.439094 (lr=7.1536e-06) (hash(x)=50539149)
+32400 val loss 5.8469
+32400 val perplexity 346.1570
+32400 train 5.770798 (lr=1.0500e-05) (hash(x)=47269028)
+32200 val loss 5.7989
+32200 val perplexity 329.9398
+32200 train 5.747228 (lr=7.1024e-06) (hash(x)=50998615)
+32500 val loss 5.8409
+32500 val perplexity 344.0915
+32500 train 5.670961 (lr=1.0424e-05) (hash(x)=50128289)
+32300 val loss 5.7983
+32300 val perplexity 329.7278
+32300 train 5.639364 (lr=7.0513e-06) (hash(x)=44030191)
+32600 val loss 5.8438
+32600 val perplexity 345.0757
+32600 train 5.629930 (lr=1.0348e-05) (hash(x)=42781351)
+32400 val loss 5.7970
+32400 val perplexity 329.2990
+32400 train 5.731838 (lr=7.0003e-06) (hash(x)=47269028)
+32700 val loss 5.8478
+32700 val perplexity 346.4788
+32700 train 5.918962 (lr=1.0273e-05) (hash(x)=51712117)
+32500 val loss 5.7970
+32500 val perplexity 329.3004
+32500 train 5.629797 (lr=6.9495e-06) (hash(x)=50128289)
+32800 val loss 5.8370
+32800 val perplexity 342.7430
+32800 train 5.933241 (lr=1.0197e-05) (hash(x)=52478104)
+32600 val loss 5.7984
+32600 val perplexity 329.7833
+32600 train 5.589591 (lr=6.8989e-06) (hash(x)=42781351)
+32900 val loss 5.8353
+32900 val perplexity 342.1809
+32900 train 5.771452 (lr=1.0122e-05) (hash(x)=53588694)
+32700 val loss 5.7991
+32700 val perplexity 330.0068
+32700 train 5.885045 (lr=6.8485e-06) (hash(x)=51712117)
+33000 val loss 5.8312
+33000 val perplexity 340.7601
+33000 train 5.713400 (lr=1.0047e-05) (hash(x)=50821436)
+32800 val loss 5.7903
+32800 val perplexity 327.0969
+32800 train 5.895542 (lr=6.7982e-06) (hash(x)=52478104)
+33100 val loss 5.8304
+33100 val perplexity 340.4898
+33100 train 5.780435 (lr=9.9725e-06) (hash(x)=50371199)
+32900 val loss 5.7882
+32900 val perplexity 326.4408
+32900 train 5.734493 (lr=6.7480e-06) (hash(x)=53588694)
+33200 val loss 5.8289
+33200 val perplexity 339.9765
+33200 train 6.017346 (lr=9.8980e-06) (hash(x)=54544623)
+33000 val loss 5.7875
+33000 val perplexity 326.2125
+33000 train 5.677229 (lr=6.6981e-06) (hash(x)=50821436)
+33300 val loss 5.8271
+33300 val perplexity 339.3638
+33300 train 5.652477 (lr=9.8239e-06) (hash(x)=55456921)
+33100 val loss 5.7854
+33100 val perplexity 325.5229
+33100 train 5.750986 (lr=6.6483e-06) (hash(x)=50371199)
+33400 val loss 5.8300
+33400 val perplexity 340.3557
+33400 train 5.386091 (lr=9.7500e-06) (hash(x)=37240412)
+33200 val loss 5.7845
+33200 val perplexity 325.2103
+33200 train 6.002258 (lr=6.5987e-06) (hash(x)=54544623)
+33500 val loss 5.8297
+33500 val perplexity 340.2451
+33500 train 6.068254 (lr=9.6764e-06) (hash(x)=56551894)
+33300 val loss 5.7820
+33300 val perplexity 324.4020
+33300 train 5.610635 (lr=6.5493e-06) (hash(x)=55456921)
+33600 val loss 5.8250
+33600 val perplexity 338.6469
+33600 train 5.749243 (lr=9.6030e-06) (hash(x)=47514225)
+33400 val loss 5.7853
+33400 val perplexity 325.4798
+33400 train 5.383198 (lr=6.5000e-06) (hash(x)=37240412)
+33700 val loss 5.8209
+33700 val perplexity 337.2920
+33700 train 5.674240 (lr=9.5300e-06) (hash(x)=51164416)
+33500 val loss 5.7830
+33500 val perplexity 324.7322
+33500 train 6.015255 (lr=6.4509e-06) (hash(x)=56551894)
+33800 val loss 5.8219
+33800 val perplexity 337.6111
+33800 train 5.662504 (lr=9.4572e-06) (hash(x)=50660204)
+33600 val loss 5.7833
+33600 val perplexity 324.8434
+33600 train 5.694076 (lr=6.4020e-06) (hash(x)=47514225)
+33900 val loss 5.8225
+33900 val perplexity 337.8138
+33900 train 5.805346 (lr=9.3846e-06) (hash(x)=48511113)
+33700 val loss 5.7813
+33700 val perplexity 324.1955
+33700 train 5.624909 (lr=6.3533e-06) (hash(x)=51164416)
+34000 val loss 5.8232
+34000 val perplexity 338.0357
+34000 train 5.426359 (lr=9.3124e-06) (hash(x)=41560072)
+33800 val loss 5.7809
+33800 val perplexity 324.0492
+33800 train 5.614145 (lr=6.3048e-06) (hash(x)=50660204)
+34100 val loss 5.8196
+34100 val perplexity 336.8347
+34100 train 5.465456 (lr=9.2405e-06) (hash(x)=50354364)
+33900 val loss 5.7799
+33900 val perplexity 323.7236
+33900 train 5.764244 (lr=6.2564e-06) (hash(x)=48511113)
+34200 val loss 5.8211
+34200 val perplexity 337.3589
+34200 train 6.009044 (lr=9.1688e-06) (hash(x)=54291562)
+34000 val loss 5.7828
+34000 val perplexity 324.6672
+34000 train 5.391787 (lr=6.2083e-06) (hash(x)=41560072)
+34300 val loss 5.8237
+34300 val perplexity 338.2345
+34300 train 6.208984 (lr=9.0974e-06) (hash(x)=53340739)
+34100 val loss 5.7790
+34100 val perplexity 323.4204
+34100 train 5.414715 (lr=6.1603e-06) (hash(x)=50354364)
+34400 val loss 5.8184
+34400 val perplexity 336.4473
+34400 train 5.461709 (lr=9.0264e-06) (hash(x)=44872469)
+34200 val loss 5.7818
+34200 val perplexity 324.3537
+34200 train 5.975448 (lr=6.1125e-06) (hash(x)=54291562)
+34500 val loss 5.8135
+34500 val perplexity 334.8054
+34500 train 4.838446 (lr=8.9556e-06) (hash(x)=23649625)
+34300 val loss 5.7864
+34300 val perplexity 325.8462
+34300 train 6.173049 (lr=6.0650e-06) (hash(x)=53340739)
+34600 val loss 5.8248
+34600 val perplexity 338.6017
+34600 train 5.773252 (lr=8.8851e-06) (hash(x)=46230721)
+34400 val loss 5.7770
+34400 val perplexity 322.7978
+34400 train 5.405843 (lr=6.0176e-06) (hash(x)=44872469)
+34700 val loss 5.8071
+34700 val perplexity 332.6449
+34700 train 5.586082 (lr=8.8150e-06) (hash(x)=44277516)
+34500 val loss 5.7769
+34500 val perplexity 322.7555
+34500 train 4.835846 (lr=5.9704e-06) (hash(x)=23649625)
+34800 val loss 5.8076
+34800 val perplexity 332.8135
+34800 train 5.820089 (lr=8.7451e-06) (hash(x)=50346875)
+34600 val loss 5.7797
+34600 val perplexity 323.6566
+34600 train 5.716429 (lr=5.9234e-06) (hash(x)=46230721)
+34900 val loss 5.8061
+34900 val perplexity 332.3326
+34900 train 5.742538 (lr=8.6755e-06) (hash(x)=45110550)
+34700 val loss 5.7698
+34700 val perplexity 320.4798
+34700 train 5.539550 (lr=5.8766e-06) (hash(x)=44277516)
+35000 val loss 5.8043
+35000 val perplexity 331.7315
+35000 train 5.807580 (lr=8.6063e-06) (hash(x)=46812915)
+34800 val loss 5.7704
+34800 val perplexity 320.6732
+34800 train 5.845528 (lr=5.8301e-06) (hash(x)=50346875)
+35100 val loss 5.8058
+35100 val perplexity 332.2310
+35100 train 5.967079 (lr=8.5374e-06) (hash(x)=54729285)
+34900 val loss 5.7678
+34900 val perplexity 319.8426
+34900 train 5.708534 (lr=5.7837e-06) (hash(x)=45110550)
+35200 val loss 5.8016
+35200 val perplexity 330.8221
+35200 train 5.529810 (lr=8.4688e-06) (hash(x)=49651453)
+35000 val loss 5.7671
+35000 val perplexity 319.6218
+35000 train 5.768589 (lr=5.7375e-06) (hash(x)=46812915)
+35300 val loss 5.8006
+35300 val perplexity 330.4833
+35300 train 5.719607 (lr=8.4005e-06) (hash(x)=51398605)
+35100 val loss 5.7665
+35100 val perplexity 319.4114
+35100 train 5.917636 (lr=5.6916e-06) (hash(x)=54729285)
+35400 val loss 5.8021
+35400 val perplexity 331.0047
+35400 train 5.606431 (lr=8.3325e-06) (hash(x)=52214092)
+35200 val loss 5.7680
+35200 val perplexity 319.9048
+35200 train 5.496180 (lr=5.6458e-06) (hash(x)=49651453)
+35500 val loss 5.8003
+35500 val perplexity 330.3981
+35500 train 6.136915 (lr=8.2648e-06) (hash(x)=60115839)
+35300 val loss 5.7645
+35300 val perplexity 318.7666
+35300 train 5.688906 (lr=5.6003e-06) (hash(x)=51398605)
+35600 val loss 5.8229
+35600 val perplexity 337.9421
+35600 train 6.489014 (lr=8.1975e-06) (hash(x)=53051302)
+35400 val loss 5.7637
+35400 val perplexity 318.5346
+35400 train 5.557629 (lr=5.5550e-06) (hash(x)=52214092)
+35700 val loss 5.8000
+35700 val perplexity 330.2965
+35700 train 5.523680 (lr=8.1305e-06) (hash(x)=47769916)
+35500 val loss 5.7619
+35500 val perplexity 317.9461
+35500 train 6.093183 (lr=5.5099e-06) (hash(x)=60115839)
+35800 val loss 5.7994
+35800 val perplexity 330.1055
+35800 train 5.572382 (lr=8.0639e-06) (hash(x)=48652911)
+35600 val loss 5.7786
+35600 val perplexity 323.2921
+35600 train 6.408434 (lr=5.4650e-06) (hash(x)=53051302)
+35900 val loss 5.7990
+35900 val perplexity 329.9698
+35900 train 5.742786 (lr=7.9976e-06) (hash(x)=50452504)
+35700 val loss 5.7630
+35700 val perplexity 318.3155
+35700 train 5.484651 (lr=5.4204e-06) (hash(x)=47769916)
+36000 val loss 5.7994
+36000 val perplexity 330.0861
+36000 train 5.787482 (lr=7.9316e-06) (hash(x)=44888966)
+35800 val loss 5.7635
+35800 val perplexity 318.4504
+35800 train 5.525833 (lr=5.3759e-06) (hash(x)=48652911)
+36100 val loss 5.7965
+36100 val perplexity 329.1411
+36100 train 5.678099 (lr=7.8659e-06) (hash(x)=48245730)
+35900 val loss 5.7632
+35900 val perplexity 318.3692
+35900 train 5.708165 (lr=5.3317e-06) (hash(x)=50452504)
+36200 val loss 5.7928
+36200 val perplexity 327.9344
+36200 train 5.777046 (lr=7.8006e-06) (hash(x)=51614607)
+36000 val loss 5.7645
+36000 val perplexity 318.7850
+36000 train 5.775196 (lr=5.2877e-06) (hash(x)=44888966)
+36300 val loss 5.7917
+36300 val perplexity 327.5678
+36300 train 5.851054 (lr=7.7357e-06) (hash(x)=43069600)
+36100 val loss 5.7616
+36100 val perplexity 317.8509
+36100 train 5.641047 (lr=5.2440e-06) (hash(x)=48245730)
+36400 val loss 5.7862
+36400 val perplexity 325.7805
+36400 train 5.875189 (lr=7.6711e-06) (hash(x)=52756265)
+36200 val loss 5.7606
+36200 val perplexity 317.5410
+36200 train 5.751701 (lr=5.2004e-06) (hash(x)=51614607)
+36500 val loss 5.7853
+36500 val perplexity 325.4793
+36500 train 5.589223 (lr=7.6068e-06) (hash(x)=47894210)
+36300 val loss 5.7580
+36300 val perplexity 316.7132
+36300 train 5.820355 (lr=5.1571e-06) (hash(x)=43069600)
+36600 val loss 5.7890
+36600 val perplexity 326.6882
+36600 train 5.775141 (lr=7.5429e-06) (hash(x)=46906979)
+36400 val loss 5.7553
+36400 val perplexity 315.8745
+36400 train 5.838860 (lr=5.1141e-06) (hash(x)=52756265)
+36700 val loss 5.7840
+36700 val perplexity 325.0631
+36700 train 6.541754 (lr=7.4794e-06) (hash(x)=45349010)
+36500 val loss 5.7541
+36500 val perplexity 315.4786
+36500 train 5.542435 (lr=5.0712e-06) (hash(x)=47894210)
+36800 val loss 5.7809
+36800 val perplexity 324.0665
+36800 train 5.551033 (lr=7.4162e-06) (hash(x)=40761454)
+36600 val loss 5.7570
+36600 val perplexity 316.3883
+36600 train 5.746761 (lr=5.0286e-06) (hash(x)=46906979)
+36900 val loss 5.7827
+36900 val perplexity 324.6252
+36900 train 5.917160 (lr=7.3534e-06) (hash(x)=52150265)
+36700 val loss 5.7544
+36700 val perplexity 315.5899
+36700 train 6.505106 (lr=4.9863e-06) (hash(x)=45349010)
+37000 val loss 5.7806
+37000 val perplexity 323.9630
+37000 train 5.857428 (lr=7.2909e-06) (hash(x)=48982242)
+36800 val loss 5.7512
+36800 val perplexity 314.5723
+36800 train 5.502524 (lr=4.9441e-06) (hash(x)=40761454)
+37100 val loss 5.7817
+37100 val perplexity 324.3237
+37100 train 5.744205 (lr=7.2289e-06) (hash(x)=52100108)
+36900 val loss 5.7521
+36900 val perplexity 314.8513
+36900 train 5.878228 (lr=4.9023e-06) (hash(x)=52150265)
+37200 val loss 5.7856
+37200 val perplexity 325.5642
+37200 train 5.609965 (lr=7.1671e-06) (hash(x)=47721621)
+37000 val loss 5.7524
+37000 val perplexity 314.9548
+37000 train 5.852871 (lr=4.8606e-06) (hash(x)=48982242)
+37300 val loss 5.7784
+37300 val perplexity 323.2296
+37300 train 5.590865 (lr=7.1058e-06) (hash(x)=49312812)
+37100 val loss 5.7524
+37100 val perplexity 314.9465
+37100 train 5.709177 (lr=4.8192e-06) (hash(x)=52100108)
+37400 val loss 5.7775
+37400 val perplexity 322.9358
+37400 train 5.616804 (lr=7.0448e-06) (hash(x)=48536877)
+37200 val loss 5.7536
+37200 val perplexity 315.3186
+37200 train 5.563495 (lr=4.7781e-06) (hash(x)=47721621)
+37500 val loss 5.7818
+37500 val perplexity 324.3271
+37500 train 5.732463 (lr=6.9842e-06) (hash(x)=50612814)
+37300 val loss 5.7499
+37300 val perplexity 314.1736
+37300 train 5.556634 (lr=4.7372e-06) (hash(x)=49312812)
+37600 val loss 5.7796
+37600 val perplexity 323.6248
+37600 train 6.262186 (lr=6.9240e-06) (hash(x)=60685704)
+37400 val loss 5.7495
+37400 val perplexity 314.0204
+37400 train 5.588504 (lr=4.6965e-06) (hash(x)=48536877)
+37700 val loss 5.7822
+37700 val perplexity 324.4685
+37700 train 5.844322 (lr=6.8642e-06) (hash(x)=56543272)
+37500 val loss 5.7526
+37500 val perplexity 315.0148
+37500 train 5.710283 (lr=4.6561e-06) (hash(x)=50612814)
+37800 val loss 5.7760
+37800 val perplexity 322.4611
+37800 train 5.876165 (lr=6.8047e-06) (hash(x)=56659623)
+37600 val loss 5.7537
+37600 val perplexity 315.3602
+37600 train 6.232800 (lr=4.6160e-06) (hash(x)=60685704)
+37900 val loss 5.7747
+37900 val perplexity 322.0493
+37900 train 5.465421 (lr=6.7456e-06) (hash(x)=48577952)
+37700 val loss 5.7516
+37700 val perplexity 314.7075
+37700 train 5.841077 (lr=4.5761e-06) (hash(x)=56543272)
+38000 val loss 5.7760
+38000 val perplexity 322.4732
+38000 train 5.411946 (lr=6.6870e-06) (hash(x)=46719420)
+37800 val loss 5.7478
+37800 val perplexity 313.5129
+37800 train 5.861818 (lr=4.5365e-06) (hash(x)=56659623)
+38100 val loss 5.7710
+38100 val perplexity 320.8696
+38100 train 5.692843 (lr=6.6287e-06) (hash(x)=56290069)
+37900 val loss 5.7508
+37900 val perplexity 314.4483
+37900 train 5.427722 (lr=4.4971e-06) (hash(x)=48577952)
+38200 val loss 5.7687
+38200 val perplexity 320.1238
+38200 train 5.740766 (lr=6.5708e-06) (hash(x)=50056974)
+38000 val loss 5.7490
+38000 val perplexity 313.8725
+38000 train 5.362552 (lr=4.4580e-06) (hash(x)=46719420)
+38300 val loss 5.7682
+38300 val perplexity 319.9477
+38300 train 5.813049 (lr=6.5133e-06) (hash(x)=48797079)
+38100 val loss 5.7456
+38100 val perplexity 312.8075
+38100 train 5.690714 (lr=4.4191e-06) (hash(x)=56290069)
+38400 val loss 5.7681
+38400 val perplexity 319.9292
+38400 train 5.653036 (lr=6.4562e-06) (hash(x)=49702043)
+38200 val loss 5.7421
+38200 val perplexity 311.7096
+38200 train 5.694210 (lr=4.3805e-06) (hash(x)=50056974)
+38500 val loss 5.7690
+38500 val perplexity 320.2328
+38500 train 5.794525 (lr=6.3995e-06) (hash(x)=53375877)
+38300 val loss 5.7444
+38300 val perplexity 312.4421
+38300 train 5.772757 (lr=4.3422e-06) (hash(x)=48797079)
+38600 val loss 5.7644
+38600 val perplexity 318.7521
+38600 train 5.779543 (lr=6.3431e-06) (hash(x)=52757407)
+38400 val loss 5.7425
+38400 val perplexity 311.8521
+38400 train 5.621862 (lr=4.3041e-06) (hash(x)=49702043)
+38700 val loss 5.7634
+38700 val perplexity 318.4228
+38700 train 5.838319 (lr=6.2872e-06) (hash(x)=51335882)
+38500 val loss 5.7434
+38500 val perplexity 312.1103
+38500 train 5.750005 (lr=4.2663e-06) (hash(x)=53375877)
+38800 val loss 5.7632
+38800 val perplexity 318.3765
+38800 train 5.696782 (lr=6.2318e-06) (hash(x)=46048181)
+38600 val loss 5.7391
+38600 val perplexity 310.7800
+38600 train 5.751609 (lr=4.2288e-06) (hash(x)=52757407)
+38900 val loss 5.7653
+38900 val perplexity 319.0292
+38900 train 5.757919 (lr=6.1767e-06) (hash(x)=50536450)
+38700 val loss 5.7390
+38700 val perplexity 310.7463
+38700 train 5.831838 (lr=4.1915e-06) (hash(x)=51335882)
+39000 val loss 5.7607
+39000 val perplexity 317.5667
+39000 train 5.609990 (lr=6.1220e-06) (hash(x)=49201278)
+38800 val loss 5.7393
+38800 val perplexity 310.8553
+38800 train 5.670016 (lr=4.1545e-06) (hash(x)=46048181)
+39100 val loss 5.7615
+39100 val perplexity 317.8277
+39100 train 5.856156 (lr=6.0677e-06) (hash(x)=52669740)
+38900 val loss 5.7401
+38900 val perplexity 311.1032
+38900 train 5.751210 (lr=4.1178e-06) (hash(x)=50536450)
+39200 val loss 5.7572
+39200 val perplexity 316.4665
+39000 val loss 5.7375
+39000 val perplexity 310.2974
+39200 train 5.760385 (lr=6.0139e-06) (hash(x)=45566341)
+39000 train 5.596824 (lr=4.0813e-06) (hash(x)=49201278)
+39300 val loss 5.7566
+39300 val perplexity 316.2702
+39300 train 6.370432 (lr=5.9605e-06) (hash(x)=50582342)
+39100 val loss 5.7380
+39100 val perplexity 310.4278
+39100 train 5.831585 (lr=4.0452e-06) (hash(x)=52669740)
+39400 val loss 5.7567
+39400 val perplexity 316.3073
+39400 train 5.515676 (lr=5.9075e-06) (hash(x)=43591296)
+39200 val loss 5.7349
+39200 val perplexity 309.4730
+39200 train 5.747909 (lr=4.0093e-06) (hash(x)=45566341)
+39500 val loss 5.7578
+39500 val perplexity 316.6478
+39500 train 5.564514 (lr=5.8549e-06) (hash(x)=49119520)
+39300 val loss 5.7354
+39300 val perplexity 309.6438
+39300 train 6.363035 (lr=3.9736e-06) (hash(x)=50582342)
+39600 val loss 5.7561
+39600 val perplexity 316.1039
+39600 train 5.978034 (lr=5.8027e-06) (hash(x)=50278799)
+39400 val loss 5.7352
+39400 val perplexity 309.5719
+39400 train 5.475320 (lr=3.9383e-06) (hash(x)=43591296)
+39700 val loss 5.7574
+39700 val perplexity 316.5251
+39700 train 5.677328 (lr=5.7510e-06) (hash(x)=54799713)
+39500 val loss 5.7350
+39500 val perplexity 309.5111
+39500 train 5.535816 (lr=3.9032e-06) (hash(x)=49119520)
+39800 val loss 5.7571
+39800 val perplexity 316.4443
+39800 train 5.849969 (lr=5.6997e-06) (hash(x)=46290010)
+39600 val loss 5.7353
+39600 val perplexity 309.6212
+39600 train 5.979152 (lr=3.8685e-06) (hash(x)=50278799)
+39900 val loss 5.7550
+39900 val perplexity 315.7692
+39900 train 5.418046 (lr=5.6488e-06) (hash(x)=41503413)
+39700 val loss 5.7365
+39700 val perplexity 309.9691
+39700 train 5.643690 (lr=3.8340e-06) (hash(x)=54799713)
+40000 val loss 5.7536
+40000 val perplexity 315.3214
+39800 val loss 5.7359
+39800 val perplexity 309.8057
+39800 train 5.825263 (lr=3.7998e-06) (hash(x)=46290010)
+40000 train 5.792921 (lr=5.5983e-06) (hash(x)=44949087)
+40100 val loss 5.7549
+40100 val perplexity 315.7472
+40100 train 5.321020 (lr=5.5483e-06) (hash(x)=45005676)
+39900 val loss 5.7341
+39900 val perplexity 309.2191
+39900 train 5.402589 (lr=3.7659e-06) (hash(x)=41503413)
+40200 val loss 5.7553
+40200 val perplexity 315.8712
+40200 train 5.920660 (lr=5.4987e-06) (hash(x)=51341494)
+40000 val loss 5.7323
+40000 val perplexity 308.6837
+40000 train 5.771432 (lr=3.7322e-06) (hash(x)=44949087)
+40300 val loss 5.7534
+40300 val perplexity 315.2459
+40300 train 5.484723 (lr=5.4496e-06) (hash(x)=47686820)
+40100 val loss 5.7334
+40100 val perplexity 309.0072
+40100 train 5.303647 (lr=3.6989e-06) (hash(x)=45005676)
+40400 val loss 5.7524
+40400 val perplexity 314.9382
+40400 train 5.635802 (lr=5.4009e-06) (hash(x)=47142962)
+40200 val loss 5.7340
+40200 val perplexity 309.2034
+40200 train 5.899469 (lr=3.6658e-06) (hash(x)=51341494)
+40500 val loss 5.7537
+40500 val perplexity 315.3667
+40500 train 5.523669 (lr=5.3526e-06) (hash(x)=47585616)
+40300 val loss 5.7341
+40300 val perplexity 309.2497
+40300 train 5.450000 (lr=3.6331e-06) (hash(x)=47686820)
+40600 val loss 5.7549
+40600 val perplexity 315.7328
+40600 train 5.575082 (lr=5.3048e-06) (hash(x)=50075832)
+40400 val loss 5.7335
+40400 val perplexity 309.0636
+40400 train 5.610949 (lr=3.6006e-06) (hash(x)=47142962)
+40700 val loss 5.7478
+40700 val perplexity 313.5100
+40700 train 5.810838 (lr=5.2574e-06) (hash(x)=52764040)
+40500 val loss 5.7329
+40500 val perplexity 308.8586
+40500 train 5.483225 (lr=3.5684e-06) (hash(x)=47585616)
+40800 val loss 5.7448
+40800 val perplexity 312.5674
+40800 train 6.078841 (lr=5.2105e-06) (hash(x)=56727171)
+40600 val loss 5.7343
+40600 val perplexity 309.2868
+40600 train 5.554240 (lr=3.5365e-06) (hash(x)=50075832)
+40900 val loss 5.7440
+40900 val perplexity 312.2982
+40900 train 5.795232 (lr=5.1641e-06) (hash(x)=53111899)
+40700 val loss 5.7270
+40700 val perplexity 307.0499
+40700 train 5.775858 (lr=3.5050e-06) (hash(x)=52764040)
+41000 val loss 5.7433
+41000 val perplexity 312.1035
+41000 train 5.608589 (lr=5.1180e-06) (hash(x)=45951630)
+40800 val loss 5.7245
+40800 val perplexity 306.2899
+40800 train 6.086235 (lr=3.4737e-06) (hash(x)=56727171)
+41100 val loss 5.7424
+41100 val perplexity 311.8052
+41100 train 6.126620 (lr=5.0725e-06) (hash(x)=59664516)
+40900 val loss 5.7247
+40900 val perplexity 306.3393
+40900 train 5.762385 (lr=3.4427e-06) (hash(x)=53111899)
+41200 val loss 5.7416
+41200 val perplexity 311.5561
+41200 train 5.610546 (lr=5.0273e-06) (hash(x)=50414471)
+41000 val loss 5.7226
+41000 val perplexity 305.7076
+41000 train 5.586489 (lr=3.4120e-06) (hash(x)=45951630)
+41300 val loss 5.7373
+41300 val perplexity 310.2396
+41300 train 5.010204 (lr=4.9827e-06) (hash(x)=33823056)
+41100 val loss 5.7238
+41100 val perplexity 306.0524
+41100 train 6.108809 (lr=3.3816e-06) (hash(x)=59664516)
+41400 val loss 5.7393
+41400 val perplexity 310.8571
+41400 train 5.841630 (lr=4.9385e-06) (hash(x)=52911940)
+41200 val loss 5.7234
+41200 val perplexity 305.9395
+41200 train 5.606280 (lr=3.3516e-06) (hash(x)=50414471)
+41500 val loss 5.7440
+41500 val perplexity 312.3051
+41500 train 5.314801 (lr=4.8947e-06) (hash(x)=40005218)
+41300 val loss 5.7206
+41300 val perplexity 305.0826
+41300 train 4.976900 (lr=3.3218e-06) (hash(x)=33823056)
+41600 val loss 5.7361
+41600 val perplexity 309.8464
+41600 train 6.151201 (lr=4.8515e-06) (hash(x)=55223673)
+41400 val loss 5.7221
+41400 val perplexity 305.5317
+41400 train 5.837681 (lr=3.2923e-06) (hash(x)=52911940)
+41700 val loss 5.7348
+41700 val perplexity 309.4367
+41700 train 5.650538 (lr=4.8087e-06) (hash(x)=49603559)
+41500 val loss 5.7257
+41500 val perplexity 306.6339
+41500 train 5.263692 (lr=3.2632e-06) (hash(x)=40005218)
+41800 val loss 5.7363
+41800 val perplexity 309.9119
+41800 train 5.765027 (lr=4.7663e-06) (hash(x)=51692850)
+41600 val loss 5.7181
+41600 val perplexity 304.3310
+41600 train 6.122997 (lr=3.2343e-06) (hash(x)=55223673)
+41900 val loss 5.7329
+41900 val perplexity 308.8734
+41900 train 5.509419 (lr=4.7244e-06) (hash(x)=51210799)
+41700 val loss 5.7182
+41700 val perplexity 304.3627
+41700 train 5.618549 (lr=3.2058e-06) (hash(x)=49603559)
+42000 val loss 5.7332
+42000 val perplexity 308.9662
+42000 train 6.310209 (lr=4.6830e-06) (hash(x)=60592263)
+41800 val loss 5.7177
+41800 val perplexity 304.2105
+41800 train 5.747568 (lr=3.1775e-06) (hash(x)=51692850)
+42100 val loss 5.7341
+42100 val perplexity 309.2281
+42100 train 5.618619 (lr=4.6421e-06) (hash(x)=48159119)
+41900 val loss 5.7176
+41900 val perplexity 304.1701
+41900 train 5.481105 (lr=3.1496e-06) (hash(x)=51210799)
+42200 val loss 5.7323
+42200 val perplexity 308.6851
+42200 train 5.615604 (lr=4.6016e-06) (hash(x)=48619712)
+42000 val loss 5.7172
+42000 val perplexity 304.0639
+42000 train 6.293019 (lr=3.1220e-06) (hash(x)=60592263)
+42300 val loss 5.7312
+42300 val perplexity 308.3441
+42300 train 5.503745 (lr=4.5616e-06) (hash(x)=56574312)
+42100 val loss 5.7177
+42100 val perplexity 304.2097
+42100 train 5.597538 (lr=3.0947e-06) (hash(x)=48159119)
+42400 val loss 5.7328
+42400 val perplexity 308.8359
+42400 train 5.832663 (lr=4.5221e-06) (hash(x)=47020520)
+42200 val loss 5.7172
+42200 val perplexity 304.0652
+42200 train 5.618904 (lr=3.0677e-06) (hash(x)=48619712)
+42500 val loss 5.7332
+42500 val perplexity 308.9474
+42500 train 5.639012 (lr=4.4830e-06) (hash(x)=48787571)
+42300 val loss 5.7172
+42300 val perplexity 304.0536
+42300 train 5.492298 (lr=3.0411e-06) (hash(x)=56574312)
+42600 val loss 5.7348
+42600 val perplexity 309.4664
+42600 train 5.823083 (lr=4.4445e-06) (hash(x)=53734398)
+42400 val loss 5.7172
+42400 val perplexity 304.0675
+42400 train 5.830759 (lr=3.0147e-06) (hash(x)=47020520)
+42700 val loss 5.7352
+42700 val perplexity 309.5734
+42700 train 5.524917 (lr=4.4064e-06) (hash(x)=52179072)
+42500 val loss 5.7171
+42500 val perplexity 304.0252
+42500 train 5.610551 (lr=2.9887e-06) (hash(x)=48787571)
+42800 val loss 5.7345
+42800 val perplexity 309.3727
+42800 train 5.481721 (lr=4.3688e-06) (hash(x)=45303025)
+42600 val loss 5.7200
+42600 val perplexity 304.9117
+42600 train 5.809055 (lr=2.9630e-06) (hash(x)=53734398)
+42900 val loss 5.7320
+42900 val perplexity 308.5905
+42900 train 5.877319 (lr=4.3316e-06) (hash(x)=53649191)
+42700 val loss 5.7213
+42700 val perplexity 305.2968
+42700 train 5.520309 (lr=2.9376e-06) (hash(x)=52179072)
+43000 val loss 5.7318
+43000 val perplexity 308.5243
+43000 train 6.193897 (lr=4.2950e-06) (hash(x)=61985066)
+42800 val loss 5.7193
+42800 val perplexity 304.6996
+42800 train 5.453915 (lr=2.9125e-06) (hash(x)=45303025)
+43100 val loss 5.7321
+43100 val perplexity 308.6307
+43100 train 5.704926 (lr=4.2589e-06) (hash(x)=48533037)
+42900 val loss 5.7185
+42900 val perplexity 304.4348
+42900 train 5.854225 (lr=2.8878e-06) (hash(x)=53649191)
+43200 val loss 5.7273
+43200 val perplexity 307.1344
+43200 train 5.826184 (lr=4.2232e-06) (hash(x)=50630327)
+43000 val loss 5.7191
+43000 val perplexity 304.6280
+43000 train 6.188183 (lr=2.8633e-06) (hash(x)=61985066)
+43300 val loss 5.7337
+43300 val perplexity 309.1008
+43300 train 5.833436 (lr=4.1880e-06) (hash(x)=54845917)
+43100 val loss 5.7180
+43100 val perplexity 304.2979
+43100 train 5.671202 (lr=2.8392e-06) (hash(x)=48533037)
+43400 val loss 5.7289
+43400 val perplexity 307.6429
+43400 train 5.659473 (lr=4.1533e-06) (hash(x)=52858739)
+43200 val loss 5.7156
+43200 val perplexity 303.5594
+43200 train 5.817543 (lr=2.8155e-06) (hash(x)=50630327)
+43500 val loss 5.7241
+43500 val perplexity 306.1530
+43500 train 5.824585 (lr=4.1191e-06) (hash(x)=49761319)
+43300 val loss 5.7203
+43300 val perplexity 304.9985
+43300 train 5.817310 (lr=2.7920e-06) (hash(x)=54845917)
+43600 val loss 5.7227
+43600 val perplexity 305.7173
+43600 train 6.047313 (lr=4.0854e-06) (hash(x)=64879186)
+43400 val loss 5.7149
+43400 val perplexity 303.3579
+43400 train 5.632379 (lr=2.7689e-06) (hash(x)=52858739)
+43700 val loss 5.7214
+43700 val perplexity 305.3380
+43700 train 5.947082 (lr=4.0522e-06) (hash(x)=55246427)
+43500 val loss 5.7115
+43500 val perplexity 302.3149
+43500 train 5.796209 (lr=2.7461e-06) (hash(x)=49761319)
+43800 val loss 5.7221
+43800 val perplexity 305.5568
+43800 train 5.646681 (lr=4.0195e-06) (hash(x)=51238313)
+43600 val loss 5.7101
+43600 val perplexity 301.8918
+43600 train 6.012553 (lr=2.7236e-06) (hash(x)=64879186)
+43900 val loss 5.7202
+43900 val perplexity 304.9778
+43900 train 5.774496 (lr=3.9873e-06) (hash(x)=51664512)
+43700 val loss 5.7101
+43700 val perplexity 301.8884
+43700 train 5.952106 (lr=2.7015e-06) (hash(x)=55246427)
+44000 val loss 5.7344
+44000 val perplexity 309.3282
+44000 train 6.035647 (lr=3.9556e-06) (hash(x)=64172296)
+43800 val loss 5.7098
+43800 val perplexity 301.8050
+43800 train 5.635701 (lr=2.6797e-06) (hash(x)=51238313)
+44100 val loss 5.7198
+44100 val perplexity 304.8503
+44100 train 5.632649 (lr=3.9243e-06) (hash(x)=50205960)
+43900 val loss 5.7085
+43900 val perplexity 301.4081
+43900 train 5.746950 (lr=2.6582e-06) (hash(x)=51664512)
+44200 val loss 5.7195
+44200 val perplexity 304.7383
+44200 train 5.764534 (lr=3.8936e-06) (hash(x)=51065522)
+44000 val loss 5.7183
+44000 val perplexity 304.3899
+44000 train 6.055036 (lr=2.6370e-06) (hash(x)=64172296)
+44300 val loss 5.7200
+44300 val perplexity 304.9142
+44300 train 5.866989 (lr=3.8634e-06) (hash(x)=56163183)
+44100 val loss 5.7065
+44100 val perplexity 300.8313
+44100 train 5.611526 (lr=2.6162e-06) (hash(x)=50205960)
+44400 val loss 5.7208
+44400 val perplexity 305.1565
+44400 train 5.743508 (lr=3.8337e-06) (hash(x)=50818020)
+44200 val loss 5.7060
+44200 val perplexity 300.6648
+44200 train 5.760792 (lr=2.5957e-06) (hash(x)=51065522)
+44500 val loss 5.7186
+44500 val perplexity 304.4919
+44500 train 5.820117 (lr=3.8045e-06) (hash(x)=52855559)
+44300 val loss 5.7080
+44300 val perplexity 301.2689
+44300 train 5.860295 (lr=2.5756e-06) (hash(x)=56163183)
+44600 val loss 5.7191
+44600 val perplexity 304.6343
+44600 train 5.603107 (lr=3.7758e-06) (hash(x)=44014306)
+44400 val loss 5.7086
+44400 val perplexity 301.4344
+44400 train 5.734999 (lr=2.5558e-06) (hash(x)=50818020)
+44700 val loss 5.7178
+44700 val perplexity 304.2207
+44700 train 5.851013 (lr=3.7476e-06) (hash(x)=53648768)
+44500 val loss 5.7087
+44500 val perplexity 301.4689
+44500 train 5.824543 (lr=2.5363e-06) (hash(x)=52855559)
+44800 val loss 5.7179
+44800 val perplexity 304.2599
+44800 train 5.637219 (lr=3.7199e-06) (hash(x)=42436477)
+44600 val loss 5.7072
+44600 val perplexity 301.0395
+44600 train 5.598558 (lr=2.5172e-06) (hash(x)=44014306)
+44900 val loss 5.7214
+44900 val perplexity 305.3320
+44900 train 5.442779 (lr=3.6927e-06) (hash(x)=48409149)
+44700 val loss 5.7069
+44700 val perplexity 300.9479
+44700 train 5.852466 (lr=2.4984e-06) (hash(x)=53648768)
+45000 val loss 5.7181
+45000 val perplexity 304.3320
+45000 train 5.635482 (lr=3.6660e-06) (hash(x)=53084805)
+44800 val loss 5.7068
+44800 val perplexity 300.9056
+44800 train 5.628494 (lr=2.4799e-06) (hash(x)=42436477)
+45100 val loss 5.7176
+45100 val perplexity 304.1659
+45100 train 5.607561 (lr=3.6398e-06) (hash(x)=56386155)
+44900 val loss 5.7075
+44900 val perplexity 301.1041
+44900 train 5.428924 (lr=2.4618e-06) (hash(x)=48409149)
+45200 val loss 5.7182
+45200 val perplexity 304.3688
+45200 train 5.758592 (lr=3.6142e-06) (hash(x)=53389864)
+45000 val loss 5.7071
+45000 val perplexity 301.0114
+45000 train 5.631902 (lr=2.4440e-06) (hash(x)=53084805)
+45300 val loss 5.7180
+45300 val perplexity 304.2943
+45300 train 5.732639 (lr=3.5891e-06) (hash(x)=50817456)
+45100 val loss 5.7085
+45100 val perplexity 301.4130
+45100 train 5.579370 (lr=2.4266e-06) (hash(x)=56386155)
+45400 val loss 5.7182
+45400 val perplexity 304.3642
+45400 train 5.744096 (lr=3.5644e-06) (hash(x)=50081660)
+45200 val loss 5.7084
+45200 val perplexity 301.3900
+45200 train 5.728960 (lr=2.4095e-06) (hash(x)=53389864)
+45500 val loss 5.7181
+45500 val perplexity 304.3163
+45500 train 5.472034 (lr=3.5403e-06) (hash(x)=55567121)
+45300 val loss 5.7085
+45300 val perplexity 301.4131
+45300 train 5.727216 (lr=2.3927e-06) (hash(x)=50817456)
+45600 val loss 5.7136
+45600 val perplexity 302.9513
+45600 train 5.776908 (lr=3.5167e-06) (hash(x)=48902854)
+45400 val loss 5.7090
+45400 val perplexity 301.5839
+45400 train 5.750453 (lr=2.3763e-06) (hash(x)=50081660)
+45700 val loss 5.7116
+45700 val perplexity 302.3618
+45700 train 5.966445 (lr=3.4936e-06) (hash(x)=53803301)
+45500 val loss 5.7081
+45500 val perplexity 301.2970
+45500 train 5.466743 (lr=2.3602e-06) (hash(x)=55567121)
+45800 val loss 5.7125
+45800 val perplexity 302.6349
+45800 train 5.750171 (lr=3.4711e-06) (hash(x)=47576606)
+45600 val loss 5.7040
+45600 val perplexity 300.0774
+45600 train 5.765701 (lr=2.3445e-06) (hash(x)=48902854)
+45900 val loss 5.7091
+45900 val perplexity 301.5849
+45900 train 5.560674 (lr=3.4490e-06) (hash(x)=56276981)
+45700 val loss 5.7026
+45700 val perplexity 299.6425
+45700 train 5.949104 (lr=2.3291e-06) (hash(x)=53803301)
+46000 val loss 5.7089
+46000 val perplexity 301.5357
+46000 train 5.658870 (lr=3.4275e-06) (hash(x)=52174902)
+45800 val loss 5.7017
+45800 val perplexity 299.3757
+45800 train 5.737519 (lr=2.3141e-06) (hash(x)=47576606)
+46100 val loss 5.7109
+46100 val perplexity 302.1516
+45900 val loss 5.6995
+45900 val perplexity 298.7207
+46100 train 5.814061 (lr=3.4065e-06) (hash(x)=52155490)
+45900 train 5.551755 (lr=2.2994e-06) (hash(x)=56276981)
+46200 val loss 5.7076
+46200 val perplexity 301.1525
+46200 train 5.524827 (lr=3.3860e-06) (hash(x)=46489984)
+46000 val loss 5.7003
+46000 val perplexity 298.9677
+46000 train 5.649840 (lr=2.2850e-06) (hash(x)=52174902)
+46300 val loss 5.7084
+46300 val perplexity 301.4018
+46300 train 5.700564 (lr=3.3661e-06) (hash(x)=56603976)
+46100 val loss 5.7002
+46100 val perplexity 298.9251
+46100 train 5.820592 (lr=2.2710e-06) (hash(x)=52155490)
+46400 val loss 5.7074
+46400 val perplexity 301.0876
+46400 train 5.516274 (lr=3.3466e-06) (hash(x)=51228412)
+46200 val loss 5.6988
+46200 val perplexity 298.5126
+46200 train 5.524873 (lr=2.2574e-06) (hash(x)=46489984)
+46500 val loss 5.7059
+46500 val perplexity 300.6437
+46500 train 5.528039 (lr=3.3277e-06) (hash(x)=48511186)
+46300 val loss 5.7005
+46300 val perplexity 299.0136
+46300 train 5.697016 (lr=2.2441e-06) (hash(x)=56603976)
+46600 val loss 5.7063
+46600 val perplexity 300.7494
+46600 train 5.747422 (lr=3.3093e-06) (hash(x)=48832039)
+46400 val loss 5.6991
+46400 val perplexity 298.6001
+46400 train 5.501826 (lr=2.2311e-06) (hash(x)=51228412)
+46700 val loss 5.7060
+46700 val perplexity 300.6601
+46700 train 6.375529 (lr=3.2915e-06) (hash(x)=65821271)
+46500 val loss 5.6980
+46500 val perplexity 298.2724
+46500 train 5.519785 (lr=2.2185e-06) (hash(x)=48511186)
+46800 val loss 5.7064
+46800 val perplexity 300.7738
+46800 train 5.422392 (lr=3.2741e-06) (hash(x)=48005593)
+46600 val loss 5.6987
+46600 val perplexity 298.4899
+46600 train 5.737981 (lr=2.2062e-06) (hash(x)=48832039)
+46900 val loss 5.7060
+46900 val perplexity 300.6533
+46900 train 5.976987 (lr=3.2573e-06) (hash(x)=53182453)
+46700 val loss 5.6983
+46700 val perplexity 298.3579
+46700 train 6.379213 (lr=2.1943e-06) (hash(x)=65821271)
+47000 val loss 5.7071
+47000 val perplexity 301.0095
+47000 train 5.630270 (lr=3.2410e-06) (hash(x)=49385983)
+46800 val loss 5.6972
+46800 val perplexity 298.0431
+47100 val loss 5.7081
+47100 val perplexity 301.2980
+46800 train 5.395824 (lr=2.1828e-06) (hash(x)=48005593)
+47100 train 5.697306 (lr=3.2253e-06) (hash(x)=48801622)
+47200 val loss 5.7065
+47200 val perplexity 300.8086
+47200 train 5.438989 (lr=3.2101e-06) (hash(x)=45216690)
+46900 val loss 5.6982
+46900 val perplexity 298.3284
+46900 train 5.965557 (lr=2.1716e-06) (hash(x)=53182453)
+47300 val loss 5.7064
+47300 val perplexity 300.7990
+47300 train 5.523960 (lr=3.1954e-06) (hash(x)=51185601)
+47000 val loss 5.6980
+47000 val perplexity 298.2556
+47000 train 5.599510 (lr=2.1607e-06) (hash(x)=49385983)
+47400 val loss 5.7054
+47400 val perplexity 300.4760
+47400 train 5.405930 (lr=3.1812e-06) (hash(x)=47743508)
+47100 val loss 5.6990
+47100 val perplexity 298.5797
+47100 train 5.698075 (lr=2.1502e-06) (hash(x)=48801622)
+47500 val loss 5.7060
+47500 val perplexity 300.6675
+47500 train 5.466727 (lr=3.1675e-06) (hash(x)=49318833)
+47200 val loss 5.6988
+47200 val perplexity 298.5047
+47200 train 5.409289 (lr=2.1400e-06) (hash(x)=45216690)
+47600 val loss 5.7089
+47600 val perplexity 301.5490
+47600 train 5.646415 (lr=3.1544e-06) (hash(x)=50430826)
+47300 val loss 5.6986
+47300 val perplexity 298.4404
+47300 train 5.517011 (lr=2.1302e-06) (hash(x)=51185601)
+47700 val loss 5.7062
+47700 val perplexity 300.7164
+47700 train 5.645108 (lr=3.1419e-06) (hash(x)=51966931)
+47400 val loss 5.6997
+47400 val perplexity 298.7687
+47400 train 5.395707 (lr=2.1208e-06) (hash(x)=47743508)
+47800 val loss 5.7064
+47800 val perplexity 300.7901
+47800 train 5.464528 (lr=3.1298e-06) (hash(x)=49517209)
+47500 val loss 5.6991
+47500 val perplexity 298.6126
+47500 train 5.463457 (lr=2.1117e-06) (hash(x)=49318833)
+47900 val loss 5.7087
+47900 val perplexity 301.4664
+47900 train 5.506170 (lr=3.1183e-06) (hash(x)=49749702)
+47600 val loss 5.7007
+47600 val perplexity 299.0796
+47600 train 5.636269 (lr=2.1030e-06) (hash(x)=50430826)
+48000 val loss 5.7092
+48000 val perplexity 301.6447
+48000 train 5.590940 (lr=3.1073e-06) (hash(x)=52535448)
+47700 val loss 5.6994
+47700 val perplexity 298.6739
+47700 train 5.638739 (lr=2.0946e-06) (hash(x)=51966931)
+48100 val loss 5.7022
+48100 val perplexity 299.5289
+48100 train 6.424832 (lr=3.0969e-06) (hash(x)=53739181)
+47800 val loss 5.6987
+47800 val perplexity 298.4743
+47800 train 5.455708 (lr=2.0865e-06) (hash(x)=49517209)
+48200 val loss 5.7011
+48200 val perplexity 299.1970
+48200 train 5.682243 (lr=3.0869e-06) (hash(x)=48086710)
+48300 val loss 5.7004
+48300 val perplexity 298.9868
+48300 train 5.658418 (lr=3.0776e-06) (hash(x)=43991942)
+47900 val loss 5.7004
+47900 val perplexity 298.9963
+47900 train 5.512270 (lr=2.0789e-06) (hash(x)=49749702)
+48400 val loss 5.6983
+48400 val perplexity 298.3570
+48400 train 5.569871 (lr=3.0687e-06) (hash(x)=50801906)
+48000 val loss 5.7011
+48000 val perplexity 299.1924
+48000 train 5.580304 (lr=2.0715e-06) (hash(x)=52535448)
+48500 val loss 5.6973
+48500 val perplexity 298.0691
+48500 train 5.756511 (lr=3.0604e-06) (hash(x)=48915599)
+48100 val loss 5.6955
+48100 val perplexity 297.5303
+48100 train 6.399620 (lr=2.0646e-06) (hash(x)=53739181)
+48600 val loss 5.6971
+48600 val perplexity 297.9879
+48600 train 5.891415 (lr=3.0526e-06) (hash(x)=54450172)
+48200 val loss 5.6930
+48200 val perplexity 296.7915
+48200 train 5.668230 (lr=2.0580e-06) (hash(x)=48086710)
+48700 val loss 5.6970
+48700 val perplexity 297.9729
+48700 train 5.604166 (lr=3.0454e-06) (hash(x)=48755509)
+48300 val loss 5.6934
+48300 val perplexity 296.9073
+48300 train 5.647903 (lr=2.0517e-06) (hash(x)=43991942)
+48800 val loss 5.7023
+48800 val perplexity 299.5513
+48800 train 5.592684 (lr=3.0387e-06) (hash(x)=51380096)
+48400 val loss 5.6922
+48400 val perplexity 296.5393
+48400 train 5.563245 (lr=2.0458e-06) (hash(x)=50801906)
+48900 val loss 5.6976
+48900 val perplexity 298.1454
+48900 train 5.563201 (lr=3.0325e-06) (hash(x)=43404218)
+48500 val loss 5.6916
+48500 val perplexity 296.3804
+48500 train 5.755304 (lr=2.0403e-06) (hash(x)=48915599)
+49000 val loss 5.6970
+49000 val perplexity 297.9710
+49000 train 5.694257 (lr=3.0269e-06) (hash(x)=55059739)
+48600 val loss 5.6921
+48600 val perplexity 296.5266
+48600 train 5.882404 (lr=2.0351e-06) (hash(x)=54450172)
+49100 val loss 5.6986
+49100 val perplexity 298.4380
+49100 train 5.894599 (lr=3.0218e-06) (hash(x)=47889309)
+48700 val loss 5.6903
+48700 val perplexity 295.9940
+48700 train 5.589544 (lr=2.0302e-06) (hash(x)=48755509)
+49200 val loss 5.6964
+49200 val perplexity 297.7977
+49200 train 5.428235 (lr=3.0172e-06) (hash(x)=46202589)
+48800 val loss 5.6949
+48800 val perplexity 297.3359
+48800 train 5.587642 (lr=2.0258e-06) (hash(x)=51380096)
+49300 val loss 5.6980
+49300 val perplexity 298.2572
+49300 train 5.655526 (lr=3.0132e-06) (hash(x)=49161813)
+49400 val loss 5.6949
+49400 val perplexity 297.3601
+48900 val loss 5.6925
+48900 val perplexity 296.6438
+49400 train 5.521633 (lr=3.0097e-06) (hash(x)=49505044)
+48900 train 5.546479 (lr=2.0217e-06) (hash(x)=43404218)
+49500 val loss 5.6955
+49500 val perplexity 297.5116
+49500 train 5.476085 (lr=3.0067e-06) (hash(x)=46610327)
+49000 val loss 5.6923
+49000 val perplexity 296.5789
+49000 train 5.675298 (lr=2.0179e-06) (hash(x)=55059739)
+49600 val loss 5.6959
+49600 val perplexity 297.6451
+49600 train 6.046562 (lr=3.0043e-06) (hash(x)=51188240)
+49100 val loss 5.6930
+49100 val perplexity 296.7973
+49100 train 5.911443 (lr=2.0145e-06) (hash(x)=47889309)
+49700 val loss 5.6950
+49700 val perplexity 297.3770
+49700 train 5.489672 (lr=3.0024e-06) (hash(x)=41492016)
+49200 val loss 5.6921
+49200 val perplexity 296.5167
+49200 train 5.431955 (lr=2.0115e-06) (hash(x)=46202589)
+49800 val loss 5.6948
+49800 val perplexity 297.3244
+49800 train 5.648407 (lr=3.0011e-06) (hash(x)=53488833)
+49300 val loss 5.6936
+49300 val perplexity 296.9700
+49300 train 5.645440 (lr=2.0088e-06) (hash(x)=49161813)
+49900 val loss 5.6933
+49900 val perplexity 296.8729
+49900 train 6.041644 (lr=3.0003e-06) (hash(x)=60260935)
+49400 val loss 5.6911
+49400 val perplexity 296.2114
+49400 train 5.506441 (lr=2.0064e-06) (hash(x)=49505044)
+49999 val loss 5.6907
+49999 val perplexity 296.1120
+49500 val loss 5.6921
+49500 val perplexity 296.5128
+49500 train 5.465817 (lr=2.0045e-06) (hash(x)=46610327)
+49600 val loss 5.6932
+49600 val perplexity 296.8287
+49600 train 6.044370 (lr=2.0029e-06) (hash(x)=51188240)
+49700 val loss 5.6911
+49700 val perplexity 296.2107
+49700 train 5.481483 (lr=2.0016e-06) (hash(x)=41492016)
+49800 val loss 5.6913
+49800 val perplexity 296.2769
+49800 train 5.630619 (lr=2.0007e-06) (hash(x)=53488833)
+49900 val loss 5.6909
+49900 val perplexity 296.1589
+49900 train 6.038949 (lr=2.0002e-06) (hash(x)=60260935)
+49999 val loss 5.6894
+49999 val perplexity 295.7296
diff --git a/attention_kindselective_n_heads4_seed1341/model_02500.pt b/attention_kindselective_n_heads4_seed1341/model_02500.pt
index b6a04d5c1eaf7ff07c19e1fbde8a92d829027d20..60eb8f1aa8b4adc3913927517edf1f0835c80be0 100644
--- a/attention_kindselective_n_heads4_seed1341/model_02500.pt
+++ b/attention_kindselective_n_heads4_seed1341/model_02500.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c484aa037e9b115403161e2fe304d0c7a89dc6b79d216e3cdfd363f6ca13d25d
+oid sha256:9be0ae6a1500ccdd6fb25290b75c4aa5cad5df4ff202df8d30bf64fb32830a49
 size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_05000.pt b/attention_kindselective_n_heads4_seed1341/model_05000.pt
index 0c272a0e9ac0181f54fe8fe91ad5eb2c26b07354..764d0fe80b5d57995bf1afb6317018ee184c6efb 100644
--- a/attention_kindselective_n_heads4_seed1341/model_05000.pt
+++ b/attention_kindselective_n_heads4_seed1341/model_05000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e58ae40f92461f191f2a2c7ae689b02053d1241f805b9a567e39cb8e5277a851
+oid sha256:0ed8c056540906461d7125570140a49c884941a998e834c885f919773ed5a7f5
 size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_07500.pt b/attention_kindselective_n_heads4_seed1341/model_07500.pt
index 3629e50bb0a0f563c77b32ae9cdf2c0c1de45378..ff01942a8729dd840f072fe14da7e6391d1b8d7a 100644
--- a/attention_kindselective_n_heads4_seed1341/model_07500.pt
+++ b/attention_kindselective_n_heads4_seed1341/model_07500.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d3bfda09d2b22fdab50914f273bc586412124529cdfa64474e214c7816f9e627
+oid sha256:0c38e2493098f850413b9bf630b581247017cd6934de174dd3a28f6665ba47c9
 size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_10000.pt b/attention_kindselective_n_heads4_seed1341/model_10000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ebb869fa002e745cebd1cf7f4e05d3344d38d04c
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_10000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59bddcb60af0e23b41bda959258fd176c4b8766360510deff267004e57a4e0de
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_12500.pt b/attention_kindselective_n_heads4_seed1341/model_12500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6c5892ba22cc77a0e0363270f0f0385bb6486a35
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_12500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:875176bfdf881b5fc9ff2f8f606bc10d742f2e775b58bd7e1944d9959e2f90d4
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_15000.pt b/attention_kindselective_n_heads4_seed1341/model_15000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f8a9866c7b3f8a82223f2302e05bea227ba3bd1a
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_15000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d6fbafd23666846f3d49b9d6780b4c38f54201cb1490921243e9fe4f416756c
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_17500.pt b/attention_kindselective_n_heads4_seed1341/model_17500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dce514d6964052f957e44db777d6090789eac103
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_17500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f65285a4ec29e037c8b40770cb285d59d90c1bb9419f3a115e9f50dfff9e08e3
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_20000.pt b/attention_kindselective_n_heads4_seed1341/model_20000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4341747d1405efa093cb52d5f9b81b7877931ce2
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_20000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86bb24316fbd6b5f53481c0404e322d0a292473e3c8b601e89258863c07923a8
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_22500.pt b/attention_kindselective_n_heads4_seed1341/model_22500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..33f832b94a66643a8b80e4f2b889243674d09866
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_22500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:330f600120c8929427e542292dabccaba368ed2d02fbea4f5313f57356164b97
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_25000.pt b/attention_kindselective_n_heads4_seed1341/model_25000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3a4d26d45ba69739ceda711a1d0949dfac8f096d
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_25000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c85eae63babcc7b9606b1d62018b1d2237424a7b2edfc5fe5baf38bd7280c72c
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_27500.pt b/attention_kindselective_n_heads4_seed1341/model_27500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..88fab782ed73284f8bdcfcea3e38ae20298b3972
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_27500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a875e21c8b8935d625b5bb183f5c1a2f0295ce88fce2d9132933cb0639c9f61f
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_30000.pt b/attention_kindselective_n_heads4_seed1341/model_30000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..89cc53f12017a6dec3fbcb7f624671580570eb33
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_30000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:397c8ae3f6fb7d1c08b98c981d7f7d0659e1859fad7eec236476dcc7a0176dc3
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_32500.pt b/attention_kindselective_n_heads4_seed1341/model_32500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4053ed8111c68da4e6af63a732c2ecede56495b4
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_32500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd9829080b4f893600ed7acd641838505198b9bf3545a8db6d7dcf4a3646c6d0
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_35000.pt b/attention_kindselective_n_heads4_seed1341/model_35000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..912d0e7adbd8c074ff9ece7fce715a7251bc6e69
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_35000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0326841882dbaa3f7541f4074e0a56cc4d87c35e1c36fa8a319f4a69566f7b6f
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_37500.pt b/attention_kindselective_n_heads4_seed1341/model_37500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..aa3fabc5ed02fbb1db15872f7eef64ea0dc3c618
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_37500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53e08e519e33fb6431360809b1849e6e3eccba88f6aaf88fa80953369f374ed7
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_40000.pt b/attention_kindselective_n_heads4_seed1341/model_40000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b36cec59e3264ac73080ffad97a2628637c16c68
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_40000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:952d7e8d936be733c6adbeae4cda1c8ad6f6eb1f5cfa924c171a1c4c9ac105ca
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_42500.pt b/attention_kindselective_n_heads4_seed1341/model_42500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4570dbb359038d955db9961d1c6f2a2e82592b7f
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_42500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2ea54fef770a7d9d826f6492ef69a9edbd289cddb8a966556901e0a42a142f4
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_45000.pt b/attention_kindselective_n_heads4_seed1341/model_45000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..573aebe572d3e6296bd0c2ce4aa19cacb8bc88b8
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/model_45000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f61615cd5739917fc974acc450587866f3e3c8a6255c17ea22349e1c274b1d6
+size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_47500.pt b/attention_kindselective_n_heads4_seed1341/model_47500.pt
index 5ec35ca0750c3ead6018f10adb3b5f1dfc2f9a4e..cc711e1a3a054579d1441670e8fd8c2f7a19bab2 100644
--- a/attention_kindselective_n_heads4_seed1341/model_47500.pt
+++ b/attention_kindselective_n_heads4_seed1341/model_47500.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6aeb2a7a2351f8f16532e3f96f1535852348a874b6e53498f54dbb2dbea87f2e
+oid sha256:9eae84098b96e876b98aac32a8c36f77fc8790952a7e382ce8909bb0c616ec76
 size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/model_49999.pt b/attention_kindselective_n_heads4_seed1341/model_49999.pt
index 87046dc5d2ec661b84c7e7b3feb83b6fda35c4dc..6bea6628cf28bac65b2a304b7765f1e99ce11edc 100644
--- a/attention_kindselective_n_heads4_seed1341/model_49999.pt
+++ b/attention_kindselective_n_heads4_seed1341/model_49999.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a27f8d94a91205f44b4097397cbe96f34c7125f7acaf12a27ed77f3f225c47c
+oid sha256:f43b466896f202b1c512cc7966ed31153489bec9e66730115b4391d2c528b648
 size 92843394
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_02500.pt b/attention_kindselective_n_heads4_seed1341/optimizer_02500.pt
index 0403d3984467a1ac34b44ee0351e9c1adf82fd77..7b04c1e3f110e47beb86c114c76bfe8532a39359 100644
--- a/attention_kindselective_n_heads4_seed1341/optimizer_02500.pt
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_02500.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c5a91ef2bc89f32f183c058e5eeb515108d458e3939109aed168281f8270a000
+oid sha256:4451193db27fcfd259ca86985ca50079097afcda3de8ef12899aad3deb7b3a8b
 size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_05000.pt b/attention_kindselective_n_heads4_seed1341/optimizer_05000.pt
index f2b4d42bea569bba0a204f8d986b4a1df87f2fda..9c73ad2aa4e147172c7d33d80ea9f2b93f635bbd 100644
--- a/attention_kindselective_n_heads4_seed1341/optimizer_05000.pt
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_05000.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4eaac3f7ff71b93997f325d4f277c4c0ec68fbf025793ee4565dbfdba779a6e
+oid sha256:16e9f5de4822a6da4a7c5ebc4b1be114be696f40cb71ad5094f4a5036edbbc1f
 size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_07500.pt b/attention_kindselective_n_heads4_seed1341/optimizer_07500.pt
index 275e16bbd7fc3fbe5dd0f6b4fc165c3ddd1f9af8..7479e4606ff00deaadb44333a89caa155b153492 100644
--- a/attention_kindselective_n_heads4_seed1341/optimizer_07500.pt
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_07500.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:553854895d5cc504a01b0c2b72a8825e26e1f903273fbf8c71f350fa796a32d4
+oid sha256:3aa6a192b836fd2fea2915b9f3875f2f27ae191db2f3b5fa7b11949e0699cac9
 size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_10000.pt b/attention_kindselective_n_heads4_seed1341/optimizer_10000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..256f53566727035d5b02e0e5c48689668aff45ad
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_10000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04fab138ca5dab326197fa18fbc6866b5d180a3a5cc824e9e21a5e775ff66c78
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_12500.pt b/attention_kindselective_n_heads4_seed1341/optimizer_12500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..59249d387d027849c3a8e9495a1e2dfb0fbc6d97
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_12500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a9a870a04c9b0968802417299b0b927cb8f166659f0b572f3d98536c7a518d9
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_15000.pt b/attention_kindselective_n_heads4_seed1341/optimizer_15000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d490d660df5a37ade8ae895c9608b65f1fdbddd3
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_15000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1522c09d3659aff31223a5f7791712dc2a8da4083f4fbef02a4f8c835ac2f58f
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_17500.pt b/attention_kindselective_n_heads4_seed1341/optimizer_17500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c2f4b300a6c91cd89c5c4a00783e5bbb45ae4134
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_17500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2130ab0509e9b1b7a7a739dfc5060d46378b7a1646a02bd49fcd0502934dfdd
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_20000.pt b/attention_kindselective_n_heads4_seed1341/optimizer_20000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7e047687c939fc1e20805b1030edb68521b9e065
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_20000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98450d8f0814e9228347e84c6a7a1df848dc332264a30a45534a5b82df90a106
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_22500.pt b/attention_kindselective_n_heads4_seed1341/optimizer_22500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0d39d57b6785c8cb2e83750b504b5b694adaa046
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_22500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70de55ca9f452f3956a15f9e4603ea849c6cdd46d4515dd312c1be347f6f27bf
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_25000.pt b/attention_kindselective_n_heads4_seed1341/optimizer_25000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1c6e3a18a4c4d6035369eb3e01784dd5adb36be8
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_25000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93781efa827024d34e0380279a80a98957319bd1b649af26527709d924a2fbf0
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_27500.pt b/attention_kindselective_n_heads4_seed1341/optimizer_27500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..944b6529eac13fd848e68d629c84ec39b0d847cb
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_27500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0bd709741029562525a20e6b381301456b6720804a9654b9a4a9f7565e7fd60
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_30000.pt b/attention_kindselective_n_heads4_seed1341/optimizer_30000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..975091900f3e33f2a7c49c7481ea79331038a239
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_30000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:291fe6b9b42cd58551655044bd0bbf4d8f5c81ce97898d00c74cec51fd6bc16c
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_32500.pt b/attention_kindselective_n_heads4_seed1341/optimizer_32500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..89b89fd394d25aa05f544bce0667f2f0a1017cd1
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_32500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5433712b10d9d0791248973c2582f820c5d035a55dc1a26d4626664ffbdaed7
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_35000.pt b/attention_kindselective_n_heads4_seed1341/optimizer_35000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..378a995a028b5f4156cf552b6d4483a2b4d09edb
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_35000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd30daac21b0e61bfe4fc0583506422e278a276d7a8263cddc03d44805969a61
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_37500.pt b/attention_kindselective_n_heads4_seed1341/optimizer_37500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..de88b3e564c6c972441f81d5aebcbf857969cd16
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_37500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6159f6b9abcd4d4a848a0e34110a89d943e73e5dad5c574dca2f67ec6f411a83
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_40000.pt b/attention_kindselective_n_heads4_seed1341/optimizer_40000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d603fef2ccbab66a0204bb15b5fdd7788be65882
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_40000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f52a6f9c8703921a6ab54273739b6e256f6416e6395c879330fe7d4cb80e413
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_42500.pt b/attention_kindselective_n_heads4_seed1341/optimizer_42500.pt
new file mode 100644
index 0000000000000000000000000000000000000000..873bb00133e090229feba1eeada3e265fb79a24b
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_42500.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4a6847f957f8c8b2500fd65b4015f90759466f2874adad2a330fedba4d5e043
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_45000.pt b/attention_kindselective_n_heads4_seed1341/optimizer_45000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea1cfc291a22cdd980e4d968297d62800531d621
--- /dev/null
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_45000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc17ab53e8a46df9484138aeb78b79def9cd24dfde54c9f250b32037af7f3ae9
+size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_47500.pt b/attention_kindselective_n_heads4_seed1341/optimizer_47500.pt
index c6a98f08b8d6bd49c7a066f864dcd0245d148cde..d080a56873a4a595686a99845c33e4c1a62220e9 100644
--- a/attention_kindselective_n_heads4_seed1341/optimizer_47500.pt
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_47500.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:142ca7b4714d69fb474f9d95e2cb07eb349cb18b2cb2b88c2fcae9ebdf2d59c3
+oid sha256:60dbf9d217845dcc35cf4bd21ae3c00762055f66c3046bce481c9ca9b7aa6f9a
 size 179406214
diff --git a/attention_kindselective_n_heads4_seed1341/optimizer_49999.pt b/attention_kindselective_n_heads4_seed1341/optimizer_49999.pt
index 0b968ea63cc54eb5639619282efecc8ed115b6fa..8b30e67c802affb6cc64f0b6a4106f561c306ac7 100644
--- a/attention_kindselective_n_heads4_seed1341/optimizer_49999.pt
+++ b/attention_kindselective_n_heads4_seed1341/optimizer_49999.pt
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:853ea0e957cdc71f93c4e51e26d64a9edbc8edb91db6673397e9508908e6913c
+oid sha256:ab159ee0f4ac2589b15d26fc0995a3826ac51df65de5fa438e5bed4a5bea1fa5
 size 179406214