Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1344/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1344", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.75e-5_30720_4_1344", "n_embd": 256}

attention_kindselective_n_heads4_seed1344/log2.txt

@@ -1,528 +1,1051 @@
 max_steps: 17500
+max_steps: 17500
+0 val loss 11.2848
+0 val perplexity 79602.4766
 0 val loss 11.2848
 0 val perplexity 79602.4766
-0 train 11.283560 (lr=1.3750e-07) (hash(x)=145830960)
-100 val loss 9.9832
-100 val perplexity 21659.9883
-100 train 10.001309 (lr=1.3888e-05) (hash(x)=156335112)
-200 val loss 9.2596
-200 val perplexity 10504.6865
-200 train 9.321742 (lr=2.7500e-05) (hash(x)=144157453)
-300 val loss 8.2356
-300 val perplexity 3772.9011
-300 train 8.243886 (lr=2.7498e-05) (hash(x)=149245414)
-400 val loss 7.7924
-400 val perplexity 2422.0842
-400 train 7.838326 (lr=2.7492e-05) (hash(x)=146764602)
-500 val loss 7.6107
-500 val perplexity 2019.6174
-500 train 7.567578 (lr=2.7482e-05) (hash(x)=152540261)
-600 val loss 7.5276
-600 val perplexity 1858.6896
-600 train 7.308209 (lr=2.7467e-05) (hash(x)=140559124)
-700 val loss 7.4688
-700 val perplexity 1752.4941
-700 train 7.520562 (lr=2.7449e-05) (hash(x)=156003556)
-800 val loss 7.4240
-800 val perplexity 1675.7202
-800 train 7.942647 (lr=2.7427e-05) (hash(x)=166024176)
-900 val loss 7.3916
-900 val perplexity 1622.2867
-900 train 7.435807 (lr=2.7400e-05) (hash(x)=156196976)
-1000 val loss 7.3601
-1000 val perplexity 1571.9177
-1000 train 7.434725 (lr=2.7370e-05) (hash(x)=161040668)
-1100 val loss 7.3324
-1100 val perplexity 1529.0920
-1100 train 7.411966 (lr=2.7335e-05) (hash(x)=175899671)
-1200 val loss 7.3054
-1200 val perplexity 1488.2810
-1200 train 7.321064 (lr=2.7297e-05) (hash(x)=148878990)
-1300 val loss 7.2732
-1300 val perplexity 1441.1879
-1300 train 7.264565 (lr=2.7254e-05) (hash(x)=147253797)
-1400 val loss 7.2508
-1400 val perplexity 1409.1924
-1400 train 7.270267 (lr=2.7207e-05) (hash(x)=150420695)
-1500 val loss 7.2080
-1500 val perplexity 1350.1888
-1500 train 7.153911 (lr=2.7157e-05) (hash(x)=146933283)
-1600 val loss 7.1696
-1600 val perplexity 1299.3105
-1600 train 7.137910 (lr=2.7102e-05) (hash(x)=151995229)
-1700 val loss 7.1428
-1700 val perplexity 1265.0243
-1700 train 7.378381 (lr=2.7044e-05) (hash(x)=154468626)
-1800 val loss 7.1122
-1800 val perplexity 1226.8884
-1800 train 6.814721 (lr=2.6981e-05) (hash(x)=139853932)
-1900 val loss 7.0736
-1900 val perplexity 1180.3677
-1900 train 6.873425 (lr=2.6915e-05) (hash(x)=142492588)
-2000 val loss 7.0344
-2000 val perplexity 1135.0413
-2000 train 7.389661 (lr=2.6845e-05) (hash(x)=174104207)
-2100 val loss 7.0015
-2100 val perplexity 1098.2297
-2100 train 6.985461 (lr=2.6771e-05) (hash(x)=151046997)
-2200 val loss 6.9589
-2200 val perplexity 1052.4565
-2200 train 6.823278 (lr=2.6693e-05) (hash(x)=146275038)
-2300 val loss 6.9287
-2300 val perplexity 1021.2055
-2300 train 6.849849 (lr=2.6611e-05) (hash(x)=147779512)
-2400 val loss 6.8994
-2400 val perplexity 991.6996
-2400 train 6.756907 (lr=2.6525e-05) (hash(x)=141403655)
-2500 val loss 6.8715
-2500 val perplexity 964.3752
-2500 train 6.857760 (lr=2.6436e-05) (hash(x)=160143436)
-2600 val loss 6.8308
-2600 val perplexity 925.9529
-2600 train 7.097726 (lr=2.6343e-05) (hash(x)=173675040)
-2700 val loss 6.7961
-2700 val perplexity 894.3893
-2700 train 6.822687 (lr=2.6246e-05) (hash(x)=150124772)
-2800 val loss 6.7634
-2800 val perplexity 865.5720
-2800 train 6.884556 (lr=2.6146e-05) (hash(x)=162475906)
-2900 val loss 6.7419
-2900 val perplexity 847.1724
-2900 train 6.668697 (lr=2.6042e-05) (hash(x)=152789231)
-3000 val loss 6.7228
-3000 val perplexity 831.1171
-3000 train 6.648977 (lr=2.5934e-05) (hash(x)=152785612)
-3100 val loss 6.6874
-3100 val perplexity 802.2723
-3100 train 6.718865 (lr=2.5823e-05) (hash(x)=156140235)
-3200 val loss 6.6601
-3200 val perplexity 780.6371
-3200 train 6.678284 (lr=2.5709e-05) (hash(x)=151987383)
-3300 val loss 6.6303
-3300 val perplexity 757.7012
-3300 train 6.861026 (lr=2.5590e-05) (hash(x)=176259299)
-3400 val loss 6.6014
-3400 val perplexity 736.1032
-3400 train 6.427187 (lr=2.5469e-05) (hash(x)=144540187)
-3500 val loss 6.5812
-3500 val perplexity 721.3984
-3500 train 6.781282 (lr=2.5344e-05) (hash(x)=156773596)
-3600 val loss 6.5557
-3600 val perplexity 703.2559
-3600 train 6.781649 (lr=2.5215e-05) (hash(x)=160934110)
-3700 val loss 6.5336
-3700 val perplexity 687.8378
-3700 train 6.647962 (lr=2.5083e-05) (hash(x)=156100357)
-3800 val loss 6.5107
-3800 val perplexity 672.3105
-3800 train 6.634124 (lr=2.4948e-05) (hash(x)=158048867)
-3900 val loss 6.5010
-3900 val perplexity 665.8036
-3900 train 6.329262 (lr=2.4810e-05) (hash(x)=148195960)
-4000 val loss 6.4667
-4000 val perplexity 643.3888
-4000 train 6.454521 (lr=2.4669e-05) (hash(x)=151976963)
-4100 val loss 6.4453
-4100 val perplexity 629.7110
-4100 train 6.413161 (lr=2.4524e-05) (hash(x)=157598041)
-4200 val loss 6.4264
-4200 val perplexity 617.9375
-4200 train 6.282009 (lr=2.4376e-05) (hash(x)=139121993)
-4300 val loss 6.4095
-4300 val perplexity 607.5652
-4300 train 6.509076 (lr=2.4226e-05) (hash(x)=154546719)
-4400 val loss 6.3961
-4400 val perplexity 599.5090
-4400 train 6.335598 (lr=2.4072e-05) (hash(x)=148310119)
-4500 val loss 6.3747
-4500 val perplexity 586.8069
-4500 train 6.261201 (lr=2.3915e-05) (hash(x)=147070971)
-4600 val loss 6.3689
-4600 val perplexity 583.3914
-4600 train 7.790611 (lr=2.3755e-05) (hash(x)=232298831)
-4700 val loss 6.3547
-4700 val perplexity 575.1835
-4700 train 6.308467 (lr=2.3593e-05) (hash(x)=149549680)
-4800 val loss 6.3372
-4800 val perplexity 565.2316
-4800 train 6.495251 (lr=2.3428e-05) (hash(x)=155703970)
-4900 val loss 6.3163
-4900 val perplexity 553.5396
-4900 train 6.312998 (lr=2.3260e-05) (hash(x)=158725785)
-5000 val loss 6.3036
-5000 val perplexity 546.5110
-5000 train 6.237771 (lr=2.3089e-05) (hash(x)=146983088)
-5100 val loss 6.2881
-5100 val perplexity 538.1198
-5100 train 6.219689 (lr=2.2916e-05) (hash(x)=144882522)
-5200 val loss 6.2803
-5200 val perplexity 533.9525
-5200 train 6.160968 (lr=2.2740e-05) (hash(x)=145880132)
-5300 val loss 6.2636
-5300 val perplexity 525.0846
-5300 train 6.071547 (lr=2.2561e-05) (hash(x)=140482543)
-5400 val loss 6.2555
-5400 val perplexity 520.8649
-5400 train 6.101057 (lr=2.2381e-05) (hash(x)=148902645)
-5500 val loss 6.2465
-5500 val perplexity 516.1917
-5500 train 6.002904 (lr=2.2197e-05) (hash(x)=149411471)
-5600 val loss 6.2280
-5600 val perplexity 506.7292
-5600 train 6.174450 (lr=2.2012e-05) (hash(x)=148673383)
-5700 val loss 6.2122
-5700 val perplexity 498.8032
-5700 train 6.277672 (lr=2.1824e-05) (hash(x)=157502794)
-5800 val loss 6.2017
-5800 val perplexity 493.5743
-5800 train 6.055883 (lr=2.1634e-05) (hash(x)=148057218)
-5900 val loss 6.1857
-5900 val perplexity 485.7591
-5900 train 6.024046 (lr=2.1442e-05) (hash(x)=141239696)
-6000 val loss 6.1754
-6000 val perplexity 480.7598
-6000 train 6.262658 (lr=2.1248e-05) (hash(x)=152338728)
-6100 val loss 6.1686
-6100 val perplexity 477.4951
-6100 train 5.851945 (lr=2.1051e-05) (hash(x)=143749713)
-6200 val loss 6.1634
-6200 val perplexity 475.0218
-6200 train 6.232953 (lr=2.0853e-05) (hash(x)=152128518)
-6300 val loss 6.1582
-6300 val perplexity 472.5694
-6300 train 6.046199 (lr=2.0653e-05) (hash(x)=152992195)
-6400 val loss 6.1448
-6400 val perplexity 466.2785
-6400 train 6.042677 (lr=2.0451e-05) (hash(x)=143799591)
-6500 val loss 6.1241
-6500 val perplexity 456.7129
-6500 train 6.219780 (lr=2.0247e-05) (hash(x)=144266888)
-6600 val loss 6.1138
-6600 val perplexity 452.0628
-6600 train 6.387853 (lr=2.0042e-05) (hash(x)=167374028)
-6700 val loss 6.1144
-6700 val perplexity 452.3122
-6700 train 6.088706 (lr=1.9835e-05) (hash(x)=151301787)
-6800 val loss 6.1000
-6800 val perplexity 445.8788
-6800 train 5.931244 (lr=1.9626e-05) (hash(x)=143204898)
-6900 val loss 6.0944
-6900 val perplexity 443.3757
-6900 train 6.175731 (lr=1.9416e-05) (hash(x)=148058660)
-7000 val loss 6.0901
-7000 val perplexity 441.4510
-7000 train 6.061460 (lr=1.9205e-05) (hash(x)=157509461)
-7100 val loss 6.0809
-7100 val perplexity 437.4422
-7100 train 6.422208 (lr=1.8992e-05) (hash(x)=171579342)
-7200 val loss 6.0689
-7200 val perplexity 432.2213
-7200 train 6.070595 (lr=1.8778e-05) (hash(x)=158180787)
-7300 val loss 6.0610
-7300 val perplexity 428.8191
-7300 train 5.927678 (lr=1.8562e-05) (hash(x)=150122338)
-7400 val loss 6.0578
-7400 val perplexity 427.4535
-7400 train 5.988640 (lr=1.8346e-05) (hash(x)=147561852)
-7500 val loss 6.0510
-7500 val perplexity 424.5388
-7500 train 6.033875 (lr=1.8128e-05) (hash(x)=149353924)
-7600 val loss 6.0436
-7600 val perplexity 421.4078
-7600 train 6.076441 (lr=1.7910e-05) (hash(x)=144545768)
-7700 val loss 6.0334
-7700 val perplexity 417.1183
-7700 train 6.038865 (lr=1.7691e-05) (hash(x)=142691770)
-7800 val loss 6.0264
-7800 val perplexity 414.2370
-7800 train 6.139307 (lr=1.7470e-05) (hash(x)=158378295)
-7900 val loss 6.0266
-7900 val perplexity 414.3207
-7900 train 5.887660 (lr=1.7249e-05) (hash(x)=150691918)
-8000 val loss 6.0117
-8000 val perplexity 408.1709
-8000 train 5.894296 (lr=1.7028e-05) (hash(x)=147837896)
-8100 val loss 6.0090
-8100 val perplexity 407.0768
-8100 train 5.975699 (lr=1.6805e-05) (hash(x)=144751226)
-8200 val loss 6.0030
-8200 val perplexity 404.6476
-8200 train 5.773884 (lr=1.6582e-05) (hash(x)=139168726)
-8300 val loss 5.9937
-8300 val perplexity 400.9057
-8300 train 5.921565 (lr=1.6359e-05) (hash(x)=154279434)
-8400 val loss 5.9857
-8400 val perplexity 397.6878
-8400 train 5.840742 (lr=1.6135e-05) (hash(x)=140111678)
-8500 val loss 5.9783
-8500 val perplexity 394.7612
-8500 train 5.952585 (lr=1.5911e-05) (hash(x)=152124088)
-8600 val loss 5.9724
-8600 val perplexity 392.4519
-8600 train 5.750079 (lr=1.5687e-05) (hash(x)=142869347)
-8700 val loss 5.9666
-8700 val perplexity 390.1960
-8700 train 5.865661 (lr=1.5462e-05) (hash(x)=145697606)
-8800 val loss 5.9649
-8800 val perplexity 389.5243
-8800 train 5.930559 (lr=1.5237e-05) (hash(x)=157314231)
-8900 val loss 5.9548
-8900 val perplexity 385.5879
-8900 train 5.938760 (lr=1.5013e-05) (hash(x)=149357509)
-9000 val loss 5.9482
-9000 val perplexity 383.0534
-9000 train 5.865352 (lr=1.4788e-05) (hash(x)=140434750)
-9100 val loss 5.9425
-9100 val perplexity 380.8907
-9100 train 5.936190 (lr=1.4563e-05) (hash(x)=144954543)
-9200 val loss 5.9361
-9200 val perplexity 378.4528
-9200 train 5.861468 (lr=1.4339e-05) (hash(x)=156329199)
-9300 val loss 5.9322
-9300 val perplexity 376.9731
-9300 train 5.719663 (lr=1.4115e-05) (hash(x)=137925654)
-9400 val loss 5.9339
-9400 val perplexity 377.6240
-9400 train 5.817344 (lr=1.3891e-05) (hash(x)=146385919)
-9500 val loss 5.9215
-9500 val perplexity 372.9842
-9500 train 5.854023 (lr=1.3668e-05) (hash(x)=144400019)
-9600 val loss 5.9250
-9600 val perplexity 374.2687
-9600 train 5.854084 (lr=1.3445e-05) (hash(x)=130562204)
-9700 val loss 5.9090
-9700 val perplexity 368.3387
-9700 train 5.847058 (lr=1.3222e-05) (hash(x)=152732805)
-9800 val loss 5.9033
-9800 val perplexity 366.2617
-9800 train 5.970324 (lr=1.3001e-05) (hash(x)=151860383)
-9900 val loss 5.9004
-9900 val perplexity 365.1756
-9900 train 5.977099 (lr=1.2780e-05) (hash(x)=156732259)
-10000 val loss 5.8938
-10000 val perplexity 362.7967
-10000 train 5.818756 (lr=1.2559e-05) (hash(x)=139943350)
-10100 val loss 5.8912
-10100 val perplexity 361.8531
-10100 train 5.707589 (lr=1.2340e-05) (hash(x)=145593924)
-10200 val loss 5.8867
-10200 val perplexity 360.2231
-10200 train 5.756473 (lr=1.2122e-05) (hash(x)=153996667)
-10300 val loss 5.8922
-10300 val perplexity 362.1841
-10300 train 5.796888 (lr=1.1904e-05) (hash(x)=154200987)
-10400 val loss 5.8867
-10400 val perplexity 360.2222
-10400 train 5.700641 (lr=1.1688e-05) (hash(x)=156872686)
-10500 val loss 5.8742
-10500 val perplexity 355.7265
-10500 train 5.959209 (lr=1.1472e-05) (hash(x)=155008723)
-10600 val loss 5.8645
-10600 val perplexity 352.3051
-10600 train 5.802585 (lr=1.1258e-05) (hash(x)=152715441)
-10700 val loss 5.8600
-10700 val perplexity 350.7289
-10700 train 6.017454 (lr=1.1045e-05) (hash(x)=166668700)
-10800 val loss 5.8566
-10800 val perplexity 349.5440
-10800 train 5.859518 (lr=1.0834e-05) (hash(x)=150698847)
-10900 val loss 5.8535
-10900 val perplexity 348.4593
-10900 train 5.662449 (lr=1.0624e-05) (hash(x)=137094574)
-11000 val loss 5.8504
-11000 val perplexity 347.3706
-11000 train 5.693173 (lr=1.0415e-05) (hash(x)=144225378)
-11100 val loss 5.8551
-11100 val perplexity 349.0094
-11100 train 5.766787 (lr=1.0208e-05) (hash(x)=151666772)
-11200 val loss 5.8512
-11200 val perplexity 347.6467
-11200 train 5.861551 (lr=1.0003e-05) (hash(x)=157843445)
-11300 val loss 5.8398
-11300 val perplexity 343.7015
-11300 train 5.745340 (lr=9.7991e-06) (hash(x)=140305735)
-11400 val loss 5.8349
-11400 val perplexity 342.0187
-11400 train 5.751746 (lr=9.5972e-06) (hash(x)=143435929)
-11500 val loss 5.8314
-11500 val perplexity 340.8513
-11500 train 5.855046 (lr=9.3970e-06) (hash(x)=150858606)
-11600 val loss 5.8269
-11600 val perplexity 339.3110
-11600 train 5.643163 (lr=9.1988e-06) (hash(x)=141643887)
-11700 val loss 5.8253
-11700 val perplexity 338.7520
-11700 train 5.690266 (lr=9.0025e-06) (hash(x)=151097313)
-11800 val loss 5.8228
-11800 val perplexity 337.9084
-11800 train 5.633714 (lr=8.8082e-06) (hash(x)=150639616)
-11900 val loss 5.8246
-11900 val perplexity 338.5177
-11900 train 5.521413 (lr=8.6160e-06) (hash(x)=154557688)
-12000 val loss 5.8197
-12000 val perplexity 336.8878
-12000 train 5.955412 (lr=8.4260e-06) (hash(x)=159324159)
-12100 val loss 5.8112
-12100 val perplexity 334.0166
-12100 train 5.856526 (lr=8.2381e-06) (hash(x)=151185357)
-12200 val loss 5.8070
-12200 val perplexity 332.6033
-12200 train 5.682366 (lr=8.0526e-06) (hash(x)=137457899)
-12300 val loss 5.8042
-12300 val perplexity 331.6975
-12300 train 5.861910 (lr=7.8694e-06) (hash(x)=146370056)
-12400 val loss 5.8006
-12400 val perplexity 330.5125
-12400 train 5.651220 (lr=7.6885e-06) (hash(x)=148352898)
-12500 val loss 5.7993
-12500 val perplexity 330.0701
-12500 train 5.785968 (lr=7.5101e-06) (hash(x)=161514681)
-12600 val loss 5.7992
-12600 val perplexity 330.0287
-12600 train 5.666851 (lr=7.3342e-06) (hash(x)=146207280)
-12700 val loss 5.7996
-12700 val perplexity 330.1529
-12700 train 5.631870 (lr=7.1609e-06) (hash(x)=142615619)
-12800 val loss 5.7938
-12800 val perplexity 328.2441
-12800 train 5.813536 (lr=6.9903e-06) (hash(x)=144604802)
-12900 val loss 5.7887
-12900 val perplexity 326.5765
-12900 train 5.767271 (lr=6.8223e-06) (hash(x)=143212941)
-13000 val loss 5.7821
-13000 val perplexity 324.4371
-13000 train 5.861295 (lr=6.6570e-06) (hash(x)=155987866)
-13100 val loss 5.7809
-13100 val perplexity 324.0486
-13100 train 6.063510 (lr=6.4945e-06) (hash(x)=155920713)
-13200 val loss 5.7777
-13200 val perplexity 323.0061
-13200 train 5.963966 (lr=6.3349e-06) (hash(x)=152145878)
-13300 val loss 5.7759
-13300 val perplexity 322.4263
-13300 train 5.584373 (lr=6.1782e-06) (hash(x)=134141709)
-13400 val loss 5.7780
-13400 val perplexity 323.1029
-13400 train 5.699019 (lr=6.0244e-06) (hash(x)=146096299)
-13500 val loss 5.7766
-13500 val perplexity 322.6459
-13500 train 5.536699 (lr=5.8737e-06) (hash(x)=144011431)
-13600 val loss 5.7686
-13600 val perplexity 320.1033
-13600 train 6.021538 (lr=5.7259e-06) (hash(x)=163950932)
-13700 val loss 5.7658
-13700 val perplexity 319.2066
-13700 train 5.906261 (lr=5.5813e-06) (hash(x)=156652406)
-13800 val loss 5.7618
-13800 val perplexity 317.9240
-13800 train 5.673179 (lr=5.4398e-06) (hash(x)=145753166)
-13900 val loss 5.7605
-13900 val perplexity 317.4920
-13900 train 5.634323 (lr=5.3016e-06) (hash(x)=147065551)
-14000 val loss 5.7587
-14000 val perplexity 316.9253
-14000 train 6.219856 (lr=5.1665e-06) (hash(x)=179077542)
-14100 val loss 5.7573
-14100 val perplexity 316.5054
-14100 train 5.854602 (lr=5.0348e-06) (hash(x)=171962802)
-14200 val loss 5.7582
-14200 val perplexity 316.7881
-14200 train 5.827171 (lr=4.9063e-06) (hash(x)=153299913)
-14300 val loss 5.7577
-14300 val perplexity 316.6262
-14300 train 5.685266 (lr=4.7813e-06) (hash(x)=154750565)
-14400 val loss 5.7549
-14400 val perplexity 315.7469
-14400 train 6.028841 (lr=4.6596e-06) (hash(x)=154536694)
-14500 val loss 5.7505
-14500 val perplexity 314.3393
-14500 train 5.796143 (lr=4.5414e-06) (hash(x)=151236383)
-14600 val loss 5.7472
-14600 val perplexity 313.3142
-14600 train 5.765508 (lr=4.4267e-06) (hash(x)=152584272)
-14700 val loss 5.7450
-14700 val perplexity 312.6319
-14700 train 5.759209 (lr=4.3155e-06) (hash(x)=175301826)
-14800 val loss 5.7445
-14800 val perplexity 312.4728
-14800 train 5.673807 (lr=4.2079e-06) (hash(x)=148729408)
-14900 val loss 5.7430
-14900 val perplexity 312.0077
-14900 train 5.921260 (lr=4.1039e-06) (hash(x)=150978283)
-15000 val loss 5.7445
-15000 val perplexity 312.4808
-15000 train 5.690734 (lr=4.0035e-06) (hash(x)=153621389)
-15100 val loss 5.7436
-15100 val perplexity 312.1883
-15100 train 5.670405 (lr=3.9068e-06) (hash(x)=150653793)
-15200 val loss 5.7412
-15200 val perplexity 311.4480
-15200 train 5.897426 (lr=3.8138e-06) (hash(x)=140802934)
-15300 val loss 5.7364
-15300 val perplexity 309.9620
-15300 train 5.711238 (lr=3.7245e-06) (hash(x)=152007556)
-15400 val loss 5.7352
-15400 val perplexity 309.5629
-15400 train 5.682077 (lr=3.6390e-06) (hash(x)=149057446)
-15500 val loss 5.7342
-15500 val perplexity 309.2806
-15500 train 5.655005 (lr=3.5572e-06) (hash(x)=147521774)
-15600 val loss 5.7330
-15600 val perplexity 308.8904
-15600 train 5.700313 (lr=3.4793e-06) (hash(x)=148944405)
-15700 val loss 5.7316
-15700 val perplexity 308.4541
-15700 train 5.686961 (lr=3.4052e-06) (hash(x)=148472532)
-15800 val loss 5.7311
-15800 val perplexity 308.3131
-15800 train 5.509686 (lr=3.3350e-06) (hash(x)=149048780)
-15900 val loss 5.7322
-15900 val perplexity 308.6458
-15900 train 5.696492 (lr=3.2687e-06) (hash(x)=150175778)
-16000 val loss 5.7336
-16000 val perplexity 309.0713
-16000 train 5.528355 (lr=3.2063e-06) (hash(x)=145943831)
-16100 val loss 5.7262
-16100 val perplexity 306.7892
-16100 train 5.899349 (lr=3.1478e-06) (hash(x)=134314842)
-16200 val loss 5.7247
-16200 val perplexity 306.3504
-16200 train 5.751829 (lr=3.0932e-06) (hash(x)=153755135)
-16300 val loss 5.7249
-16300 val perplexity 306.4005
-16300 train 5.447047 (lr=3.0427e-06) (hash(x)=140070802)
-16400 val loss 5.7231
-16400 val perplexity 305.8436
-16400 train 5.745318 (lr=2.9961e-06) (hash(x)=151132329)
-16500 val loss 5.7205
-16500 val perplexity 305.0458
-16500 train 5.751915 (lr=2.9535e-06) (hash(x)=151585550)
-16600 val loss 5.7193
-16600 val perplexity 304.7025
-16600 train 5.882590 (lr=2.9149e-06) (hash(x)=164389325)
-16700 val loss 5.7181
-16700 val perplexity 304.3339
-16700 train 5.623256 (lr=2.8804e-06) (hash(x)=146016298)
-16800 val loss 5.7181
-16800 val perplexity 304.3256
-16800 train 5.625247 (lr=2.8498e-06) (hash(x)=147668192)
-16900 val loss 5.7173
-16900 val perplexity 304.0757
-16900 train 5.694380 (lr=2.8234e-06) (hash(x)=155692530)
-17000 val loss 5.7186
-17000 val perplexity 304.4734
-17000 train 5.640749 (lr=2.8010e-06) (hash(x)=156598351)
-17100 val loss 5.7186
-17100 val perplexity 304.4753
-17100 train 5.630686 (lr=2.7826e-06) (hash(x)=157344236)
-17200 val loss 5.7154
-17200 val perplexity 303.5097
-17200 train 5.522198 (lr=2.7684e-06) (hash(x)=126288656)
-17300 val loss 5.7130
-17300 val perplexity 302.7687
-17300 train 5.754097 (lr=2.7582e-06) (hash(x)=146365959)
-17400 val loss 5.7112
-17400 val perplexity 302.2420
-17400 train 5.698453 (lr=2.7520e-06) (hash(x)=152729388)
-17499 val loss 5.7100
-17499 val perplexity 301.8719
+0 train 11.283560 (lr=2.0000e-07) (hash(x)=145830960)
+0 train 11.283560 (lr=1.8750e-07) (hash(x)=145830960)
+100 val loss 9.9044
+100 val perplexity 20018.9609
+100 train 9.926474 (lr=2.0200e-05) (hash(x)=156335112)
+100 val loss 9.9188
+100 val perplexity 20308.7480
+100 train 9.939997 (lr=1.8937e-05) (hash(x)=156335112)
+200 val loss 8.8890
+200 val perplexity 7251.4038
+200 train 8.946302 (lr=4.0000e-05) (hash(x)=144157453)
+200 val loss 8.7407
+200 val perplexity 6251.9893
+200 train 8.794454 (lr=3.7500e-05) (hash(x)=144157453)
+300 val loss 8.0152
+300 val perplexity 3026.6396
+300 train 8.012555 (lr=3.9997e-05) (hash(x)=149245414)
+300 val loss 7.7699
+300 val perplexity 2368.1641
+300 train 7.755411 (lr=3.7497e-05) (hash(x)=149245414)
+400 val loss 7.6736
+400 val perplexity 2150.8848
+400 train 7.720870 (lr=3.9988e-05) (hash(x)=146764602)
+400 val loss 7.5542
+400 val perplexity 1908.7576
+400 train 7.598252 (lr=3.7489e-05) (hash(x)=146764602)
+500 val loss 7.5537
+500 val perplexity 1907.7357
+500 train 7.510151 (lr=3.9973e-05) (hash(x)=152540261)
+500 val loss 7.4764
+500 val perplexity 1765.8585
+500 train 7.437222 (lr=3.7475e-05) (hash(x)=152540261)
+600 val loss 7.4987
+600 val perplexity 1805.6774
+600 train 7.275739 (lr=3.9953e-05) (hash(x)=140559124)
+600 val loss 7.4140
+600 val perplexity 1658.9836
+600 train 7.183785 (lr=3.7456e-05) (hash(x)=140559124)
+700 val loss 7.4458
+700 val perplexity 1712.5868
+700 train 7.495838 (lr=3.9926e-05) (hash(x)=156003556)
+700 val loss 7.3484
+700 val perplexity 1553.6969
+700 train 7.405344 (lr=3.7430e-05) (hash(x)=156003556)
+800 val loss 7.3984
+800 val perplexity 1633.4504
+800 train 7.925805 (lr=3.9893e-05) (hash(x)=166024176)
+800 val loss 7.2684
+800 val perplexity 1434.2384
+800 train 7.818190 (lr=3.7400e-05) (hash(x)=166024176)
+900 val loss 7.3648
+900 val perplexity 1579.3723
+900 train 7.414095 (lr=3.9855e-05) (hash(x)=156196976)
+900 val loss 7.2047
+900 val perplexity 1345.7717
+900 train 7.264163 (lr=3.7364e-05) (hash(x)=156196976)
+1000 val loss 7.3310
+1000 val perplexity 1526.9521
+1000 train 7.409344 (lr=3.9810e-05) (hash(x)=161040668)
+1000 val loss 7.1463
+1000 val perplexity 1269.3542
+1000 train 7.210616 (lr=3.7322e-05) (hash(x)=161040668)
+1100 val loss 7.3047
+1100 val perplexity 1487.2643
+1100 train 7.379783 (lr=3.9760e-05) (hash(x)=175899671)
+1100 val loss 7.0888
+1100 val perplexity 1198.4373
+1100 train 7.141298 (lr=3.7275e-05) (hash(x)=175899671)
+1200 val loss 7.2873
+1200 val perplexity 1461.6686
+1200 train 7.304779 (lr=3.9704e-05) (hash(x)=148878990)
+1200 val loss 7.0389
+1200 val perplexity 1140.1115
+1200 train 7.047750 (lr=3.7223e-05) (hash(x)=148878990)
+1300 val loss 7.2525
+1300 val perplexity 1411.5596
+1300 train 7.244370 (lr=3.9642e-05) (hash(x)=147253797)
+1300 val loss 6.9872
+1300 val perplexity 1082.6766
+1300 train 6.986557 (lr=3.7164e-05) (hash(x)=147253797)
+1400 val loss 7.2275
+1400 val perplexity 1376.8248
+1400 train 7.253474 (lr=3.9574e-05) (hash(x)=150420695)
+1400 val loss 6.9456
+1400 val perplexity 1038.6096
+1400 train 6.970796 (lr=3.7101e-05) (hash(x)=150420695)
+1500 val loss 7.2060
+1500 val perplexity 1347.4380
+1500 train 7.145587 (lr=3.9501e-05) (hash(x)=146933283)
+1500 val loss 6.8918
+1500 val perplexity 984.1647
+1500 train 6.834597 (lr=3.7032e-05) (hash(x)=146933283)
+1600 val loss 7.1723
+1600 val perplexity 1302.8394
+1600 train 7.143783 (lr=3.9421e-05) (hash(x)=151995229)
+1600 val loss 6.8399
+1600 val perplexity 934.4082
+1600 train 6.804800 (lr=3.6958e-05) (hash(x)=151995229)
+1700 val loss 7.1669
+1700 val perplexity 1295.8228
+1700 train 7.412879 (lr=3.9336e-05) (hash(x)=154468626)
+1700 val loss 6.7931
+1700 val perplexity 891.7172
+1700 train 7.042243 (lr=3.6878e-05) (hash(x)=154468626)
+1800 val loss 7.1470
+1800 val perplexity 1270.3074
+1800 train 6.852347 (lr=3.9246e-05) (hash(x)=139853932)
+1800 val loss 6.7603
+1800 val perplexity 862.8633
+1800 train 6.456750 (lr=3.6793e-05) (hash(x)=139853932)
+1900 val loss 7.1137
+1900 val perplexity 1228.6471
+1900 train 6.909775 (lr=3.9149e-05) (hash(x)=142492588)
+1900 val loss 6.7137
+1900 val perplexity 823.6142
+1900 train 6.496400 (lr=3.6702e-05) (hash(x)=142492588)
+2000 val loss 7.0824
+2000 val perplexity 1190.7944
+2000 train 7.416164 (lr=3.9047e-05) (hash(x)=174104207)
+2000 val loss 6.6657
+2000 val perplexity 785.0412
+2000 train 6.995862 (lr=3.6606e-05) (hash(x)=174104207)
+2100 val loss 7.0585
+2100 val perplexity 1162.6946
+2100 train 7.041966 (lr=3.8939e-05) (hash(x)=151046997)
+2100 val loss 6.6230
+2100 val perplexity 752.2009
+2100 train 6.615501 (lr=3.6505e-05) (hash(x)=151046997)
+2200 val loss 7.0363
+2200 val perplexity 1137.2002
+2200 train 6.895342 (lr=3.8826e-05) (hash(x)=146275038)
+2200 val loss 6.5858
+2200 val perplexity 724.7511
+2200 train 6.437511 (lr=3.6399e-05) (hash(x)=146275038)
+2300 val loss 7.0187
+2300 val perplexity 1117.3204
+2300 train 6.941658 (lr=3.8707e-05) (hash(x)=147779512)
+2300 val loss 6.5557
+2300 val perplexity 703.2089
+2300 train 6.495374 (lr=3.6288e-05) (hash(x)=147779512)
+2400 val loss 6.9968
+2400 val perplexity 1093.1711
+2400 train 6.854197 (lr=3.8583e-05) (hash(x)=141403655)
+2400 val loss 6.5301
+2400 val perplexity 685.4817
+2400 train 6.409213 (lr=3.6171e-05) (hash(x)=141403655)
+2500 val loss 6.9711
+2500 val perplexity 1065.4285
+2500 train 6.946971 (lr=3.8453e-05) (hash(x)=160143436)
+2500 val loss 6.4906
+2500 val perplexity 658.9385
+2500 train 6.469748 (lr=3.6049e-05) (hash(x)=160143436)
+2600 val loss 6.9418
+2600 val perplexity 1034.5851
+2600 train 7.218960 (lr=3.8317e-05) (hash(x)=173675040)
+2600 val loss 6.4649
+2600 val perplexity 642.1693
+2600 train 6.741949 (lr=3.5923e-05) (hash(x)=173675040)
+2700 val loss 6.9190
+2700 val perplexity 1011.3455
+2700 train 6.941751 (lr=3.8177e-05) (hash(x)=150124772)
+2700 val loss 6.4364
+2700 val perplexity 624.1826
+2700 train 6.470803 (lr=3.5791e-05) (hash(x)=150124772)
+2800 val loss 6.9017
+2800 val perplexity 993.9711
+2800 train 7.019645 (lr=3.8031e-05) (hash(x)=162475906)
+2800 val loss 6.4127
+2800 val perplexity 609.5420
+2800 train 6.538065 (lr=3.5654e-05) (hash(x)=162475906)
+2900 val loss 6.8824
+2900 val perplexity 974.9811
+2900 train 6.812986 (lr=3.7879e-05) (hash(x)=152789231)
+2900 val loss 6.3965
+2900 val perplexity 599.7406
+2900 train 6.333413 (lr=3.5512e-05) (hash(x)=152789231)
+3000 val loss 6.8730
+3000 val perplexity 965.8179
+3000 train 6.801472 (lr=3.7723e-05) (hash(x)=152785612)
+3000 val loss 6.3856
+3000 val perplexity 593.2245
+3000 train 6.316334 (lr=3.5365e-05) (hash(x)=152785612)
+3100 val loss 6.8487
+3100 val perplexity 942.6268
+3100 train 6.896217 (lr=3.7561e-05) (hash(x)=156140235)
+3100 val loss 6.3475
+3100 val perplexity 571.0749
+3100 train 6.358536 (lr=3.5214e-05) (hash(x)=156140235)
+3200 val loss 6.8199
+3200 val perplexity 915.9048
+3200 train 6.843349 (lr=3.7394e-05) (hash(x)=151987383)
+3200 val loss 6.3233
+3200 val perplexity 557.4070
+3200 train 6.336439 (lr=3.5057e-05) (hash(x)=151987383)
+3300 val loss 6.7990
+3300 val perplexity 896.9172
+3300 train 7.026584 (lr=3.7222e-05) (hash(x)=176259299)
+3300 val loss 6.3036
+3300 val perplexity 546.5593
+3300 train 6.495580 (lr=3.4896e-05) (hash(x)=176259299)
+3400 val loss 6.7766
+3400 val perplexity 877.1120
+3400 train 6.601028 (lr=3.7045e-05) (hash(x)=144540187)
+3400 val loss 6.2814
+3400 val perplexity 534.5234
+3400 train 6.098760 (lr=3.4730e-05) (hash(x)=144540187)
+3500 val loss 6.7574
+3500 val perplexity 860.3961
+3500 train 6.949420 (lr=3.6864e-05) (hash(x)=156773596)
+3500 val loss 6.2641
+3500 val perplexity 525.3843
+3500 train 6.478076 (lr=3.4560e-05) (hash(x)=156773596)
+3600 val loss 6.7376
+3600 val perplexity 843.5590
+3600 train 6.961821 (lr=3.6677e-05) (hash(x)=160934110)
+3600 val loss 6.2582
+3600 val perplexity 522.2574
+3600 train 6.484216 (lr=3.4384e-05) (hash(x)=160934110)
+3700 val loss 6.7187
+3700 val perplexity 827.7170
+3700 train 6.826984 (lr=3.6485e-05) (hash(x)=156100357)
+3700 val loss 6.2357
+3700 val perplexity 510.6497
+3700 train 6.348337 (lr=3.4205e-05) (hash(x)=156100357)
+3800 val loss 6.6926
+3800 val perplexity 806.4541
+3800 train 6.832241 (lr=3.6289e-05) (hash(x)=158048867)
+3800 val loss 6.2221
+3800 val perplexity 503.7738
+3800 train 6.322408 (lr=3.4021e-05) (hash(x)=158048867)
+3900 val loss 6.6789
+3900 val perplexity 795.4177
+3900 train 6.509211 (lr=3.6088e-05) (hash(x)=148195960)
+3900 val loss 6.2207
+3900 val perplexity 503.0765
+3900 train 6.062790 (lr=3.3832e-05) (hash(x)=148195960)
+4000 val loss 6.6530
+4000 val perplexity 775.0702
+4000 train 6.649765 (lr=3.5882e-05) (hash(x)=151976963)
+4000 val loss 6.1855
+4000 val perplexity 485.6660
+4000 train 6.187091 (lr=3.3639e-05) (hash(x)=151976963)
+4100 val loss 6.6441
+4100 val perplexity 768.2366
+4100 train 6.624808 (lr=3.5671e-05) (hash(x)=157598041)
+4100 val loss 6.1726
+4100 val perplexity 479.4286
+4100 train 6.136642 (lr=3.3442e-05) (hash(x)=157598041)
+4200 val loss 6.6148
+4200 val perplexity 746.0751
+4200 train 6.473064 (lr=3.5457e-05) (hash(x)=139121993)
+4200 val loss 6.1542
+4200 val perplexity 470.6697
+4200 train 6.025104 (lr=3.3240e-05) (hash(x)=139121993)
+4300 val loss 6.5934
+4300 val perplexity 730.2327
+4300 train 6.699790 (lr=3.5237e-05) (hash(x)=154546719)
+4300 val loss 6.1504
+4300 val perplexity 468.9068
+4300 train 6.234024 (lr=3.3035e-05) (hash(x)=154546719)
+4400 val loss 6.5787
+4400 val perplexity 719.6163
+4400 train 6.507167 (lr=3.5014e-05) (hash(x)=148310119)
+4400 val loss 6.1401
+4400 val perplexity 464.0887
+4400 train 6.076564 (lr=3.2825e-05) (hash(x)=148310119)
+4500 val loss 6.5465
+4500 val perplexity 696.7703
+4500 train 6.420502 (lr=3.4786e-05) (hash(x)=147070971)
+4500 val loss 6.1201
+4500 val perplexity 454.8887
+4500 train 6.007422 (lr=3.2611e-05) (hash(x)=147070971)
+4600 val loss 6.5343
+4600 val perplexity 688.3343
+4600 train 7.890304 (lr=3.4553e-05) (hash(x)=232298831)
+4600 val loss 6.1113
+4600 val perplexity 450.9316
+4600 train 7.637537 (lr=3.2394e-05) (hash(x)=232298831)
+4700 val loss 6.5190
+4700 val perplexity 677.8763
+4700 train 6.470416 (lr=3.4317e-05) (hash(x)=149549680)
+4700 val loss 6.1063
+4700 val perplexity 448.6618
+4700 train 6.076235 (lr=3.2172e-05) (hash(x)=149549680)
+4800 val loss 6.4954
+4800 val perplexity 662.0760
+4800 train 6.649278 (lr=3.4077e-05) (hash(x)=155703970)
+4800 val loss 6.0901
+4800 val perplexity 441.4716
+4800 train 6.233325 (lr=3.1947e-05) (hash(x)=155703970)
+4900 val loss 6.4744
+4900 val perplexity 648.3574
+4900 train 6.474796 (lr=3.3832e-05) (hash(x)=158725785)
+4900 val loss 6.0749
+4900 val perplexity 434.7963
+4900 train 6.072459 (lr=3.1718e-05) (hash(x)=158725785)
+5000 val loss 6.4524
+5000 val perplexity 634.2155
+5000 train 6.378806 (lr=3.3584e-05) (hash(x)=146983088)
+5000 val loss 6.0573
+5000 val perplexity 427.2005
+5000 train 6.008521 (lr=3.1485e-05) (hash(x)=146983088)
+5100 val loss 6.4402
+5100 val perplexity 626.5538
+5100 train 6.354009 (lr=3.3332e-05) (hash(x)=144882522)
+5100 val loss 6.0485
+5100 val perplexity 423.4852
+5100 train 5.987097 (lr=3.1249e-05) (hash(x)=144882522)
+5200 val loss 6.4258
+5200 val perplexity 617.5587
+5200 train 6.304364 (lr=3.3076e-05) (hash(x)=145880132)
+5200 val loss 6.0410
+5200 val perplexity 420.3193
+5200 train 5.924924 (lr=3.1009e-05) (hash(x)=145880132)
+5300 val loss 6.4094
+5300 val perplexity 607.5372
+5300 train 6.206413 (lr=3.2817e-05) (hash(x)=140482543)
+5300 val loss 6.0300
+5300 val perplexity 415.7256
+5300 train 5.866216 (lr=3.0766e-05) (hash(x)=140482543)
+5400 val loss 6.3948
+5400 val perplexity 598.7214
+5400 train 6.235331 (lr=3.2554e-05) (hash(x)=148902645)
+5400 val loss 6.0197
+5400 val perplexity 411.4583
+5400 train 5.869787 (lr=3.0519e-05) (hash(x)=148902645)
+5500 val loss 6.3929
+5500 val perplexity 597.5919
+5500 train 6.158656 (lr=3.2287e-05) (hash(x)=149411471)
+5500 val loss 6.0142
+5500 val perplexity 409.1789
+5500 train 5.757872 (lr=3.0269e-05) (hash(x)=149411471)
+5600 val loss 6.3688
+5600 val perplexity 583.3516
+5600 train 6.299661 (lr=3.2017e-05) (hash(x)=148673383)
+5600 val loss 5.9975
+5600 val perplexity 402.4389
+5600 train 5.953882 (lr=3.0016e-05) (hash(x)=148673383)
+5700 val loss 6.3467
+5700 val perplexity 570.5956
+5700 train 6.413996 (lr=3.1744e-05) (hash(x)=157502794)
+5700 val loss 5.9857
+5700 val perplexity 397.7166
+5700 train 6.040728 (lr=2.9760e-05) (hash(x)=157502794)
+5800 val loss 6.3356
+5800 val perplexity 564.3046
+5800 train 6.190205 (lr=3.1468e-05) (hash(x)=148057218)
+5800 val loss 5.9765
+5800 val perplexity 394.0467
+5800 train 5.827758 (lr=2.9501e-05) (hash(x)=148057218)
+5900 val loss 6.3211
+5900 val perplexity 556.1605
+5900 train 6.147617 (lr=3.1188e-05) (hash(x)=141239696)
+5900 val loss 5.9651
+5900 val perplexity 389.6008
+5900 train 5.811947 (lr=2.9239e-05) (hash(x)=141239696)
+6000 val loss 6.3061
+6000 val perplexity 547.8906
+6000 train 6.375755 (lr=3.0905e-05) (hash(x)=152338728)
+6000 val loss 5.9551
+6000 val perplexity 385.7028
+6000 train 6.047725 (lr=2.8974e-05) (hash(x)=152338728)
+6100 val loss 6.2970
+6100 val perplexity 542.9590
+6100 train 5.965464 (lr=3.0620e-05) (hash(x)=143749713)
+6100 val loss 5.9436
+6100 val perplexity 381.2927
+6100 train 5.640822 (lr=2.8706e-05) (hash(x)=143749713)
+6200 val loss 6.2850
+6200 val perplexity 536.4752
+6200 train 6.336672 (lr=3.0332e-05) (hash(x)=152128518)
+6200 val loss 5.9422
+6200 val perplexity 380.7649
+6200 train 5.984580 (lr=2.8436e-05) (hash(x)=152128518)
+6300 val loss 6.2827
+6300 val perplexity 535.2520
+6300 train 6.175468 (lr=3.0040e-05) (hash(x)=152992195)
+6300 val loss 5.9403
+6300 val perplexity 380.0469
+6300 train 5.807545 (lr=2.8163e-05) (hash(x)=152992195)
+6400 val loss 6.2684
+6400 val perplexity 527.6567
+6400 train 6.160597 (lr=2.9747e-05) (hash(x)=143799591)
+6400 val loss 5.9212
+6400 val perplexity 372.8627
+6400 train 5.829625 (lr=2.7888e-05) (hash(x)=143799591)
+6500 val loss 6.2418
+6500 val perplexity 513.7905
+6500 train 6.299383 (lr=2.9450e-05) (hash(x)=144266888)
+6500 val loss 5.9016
+6500 val perplexity 365.6378
+6500 train 6.044479 (lr=2.7610e-05) (hash(x)=144266888)
+6600 val loss 6.2254
+6600 val perplexity 505.4254
+6600 train 6.497571 (lr=2.9152e-05) (hash(x)=167374028)
+6600 val loss 5.8941
+6600 val perplexity 362.9050
+6600 train 6.175533 (lr=2.7330e-05) (hash(x)=167374028)
+6700 val loss 6.2230
+6700 val perplexity 504.2266
+6700 train 6.195429 (lr=2.8851e-05) (hash(x)=151301787)
+6700 val loss 5.8874
+6700 val perplexity 360.4626
+6700 train 5.867244 (lr=2.7047e-05) (hash(x)=151301787)
+6800 val loss 6.2062
+6800 val perplexity 495.8087
+6800 train 6.031265 (lr=2.8547e-05) (hash(x)=143204898)
+6800 val loss 5.8788
+6800 val perplexity 357.3724
+6800 train 5.706612 (lr=2.6763e-05) (hash(x)=143204898)
+6900 val loss 6.1989
+6900 val perplexity 492.1863
+6900 train 6.276158 (lr=2.8242e-05) (hash(x)=148058660)
+6900 val loss 5.8761
+6900 val perplexity 356.4021
+6900 train 5.967237 (lr=2.6477e-05) (hash(x)=148058660)
+7000 val loss 6.1899
+7000 val perplexity 487.7905
+7000 train 6.146046 (lr=2.7934e-05) (hash(x)=157509461)
+7000 val loss 5.8728
+7000 val perplexity 355.2281
+7000 train 5.843843 (lr=2.6188e-05) (hash(x)=157509461)
+7100 val loss 6.1766
+7100 val perplexity 481.3730
+7100 train 6.514965 (lr=2.7624e-05) (hash(x)=171579342)
+7100 val loss 5.8557
+7100 val perplexity 349.2054
+7100 train 6.162778 (lr=2.5898e-05) (hash(x)=171579342)
+7200 val loss 6.1671
+7200 val perplexity 476.7843
+7200 train 6.159540 (lr=2.7313e-05) (hash(x)=158180787)
+7200 val loss 5.8474
+7200 val perplexity 346.3492
+7200 train 5.858975 (lr=2.5606e-05) (hash(x)=158180787)
+7300 val loss 6.1536
+7300 val perplexity 470.4131
+7300 train 6.016136 (lr=2.7000e-05) (hash(x)=150122338)
+7300 val loss 5.8372
+7300 val perplexity 342.8299
+7300 train 5.706229 (lr=2.5312e-05) (hash(x)=150122338)
+7400 val loss 6.1432
+7400 val perplexity 465.5236
+7400 train 6.075076 (lr=2.6685e-05) (hash(x)=147561852)
+7400 val loss 5.8337
+7400 val perplexity 341.6235
+7400 train 5.776711 (lr=2.5017e-05) (hash(x)=147561852)
+7500 val loss 6.1402
+7500 val perplexity 464.1617
+7500 train 6.126089 (lr=2.6369e-05) (hash(x)=149353924)
+7500 val loss 5.8290
+7500 val perplexity 340.0321
+7500 train 5.815456 (lr=2.4721e-05) (hash(x)=149353924)
+7600 val loss 6.1315
+7600 val perplexity 460.1191
+7600 train 6.151715 (lr=2.6051e-05) (hash(x)=144545768)
+7600 val loss 5.8220
+7600 val perplexity 337.6349
+7600 train 5.856632 (lr=2.4423e-05) (hash(x)=144545768)
+7700 val loss 6.1194
+7700 val perplexity 454.5786
+7700 train 6.124203 (lr=2.5732e-05) (hash(x)=142691770)
+7700 val loss 5.8168
+7700 val perplexity 335.9070
+7700 train 5.845797 (lr=2.4124e-05) (hash(x)=142691770)
+7800 val loss 6.1109
+7800 val perplexity 450.7235
+7800 train 6.223000 (lr=2.5411e-05) (hash(x)=158378295)
+7800 val loss 5.8033
+7800 val perplexity 331.3946
+7800 train 5.938878 (lr=2.3823e-05) (hash(x)=158378295)
+7900 val loss 6.1134
+7900 val perplexity 451.8873
+7900 train 5.980176 (lr=2.5090e-05) (hash(x)=150691918)
+7900 val loss 5.8050
+7900 val perplexity 331.9542
+7900 train 5.657355 (lr=2.3522e-05) (hash(x)=150691918)
+8000 val loss 6.0905
+8000 val perplexity 441.6312
+8000 train 5.966156 (lr=2.4767e-05) (hash(x)=147837896)
+8000 val loss 5.7853
+8000 val perplexity 325.4928
+8000 train 5.679683 (lr=2.3219e-05) (hash(x)=147837896)
+8100 val loss 6.0906
+8100 val perplexity 441.6741
+8100 train 6.050905 (lr=2.4444e-05) (hash(x)=144751226)
+8100 val loss 5.7872
+8100 val perplexity 326.1053
+8200 val loss 6.0778
+8200 val perplexity 436.0870
+8100 train 5.754948 (lr=2.2916e-05) (hash(x)=144751226)
+8200 train 5.842104 (lr=2.4120e-05) (hash(x)=139168726)
+8300 val loss 6.0715
+8300 val perplexity 433.3241
+8200 val loss 5.7779
+8200 val perplexity 323.0799
+8300 train 6.002634 (lr=2.3795e-05) (hash(x)=154279434)
+8200 train 5.549057 (lr=2.2612e-05) (hash(x)=139168726)
+8400 val loss 6.0618
+8400 val perplexity 429.1351
+8300 val loss 5.7729
+8300 val perplexity 321.4792
+8400 train 5.905673 (lr=2.3469e-05) (hash(x)=140111678)
+8300 train 5.694819 (lr=2.2308e-05) (hash(x)=154279434)
+8500 val loss 6.0484
+8500 val perplexity 423.4420
+8500 train 6.012981 (lr=2.3143e-05) (hash(x)=152124088)
+8400 val loss 5.7613
+8400 val perplexity 317.7703
+8400 train 5.648642 (lr=2.2002e-05) (hash(x)=140111678)
+8600 val loss 6.0446
+8600 val perplexity 421.8292
+8600 train 5.821149 (lr=2.2817e-05) (hash(x)=142869347)
+8500 val loss 5.7565
+8500 val perplexity 316.2472
+8500 train 5.747245 (lr=2.1697e-05) (hash(x)=152124088)
+8700 val loss 6.0383
+8700 val perplexity 419.1920
+8700 train 5.930578 (lr=2.2490e-05) (hash(x)=145697606)
+8600 val loss 5.7549
+8600 val perplexity 315.7347
+8600 train 5.526241 (lr=2.1391e-05) (hash(x)=142869347)
+8800 val loss 6.0345
+8800 val perplexity 417.5775
+8800 train 5.998886 (lr=2.2163e-05) (hash(x)=157314231)
+8700 val loss 5.7458
+8700 val perplexity 312.8595
+8700 train 5.655713 (lr=2.1085e-05) (hash(x)=145697606)
+8900 val loss 6.0189
+8900 val perplexity 411.1378
+8900 train 6.003937 (lr=2.1837e-05) (hash(x)=149357509)
+8800 val loss 5.7423
+8800 val perplexity 311.7849
+8800 train 5.714269 (lr=2.0778e-05) (hash(x)=157314231)
+9000 val loss 6.0156
+9000 val perplexity 409.7707
+9000 train 5.924284 (lr=2.1510e-05) (hash(x)=140434750)
+8900 val loss 5.7330
+8900 val perplexity 308.9079
+8900 train 5.704731 (lr=2.0472e-05) (hash(x)=149357509)
+9100 val loss 6.0077
+9100 val perplexity 406.5369
+9100 train 6.005090 (lr=2.1183e-05) (hash(x)=144954543)
+9000 val loss 5.7268
+9000 val perplexity 306.9815
+9000 train 5.668694 (lr=2.0165e-05) (hash(x)=140434750)
+9200 val loss 6.0051
+9200 val perplexity 405.5075
+9200 train 5.929798 (lr=2.0857e-05) (hash(x)=156329199)
+9100 val loss 5.7162
+9100 val perplexity 303.7405
+9100 train 5.716055 (lr=1.9859e-05) (hash(x)=144954543)
+9300 val loss 5.9960
+9300 val perplexity 401.8012
+9300 train 5.778761 (lr=2.0531e-05) (hash(x)=137925654)
+9200 val loss 5.7145
+9200 val perplexity 303.2447
+9200 train 5.632076 (lr=1.9553e-05) (hash(x)=156329199)
+9400 val loss 5.9974
+9400 val perplexity 402.3770
+9400 train 5.875072 (lr=2.0205e-05) (hash(x)=146385919)
+9300 val loss 5.7091
+9300 val perplexity 301.5869
+9300 train 5.510180 (lr=1.9248e-05) (hash(x)=137925654)
+9500 val loss 5.9884
+9500 val perplexity 398.7888
+9500 train 5.920503 (lr=1.9880e-05) (hash(x)=144400019)
+9400 val loss 5.7069
+9400 val perplexity 300.9478
+9400 train 5.595889 (lr=1.8942e-05) (hash(x)=146385919)
+9600 val loss 5.9853
+9600 val perplexity 397.5399
+9600 train 5.919597 (lr=1.9556e-05) (hash(x)=130562204)
+9500 val loss 5.6972
+9500 val perplexity 298.0300
+9500 train 5.650381 (lr=1.8638e-05) (hash(x)=144400019)
+9700 val loss 5.9692
+9700 val perplexity 391.1943
+9700 train 5.914293 (lr=1.9233e-05) (hash(x)=152732805)
+9600 val loss 5.6982
+9600 val perplexity 298.3203
+9600 train 5.691796 (lr=1.8334e-05) (hash(x)=130562204)
+9800 val loss 5.9650
+9800 val perplexity 389.5724
+9800 train 6.037981 (lr=1.8910e-05) (hash(x)=151860383)
+9700 val loss 5.6849
+9700 val perplexity 294.3834
+9700 train 5.632656 (lr=1.8031e-05) (hash(x)=152732805)
+9900 val loss 5.9644
+9900 val perplexity 389.3280
+9900 train 6.045511 (lr=1.8589e-05) (hash(x)=156732259)
+9800 val loss 5.6820
+9800 val perplexity 293.5330
+9800 train 5.743217 (lr=1.7728e-05) (hash(x)=151860383)
+10000 val loss 5.9555
+10000 val perplexity 385.8691
+10000 train 5.873840 (lr=1.8268e-05) (hash(x)=139943350)
+9900 val loss 5.6758
+9900 val perplexity 291.7221
+9900 train 5.777194 (lr=1.7427e-05) (hash(x)=156732259)
+10100 val loss 5.9473
+10100 val perplexity 382.7348
+10100 train 5.768445 (lr=1.7949e-05) (hash(x)=145593924)
+10000 val loss 5.6722
+10000 val perplexity 290.6856
+10000 train 5.598105 (lr=1.7126e-05) (hash(x)=139943350)
+10200 val loss 5.9456
+10200 val perplexity 382.0676
+10200 train 5.815557 (lr=1.7631e-05) (hash(x)=153996667)
+10100 val loss 5.6687
+10100 val perplexity 289.6549
+10100 train 5.479349 (lr=1.6827e-05) (hash(x)=145593924)
+10300 val loss 5.9468
+10300 val perplexity 382.5385
+10300 train 5.856447 (lr=1.7315e-05) (hash(x)=154200987)
+10200 val loss 5.6647
+10200 val perplexity 288.5155
+10200 train 5.529576 (lr=1.6529e-05) (hash(x)=153996667)
+10400 val loss 5.9447
+10400 val perplexity 381.7344
+10400 train 5.747140 (lr=1.7000e-05) (hash(x)=156872686)
+10300 val loss 5.6672
+10300 val perplexity 289.2102
+10300 train 5.558901 (lr=1.6233e-05) (hash(x)=154200987)
+10500 val loss 5.9323
+10500 val perplexity 377.0121
+10500 train 6.021731 (lr=1.6687e-05) (hash(x)=155008723)
+10400 val loss 5.6718
+10400 val perplexity 290.5699
+10400 train 5.467068 (lr=1.5938e-05) (hash(x)=156872686)
+10600 val loss 5.9234
+10600 val perplexity 373.6731
+10600 train 5.856106 (lr=1.6376e-05) (hash(x)=152715441)
+10500 val loss 5.6540
+10500 val perplexity 285.4170
+10500 train 5.746810 (lr=1.5644e-05) (hash(x)=155008723)
+10700 val loss 5.9197
+10700 val perplexity 372.2960
+10700 train 6.072061 (lr=1.6066e-05) (hash(x)=166668700)
+10600 val loss 5.6444
+10600 val perplexity 282.7000
+10600 train 5.579328 (lr=1.5352e-05) (hash(x)=152715441)
+10800 val loss 5.9160
+10800 val perplexity 370.9321
+10800 train 5.912136 (lr=1.5758e-05) (hash(x)=150698847)
+10700 val loss 5.6417
+10700 val perplexity 281.9391
+10700 train 5.783310 (lr=1.5062e-05) (hash(x)=166668700)
+10900 val loss 5.9112
+10900 val perplexity 369.1431
+10900 train 5.722392 (lr=1.5453e-05) (hash(x)=137094574)
+10800 val loss 5.6375
+10800 val perplexity 280.7500
+10800 train 5.650374 (lr=1.4773e-05) (hash(x)=150698847)
+11000 val loss 5.9065
+11000 val perplexity 367.4219
+11000 train 5.755096 (lr=1.5149e-05) (hash(x)=144225378)
+10900 val loss 5.6364
+10900 val perplexity 280.4449
+10900 train 5.467975 (lr=1.4487e-05) (hash(x)=137094574)
+11100 val loss 5.9076
+11100 val perplexity 367.8390
+11100 train 5.820272 (lr=1.4848e-05) (hash(x)=151666772)
+11000 val loss 5.6322
+11000 val perplexity 279.2654
+11000 train 5.472770 (lr=1.4203e-05) (hash(x)=144225378)
+11200 val loss 5.9051
+11200 val perplexity 366.9198
+11200 train 5.916715 (lr=1.4550e-05) (hash(x)=157843445)
+11100 val loss 5.6363
+11100 val perplexity 280.4174
+11100 train 5.550351 (lr=1.3920e-05) (hash(x)=151666772)
+11300 val loss 5.8938
+11300 val perplexity 362.7856
+11300 train 5.799849 (lr=1.4253e-05) (hash(x)=140305735)
+11200 val loss 5.6338
+11200 val perplexity 279.7276
+11200 train 5.625057 (lr=1.3640e-05) (hash(x)=157843445)
+11400 val loss 5.8892
+11400 val perplexity 361.1194
+11400 train 5.798059 (lr=1.3960e-05) (hash(x)=143435929)
+11300 val loss 5.6215
+11300 val perplexity 276.3054
+11300 train 5.523235 (lr=1.3362e-05) (hash(x)=140305735)
+11500 val loss 5.8852
+11500 val perplexity 359.6635
+11500 train 5.915758 (lr=1.3668e-05) (hash(x)=150858606)
+11400 val loss 5.6203
+11400 val perplexity 275.9832
+11400 train 5.541306 (lr=1.3087e-05) (hash(x)=143435929)
+11600 val loss 5.8805
+11600 val perplexity 357.9943
+11600 train 5.694943 (lr=1.3380e-05) (hash(x)=141643887)
+11500 val loss 5.6158
+11500 val perplexity 274.7440
+11500 train 5.652623 (lr=1.2814e-05) (hash(x)=150858606)
+11700 val loss 5.8761
+11700 val perplexity 356.4077
+11700 train 5.735241 (lr=1.3095e-05) (hash(x)=151097313)
+11600 val loss 5.6109
+11600 val perplexity 273.3787
+11600 train 5.426761 (lr=1.2544e-05) (hash(x)=141643887)
+11800 val loss 5.8746
+11800 val perplexity 355.8955
+11800 train 5.681262 (lr=1.2812e-05) (hash(x)=150639616)
+11700 val loss 5.6099
+11700 val perplexity 273.1186
+11700 train 5.469909 (lr=1.2276e-05) (hash(x)=151097313)
+11900 val loss 5.8748
+11900 val perplexity 355.9547
+11900 train 5.567788 (lr=1.2532e-05) (hash(x)=154557688)
+11800 val loss 5.6071
+11800 val perplexity 272.3646
+11800 train 5.409575 (lr=1.2011e-05) (hash(x)=150639616)
+12000 val loss 5.8698
+12000 val perplexity 354.1915
+12000 train 6.008741 (lr=1.2256e-05) (hash(x)=159324159)
+11900 val loss 5.6111
+11900 val perplexity 273.4509
+11900 train 5.281753 (lr=1.1749e-05) (hash(x)=154557688)
+12100 val loss 5.8587
+12100 val perplexity 350.2790
+12100 train 5.901215 (lr=1.1983e-05) (hash(x)=151185357)
+12000 val loss 5.6054
+12000 val perplexity 271.8920
+12000 train 5.735380 (lr=1.1490e-05) (hash(x)=159324159)
+12200 val loss 5.8574
+12200 val perplexity 349.8243
+12200 train 5.739561 (lr=1.1713e-05) (hash(x)=137457899)
+12100 val loss 5.5982
+12100 val perplexity 269.9411
+12100 train 5.653167 (lr=1.1234e-05) (hash(x)=151185357)
+12300 val loss 5.8519
+12300 val perplexity 347.8885
+12300 train 5.907245 (lr=1.1446e-05) (hash(x)=146370056)
+12200 val loss 5.5924
+12200 val perplexity 268.3843
+12200 train 5.485268 (lr=1.0981e-05) (hash(x)=137457899)
+12400 val loss 5.8493
+12400 val perplexity 346.9917
+12400 train 5.691209 (lr=1.1183e-05) (hash(x)=148352898)
+12300 val loss 5.5894
+12300 val perplexity 267.5695
+12300 train 5.655680 (lr=1.0731e-05) (hash(x)=146370056)
+12500 val loss 5.8428
+12500 val perplexity 344.7307
+12500 train 5.838378 (lr=1.0924e-05) (hash(x)=161514681)
+12400 val loss 5.5867
+12400 val perplexity 266.8641
+12400 train 5.433014 (lr=1.0484e-05) (hash(x)=148352898)
+12600 val loss 5.8424
+12600 val perplexity 344.6033
+12600 train 5.712890 (lr=1.0668e-05) (hash(x)=146207280)
+12500 val loss 5.5863
+12500 val perplexity 266.7467
+12500 train 5.551850 (lr=1.0241e-05) (hash(x)=161514681)
+12700 val loss 5.8435
+12700 val perplexity 344.9972
+12700 train 5.668590 (lr=1.0416e-05) (hash(x)=142615619)
+12600 val loss 5.5866
+12600 val perplexity 266.8208
+12600 train 5.446975 (lr=1.0001e-05) (hash(x)=146207280)
+12800 val loss 5.8373
+12800 val perplexity 342.8404
+12800 train 5.855917 (lr=1.0168e-05) (hash(x)=144604802)
+12700 val loss 5.5854
+12700 val perplexity 266.4956
+12700 train 5.415060 (lr=9.7649e-06) (hash(x)=142615619)
+12900 val loss 5.8297
+12900 val perplexity 340.2576
+12900 train 5.814535 (lr=9.9233e-06) (hash(x)=143212941)
+12800 val loss 5.5805
+12800 val perplexity 265.2023
+12800 train 5.610810 (lr=9.5322e-06) (hash(x)=144604802)
+13000 val loss 5.8220
+13000 val perplexity 337.6538
+13000 train 5.901246 (lr=9.6829e-06) (hash(x)=155987866)
+12900 val loss 5.5757
+12900 val perplexity 263.9453
+12900 train 5.578447 (lr=9.3031e-06) (hash(x)=143212941)
+13100 val loss 5.8229
+13100 val perplexity 337.9358
+13100 train 6.100163 (lr=9.4466e-06) (hash(x)=155920713)
+13000 val loss 5.5680
+13000 val perplexity 261.9206
+13000 train 5.646559 (lr=9.0777e-06) (hash(x)=155987866)
+13200 val loss 5.8179
+13200 val perplexity 336.2795
+13200 train 6.005582 (lr=9.2144e-06) (hash(x)=152145878)
+13100 val loss 5.5670
+13100 val perplexity 261.6476
+13100 train 5.860734 (lr=8.8562e-06) (hash(x)=155920713)
+13300 val loss 5.8157
+13300 val perplexity 335.5233
+13300 train 5.617593 (lr=8.9865e-06) (hash(x)=134141709)
+13200 val loss 5.5661
+13200 val perplexity 261.4016
+13200 train 5.760226 (lr=8.6385e-06) (hash(x)=152145878)
+13400 val loss 5.8184
+13400 val perplexity 336.4338
+13400 train 5.728613 (lr=8.7628e-06) (hash(x)=146096299)
+13300 val loss 5.5639
+13300 val perplexity 260.8471
+13300 train 5.397263 (lr=8.4248e-06) (hash(x)=134141709)
+13500 val loss 5.8142
+13500 val perplexity 335.0111
+13500 train 5.572219 (lr=8.5435e-06) (hash(x)=144011431)
+13400 val loss 5.5668
+13400 val perplexity 261.5923
+13400 train 5.490836 (lr=8.2151e-06) (hash(x)=146096299)
+13600 val loss 5.8067
+13600 val perplexity 332.5340
+13600 train 6.047503 (lr=8.3286e-06) (hash(x)=163950932)
+13500 val loss 5.5627
+13500 val perplexity 260.5127
+13500 train 5.317993 (lr=8.0095e-06) (hash(x)=144011431)
+13700 val loss 5.8025
+13700 val perplexity 331.1383
+13700 train 5.944914 (lr=8.1183e-06) (hash(x)=156652406)
+13600 val loss 5.5564
+13600 val perplexity 258.8997
+13600 train 5.804774 (lr=7.8081e-06) (hash(x)=163950932)
+13800 val loss 5.7986
+13800 val perplexity 329.8388
+13800 train 5.708843 (lr=7.9125e-06) (hash(x)=145753166)
+13700 val loss 5.5531
+13700 val perplexity 258.0291
+13700 train 5.690301 (lr=7.6109e-06) (hash(x)=156652406)
+13900 val loss 5.7978
+13900 val perplexity 329.5803
+13900 train 5.670137 (lr=7.7114e-06) (hash(x)=147065551)
+13800 val loss 5.5490
+13800 val perplexity 256.9872
+13800 train 5.473596 (lr=7.4180e-06) (hash(x)=145753166)
+14000 val loss 5.7940
+14000 val perplexity 328.3198
+14000 train 6.254804 (lr=7.5149e-06) (hash(x)=179077542)
+13900 val loss 5.5482
+13900 val perplexity 256.7810
+13900 train 5.421669 (lr=7.2294e-06) (hash(x)=147065551)
+14100 val loss 5.7931
+14100 val perplexity 328.0438
+14100 train 5.873286 (lr=7.3233e-06) (hash(x)=171962802)
+14000 val loss 5.5460
+14000 val perplexity 256.2204
+14000 train 6.017848 (lr=7.0452e-06) (hash(x)=179077542)
+14200 val loss 5.7917
+14200 val perplexity 327.5798
+14200 train 5.855141 (lr=7.1365e-06) (hash(x)=153299913)
+14100 val loss 5.5449
+14100 val perplexity 255.9265
+14100 train 5.615407 (lr=6.8656e-06) (hash(x)=171962802)
+14300 val loss 5.7920
+14300 val perplexity 327.6578
+14300 train 5.718112 (lr=6.9546e-06) (hash(x)=154750565)
+14200 val loss 5.5463
+14200 val perplexity 256.2790
+14200 train 5.620974 (lr=6.6904e-06) (hash(x)=153299913)
+14400 val loss 5.7895
+14400 val perplexity 326.8647
+14400 train 6.064641 (lr=6.7776e-06) (hash(x)=154536694)
+14300 val loss 5.5460
+14300 val perplexity 256.2026
+14300 train 5.465533 (lr=6.5199e-06) (hash(x)=154750565)
+14500 val loss 5.7828
+14500 val perplexity 324.6520
+14500 train 5.826786 (lr=6.6057e-06) (hash(x)=151236383)
+14400 val loss 5.5425
+14400 val perplexity 255.3064
+14400 train 5.813636 (lr=6.3540e-06) (hash(x)=154536694)
+14600 val loss 5.7797
+14600 val perplexity 323.6683
+14600 train 5.793256 (lr=6.4389e-06) (hash(x)=152584272)
+14500 val loss 5.5382
+14500 val perplexity 254.2301
+14500 train 5.602794 (lr=6.1928e-06) (hash(x)=151236383)
+14700 val loss 5.7792
+14700 val perplexity 323.4995
+14700 train 5.790602 (lr=6.2771e-06) (hash(x)=175301826)
+14600 val loss 5.5341
+14600 val perplexity 253.1697
+14600 train 5.567681 (lr=6.0364e-06) (hash(x)=152584272)
+14800 val loss 5.7774
+14800 val perplexity 322.9174
+14800 train 5.708154 (lr=6.1206e-06) (hash(x)=148729408)
+14700 val loss 5.5323
+14700 val perplexity 252.7340
+14700 train 5.513443 (lr=5.8848e-06) (hash(x)=175301826)
+14900 val loss 5.7767
+14900 val perplexity 322.6887
+14900 train 5.951231 (lr=5.9693e-06) (hash(x)=150978283)
+14800 val loss 5.5331
+14800 val perplexity 252.9163
+14800 train 5.460180 (lr=5.7381e-06) (hash(x)=148729408)
+15000 val loss 5.7754
+15000 val perplexity 322.2623
+15000 train 5.714064 (lr=5.8233e-06) (hash(x)=153621389)
+14900 val loss 5.5314
+14900 val perplexity 252.4985
+14900 train 5.730659 (lr=5.5962e-06) (hash(x)=150978283)
+15100 val loss 5.7745
+15100 val perplexity 321.9966
+15100 train 5.690768 (lr=5.6826e-06) (hash(x)=150653793)
+15000 val loss 5.5320
+15000 val perplexity 252.6417
+15000 train 5.474362 (lr=5.4593e-06) (hash(x)=153621389)
+15200 val loss 5.7715
+15200 val perplexity 321.0341
+15200 train 5.935984 (lr=5.5473e-06) (hash(x)=140802934)
+15100 val loss 5.5312
+15100 val perplexity 252.4400
+15100 train 5.453715 (lr=5.3275e-06) (hash(x)=150653793)
+15300 val loss 5.7656
+15300 val perplexity 319.1288
+15300 train 5.742896 (lr=5.4175e-06) (hash(x)=152007556)
+15200 val loss 5.5296
+15200 val perplexity 252.0420
+15200 train 5.721358 (lr=5.2006e-06) (hash(x)=140802934)
+15400 val loss 5.7647
+15400 val perplexity 318.8434
+15400 train 5.719880 (lr=5.2931e-06) (hash(x)=149057446)
+15300 val loss 5.5237
+15300 val perplexity 250.5698
+15300 train 5.531915 (lr=5.0789e-06) (hash(x)=152007556)
+15500 val loss 5.7632
+15500 val perplexity 318.3530
+15500 train 5.682706 (lr=5.1742e-06) (hash(x)=147521774)
+15400 val loss 5.5227
+15400 val perplexity 250.3170
+15400 train 5.471367 (lr=4.9622e-06) (hash(x)=149057446)
+15600 val loss 5.7608
+15600 val perplexity 317.5959
+15600 train 5.726193 (lr=5.0608e-06) (hash(x)=148944405)
+15500 val loss 5.5217
+15500 val perplexity 250.0559
+15500 train 5.423968 (lr=4.8508e-06) (hash(x)=147521774)
+15700 val loss 5.7601
+15700 val perplexity 317.3654
+15700 train 5.717175 (lr=4.9531e-06) (hash(x)=148472532)
+15600 val loss 5.5211
+15600 val perplexity 249.9127
+15600 train 5.498367 (lr=4.7445e-06) (hash(x)=148944405)
+15800 val loss 5.7592
+15800 val perplexity 317.1059
+15800 train 5.532694 (lr=4.8509e-06) (hash(x)=149048780)
+15700 val loss 5.5208
+15700 val perplexity 249.8366
+15700 train 5.467693 (lr=4.6435e-06) (hash(x)=148472532)
+15900 val loss 5.7597
+15900 val perplexity 317.2405
+15900 train 5.722683 (lr=4.7545e-06) (hash(x)=150175778)
+15800 val loss 5.5196
+15800 val perplexity 249.5341
+15800 train 5.275207 (lr=4.5478e-06) (hash(x)=149048780)
+16000 val loss 5.7607
+16000 val perplexity 317.5803
+16000 train 5.559212 (lr=4.6637e-06) (hash(x)=145943831)
+15900 val loss 5.5204
+15900 val perplexity 249.7436
+15900 train 5.489237 (lr=4.4573e-06) (hash(x)=150175778)
+16100 val loss 5.7531
+16100 val perplexity 315.1729
+16100 train 5.913213 (lr=4.5786e-06) (hash(x)=134314842)
+16000 val loss 5.5214
+16000 val perplexity 249.9827
+16000 train 5.292845 (lr=4.3722e-06) (hash(x)=145943831)
+16200 val loss 5.7523
+16200 val perplexity 314.9002
+16200 train 5.772738 (lr=4.4993e-06) (hash(x)=153755135)
+16100 val loss 5.5150
+16100 val perplexity 248.3849
+16100 train 5.714660 (lr=4.2924e-06) (hash(x)=134314842)
+16300 val loss 5.7506
+16300 val perplexity 314.3870
+16300 train 5.471520 (lr=4.4257e-06) (hash(x)=140070802)
+16200 val loss 5.5128
+16200 val perplexity 247.8334
+16200 train 5.547480 (lr=4.2180e-06) (hash(x)=153755135)
+16400 val loss 5.7491
+16400 val perplexity 313.9213
+16400 train 5.768837 (lr=4.3579e-06) (hash(x)=151132329)
+16300 val loss 5.5134
+16300 val perplexity 247.9932
+16300 train 5.221501 (lr=4.1491e-06) (hash(x)=140070802)
+16500 val loss 5.7457
+16500 val perplexity 312.8365
+16500 train 5.782087 (lr=4.2960e-06) (hash(x)=151585550)
+16400 val loss 5.5114
+16400 val perplexity 247.5039
+16400 train 5.572089 (lr=4.0856e-06) (hash(x)=151132329)
+16600 val loss 5.7444
+16600 val perplexity 312.4376
+16600 train 5.902431 (lr=4.2399e-06) (hash(x)=164389325)
+16500 val loss 5.5089
+16500 val perplexity 246.8896
+16500 train 5.538912 (lr=4.0275e-06) (hash(x)=151585550)
+16700 val loss 5.7423
+16700 val perplexity 311.7658
+16700 train 5.646008 (lr=4.1896e-06) (hash(x)=146016298)
+16600 val loss 5.5080
+16600 val perplexity 246.6526
+16600 train 5.663129 (lr=3.9749e-06) (hash(x)=164389325)
+16800 val loss 5.7435
+16800 val perplexity 312.1398
+16800 train 5.650313 (lr=4.1452e-06) (hash(x)=147668192)
+16700 val loss 5.5073
+16700 val perplexity 246.4835
+16700 train 5.417647 (lr=3.9278e-06) (hash(x)=146016298)
+16900 val loss 5.7422
+16900 val perplexity 311.7538
+16900 train 5.716069 (lr=4.1067e-06) (hash(x)=155692530)
+16800 val loss 5.5070
+16800 val perplexity 246.4051
+16800 train 5.414197 (lr=3.8862e-06) (hash(x)=147668192)
+17000 val loss 5.7428
+17000 val perplexity 311.9516
+17000 train 5.666575 (lr=4.0741e-06) (hash(x)=156598351)
+16900 val loss 5.5057
+16900 val perplexity 246.0977
+16900 train 5.467527 (lr=3.8501e-06) (hash(x)=155692530)
+17100 val loss 5.7430
+17100 val perplexity 312.0139
+17100 train 5.649395 (lr=4.0475e-06) (hash(x)=157344236)
+17000 val loss 5.5077
+17000 val perplexity 246.5890
+17000 train 5.406383 (lr=3.8195e-06) (hash(x)=156598351)
+17200 val loss 5.7387
+17200 val perplexity 310.6666
+17200 train 5.541557 (lr=4.0267e-06) (hash(x)=126288656)
+17100 val loss 5.5084
+17100 val perplexity 246.7521
+17100 train 5.397768 (lr=3.7945e-06) (hash(x)=157344236)
+17300 val loss 5.7355
+17300 val perplexity 309.6783
+17300 train 5.773484 (lr=4.0119e-06) (hash(x)=146365959)
+17200 val loss 5.5051
+17200 val perplexity 245.9518
+17200 train 5.334321 (lr=3.7750e-06) (hash(x)=126288656)
+17400 val loss 5.7335
+17400 val perplexity 309.0390
+17400 train 5.708815 (lr=4.0030e-06) (hash(x)=152729388)
+17300 val loss 5.5021
+17300 val perplexity 245.2001
+17300 train 5.543229 (lr=3.7611e-06) (hash(x)=146365959)
+17499 val loss 5.7321
+17499 val perplexity 308.6043

attention_kindselective_n_heads4_seed1344/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e756f7d9491da10e8b323fb0adc65f8e710af9bae581acc742915d1250d7106
+oid sha256:47a519a6470ae2abbd6b4fc856848a2ce1fde73d28c8d14009a94977e34cb91e
 size 92843394

attention_kindselective_n_heads4_seed1344/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff2b75f5d4306450c1c59182852607172124dd504534b1a18fffba7ed3e8c308
+oid sha256:4ec56e4c6edc0609577cc85b9ad3e3d86c5de6c63bdd781bbce05f5230f85d9e
 size 179406214