Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1345/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_30720_4_1345", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_30720_4_1345", "n_embd": 256}

attention_kindselective_n_heads4_seed1345/log2.txt

@@ -1,528 +1,1054 @@
 max_steps: 17500
+max_steps: 17500
+0 val loss 11.2682
+0 val perplexity 78293.8047
 0 val loss 11.2682
 0 val perplexity 78293.8047
-0 train 11.260221 (lr=1.7500e-07) (hash(x)=134751525)
-100 val loss 9.9291
-100 val perplexity 20519.6289
-100 train 9.985161 (lr=1.7675e-05) (hash(x)=144692155)
-200 val loss 8.9508
-200 val perplexity 7714.3545
-200 train 8.941199 (lr=3.5000e-05) (hash(x)=150358957)
-300 val loss 7.9899
-300 val perplexity 2951.1233
-300 train 7.977305 (lr=3.4997e-05) (hash(x)=150268299)
-400 val loss 7.6587
-400 val perplexity 2119.0889
-400 train 7.523974 (lr=3.4990e-05) (hash(x)=126319983)
-500 val loss 7.5258
-500 val perplexity 1855.2982
-500 train 7.562843 (lr=3.4977e-05) (hash(x)=153934799)
-600 val loss 7.4580
-600 val perplexity 1733.6735
-600 train 7.496515 (lr=3.4958e-05) (hash(x)=159305874)
-700 val loss 7.3982
-700 val perplexity 1633.0657
-700 train 7.247789 (lr=3.4935e-05) (hash(x)=141986303)
-800 val loss 7.3487
-800 val perplexity 1554.2023
-800 train 7.334450 (lr=3.4907e-05) (hash(x)=149855631)
-900 val loss 7.3277
-900 val perplexity 1521.9357
-900 train 7.209392 (lr=3.4873e-05) (hash(x)=149870240)
-1000 val loss 7.2682
-1000 val perplexity 1434.0243
-1000 train 7.337144 (lr=3.4834e-05) (hash(x)=150706760)
-1100 val loss 7.2189
-1100 val perplexity 1364.9252
-1100 train 7.050467 (lr=3.4790e-05) (hash(x)=139020621)
-1200 val loss 7.1836
-1200 val perplexity 1317.6602
-1200 train 7.040086 (lr=3.4741e-05) (hash(x)=146858163)
-1300 val loss 7.1446
-1300 val perplexity 1267.2871
-1300 train 7.213032 (lr=3.4687e-05) (hash(x)=156383408)
-1400 val loss 7.1153
-1400 val perplexity 1230.6442
-1400 train 7.043497 (lr=3.4628e-05) (hash(x)=144262072)
-1500 val loss 7.0863
-1500 val perplexity 1195.4648
-1500 train 6.938719 (lr=3.4563e-05) (hash(x)=144528048)
-1600 val loss 7.0442
-1600 val perplexity 1146.2073
-1600 train 6.957266 (lr=3.4494e-05) (hash(x)=135443880)
-1700 val loss 7.0055
-1700 val perplexity 1102.6990
-1700 train 6.924539 (lr=3.4419e-05) (hash(x)=145067496)
-1800 val loss 6.9669
-1800 val perplexity 1060.9297
-1800 train 6.931795 (lr=3.4340e-05) (hash(x)=153147388)
-1900 val loss 6.9350
-1900 val perplexity 1027.5972
-1900 train 6.825403 (lr=3.4255e-05) (hash(x)=151331979)
-2000 val loss 6.9077
-2000 val perplexity 999.9653
-2000 train 6.702426 (lr=3.4166e-05) (hash(x)=151448445)
-2100 val loss 6.8685
-2100 val perplexity 961.4578
-2100 train 6.773091 (lr=3.4072e-05) (hash(x)=157223072)
-2200 val loss 6.8307
-2200 val perplexity 925.7949
-2200 train 6.842752 (lr=3.3973e-05) (hash(x)=153161010)
-2300 val loss 6.7902
-2300 val perplexity 889.0865
-2300 train 6.879936 (lr=3.3869e-05) (hash(x)=156906742)
-2400 val loss 6.7585
-2400 val perplexity 861.3398
-2400 train 6.646911 (lr=3.3760e-05) (hash(x)=143227423)
-2500 val loss 6.7132
-2500 val perplexity 823.2204
-2500 train 6.645422 (lr=3.3646e-05) (hash(x)=151827079)
-2600 val loss 6.6785
-2600 val perplexity 795.1075
-2600 train 7.156397 (lr=3.3528e-05) (hash(x)=176373796)
-2700 val loss 6.6477
-2700 val perplexity 771.0331
-2700 train 6.591852 (lr=3.3405e-05) (hash(x)=148437519)
-2800 val loss 6.6107
-2800 val perplexity 743.0215
-2800 train 6.601998 (lr=3.3277e-05) (hash(x)=155989503)
-2900 val loss 6.5787
-2900 val perplexity 719.5923
-2900 train 6.717565 (lr=3.3144e-05) (hash(x)=159227845)
-3000 val loss 6.5504
-3000 val perplexity 699.5578
-3000 train 6.573480 (lr=3.3008e-05) (hash(x)=156507542)
-3100 val loss 6.5291
-3100 val perplexity 684.7741
-3100 train 6.512413 (lr=3.2866e-05) (hash(x)=155771797)
-3200 val loss 6.5010
-3200 val perplexity 665.7867
-3200 train 6.423993 (lr=3.2720e-05) (hash(x)=156078901)
-3300 val loss 6.4754
-3300 val perplexity 648.9933
-3300 train 6.343053 (lr=3.2570e-05) (hash(x)=143722886)
-3400 val loss 6.4531
-3400 val perplexity 634.6980
-3400 train 6.378851 (lr=3.2415e-05) (hash(x)=143793373)
-3500 val loss 6.4428
-3500 val perplexity 628.1869
-3500 train 6.698905 (lr=3.2256e-05) (hash(x)=147278080)
-3600 val loss 6.4188
-3600 val perplexity 613.2518
-3600 train 7.153553 (lr=3.2092e-05) (hash(x)=175299205)
-3700 val loss 6.3935
-3700 val perplexity 597.9476
-3700 train 6.438568 (lr=3.1924e-05) (hash(x)=150047891)
-3800 val loss 6.3737
-3800 val perplexity 586.1958
-3800 train 6.409956 (lr=3.1753e-05) (hash(x)=153419668)
-3900 val loss 6.3563
-3900 val perplexity 576.1237
-3900 train 6.463406 (lr=3.1577e-05) (hash(x)=153147630)
-4000 val loss 6.3455
-4000 val perplexity 569.9233
-4000 train 6.182364 (lr=3.1397e-05) (hash(x)=142428269)
-4100 val loss 6.3320
-4100 val perplexity 562.2965
-4100 train 6.217992 (lr=3.1212e-05) (hash(x)=148836993)
-4200 val loss 6.3046
-4200 val perplexity 547.0716
-4200 train 6.344437 (lr=3.1024e-05) (hash(x)=159557219)
-4300 val loss 6.2892
-4300 val perplexity 538.7408
-4300 train 6.412379 (lr=3.0833e-05) (hash(x)=154137450)
-4400 val loss 6.2695
-4400 val perplexity 528.2305
-4400 train 6.423839 (lr=3.0637e-05) (hash(x)=153336405)
-4500 val loss 6.2527
-4500 val perplexity 519.4207
-4500 train 6.222682 (lr=3.0437e-05) (hash(x)=145729145)
-4600 val loss 6.2462
-4600 val perplexity 516.0644
-4600 train 6.244120 (lr=3.0234e-05) (hash(x)=157789272)
-4700 val loss 6.2365
-4700 val perplexity 511.0797
-4700 train 6.209208 (lr=3.0027e-05) (hash(x)=146146781)
-4800 val loss 6.2180
-4800 val perplexity 501.7088
-4800 train 6.228053 (lr=2.9817e-05) (hash(x)=149454469)
-4900 val loss 6.1962
-4900 val perplexity 490.8600
-4900 train 6.280124 (lr=2.9603e-05) (hash(x)=155966467)
-5000 val loss 6.1919
-5000 val perplexity 488.7705
-5000 train 6.148453 (lr=2.9386e-05) (hash(x)=147184546)
-5100 val loss 6.1817
-5100 val perplexity 483.8053
-5100 train 6.151196 (lr=2.9166e-05) (hash(x)=149019501)
-5200 val loss 6.1647
-5200 val perplexity 475.6766
-5200 train 6.073504 (lr=2.8942e-05) (hash(x)=143521586)
-5300 val loss 6.1529
-5300 val perplexity 470.0709
-5300 train 6.013883 (lr=2.8715e-05) (hash(x)=143870567)
-5400 val loss 6.1355
-5400 val perplexity 461.9546
-5400 train 6.160471 (lr=2.8484e-05) (hash(x)=150658725)
-5500 val loss 6.1215
-5500 val perplexity 455.5701
-5500 train 6.147098 (lr=2.8251e-05) (hash(x)=147645530)
-5600 val loss 6.1134
-5600 val perplexity 451.8677
-5600 train 6.141778 (lr=2.8015e-05) (hash(x)=146088410)
-5700 val loss 6.1025
-5700 val perplexity 446.9525
-5700 train 6.201876 (lr=2.7776e-05) (hash(x)=151193083)
-5800 val loss 6.0930
-5800 val perplexity 442.7527
-5800 train 5.860562 (lr=2.7534e-05) (hash(x)=137474865)
-5900 val loss 6.0883
-5900 val perplexity 440.6581
-5900 train 6.032021 (lr=2.7290e-05) (hash(x)=142426763)
-6000 val loss 6.0665
-6000 val perplexity 431.1655
-6000 train 5.839445 (lr=2.7042e-05) (hash(x)=140492670)
-6100 val loss 6.0555
-6100 val perplexity 426.4716
-6100 train 6.086936 (lr=2.6792e-05) (hash(x)=175687796)
-6200 val loss 6.0457
-6200 val perplexity 422.3048
-6200 train 6.098376 (lr=2.6540e-05) (hash(x)=165841139)
-6300 val loss 6.0434
-6300 val perplexity 421.3156
-6300 train 6.097427 (lr=2.6285e-05) (hash(x)=145328367)
-6400 val loss 6.0373
-6400 val perplexity 418.7551
-6400 train 6.018971 (lr=2.6028e-05) (hash(x)=148213323)
-6500 val loss 6.0263
-6500 val perplexity 414.1758
-6500 train 6.114561 (lr=2.5769e-05) (hash(x)=153344426)
-6600 val loss 6.0166
-6600 val perplexity 410.1727
-6600 train 5.989990 (lr=2.5508e-05) (hash(x)=145840902)
-6700 val loss 6.0059
-6700 val perplexity 405.8290
-6700 train 5.838514 (lr=2.5244e-05) (hash(x)=142812723)
-6800 val loss 6.0015
-6800 val perplexity 404.0437
-6800 train 5.850190 (lr=2.4979e-05) (hash(x)=147420964)
-6900 val loss 5.9949
-6900 val perplexity 401.3853
-6900 train 6.030012 (lr=2.4711e-05) (hash(x)=152637292)
-7000 val loss 5.9816
-7000 val perplexity 396.0677
-7000 train 5.982900 (lr=2.4442e-05) (hash(x)=144728159)
-7100 val loss 5.9738
-7100 val perplexity 393.0115
-7100 train 5.948391 (lr=2.4171e-05) (hash(x)=143450899)
-7200 val loss 5.9683
-7200 val perplexity 390.8412
-7200 train 6.040811 (lr=2.3899e-05) (hash(x)=156229897)
-7300 val loss 5.9593
-7300 val perplexity 387.3397
-7300 train 5.835092 (lr=2.3625e-05) (hash(x)=147447250)
-7400 val loss 5.9541
-7400 val perplexity 385.3466
-7400 train 5.702358 (lr=2.3349e-05) (hash(x)=134836112)
-7500 val loss 5.9463
-7500 val perplexity 382.3476
-7500 train 5.964333 (lr=2.3073e-05) (hash(x)=155027499)
-7600 val loss 5.9366
-7600 val perplexity 378.6516
-7600 train 5.813637 (lr=2.2795e-05) (hash(x)=145744849)
-7700 val loss 5.9278
-7700 val perplexity 375.3369
-7700 train 6.050231 (lr=2.2515e-05) (hash(x)=158082973)
-7800 val loss 5.9205
-7800 val perplexity 372.6146
-7800 train 6.062943 (lr=2.2235e-05) (hash(x)=161538768)
-7900 val loss 5.9117
-7900 val perplexity 369.3188
-7900 train 6.005617 (lr=2.1954e-05) (hash(x)=152103064)
-8000 val loss 5.9067
-8000 val perplexity 367.5013
-8000 train 5.901193 (lr=2.1671e-05) (hash(x)=152409525)
-8100 val loss 5.8979
-8100 val perplexity 364.2686
-8100 train 5.867712 (lr=2.1388e-05) (hash(x)=150986182)
-8200 val loss 5.8891
-8200 val perplexity 361.0702
-8200 train 5.889381 (lr=2.1105e-05) (hash(x)=159555650)
-8300 val loss 5.8858
-8300 val perplexity 359.8856
-8300 train 6.054835 (lr=2.0820e-05) (hash(x)=139602404)
-8400 val loss 5.8738
-8400 val perplexity 355.5927
-8400 train 5.773792 (lr=2.0536e-05) (hash(x)=154432574)
-8500 val loss 5.8668
-8500 val perplexity 353.1323
-8500 train 5.813470 (lr=2.0250e-05) (hash(x)=142554189)
-8600 val loss 5.8625
-8600 val perplexity 351.6102
-8600 train 5.813890 (lr=1.9965e-05) (hash(x)=151300791)
-8700 val loss 5.8597
-8700 val perplexity 350.6020
-8700 train 5.857419 (lr=1.9679e-05) (hash(x)=148488790)
-8800 val loss 5.8626
-8800 val perplexity 351.6325
-8800 train 5.888811 (lr=1.9393e-05) (hash(x)=156797847)
-8900 val loss 5.8404
-8900 val perplexity 343.9224
-8900 train 5.656487 (lr=1.9107e-05) (hash(x)=143919631)
-9000 val loss 5.8365
-9000 val perplexity 342.5618
-9000 train 5.783753 (lr=1.8821e-05) (hash(x)=148706835)
-9100 val loss 5.8337
-9100 val perplexity 341.6362
-9100 train 5.852053 (lr=1.8535e-05) (hash(x)=153074259)
-9200 val loss 5.8267
-9200 val perplexity 339.2331
-9200 train 5.757070 (lr=1.8250e-05) (hash(x)=151127077)
-9300 val loss 5.8228
-9300 val perplexity 337.9189
-9300 train 5.711324 (lr=1.7964e-05) (hash(x)=152170602)
-9400 val loss 5.8316
-9400 val perplexity 340.9194
-9400 train 5.859845 (lr=1.7680e-05) (hash(x)=153874803)
-9500 val loss 5.8074
-9500 val perplexity 332.7599
-9500 train 5.811232 (lr=1.7395e-05) (hash(x)=157021298)
-9600 val loss 5.8003
-9600 val perplexity 330.3847
-9600 train 5.644323 (lr=1.7112e-05) (hash(x)=135620494)
-9700 val loss 5.7963
-9700 val perplexity 329.0877
-9700 train 5.519238 (lr=1.6829e-05) (hash(x)=142434463)
-9800 val loss 5.7950
-9800 val perplexity 328.6615
-9800 train 5.653191 (lr=1.6546e-05) (hash(x)=149075541)
-9900 val loss 5.7901
-9900 val perplexity 327.0433
-9900 train 5.663763 (lr=1.6265e-05) (hash(x)=141901425)
-10000 val loss 5.8030
-10000 val perplexity 331.2853
-10000 train 5.787296 (lr=1.5985e-05) (hash(x)=133062699)
-10100 val loss 5.7769
-10100 val perplexity 322.7687
-10100 train 5.801806 (lr=1.5705e-05) (hash(x)=157339196)
-10200 val loss 5.7711
-10200 val perplexity 320.8841
-10200 train 5.535702 (lr=1.5427e-05) (hash(x)=131082121)
-10300 val loss 5.7706
-10300 val perplexity 320.7148
-10300 train 5.882634 (lr=1.5151e-05) (hash(x)=144553757)
-10400 val loss 5.7630
-10400 val perplexity 318.2991
-10400 train 5.552830 (lr=1.4875e-05) (hash(x)=143398861)
-10500 val loss 5.7607
-10500 val perplexity 317.5827
-10500 train 5.759555 (lr=1.4601e-05) (hash(x)=157633809)
-10600 val loss 5.7540
-10600 val perplexity 315.4557
-10600 train 5.753001 (lr=1.4329e-05) (hash(x)=147294315)
-10700 val loss 5.7478
-10700 val perplexity 313.4893
-10700 train 5.864288 (lr=1.4058e-05) (hash(x)=157735028)
-10800 val loss 5.7406
-10800 val perplexity 311.2406
-10800 train 5.716289 (lr=1.3789e-05) (hash(x)=149846113)
-10900 val loss 5.7385
-10900 val perplexity 310.5964
-10900 train 6.123055 (lr=1.3521e-05) (hash(x)=157353556)
-11000 val loss 5.7349
-11000 val perplexity 309.4757
-11000 train 6.046704 (lr=1.3256e-05) (hash(x)=160672627)
-11100 val loss 5.7331
-11100 val perplexity 308.9318
-11100 train 5.941358 (lr=1.2992e-05) (hash(x)=166692335)
-11200 val loss 5.7280
-11200 val perplexity 307.3401
-11200 train 5.787968 (lr=1.2731e-05) (hash(x)=158899869)
-11300 val loss 5.7211
-11300 val perplexity 305.2282
-11300 train 6.013842 (lr=1.2472e-05) (hash(x)=166306828)
-11400 val loss 5.7178
-11400 val perplexity 304.2259
-11400 train 5.641149 (lr=1.2215e-05) (hash(x)=149545840)
-11500 val loss 5.7135
-11500 val perplexity 302.9425
-11500 train 5.681398 (lr=1.1960e-05) (hash(x)=139076910)
-11600 val loss 5.7133
-11600 val perplexity 302.8682
-11600 train 5.548331 (lr=1.1708e-05) (hash(x)=151871279)
-11700 val loss 5.7086
-11700 val perplexity 301.4414
-11700 train 5.653671 (lr=1.1458e-05) (hash(x)=152940196)
-11800 val loss 5.7036
-11800 val perplexity 299.9490
-11800 train 5.792050 (lr=1.1210e-05) (hash(x)=150754129)
-11900 val loss 5.6992
-11900 val perplexity 298.6337
-11900 train 5.654656 (lr=1.0966e-05) (hash(x)=148982597)
-12000 val loss 5.6949
-12000 val perplexity 297.3535
-12000 train 5.737731 (lr=1.0724e-05) (hash(x)=167798390)
-12100 val loss 5.6965
-12100 val perplexity 297.8293
-12100 train 5.777075 (lr=1.0485e-05) (hash(x)=146590810)
-12200 val loss 5.6904
-12200 val perplexity 296.0153
-12200 train 5.920342 (lr=1.0249e-05) (hash(x)=159875780)
-12300 val loss 5.6880
-12300 val perplexity 295.3060
-12300 train 5.556569 (lr=1.0016e-05) (hash(x)=135887861)
-12400 val loss 5.6812
-12400 val perplexity 293.2936
-12400 train 5.745451 (lr=9.7854e-06) (hash(x)=150484221)
-12500 val loss 5.6781
-12500 val perplexity 292.3839
-12500 train 5.679491 (lr=9.5583e-06) (hash(x)=157629455)
-12600 val loss 5.6770
-12600 val perplexity 292.0846
-12600 train 5.581851 (lr=9.3345e-06) (hash(x)=147308692)
-12700 val loss 5.6747
-12700 val perplexity 291.4074
-12700 train 5.506831 (lr=9.1139e-06) (hash(x)=144614722)
-12800 val loss 5.6733
-12800 val perplexity 290.9929
-12800 train 5.462621 (lr=8.8967e-06) (hash(x)=137633606)
-12900 val loss 5.6713
-12900 val perplexity 290.4078
-12900 train 5.713201 (lr=8.6829e-06) (hash(x)=155534772)
-13000 val loss 5.6660
-13000 val perplexity 288.8828
-13000 train 5.678630 (lr=8.4726e-06) (hash(x)=148501470)
-13100 val loss 5.6692
-13100 val perplexity 289.7993
-13100 train 5.717379 (lr=8.2658e-06) (hash(x)=154570618)
-13200 val loss 5.6652
-13200 val perplexity 288.6525
-13200 train 5.764972 (lr=8.0626e-06) (hash(x)=154153766)
-13300 val loss 5.6603
-13300 val perplexity 287.2264
-13300 train 5.434469 (lr=7.8632e-06) (hash(x)=162760888)
-13400 val loss 5.6548
-13400 val perplexity 285.6629
-13400 train 5.478197 (lr=7.6674e-06) (hash(x)=143223528)
-13500 val loss 5.6637
-13500 val perplexity 288.2151
-13500 train 5.808773 (lr=7.4756e-06) (hash(x)=156416986)
-13600 val loss 5.6558
-13600 val perplexity 285.9459
-13600 train 5.660703 (lr=7.2875e-06) (hash(x)=147521418)
-13700 val loss 5.6516
-13700 val perplexity 284.7564
-13700 train 5.621961 (lr=7.1035e-06) (hash(x)=151127796)
-13800 val loss 5.6498
-13800 val perplexity 284.2325
-13800 train 5.711004 (lr=6.9234e-06) (hash(x)=156863042)
-13900 val loss 5.6459
-13900 val perplexity 283.1349
-13900 train 5.650376 (lr=6.7474e-06) (hash(x)=170986545)
-14000 val loss 5.6442
-14000 val perplexity 282.6507
-14000 train 6.113935 (lr=6.5756e-06) (hash(x)=170744741)
-14100 val loss 5.6412
-14100 val perplexity 281.7943
-14100 train 6.038649 (lr=6.4079e-06) (hash(x)=142328767)
-14200 val loss 5.6403
-14200 val perplexity 281.5575
-14200 train 5.787434 (lr=6.2444e-06) (hash(x)=156141076)
-14300 val loss 5.6435
-14300 val perplexity 282.4610
-14300 train 5.623202 (lr=6.0852e-06) (hash(x)=155447298)
-14400 val loss 5.6371
-14400 val perplexity 280.6448
-14400 train 5.566492 (lr=5.9304e-06) (hash(x)=151991844)
-14500 val loss 5.6329
-14500 val perplexity 279.4600
-14500 train 5.607948 (lr=5.7800e-06) (hash(x)=149773009)
-14600 val loss 5.6308
-14600 val perplexity 278.8895
-14600 train 5.581574 (lr=5.6340e-06) (hash(x)=153836753)
-14700 val loss 5.6302
-14700 val perplexity 278.7067
-14700 train 5.764620 (lr=5.4925e-06) (hash(x)=155466746)
-14800 val loss 5.6292
-14800 val perplexity 278.4420
-14800 train 5.567979 (lr=5.3555e-06) (hash(x)=150110933)
-14900 val loss 5.6296
-14900 val perplexity 278.5507
-14900 train 5.516385 (lr=5.2231e-06) (hash(x)=144558528)
-15000 val loss 5.6273
-15000 val perplexity 277.9196
-15000 train 5.569795 (lr=5.0954e-06) (hash(x)=148315702)
-15100 val loss 5.6239
-15100 val perplexity 276.9653
-15100 train 5.474117 (lr=4.9723e-06) (hash(x)=148452301)
-15200 val loss 5.6229
-15200 val perplexity 276.6985
-15200 train 5.878456 (lr=4.8539e-06) (hash(x)=162181769)
-15300 val loss 5.6205
-15300 val perplexity 276.0157
-15300 train 5.547765 (lr=4.7403e-06) (hash(x)=147256707)
-15400 val loss 5.6195
-15400 val perplexity 275.7474
-15400 train 5.726286 (lr=4.6314e-06) (hash(x)=154947797)
-15500 val loss 5.6196
-15500 val perplexity 275.7789
-15500 train 5.640979 (lr=4.5274e-06) (hash(x)=164200136)
-15600 val loss 5.6190
-15600 val perplexity 275.6071
-15600 train 5.492680 (lr=4.4282e-06) (hash(x)=145086131)
-15700 val loss 5.6139
-15700 val perplexity 274.2110
-15700 train 5.548183 (lr=4.3339e-06) (hash(x)=152070006)
-15800 val loss 5.6117
-15800 val perplexity 273.6021
-15800 train 5.638015 (lr=4.2446e-06) (hash(x)=150969760)
-15900 val loss 5.6115
-15900 val perplexity 273.5452
-15900 train 5.627972 (lr=4.1601e-06) (hash(x)=149495681)
-16000 val loss 5.6105
-16000 val perplexity 273.2919
-16000 train 5.729979 (lr=4.0807e-06) (hash(x)=159383533)
-16100 val loss 5.6109
-16100 val perplexity 273.3976
-16100 train 5.485047 (lr=4.0063e-06) (hash(x)=154365611)
-16200 val loss 5.6093
-16200 val perplexity 272.9447
-16200 train 5.772431 (lr=3.9368e-06) (hash(x)=154284815)
-16300 val loss 5.6082
-16300 val perplexity 272.6483
-16300 train 5.598111 (lr=3.8725e-06) (hash(x)=148416573)
-16400 val loss 5.6056
-16400 val perplexity 271.9573
-16400 train 5.550362 (lr=3.8132e-06) (hash(x)=150447087)
-16500 val loss 5.6063
-16500 val perplexity 272.1422
-16500 train 5.601939 (lr=3.7590e-06) (hash(x)=156112678)
-16600 val loss 5.6044
-16600 val perplexity 271.6206
-16600 train 5.548048 (lr=3.7099e-06) (hash(x)=150449728)
-16700 val loss 5.6020
-16700 val perplexity 270.9666
-16700 train 5.661832 (lr=3.6659e-06) (hash(x)=153905935)
-16800 val loss 5.5997
-16800 val perplexity 270.3404
-16800 train 5.565632 (lr=3.6271e-06) (hash(x)=144086573)
-16900 val loss 5.5996
-16900 val perplexity 270.3120
-16900 train 5.573380 (lr=3.5934e-06) (hash(x)=145863379)
-17000 val loss 5.5997
-17000 val perplexity 270.3401
-17000 train 5.662974 (lr=3.5649e-06) (hash(x)=149054226)
-17100 val loss 5.6001
-17100 val perplexity 270.4409
-17100 train 5.507200 (lr=3.5415e-06) (hash(x)=147109203)
-17200 val loss 5.5970
-17200 val perplexity 269.6129
-17200 train 5.529955 (lr=3.5234e-06) (hash(x)=141837353)
-17300 val loss 5.5958
-17300 val perplexity 269.2928
-17300 train 5.607386 (lr=3.5104e-06) (hash(x)=148542064)
-17400 val loss 5.5940
-17400 val perplexity 268.8177
-17400 train 5.431786 (lr=3.5026e-06) (hash(x)=140939390)
-17499 val loss 5.5938
-17499 val perplexity 268.7424
+0 train 11.260221 (lr=1.8750e-07) (hash(x)=134751525)
+0 train 11.260221 (lr=2.0000e-07) (hash(x)=134751525)
+100 val loss 9.9224
+100 val perplexity 20381.3145
+100 train 9.978566 (lr=1.8937e-05) (hash(x)=144692155)
+100 val loss 9.9026
+100 val perplexity 19981.9570
+100 train 9.958472 (lr=2.0200e-05) (hash(x)=144692155)
+200 val loss 8.7106
+200 val perplexity 6066.6841
+200 train 8.693931 (lr=3.7500e-05) (hash(x)=150358957)
+200 val loss 8.8364
+200 val perplexity 6880.2979
+200 train 8.824292 (lr=4.0000e-05) (hash(x)=150358957)
+300 val loss 7.7947
+300 val perplexity 2427.6169
+300 train 7.770412 (lr=3.7497e-05) (hash(x)=150268299)
+300 val loss 7.9373
+300 val perplexity 2799.7073
+300 train 7.926304 (lr=3.9997e-05) (hash(x)=150268299)
+400 val loss 7.5322
+400 val perplexity 1867.2103
+400 train 7.378184 (lr=3.7489e-05) (hash(x)=126319983)
+400 val loss 7.6340
+400 val perplexity 2067.3862
+400 train 7.495752 (lr=3.9988e-05) (hash(x)=126319983)
+500 val loss 7.4340
+500 val perplexity 1692.6014
+500 train 7.467863 (lr=3.7475e-05) (hash(x)=153934799)
+500 val loss 7.5062
+500 val perplexity 1819.3467
+500 train 7.542481 (lr=3.9973e-05) (hash(x)=153934799)
+600 val loss 7.3549
+600 val perplexity 1563.9165
+600 train 7.397041 (lr=3.7456e-05) (hash(x)=159305874)
+600 val loss 7.4469
+600 val perplexity 1714.4636
+600 train 7.486856 (lr=3.9953e-05) (hash(x)=159305874)
+700 val loss 7.2917
+700 val perplexity 1468.0907
+700 train 7.144725 (lr=3.7430e-05) (hash(x)=141986303)
+700 val loss 7.3974
+700 val perplexity 1631.6576
+700 train 7.242340 (lr=3.9926e-05) (hash(x)=141986303)
+800 val loss 7.2317
+800 val perplexity 1382.5919
+800 train 7.207565 (lr=3.7400e-05) (hash(x)=149855631)
+800 val loss 7.3438
+800 val perplexity 1546.6282
+800 train 7.328330 (lr=3.9893e-05) (hash(x)=149855631)
+900 val loss 7.1985
+900 val perplexity 1337.3608
+900 train 7.079328 (lr=3.7364e-05) (hash(x)=149870240)
+900 val loss 7.3186
+900 val perplexity 1508.1108
+900 train 7.203938 (lr=3.9855e-05) (hash(x)=149870240)
+1000 val loss 7.1332
+1000 val perplexity 1252.9331
+1000 train 7.201777 (lr=3.7322e-05) (hash(x)=150706760)
+1000 val loss 7.2615
+1000 val perplexity 1424.3921
+1000 train 7.330913 (lr=3.9810e-05) (hash(x)=150706760)
+1100 val loss 7.0820
+1100 val perplexity 1190.3313
+1100 train 6.918931 (lr=3.7275e-05) (hash(x)=139020621)
+1100 val loss 7.2093
+1100 val perplexity 1351.9862
+1100 train 7.051281 (lr=3.9760e-05) (hash(x)=139020621)
+1200 val loss 7.0396
+1200 val perplexity 1140.9326
+1200 train 6.899639 (lr=3.7223e-05) (hash(x)=146858163)
+1200 val loss 7.1719
+1200 val perplexity 1302.3617
+1200 train 7.031198 (lr=3.9704e-05) (hash(x)=146858163)
+1300 val loss 6.9959
+1300 val perplexity 1092.1119
+1300 train 7.059475 (lr=3.7164e-05) (hash(x)=156383408)
+1300 val loss 7.1213
+1300 val perplexity 1238.0621
+1300 train 7.194174 (lr=3.9642e-05) (hash(x)=156383408)
+1400 val loss 6.9603
+1400 val perplexity 1053.9180
+1400 train 6.890877 (lr=3.7101e-05) (hash(x)=144262072)
+1400 val loss 7.0776
+1400 val perplexity 1185.1265
+1400 train 7.007119 (lr=3.9574e-05) (hash(x)=144262072)
+1500 val loss 6.9248
+1500 val perplexity 1017.1529
+1500 train 6.774604 (lr=3.7032e-05) (hash(x)=144528048)
+1500 val loss 7.0352
+1500 val perplexity 1135.9320
+1500 train 6.889645 (lr=3.9501e-05) (hash(x)=144528048)
+1600 val loss 6.8806
+1600 val perplexity 973.1651
+1600 train 6.797646 (lr=3.6958e-05) (hash(x)=135443880)
+1600 val loss 6.9865
+1600 val perplexity 1081.8743
+1600 train 6.900005 (lr=3.9421e-05) (hash(x)=135443880)
+1700 val loss 6.8431
+1700 val perplexity 937.4041
+1700 train 6.766635 (lr=3.6878e-05) (hash(x)=145067496)
+1700 val loss 6.9683
+1700 val perplexity 1062.3906
+1700 train 6.887623 (lr=3.9336e-05) (hash(x)=145067496)
+1800 val loss 6.8003
+1800 val perplexity 898.1185
+1800 train 6.762501 (lr=3.6793e-05) (hash(x)=153147388)
+1800 val loss 6.9060
+1800 val perplexity 998.2911
+1800 train 6.871210 (lr=3.9246e-05) (hash(x)=153147388)
+1900 val loss 6.7612
+1900 val perplexity 863.6940
+1900 train 6.646096 (lr=3.6702e-05) (hash(x)=151331979)
+1900 val loss 6.8692
+1900 val perplexity 962.1613
+1900 train 6.756928 (lr=3.9149e-05) (hash(x)=151331979)
+2000 val loss 6.7419
+2000 val perplexity 847.1882
+2000 train 6.551958 (lr=3.6606e-05) (hash(x)=151448445)
+2000 val loss 6.8355
+2000 val perplexity 930.3309
+2000 train 6.634272 (lr=3.9047e-05) (hash(x)=151448445)
+2100 val loss 6.6880
+2100 val perplexity 802.7498
+2100 train 6.586328 (lr=3.6505e-05) (hash(x)=157223072)
+2100 val loss 6.8038
+2100 val perplexity 901.2855
+2100 train 6.707023 (lr=3.8939e-05) (hash(x)=157223072)
+2200 val loss 6.6446
+2200 val perplexity 768.5997
+2200 train 6.666820 (lr=3.6399e-05) (hash(x)=153161010)
+2200 val loss 6.7681
+2200 val perplexity 869.6474
+2200 train 6.781376 (lr=3.8826e-05) (hash(x)=153161010)
+2300 val loss 6.6056
+2300 val perplexity 739.1978
+2300 train 6.688467 (lr=3.6288e-05) (hash(x)=156906742)
+2300 val loss 6.7296
+2300 val perplexity 836.7805
+2300 train 6.817736 (lr=3.8707e-05) (hash(x)=156906742)
+2400 val loss 6.5766
+2400 val perplexity 718.1187
+2400 train 6.470441 (lr=3.6171e-05) (hash(x)=143227423)
+2400 val loss 6.7280
+2400 val perplexity 835.5054
+2400 train 6.619276 (lr=3.8583e-05) (hash(x)=143227423)
+2500 val loss 6.5469
+2500 val perplexity 697.0684
+2500 train 6.497835 (lr=3.6049e-05) (hash(x)=151827079)
+2500 val loss 6.6696
+2500 val perplexity 788.0593
+2500 train 6.607266 (lr=3.8453e-05) (hash(x)=151827079)
+2600 val loss 6.5227
+2600 val perplexity 680.4191
+2600 train 6.965666 (lr=3.5923e-05) (hash(x)=176373796)
+2600 val loss 6.6474
+2600 val perplexity 770.7552
+2600 train 7.123261 (lr=3.8317e-05) (hash(x)=176373796)
+2700 val loss 6.4933
+2700 val perplexity 660.6890
+2700 train 6.428716 (lr=3.5791e-05) (hash(x)=148437519)
+2700 val loss 6.6203
+2700 val perplexity 750.1442
+2700 train 6.563433 (lr=3.8177e-05) (hash(x)=148437519)
+2800 val loss 6.4604
+2800 val perplexity 639.2952
+2800 train 6.459748 (lr=3.5654e-05) (hash(x)=155989503)
+2800 val loss 6.5978
+2800 val perplexity 733.4659
+2800 train 6.597178 (lr=3.8031e-05) (hash(x)=155989503)
+2900 val loss 6.4426
+2900 val perplexity 628.0563
+2900 train 6.569895 (lr=3.5512e-05) (hash(x)=159227845)
+2900 val loss 6.5750
+2900 val perplexity 716.9130
+2900 train 6.709556 (lr=3.7879e-05) (hash(x)=159227845)
+3000 val loss 6.4088
+3000 val perplexity 607.1508
+3000 train 6.432460 (lr=3.5365e-05) (hash(x)=156507542)
+3000 val loss 6.5458
+3000 val perplexity 696.3286
+3000 train 6.572825 (lr=3.7723e-05) (hash(x)=156507542)
+3100 val loss 6.3846
+3100 val perplexity 592.6638
+3100 train 6.370049 (lr=3.5214e-05) (hash(x)=155771797)
+3100 val loss 6.5187
+3100 val perplexity 677.6685
+3100 train 6.503825 (lr=3.7561e-05) (hash(x)=155771797)
+3200 val loss 6.3627
+3200 val perplexity 579.7885
+3200 train 6.287634 (lr=3.5057e-05) (hash(x)=156078901)
+3200 val loss 6.4918
+3200 val perplexity 659.7232
+3200 train 6.411802 (lr=3.7394e-05) (hash(x)=156078901)
+3300 val loss 6.3464
+3300 val perplexity 570.4557
+3300 train 6.214350 (lr=3.4896e-05) (hash(x)=143722886)
+3300 val loss 6.4636
+3300 val perplexity 641.3925
+3300 train 6.330358 (lr=3.7222e-05) (hash(x)=143722886)
+3400 val loss 6.3244
+3400 val perplexity 558.0221
+3400 train 6.250166 (lr=3.4730e-05) (hash(x)=143793373)
+3400 val loss 6.4451
+3400 val perplexity 629.5942
+3400 train 6.376811 (lr=3.7045e-05) (hash(x)=143793373)
+3500 val loss 6.3115
+3500 val perplexity 550.8678
+3500 train 6.593259 (lr=3.4560e-05) (hash(x)=147278080)
+3500 val loss 6.4321
+3500 val perplexity 621.4998
+3500 train 6.691866 (lr=3.6864e-05) (hash(x)=147278080)
+3600 val loss 6.2929
+3600 val perplexity 540.7174
+3600 train 7.051870 (lr=3.4384e-05) (hash(x)=175299205)
+3600 val loss 6.4133
+3600 val perplexity 609.8760
+3600 train 7.152054 (lr=3.6677e-05) (hash(x)=175299205)
+3700 val loss 6.2706
+3700 val perplexity 528.8101
+3700 train 6.318336 (lr=3.4205e-05) (hash(x)=150047891)
+3700 val loss 6.3813
+3700 val perplexity 590.7053
+3700 train 6.425822 (lr=3.6485e-05) (hash(x)=150047891)
+3800 val loss 6.2556
+3800 val perplexity 520.9250
+3800 train 6.285592 (lr=3.4021e-05) (hash(x)=153419668)
+3800 val loss 6.3678
+3800 val perplexity 582.7953
+3800 train 6.395144 (lr=3.6289e-05) (hash(x)=153419668)
+3900 val loss 6.2362
+3900 val perplexity 510.8896
+3900 train 6.351056 (lr=3.3832e-05) (hash(x)=153147630)
+3900 val loss 6.3476
+3900 val perplexity 571.1245
+3900 train 6.456195 (lr=3.6088e-05) (hash(x)=153147630)
+4000 val loss 6.2256
+4000 val perplexity 505.5310
+4000 train 6.058341 (lr=3.3639e-05) (hash(x)=142428269)
+4000 val loss 6.3408
+4000 val perplexity 567.2731
+4000 train 6.177061 (lr=3.5882e-05) (hash(x)=142428269)
+4100 val loss 6.2155
+4100 val perplexity 500.4389
+4100 train 6.109726 (lr=3.3442e-05) (hash(x)=148836993)
+4100 val loss 6.3292
+4100 val perplexity 560.7189
+4100 train 6.216930 (lr=3.5671e-05) (hash(x)=148836993)
+4200 val loss 6.1891
+4200 val perplexity 487.4299
+4200 train 6.215916 (lr=3.3240e-05) (hash(x)=159557219)
+4200 val loss 6.3076
+4200 val perplexity 548.7442
+4200 train 6.353669 (lr=3.5457e-05) (hash(x)=159557219)
+4300 val loss 6.1722
+4300 val perplexity 479.2188
+4300 train 6.305759 (lr=3.3035e-05) (hash(x)=154137450)
+4300 val loss 6.3024
+4300 val perplexity 545.8607
+4300 train 6.428835 (lr=3.5237e-05) (hash(x)=154137450)
+4400 val loss 6.1598
+4400 val perplexity 473.3203
+4400 train 6.318167 (lr=3.2825e-05) (hash(x)=153336405)
+4400 val loss 6.2828
+4400 val perplexity 535.2630
+4400 train 6.430600 (lr=3.5014e-05) (hash(x)=153336405)
+4500 val loss 6.1431
+4500 val perplexity 465.5147
+4500 train 6.117867 (lr=3.2611e-05) (hash(x)=145729145)
+4500 val loss 6.2663
+4500 val perplexity 526.5511
+4500 train 6.233288 (lr=3.4786e-05) (hash(x)=145729145)
+4600 val loss 6.1397
+4600 val perplexity 463.9298
+4600 train 6.154843 (lr=3.2394e-05) (hash(x)=157789272)
+4600 val loss 6.2642
+4600 val perplexity 525.4109
+4600 train 6.261403 (lr=3.4553e-05) (hash(x)=157789272)
+4700 val loss 6.1223
+4700 val perplexity 455.9128
+4700 train 6.113758 (lr=3.2172e-05) (hash(x)=146146781)
+4700 val loss 6.2476
+4700 val perplexity 516.7608
+4700 train 6.227163 (lr=3.4317e-05) (hash(x)=146146781)
+4800 val loss 6.1178
+4800 val perplexity 453.8862
+4800 train 6.142168 (lr=3.1947e-05) (hash(x)=149454469)
+4800 val loss 6.2516
+4800 val perplexity 518.8306
+4800 train 6.256900 (lr=3.4077e-05) (hash(x)=149454469)
+4900 val loss 6.0944
+4900 val perplexity 443.3552
+4900 train 6.184642 (lr=3.1718e-05) (hash(x)=155966467)
+4900 val loss 6.2232
+4900 val perplexity 504.3185
+4900 train 6.321077 (lr=3.3832e-05) (hash(x)=155966467)
+5000 val loss 6.0871
+5000 val perplexity 440.1455
+5000 train 6.048105 (lr=3.1485e-05) (hash(x)=147184546)
+5000 val loss 6.2208
+5000 val perplexity 503.0935
+5000 train 6.176530 (lr=3.3584e-05) (hash(x)=147184546)
+5100 val loss 6.0836
+5100 val perplexity 438.5894
+5100 train 6.055791 (lr=3.1249e-05) (hash(x)=149019501)
+5100 val loss 6.2195
+5100 val perplexity 502.4680
+5100 train 6.189746 (lr=3.3332e-05) (hash(x)=149019501)
+5200 val loss 6.0647
+5200 val perplexity 430.4049
+5200 train 5.981862 (lr=3.1009e-05) (hash(x)=143521586)
+5200 val loss 6.2022
+5200 val perplexity 493.8430
+5200 train 6.108462 (lr=3.3076e-05) (hash(x)=143521586)
+5300 val loss 6.0555
+5300 val perplexity 426.4686
+5300 train 5.918194 (lr=3.0766e-05) (hash(x)=143870567)
+5300 val loss 6.1932
+5300 val perplexity 489.3878
+5300 train 6.047366 (lr=3.2817e-05) (hash(x)=143870567)
+5400 val loss 6.0455
+5400 val perplexity 422.2164
+5400 train 6.074172 (lr=3.0519e-05) (hash(x)=150658725)
+5400 val loss 6.1730
+5400 val perplexity 479.6065
+5400 train 6.186630 (lr=3.2554e-05) (hash(x)=150658725)
+5500 val loss 6.0326
+5500 val perplexity 416.8026
+5500 train 6.064359 (lr=3.0269e-05) (hash(x)=147645530)
+5500 val loss 6.1636
+5500 val perplexity 475.1183
+5500 train 6.185577 (lr=3.2287e-05) (hash(x)=147645530)
+5600 val loss 6.0319
+5600 val perplexity 416.4993
+5600 train 6.074053 (lr=3.0016e-05) (hash(x)=146088410)
+5600 val loss 6.1565
+5600 val perplexity 471.7789
+5600 train 6.184332 (lr=3.2017e-05) (hash(x)=146088410)
+5700 val loss 6.0128
+5700 val perplexity 408.6373
+5700 train 6.120058 (lr=2.9760e-05) (hash(x)=151193083)
+5700 val loss 6.1430
+5700 val perplexity 465.4441
+5700 train 6.244153 (lr=3.1744e-05) (hash(x)=151193083)
+5800 val loss 6.0042
+5800 val perplexity 405.1129
+5800 train 5.773274 (lr=2.9501e-05) (hash(x)=137474865)
+5800 val loss 6.1343
+5800 val perplexity 461.3980
+5800 train 5.897436 (lr=3.1468e-05) (hash(x)=137474865)
+5900 val loss 5.9983
+5900 val perplexity 402.7342
+5900 train 5.959085 (lr=2.9239e-05) (hash(x)=142426763)
+5900 val loss 6.1302
+5900 val perplexity 459.5471
+5900 train 6.072005 (lr=3.1188e-05) (hash(x)=142426763)
+6000 val loss 5.9785
+6000 val perplexity 394.8582
+6000 train 5.758265 (lr=2.8974e-05) (hash(x)=140492670)
+6000 val loss 6.1079
+6000 val perplexity 449.4054
+6000 train 5.878311 (lr=3.0905e-05) (hash(x)=140492670)
+6100 val loss 5.9674
+6100 val perplexity 390.4943
+6100 train 5.966070 (lr=2.8706e-05) (hash(x)=175687796)
+6100 val loss 6.1001
+6100 val perplexity 445.8966
+6100 train 6.160385 (lr=3.0620e-05) (hash(x)=175687796)
+6200 val loss 5.9568
+6200 val perplexity 386.3677
+6200 train 6.023569 (lr=2.8436e-05) (hash(x)=165841139)
+6200 val loss 6.0909
+6200 val perplexity 441.8349
+6200 train 6.148386 (lr=3.0332e-05) (hash(x)=165841139)
+6300 val loss 5.9504
+6300 val perplexity 383.9209
+6300 train 6.004202 (lr=2.8163e-05) (hash(x)=145328367)
+6300 val loss 6.0847
+6300 val perplexity 439.0967
+6300 train 6.139287 (lr=3.0040e-05) (hash(x)=145328367)
+6400 val loss 5.9501
+6400 val perplexity 383.7911
+6400 train 5.937871 (lr=2.7888e-05) (hash(x)=148213323)
+6400 val loss 6.0794
+6400 val perplexity 436.7498
+6400 train 6.061519 (lr=2.9747e-05) (hash(x)=148213323)
+6500 val loss 5.9341
+6500 val perplexity 377.6818
+6500 train 6.011780 (lr=2.7610e-05) (hash(x)=153344426)
+6500 val loss 6.0682
+6500 val perplexity 431.9211
+6500 train 6.160855 (lr=2.9450e-05) (hash(x)=153344426)
+6600 val loss 5.9266
+6600 val perplexity 374.8727
+6600 train 5.903677 (lr=2.7330e-05) (hash(x)=145840902)
+6600 val loss 6.0594
+6600 val perplexity 428.1175
+6600 train 6.020699 (lr=2.9152e-05) (hash(x)=145840902)
+6700 val loss 5.9211
+6700 val perplexity 372.8060
+6700 train 5.757521 (lr=2.7047e-05) (hash(x)=142812723)
+6700 val loss 6.0461
+6700 val perplexity 422.4782
+6700 train 5.886734 (lr=2.8851e-05) (hash(x)=142812723)
+6800 val loss 5.9144
+6800 val perplexity 370.3436
+6800 train 5.761705 (lr=2.6763e-05) (hash(x)=147420964)
+6800 val loss 6.0408
+6800 val perplexity 420.2113
+6800 train 5.889901 (lr=2.8547e-05) (hash(x)=147420964)
+6900 val loss 5.9056
+6900 val perplexity 367.0766
+6900 train 5.940935 (lr=2.6477e-05) (hash(x)=152637292)
+6900 val loss 6.0352
+6900 val perplexity 417.8958
+6900 train 6.087862 (lr=2.8242e-05) (hash(x)=152637292)
+7000 val loss 5.8875
+7000 val perplexity 360.5145
+7000 train 5.894612 (lr=2.6188e-05) (hash(x)=144728159)
+7000 val loss 6.0211
+7000 val perplexity 412.0505
+7000 train 6.019482 (lr=2.7934e-05) (hash(x)=144728159)
+7100 val loss 5.8794
+7100 val perplexity 357.5976
+7100 train 5.857129 (lr=2.5898e-05) (hash(x)=143450899)
+7100 val loss 6.0116
+7100 val perplexity 408.1296
+7100 train 5.993532 (lr=2.7624e-05) (hash(x)=143450899)
+7200 val loss 5.8710
+7200 val perplexity 354.5907
+7200 train 5.953189 (lr=2.5606e-05) (hash(x)=156229897)
+7200 val loss 6.0054
+7200 val perplexity 405.6198
+7200 train 6.080842 (lr=2.7313e-05) (hash(x)=156229897)
+7300 val loss 5.8619
+7300 val perplexity 351.3760
+7300 train 5.745842 (lr=2.5312e-05) (hash(x)=147447250)
+7300 val loss 5.9921
+7300 val perplexity 400.2460
+7300 train 5.861655 (lr=2.7000e-05) (hash(x)=147447250)
+7400 val loss 5.8641
+7400 val perplexity 352.1667
+7400 train 5.624749 (lr=2.5017e-05) (hash(x)=134836112)
+7400 val loss 5.9922
+7400 val perplexity 400.2918
+7400 train 5.735302 (lr=2.6685e-05) (hash(x)=134836112)
+7500 val loss 5.8571
+7500 val perplexity 349.7002
+7500 train 5.881787 (lr=2.4721e-05) (hash(x)=155027499)
+7500 val loss 5.9880
+7500 val perplexity 398.6122
+7500 train 6.009743 (lr=2.6369e-05) (hash(x)=155027499)
+7600 val loss 5.8399
+7600 val perplexity 343.7379
+7600 train 5.726958 (lr=2.4423e-05) (hash(x)=145744849)
+7600 val loss 5.9702
+7600 val perplexity 391.5980
+7600 train 5.859680 (lr=2.6051e-05) (hash(x)=145744849)
+7700 val loss 5.8324
+7700 val perplexity 341.1719
+7700 train 5.965559 (lr=2.4124e-05) (hash(x)=158082973)
+7700 val loss 5.9659
+7700 val perplexity 389.9156
+7700 train 6.088557 (lr=2.5732e-05) (hash(x)=158082973)
+7800 val loss 5.8256
+7800 val perplexity 338.8489
+7800 train 5.963749 (lr=2.3823e-05) (hash(x)=161538768)
+7800 val loss 5.9585
+7800 val perplexity 387.0208
+7800 train 6.099160 (lr=2.5411e-05) (hash(x)=161538768)
+7900 val loss 5.8157
+7900 val perplexity 335.5297
+7900 train 5.912204 (lr=2.3522e-05) (hash(x)=152103064)
+7900 val loss 5.9481
+7900 val perplexity 383.0178
+7900 train 6.041970 (lr=2.5090e-05) (hash(x)=152103064)
+8000 val loss 5.8101
+8000 val perplexity 333.6431
+8000 train 5.795048 (lr=2.3219e-05) (hash(x)=152409525)
+8000 val loss 5.9415
+8000 val perplexity 380.4964
+8000 train 5.935862 (lr=2.4767e-05) (hash(x)=152409525)
+8100 val loss 5.8058
+8100 val perplexity 332.2193
+8100 train 5.785800 (lr=2.2916e-05) (hash(x)=150986182)
+8100 val loss 5.9357
+8100 val perplexity 378.3020
+8100 train 5.913834 (lr=2.4444e-05) (hash(x)=150986182)
+8200 val loss 5.7943
+8200 val perplexity 328.4197
+8200 train 5.795736 (lr=2.2612e-05) (hash(x)=159555650)
+8200 val loss 5.9304
+8200 val perplexity 376.2982
+8200 train 5.935581 (lr=2.4120e-05) (hash(x)=159555650)
+8300 val loss 5.7917
+8300 val perplexity 327.5845
+8300 train 5.967180 (lr=2.2308e-05) (hash(x)=139602404)
+8300 val loss 5.9256
+8300 val perplexity 374.5074
+8300 train 6.084064 (lr=2.3795e-05) (hash(x)=139602404)
+8400 val loss 5.7826
+8400 val perplexity 324.5954
+8400 train 5.679160 (lr=2.2002e-05) (hash(x)=154432574)
+8400 val loss 5.9134
+8400 val perplexity 369.9438
+8400 train 5.807495 (lr=2.3469e-05) (hash(x)=154432574)
+8500 val loss 5.7777
+8500 val perplexity 323.0103
+8500 train 5.736130 (lr=2.1697e-05) (hash(x)=142554189)
+8500 val loss 5.9038
+8500 val perplexity 366.4180
+8500 train 5.842204 (lr=2.3143e-05) (hash(x)=142554189)
+8600 val loss 5.7696
+8600 val perplexity 320.4031
+8600 train 5.718740 (lr=2.1391e-05) (hash(x)=151300791)
+8600 val loss 5.9026
+8600 val perplexity 366.0033
+8600 train 5.853130 (lr=2.2817e-05) (hash(x)=151300791)
+8700 val loss 5.7677
+8700 val perplexity 319.8089
+8700 train 5.769819 (lr=2.1085e-05) (hash(x)=148488790)
+8700 val loss 5.8968
+8700 val perplexity 363.8705
+8700 train 5.895819 (lr=2.2490e-05) (hash(x)=148488790)
+8800 val loss 5.7722
+8800 val perplexity 321.2419
+8800 train 5.793202 (lr=2.0778e-05) (hash(x)=156797847)
+8800 val loss 5.8967
+8800 val perplexity 363.8233
+8800 train 5.924081 (lr=2.2163e-05) (hash(x)=156797847)
+8900 val loss 5.7454
+8900 val perplexity 312.7444
+8900 train 5.563432 (lr=2.0472e-05) (hash(x)=143919631)
+8900 val loss 5.8758
+8900 val perplexity 356.3207
+8900 train 5.694329 (lr=2.1837e-05) (hash(x)=143919631)
+9000 val loss 5.7429
+9000 val perplexity 311.9829
+9000 train 5.697790 (lr=2.0165e-05) (hash(x)=148706835)
+9000 val loss 5.8706
+9000 val perplexity 354.4645
+9000 train 5.813400 (lr=2.1510e-05) (hash(x)=148706835)
+9100 val loss 5.7340
+9100 val perplexity 309.2187
+9100 train 5.746788 (lr=1.9859e-05) (hash(x)=153074259)
+9100 val loss 5.8633
+9100 val perplexity 351.8923
+9100 train 5.881595 (lr=2.1183e-05) (hash(x)=153074259)
+9200 val loss 5.7314
+9200 val perplexity 308.3887
+9200 train 5.648998 (lr=1.9553e-05) (hash(x)=151127077)
+9200 val loss 5.8608
+9200 val perplexity 351.0108
+9200 train 5.790888 (lr=2.0857e-05) (hash(x)=151127077)
+9300 val loss 5.7295
+9300 val perplexity 307.8009
+9300 train 5.615701 (lr=1.9248e-05) (hash(x)=152170602)
+9300 val loss 5.8557
+9300 val perplexity 349.2050
+9300 train 5.747551 (lr=2.0531e-05) (hash(x)=152170602)
+9400 val loss 5.7279
+9400 val perplexity 307.3193
+9400 train 5.753187 (lr=1.8942e-05) (hash(x)=153874803)
+9400 val loss 5.8605
+9400 val perplexity 350.9137
+9400 train 5.890156 (lr=2.0205e-05) (hash(x)=153874803)
+9500 val loss 5.7154
+9500 val perplexity 303.4984
+9500 train 5.708488 (lr=1.8638e-05) (hash(x)=157021298)
+9500 val loss 5.8374
+9500 val perplexity 342.8917
+9500 train 5.833831 (lr=1.9880e-05) (hash(x)=157021298)
+9600 val loss 5.7075
+9600 val perplexity 301.1231
+9600 train 5.551288 (lr=1.8334e-05) (hash(x)=135620494)
+9600 val loss 5.8302
+9600 val perplexity 340.4203
+9600 train 5.669508 (lr=1.9556e-05) (hash(x)=135620494)
+9700 val loss 5.7031
+9700 val perplexity 299.7987
+9700 train 5.422754 (lr=1.8031e-05) (hash(x)=142434463)
+9700 val loss 5.8272
+9700 val perplexity 339.4169
+9700 train 5.546177 (lr=1.9233e-05) (hash(x)=142434463)
+9800 val loss 5.7007
+9800 val perplexity 299.0790
+9800 train 5.567791 (lr=1.7728e-05) (hash(x)=149075541)
+9800 val loss 5.8215
+9800 val perplexity 337.4643
+9800 train 5.689050 (lr=1.8910e-05) (hash(x)=149075541)
+9900 val loss 5.6956
+9900 val perplexity 297.5502
+9900 train 5.564990 (lr=1.7427e-05) (hash(x)=141901425)
+9900 val loss 5.8226
+9900 val perplexity 337.8496
+9900 train 5.691305 (lr=1.8589e-05) (hash(x)=141901425)
+10000 val loss 5.7064
+10000 val perplexity 300.7795
+10000 train 5.710242 (lr=1.7126e-05) (hash(x)=133062699)
+10000 val loss 5.8309
+10000 val perplexity 340.6516
+10000 train 5.793503 (lr=1.8268e-05) (hash(x)=133062699)
+10100 val loss 5.6820
+10100 val perplexity 293.5367
+10100 train 5.713012 (lr=1.6827e-05) (hash(x)=157339196)
+10100 val loss 5.8057
+10100 val perplexity 332.1971
+10100 train 5.830470 (lr=1.7949e-05) (hash(x)=157339196)
+10200 val loss 5.6760
+10200 val perplexity 291.7882
+10200 train 5.457125 (lr=1.6529e-05) (hash(x)=131082121)
+10200 val loss 5.8028
+10200 val perplexity 331.2259
+10200 train 5.567506 (lr=1.7631e-05) (hash(x)=131082121)
+10300 val loss 5.6741
+10300 val perplexity 291.2323
+10300 train 5.780905 (lr=1.6233e-05) (hash(x)=144553757)
+10300 val loss 5.7975
+10300 val perplexity 329.4636
+10300 train 5.897439 (lr=1.7315e-05) (hash(x)=144553757)
+10400 val loss 5.6704
+10400 val perplexity 290.1580
+10400 train 5.475269 (lr=1.5938e-05) (hash(x)=143398861)
+10400 val loss 5.7914
+10400 val perplexity 327.4641
+10400 train 5.582107 (lr=1.7000e-05) (hash(x)=143398861)
+10500 val loss 5.6653
+10500 val perplexity 288.6734
+10500 train 5.667819 (lr=1.5644e-05) (hash(x)=157633809)
+10500 val loss 5.7876
+10500 val perplexity 326.2262
+10500 train 5.782300 (lr=1.6687e-05) (hash(x)=157633809)
+10600 val loss 5.6602
+10600 val perplexity 287.1956
+10600 train 5.656303 (lr=1.5352e-05) (hash(x)=147294315)
+10600 val loss 5.7819
+10600 val perplexity 324.3848
+10600 train 5.784723 (lr=1.6376e-05) (hash(x)=147294315)
+10700 val loss 5.6542
+10700 val perplexity 285.4958
+10700 train 5.770455 (lr=1.5062e-05) (hash(x)=157735028)
+10700 val loss 5.7786
+10700 val perplexity 323.2980
+10700 train 5.899136 (lr=1.6066e-05) (hash(x)=157735028)
+10800 val loss 5.6488
+10800 val perplexity 283.9470
+10800 train 5.618337 (lr=1.4773e-05) (hash(x)=149846113)
+10800 val loss 5.7691
+10800 val perplexity 320.2588
+10800 train 5.736881 (lr=1.5758e-05) (hash(x)=149846113)
+10900 val loss 5.6446
+10900 val perplexity 282.7607
+10900 train 6.028375 (lr=1.4487e-05) (hash(x)=157353556)
+10900 val loss 5.7660
+10900 val perplexity 319.2617
+10900 train 6.155986 (lr=1.5453e-05) (hash(x)=157353556)
+11000 val loss 5.6426
+11000 val perplexity 282.2033
+11000 train 5.940796 (lr=1.4203e-05) (hash(x)=160672627)
+11000 val loss 5.7646
+11000 val perplexity 318.8192
+11000 train 6.067262 (lr=1.5149e-05) (hash(x)=160672627)
+11100 val loss 5.6409
+11100 val perplexity 281.7201
+11100 train 5.861447 (lr=1.3920e-05) (hash(x)=166692335)
+11100 val loss 5.7601
+11100 val perplexity 317.3687
+11100 train 5.980080 (lr=1.4848e-05) (hash(x)=166692335)
+11200 val loss 5.6315
+11200 val perplexity 279.0858
+11200 train 5.687639 (lr=1.3640e-05) (hash(x)=158899869)
+11200 val loss 5.7544
+11200 val perplexity 315.5762
+11200 train 5.817999 (lr=1.4550e-05) (hash(x)=158899869)
+11300 val loss 5.6277
+11300 val perplexity 278.0259
+11300 train 5.896500 (lr=1.3362e-05) (hash(x)=166306828)
+11300 val loss 5.7467
+11300 val perplexity 313.1556
+11300 train 6.023516 (lr=1.4253e-05) (hash(x)=166306828)
+11400 val loss 5.6253
+11400 val perplexity 277.3638
+11400 train 5.544091 (lr=1.3087e-05) (hash(x)=149545840)
+11400 val loss 5.7425
+11400 val perplexity 311.8488
+11400 train 5.663081 (lr=1.3960e-05) (hash(x)=149545840)
+11500 val loss 5.6223
+11500 val perplexity 276.5126
+11500 train 5.603643 (lr=1.2814e-05) (hash(x)=139076910)
+11500 val loss 5.7426
+11500 val perplexity 311.8870
+11500 train 5.702015 (lr=1.3668e-05) (hash(x)=139076910)
+11600 val loss 5.6191
+11600 val perplexity 275.6353
+11600 train 5.457228 (lr=1.2544e-05) (hash(x)=151871279)
+11600 val loss 5.7380
+11600 val perplexity 310.4578
+11600 train 5.567941 (lr=1.3380e-05) (hash(x)=151871279)
+11700 val loss 5.6158
+11700 val perplexity 274.7234
+11700 train 5.563904 (lr=1.2276e-05) (hash(x)=152940196)
+11700 val loss 5.7336
+11700 val perplexity 309.0824
+11700 train 5.681150 (lr=1.3095e-05) (hash(x)=152940196)
+11800 val loss 5.6118
+11800 val perplexity 273.6394
+11800 train 5.697263 (lr=1.2011e-05) (hash(x)=150754129)
+11800 val loss 5.7298
+11800 val perplexity 307.9215
+11800 train 5.819008 (lr=1.2812e-05) (hash(x)=150754129)
+11900 val loss 5.6070
+11900 val perplexity 272.3301
+11900 train 5.570013 (lr=1.1749e-05) (hash(x)=148982597)
+11900 val loss 5.7252
+11900 val perplexity 306.4939
+11900 train 5.677073 (lr=1.2532e-05) (hash(x)=148982597)
+12000 val loss 5.6033
+12000 val perplexity 271.3333
+12000 train 5.634357 (lr=1.1490e-05) (hash(x)=167798390)
+12000 val loss 5.7203
+12000 val perplexity 305.0081
+12000 train 5.760694 (lr=1.2256e-05) (hash(x)=167798390)
+12100 val loss 5.6032
+12100 val perplexity 271.2964
+12100 train 5.685317 (lr=1.1234e-05) (hash(x)=146590810)
+12100 val loss 5.7247
+12100 val perplexity 306.3307
+12100 train 5.802352 (lr=1.1983e-05) (hash(x)=146590810)
+12200 val loss 5.5975
+12200 val perplexity 269.7564
+12200 train 5.834640 (lr=1.0981e-05) (hash(x)=159875780)
+12200 val loss 5.7183
+12200 val perplexity 304.3986
+12200 train 5.944863 (lr=1.1713e-05) (hash(x)=159875780)
+12300 val loss 5.5976
+12300 val perplexity 269.7828
+12300 train 5.475709 (lr=1.0731e-05) (hash(x)=135887861)
+12300 val loss 5.7144
+12300 val perplexity 303.1967
+12300 train 5.577959 (lr=1.1446e-05) (hash(x)=135887861)
+12400 val loss 5.5885
+12400 val perplexity 267.3324
+12400 train 5.658318 (lr=1.0484e-05) (hash(x)=150484221)
+12400 val loss 5.7083
+12400 val perplexity 301.3469
+12400 train 5.766685 (lr=1.1183e-05) (hash(x)=150484221)
+12500 val loss 5.5868
+12500 val perplexity 266.8819
+12500 train 5.594129 (lr=1.0241e-05) (hash(x)=157629455)
+12500 val loss 5.7041
+12500 val perplexity 300.0880
+12500 train 5.714652 (lr=1.0924e-05) (hash(x)=157629455)
+12600 val loss 5.5830
+12600 val perplexity 265.8598
+12600 train 5.483679 (lr=1.0001e-05) (hash(x)=147308692)
+12600 val loss 5.7032
+12600 val perplexity 299.8161
+12600 train 5.604730 (lr=1.0668e-05) (hash(x)=147308692)
+12700 val loss 5.5829
+12700 val perplexity 265.8332
+12700 train 5.424228 (lr=9.7649e-06) (hash(x)=144614722)
+12700 val loss 5.6991
+12700 val perplexity 298.6121
+12700 train 5.527828 (lr=1.0416e-05) (hash(x)=144614722)
+12800 val loss 5.5785
+12800 val perplexity 264.6794
+12800 train 5.373883 (lr=9.5322e-06) (hash(x)=137633606)
+12800 val loss 5.6981
+12800 val perplexity 298.2974
+12800 train 5.482248 (lr=1.0168e-05) (hash(x)=137633606)
+12900 val loss 5.5781
+12900 val perplexity 264.5634
+12900 train 5.610073 (lr=9.3031e-06) (hash(x)=155534772)
+12900 val loss 5.6962
+12900 val perplexity 297.7232
+13000 val loss 5.5729
+13000 val perplexity 263.2035
+12900 train 5.737048 (lr=9.9233e-06) (hash(x)=155534772)
+13000 train 5.583975 (lr=9.0777e-06) (hash(x)=148501470)
+13000 val loss 5.6919
+13000 val perplexity 296.4686
+13000 train 5.698692 (lr=9.6829e-06) (hash(x)=148501470)
+13100 val loss 5.5755
+13100 val perplexity 263.8708
+13100 train 5.618188 (lr=8.8562e-06) (hash(x)=154570618)
+13200 val loss 5.5710
+13200 val perplexity 262.7007
+13200 train 5.668798 (lr=8.6385e-06) (hash(x)=154153766)
+13100 val loss 5.6918
+13100 val perplexity 296.4400
+13100 train 5.738413 (lr=9.4466e-06) (hash(x)=154570618)
+13300 val loss 5.5658
+13300 val perplexity 261.3461
+13300 train 5.324534 (lr=8.4248e-06) (hash(x)=162760888)
+13200 val loss 5.6868
+13200 val perplexity 294.9540
+13200 train 5.783911 (lr=9.2144e-06) (hash(x)=154153766)
+13400 val loss 5.5625
+13400 val perplexity 260.4643
+13400 train 5.388410 (lr=8.2151e-06) (hash(x)=143223528)
+13300 val loss 5.6839
+13300 val perplexity 294.0870
+13300 train 5.475991 (lr=8.9865e-06) (hash(x)=162760888)
+13500 val loss 5.5673
+13500 val perplexity 261.7356
+13500 train 5.720043 (lr=8.0095e-06) (hash(x)=156416986)
+13400 val loss 5.6781
+13400 val perplexity 292.4062
+13400 train 5.499577 (lr=8.7628e-06) (hash(x)=143223528)
+13600 val loss 5.5630
+13600 val perplexity 260.6043
+13600 train 5.567687 (lr=7.8081e-06) (hash(x)=147521418)
+13500 val loss 5.6845
+13500 val perplexity 294.2740
+13500 train 5.829630 (lr=8.5435e-06) (hash(x)=156416986)
+13700 val loss 5.5587
+13700 val perplexity 259.4802
+13700 train 5.537975 (lr=7.6109e-06) (hash(x)=151127796)
+13600 val loss 5.6802
+13600 val perplexity 293.0212
+13600 train 5.681534 (lr=8.3286e-06) (hash(x)=147521418)
+13800 val loss 5.5565
+13800 val perplexity 258.9199
+13800 train 5.622880 (lr=7.4180e-06) (hash(x)=156863042)
+13700 val loss 5.6766
+13700 val perplexity 291.9636
+13700 train 5.645567 (lr=8.1183e-06) (hash(x)=151127796)
+13900 val loss 5.5517
+13900 val perplexity 257.6819
+13900 train 5.521236 (lr=7.2294e-06) (hash(x)=170986545)
+13800 val loss 5.6731
+13800 val perplexity 290.9450
+13800 train 5.732980 (lr=7.9125e-06) (hash(x)=156863042)
+14000 val loss 5.5492
+14000 val perplexity 257.0268
+14000 train 6.024418 (lr=7.0452e-06) (hash(x)=170744741)
+13900 val loss 5.6690
+13900 val perplexity 289.7548
+13900 train 5.673453 (lr=7.7114e-06) (hash(x)=170986545)
+14100 val loss 5.5483
+14100 val perplexity 256.8030
+14100 train 5.961041 (lr=6.8656e-06) (hash(x)=142328767)
+14000 val loss 5.6654
+14000 val perplexity 288.6993
+14000 train 6.136712 (lr=7.5149e-06) (hash(x)=170744741)
+14200 val loss 5.5467
+14200 val perplexity 256.3835
+14200 train 5.688292 (lr=6.6904e-06) (hash(x)=156141076)
+14100 val loss 5.6645
+14100 val perplexity 288.4526
+14100 train 6.060987 (lr=7.3233e-06) (hash(x)=142328767)
+14300 val loss 5.5492
+14300 val perplexity 257.0196
+14300 train 5.515243 (lr=6.5199e-06) (hash(x)=155447298)
+14200 val loss 5.6627
+14200 val perplexity 287.9176
+14200 train 5.813386 (lr=7.1365e-06) (hash(x)=156141076)
+14400 val loss 5.5428
+14400 val perplexity 255.3856
+14400 train 5.466012 (lr=6.3540e-06) (hash(x)=151991844)
+14300 val loss 5.6665
+14300 val perplexity 289.0223
+14300 train 5.643676 (lr=6.9546e-06) (hash(x)=155447298)
+14500 val loss 5.5391
+14500 val perplexity 254.4577
+14500 train 5.525082 (lr=6.1928e-06) (hash(x)=149773009)
+14400 val loss 5.6593
+14400 val perplexity 286.9611
+14400 train 5.591360 (lr=6.7776e-06) (hash(x)=151991844)
+14600 val loss 5.5368
+14600 val perplexity 253.8520
+14600 train 5.477211 (lr=6.0364e-06) (hash(x)=153836753)
+14500 val loss 5.6561
+14500 val perplexity 286.0430
+14500 train 5.631670 (lr=6.6057e-06) (hash(x)=149773009)
+14700 val loss 5.5346
+14700 val perplexity 253.2984
+14700 train 5.670206 (lr=5.8848e-06) (hash(x)=155466746)
+14600 val loss 5.6531
+14600 val perplexity 285.1770
+14600 train 5.608214 (lr=6.4389e-06) (hash(x)=153836753)
+14800 val loss 5.5342
+14800 val perplexity 253.2075
+14800 train 5.455852 (lr=5.7381e-06) (hash(x)=150110933)
+14700 val loss 5.6515
+14700 val perplexity 284.7285
+14700 train 5.781851 (lr=6.2771e-06) (hash(x)=155466746)
+14900 val loss 5.5353
+14900 val perplexity 253.4765
+14900 train 5.424331 (lr=5.5962e-06) (hash(x)=144558528)
+14800 val loss 5.6519
+14800 val perplexity 284.8269
+14800 train 5.579660 (lr=6.1206e-06) (hash(x)=150110933)
+15000 val loss 5.5324
+15000 val perplexity 252.7622
+15000 train 5.461942 (lr=5.4593e-06) (hash(x)=148315702)
+14900 val loss 5.6529
+14900 val perplexity 285.1064
+14900 train 5.540158 (lr=5.9693e-06) (hash(x)=144558528)
+15100 val loss 5.5290
+15100 val perplexity 251.8855
+15100 train 5.360836 (lr=5.3275e-06) (hash(x)=148452301)
+15000 val loss 5.6492
+15000 val perplexity 284.0755
+15000 train 5.591082 (lr=5.8233e-06) (hash(x)=148315702)
+15200 val loss 5.5284
+15200 val perplexity 251.7516
+15200 train 5.794895 (lr=5.2006e-06) (hash(x)=162181769)
+15100 val loss 5.6469
+15100 val perplexity 283.4043
+15100 train 5.490116 (lr=5.6826e-06) (hash(x)=148452301)
+15300 val loss 5.5254
+15300 val perplexity 250.9756
+15300 train 5.455128 (lr=5.0789e-06) (hash(x)=147256707)
+15200 val loss 5.6460
+15200 val perplexity 283.1549
+15200 train 5.907623 (lr=5.5473e-06) (hash(x)=162181769)
+15400 val loss 5.5246
+15400 val perplexity 250.7757
+15400 train 5.608875 (lr=4.9622e-06) (hash(x)=154947797)
+15300 val loss 5.6429
+15300 val perplexity 282.2780
+15300 train 5.572362 (lr=5.4175e-06) (hash(x)=147256707)
+15500 val loss 5.5235
+15500 val perplexity 250.5068
+15500 train 5.526404 (lr=4.8508e-06) (hash(x)=164200136)
+15400 val loss 5.6435
+15400 val perplexity 282.4595
+15400 train 5.747571 (lr=5.2931e-06) (hash(x)=154947797)
+15600 val loss 5.5229
+15600 val perplexity 250.3518
+15600 train 5.393709 (lr=4.7445e-06) (hash(x)=145086131)
+15500 val loss 5.6418
+15500 val perplexity 281.9762
+15500 train 5.676428 (lr=5.1742e-06) (hash(x)=164200136)
+15700 val loss 5.5191
+15700 val perplexity 249.4096
+15700 train 5.456976 (lr=4.6435e-06) (hash(x)=152070006)
+15600 val loss 5.6410
+15600 val perplexity 281.7361
+15600 train 5.505874 (lr=5.0608e-06) (hash(x)=145086131)
+15800 val loss 5.5173
+15800 val perplexity 248.9668
+15800 train 5.554524 (lr=4.5478e-06) (hash(x)=150969760)
+15700 val loss 5.6360
+15700 val perplexity 280.3287
+15700 train 5.572722 (lr=4.9531e-06) (hash(x)=152070006)
+15900 val loss 5.5166
+15900 val perplexity 248.7816
+15900 train 5.541933 (lr=4.4573e-06) (hash(x)=149495681)
+15800 val loss 5.6344
+15800 val perplexity 279.8850
+15800 train 5.664589 (lr=4.8509e-06) (hash(x)=150969760)
+16000 val loss 5.5158
+16000 val perplexity 248.5788
+16000 train 5.636119 (lr=4.3722e-06) (hash(x)=159383533)
+15900 val loss 5.6345
+15900 val perplexity 279.9112
+15900 train 5.652789 (lr=4.7545e-06) (hash(x)=149495681)
+16100 val loss 5.5158
+16100 val perplexity 248.5813
+16100 train 5.390795 (lr=4.2924e-06) (hash(x)=154365611)
+16000 val loss 5.6332
+16000 val perplexity 279.5606
+16000 train 5.748144 (lr=4.6637e-06) (hash(x)=159383533)
+16200 val loss 5.5139
+16200 val perplexity 248.1075
+16200 train 5.667452 (lr=4.2180e-06) (hash(x)=154284815)
+16100 val loss 5.6329
+16100 val perplexity 279.4662
+16100 train 5.498175 (lr=4.5786e-06) (hash(x)=154365611)
+16300 val loss 5.5127
+16300 val perplexity 247.8266
+16300 train 5.508811 (lr=4.1491e-06) (hash(x)=148416573)
+16200 val loss 5.6316
+16200 val perplexity 279.0958
+16200 train 5.793496 (lr=4.4993e-06) (hash(x)=154284815)
+16400 val loss 5.5107
+16400 val perplexity 247.3319
+16400 train 5.460778 (lr=4.0856e-06) (hash(x)=150447087)
+16300 val loss 5.6309
+16300 val perplexity 278.9182
+16300 train 5.620351 (lr=4.4257e-06) (hash(x)=148416573)
+16500 val loss 5.5105
+16500 val perplexity 247.2626
+16500 train 5.507597 (lr=4.0275e-06) (hash(x)=156112678)
+16400 val loss 5.6282
+16400 val perplexity 278.1597
+16400 train 5.578074 (lr=4.3579e-06) (hash(x)=150447087)
+16600 val loss 5.5083
+16600 val perplexity 246.7281
+16600 train 5.449524 (lr=3.9749e-06) (hash(x)=150449728)
+16500 val loss 5.6285
+16500 val perplexity 278.2431
+16500 train 5.620706 (lr=4.2960e-06) (hash(x)=156112678)
+16700 val loss 5.5066
+16700 val perplexity 246.3001
+16700 train 5.568088 (lr=3.9278e-06) (hash(x)=153905935)
+16600 val loss 5.6267
+16600 val perplexity 277.7373
+16600 train 5.564746 (lr=4.2399e-06) (hash(x)=150449728)
+16800 val loss 5.5031
+16800 val perplexity 245.4518
+16800 train 5.478307 (lr=3.8862e-06) (hash(x)=144086573)
+16700 val loss 5.6246
+16700 val perplexity 277.1668
+16700 train 5.682582 (lr=4.1896e-06) (hash(x)=153905935)
+16900 val loss 5.5039
+16900 val perplexity 245.6386
+16900 train 5.485327 (lr=3.8501e-06) (hash(x)=145863379)
+16800 val loss 5.6222
+16800 val perplexity 276.4983
+16800 train 5.592798 (lr=4.1452e-06) (hash(x)=144086573)
+17000 val loss 5.5038
+17000 val perplexity 245.6192
+17000 train 5.595400 (lr=3.8195e-06) (hash(x)=149054226)
+16900 val loss 5.6215
+16900 val perplexity 276.3130
+16900 train 5.600308 (lr=4.1067e-06) (hash(x)=145863379)
+17100 val loss 5.5031
+17100 val perplexity 245.4549
+17100 train 5.401844 (lr=3.7945e-06) (hash(x)=147109203)
+17000 val loss 5.6219
+17000 val perplexity 276.4100
+17000 train 5.686682 (lr=4.0741e-06) (hash(x)=149054226)
+17200 val loss 5.5015
+17200 val perplexity 245.0698
+17200 train 5.427037 (lr=3.7750e-06) (hash(x)=141837353)
+17100 val loss 5.6218
+17100 val perplexity 276.3966
+17100 train 5.527103 (lr=4.0475e-06) (hash(x)=147109203)
+17300 val loss 5.4995
+17300 val perplexity 244.5776
+17300 train 5.518600 (lr=3.7611e-06) (hash(x)=148542064)
+17200 val loss 5.6190
+17200 val perplexity 275.6230
+17200 train 5.551311 (lr=4.0267e-06) (hash(x)=141837353)
+17400 val loss 5.4973
+17400 val perplexity 244.0231
+17400 train 5.341631 (lr=3.7528e-06) (hash(x)=140939390)
+17300 val loss 5.6178
+17300 val perplexity 275.2757
+17300 train 5.632535 (lr=4.0119e-06) (hash(x)=148542064)
+17499 val loss 5.4967
+17499 val perplexity 243.8866
+17400 val loss 5.6163
+17400 val perplexity 274.8719
+17400 train 5.445556 (lr=4.0030e-06) (hash(x)=140939390)

attention_kindselective_n_heads4_seed1345/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9c03078ab35dc9ab5790c49b617751a42bea97336db9b64afcf4c778636c143
+oid sha256:77def304d5a4d5766a399670015f1443356b783f1e414d80d29ce3f454a56667
 size 92843394

attention_kindselective_n_heads4_seed1345/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b017c1973a1d488fb938785b8bf3b489ae838b210f3ebbe33c68b4339f71cfd
+oid sha256:cedb87afb700e6e774bf19f4eaa8abb5e27bc745f8f3d39a40ab734951ef407e
 size 179406214