Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_30720_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_30720_4_1340", "n_embd": 256}

attention_kindselective_n_heads4_seed1340/log2.txt

@@ -1,528 +1,1056 @@
 max_steps: 17500
+max_steps: 17500
+0 val loss 11.2714
+0 val perplexity 78545.6094
 0 val loss 11.2714
 0 val perplexity 78545.6094
-0 train 11.276529 (lr=1.7500e-07) (hash(x)=164406924)
-100 val loss 9.9266
-100 val perplexity 20467.4668
-100 train 9.971992 (lr=1.7675e-05) (hash(x)=177407419)
-200 val loss 8.9790
-200 val perplexity 7934.9438
-200 train 8.945872 (lr=3.5000e-05) (hash(x)=144903932)
-300 val loss 7.9072
-300 val perplexity 2716.8372
-300 train 8.167640 (lr=3.4997e-05) (hash(x)=173839165)
-400 val loss 7.6143
-400 val perplexity 2026.9836
-400 train 7.693684 (lr=3.4990e-05) (hash(x)=167734596)
-500 val loss 7.5204
-500 val perplexity 1845.3423
-500 train 7.531204 (lr=3.4977e-05) (hash(x)=153224076)
-600 val loss 7.4565
-600 val perplexity 1731.0846
-600 train 7.358781 (lr=3.4958e-05) (hash(x)=149619098)
-700 val loss 7.4099
-700 val perplexity 1652.3087
-700 train 7.336334 (lr=3.4935e-05) (hash(x)=146539909)
-800 val loss 7.3613
-800 val perplexity 1573.9480
-800 train 7.321278 (lr=3.4907e-05) (hash(x)=153710890)
-900 val loss 7.3258
-900 val perplexity 1519.0479
-900 train 7.265766 (lr=3.4873e-05) (hash(x)=155873620)
-1000 val loss 7.2840
-1000 val perplexity 1456.8236
-1000 train 7.201667 (lr=3.4834e-05) (hash(x)=145450636)
-1100 val loss 7.2368
-1100 val perplexity 1389.6337
-1100 train 7.307060 (lr=3.4790e-05) (hash(x)=154123388)
-1200 val loss 7.1929
-1200 val perplexity 1329.9440
-1200 train 7.065642 (lr=3.4741e-05) (hash(x)=145249251)
-1300 val loss 7.1508
-1300 val perplexity 1275.1411
-1300 train 7.040481 (lr=3.4687e-05) (hash(x)=148937127)
-1400 val loss 7.1070
-1400 val perplexity 1220.4199
-1400 train 7.223106 (lr=3.4628e-05) (hash(x)=150475545)
-1500 val loss 7.0655
-1500 val perplexity 1170.8129
-1500 train 6.988626 (lr=3.4563e-05) (hash(x)=154653428)
-1600 val loss 7.0237
-1600 val perplexity 1122.9769
-1600 train 7.005347 (lr=3.4494e-05) (hash(x)=144483776)
-1700 val loss 6.9803
-1700 val perplexity 1075.2378
-1700 train 7.135699 (lr=3.4419e-05) (hash(x)=157395496)
-1800 val loss 6.9478
-1800 val perplexity 1040.8788
-1800 train 6.965749 (lr=3.4340e-05) (hash(x)=157916369)
-1900 val loss 6.9103
-1900 val perplexity 1002.5468
-1900 train 7.111033 (lr=3.4255e-05) (hash(x)=166073923)
-2000 val loss 6.8849
-2000 val perplexity 977.3700
-2000 train 6.956383 (lr=3.4166e-05) (hash(x)=154856891)
-2100 val loss 6.8511
-2100 val perplexity 944.8918
-2100 train 6.848049 (lr=3.4072e-05) (hash(x)=151925203)
-2200 val loss 6.8133
-2200 val perplexity 909.8325
-2200 train 6.523357 (lr=3.3973e-05) (hash(x)=136191502)
-2300 val loss 6.7732
-2300 val perplexity 874.0762
-2300 train 6.939043 (lr=3.3869e-05) (hash(x)=153273362)
-2400 val loss 6.7307
-2400 val perplexity 837.6920
-2400 train 6.681937 (lr=3.3760e-05) (hash(x)=148021541)
-2500 val loss 6.7012
-2500 val perplexity 813.3631
-2500 train 6.649411 (lr=3.3646e-05) (hash(x)=141356608)
-2600 val loss 6.6608
-2600 val perplexity 781.1562
-2600 train 6.598667 (lr=3.3528e-05) (hash(x)=146005217)
-2700 val loss 6.6327
-2700 val perplexity 759.5594
-2700 train 6.472603 (lr=3.3405e-05) (hash(x)=144511718)
-2800 val loss 6.5913
-2800 val perplexity 728.7283
-2800 train 6.472763 (lr=3.3277e-05) (hash(x)=146019502)
-2900 val loss 6.5594
-2900 val perplexity 705.8724
-2900 train 6.474362 (lr=3.3144e-05) (hash(x)=146496200)
-3000 val loss 6.5411
-3000 val perplexity 693.0455
-3000 train 6.485529 (lr=3.3008e-05) (hash(x)=150127281)
-3100 val loss 6.5020
-3100 val perplexity 666.4407
-3100 train 6.481001 (lr=3.2866e-05) (hash(x)=142022255)
-3200 val loss 6.4705
-3200 val perplexity 645.7927
-3200 train 6.523926 (lr=3.2720e-05) (hash(x)=154120875)
-3300 val loss 6.4562
-3300 val perplexity 636.6580
-3300 train 6.525292 (lr=3.2570e-05) (hash(x)=153999717)
-3400 val loss 6.4321
-3400 val perplexity 621.4645
-3400 train 6.200135 (lr=3.2415e-05) (hash(x)=139694097)
-3500 val loss 6.4008
-3500 val perplexity 602.3418
-3500 train 6.595932 (lr=3.2256e-05) (hash(x)=162992732)
-3600 val loss 6.3711
-3600 val perplexity 584.6736
-3600 train 6.347825 (lr=3.2092e-05) (hash(x)=147574101)
-3700 val loss 6.3570
-3700 val perplexity 576.4910
-3700 train 6.464335 (lr=3.1924e-05) (hash(x)=157763099)
-3800 val loss 6.3282
-3800 val perplexity 560.1225
-3800 train 6.493472 (lr=3.1753e-05) (hash(x)=170800034)
-3900 val loss 6.3173
-3900 val perplexity 554.0912
-3900 train 6.395955 (lr=3.1577e-05) (hash(x)=164984528)
-4000 val loss 6.2971
-4000 val perplexity 542.9831
-4000 train 6.155862 (lr=3.1397e-05) (hash(x)=141743323)
-4100 val loss 6.2924
-4100 val perplexity 540.4485
-4100 train 6.310122 (lr=3.1212e-05) (hash(x)=153392872)
-4200 val loss 6.2781
-4200 val perplexity 532.7952
-4200 train 6.143710 (lr=3.1024e-05) (hash(x)=149074933)
-4300 val loss 6.2582
-4300 val perplexity 522.2766
-4300 train 6.652179 (lr=3.0833e-05) (hash(x)=167823423)
-4400 val loss 6.2320
-4400 val perplexity 508.7494
-4400 train 5.998601 (lr=3.0637e-05) (hash(x)=141203114)
-4500 val loss 6.2189
-4500 val perplexity 502.1626
-4500 train 6.219765 (lr=3.0437e-05) (hash(x)=146284780)
-4600 val loss 6.2007
-4600 val perplexity 493.1036
-4600 train 6.002645 (lr=3.0234e-05) (hash(x)=141126464)
-4700 val loss 6.1858
-4700 val perplexity 485.7832
-4700 train 6.216115 (lr=3.0027e-05) (hash(x)=154751926)
-4800 val loss 6.1708
-4800 val perplexity 478.5557
-4800 train 6.251617 (lr=2.9817e-05) (hash(x)=154793198)
-4900 val loss 6.1639
-4900 val perplexity 475.2658
-4900 train 5.881503 (lr=2.9603e-05) (hash(x)=139406392)
-5000 val loss 6.1723
-5000 val perplexity 479.2990
-5000 train 5.887846 (lr=2.9386e-05) (hash(x)=153548741)
-5100 val loss 6.1475
-5100 val perplexity 467.5700
-5100 train 6.233172 (lr=2.9166e-05) (hash(x)=160488568)
-5200 val loss 6.1319
-5200 val perplexity 460.3081
-5200 train 6.151352 (lr=2.8942e-05) (hash(x)=149645053)
-5300 val loss 6.1078
-5300 val perplexity 449.3437
-5300 train 6.208061 (lr=2.8715e-05) (hash(x)=155820556)
-5400 val loss 6.0966
-5400 val perplexity 444.3400
-5400 train 6.046060 (lr=2.8484e-05) (hash(x)=147538134)
-5500 val loss 6.0916
-5500 val perplexity 442.1204
-5500 train 6.301843 (lr=2.8251e-05) (hash(x)=166889307)
-5600 val loss 6.0740
-5600 val perplexity 434.4076
-5600 train 5.769093 (lr=2.8015e-05) (hash(x)=139516699)
-5700 val loss 6.0724
-5700 val perplexity 433.7223
-5700 train 5.755292 (lr=2.7776e-05) (hash(x)=140453511)
-5800 val loss 6.0626
-5800 val perplexity 429.4872
-5800 train 6.000001 (lr=2.7534e-05) (hash(x)=162964847)
-5900 val loss 6.0503
-5900 val perplexity 424.2379
-5900 train 6.053433 (lr=2.7290e-05) (hash(x)=150606634)
-6000 val loss 6.0302
-6000 val perplexity 415.8105
-6000 train 6.121263 (lr=2.7042e-05) (hash(x)=149890857)
-6100 val loss 6.0265
-6100 val perplexity 414.2556
-6100 train 6.125873 (lr=2.6792e-05) (hash(x)=173884145)
-6200 val loss 6.0170
-6200 val perplexity 410.3571
-6200 train 6.064613 (lr=2.6540e-05) (hash(x)=151987098)
-6300 val loss 6.0003
-6300 val perplexity 403.5415
-6300 train 5.954297 (lr=2.6285e-05) (hash(x)=148853562)
-6400 val loss 5.9930
-6400 val perplexity 400.5982
-6400 train 5.743896 (lr=2.6028e-05) (hash(x)=141530101)
-6500 val loss 5.9873
-6500 val perplexity 398.3281
-6500 train 5.854073 (lr=2.5769e-05) (hash(x)=142297809)
-6600 val loss 5.9702
-6600 val perplexity 391.5672
-6600 train 5.868578 (lr=2.5508e-05) (hash(x)=142447782)
-6700 val loss 5.9670
-6700 val perplexity 390.3195
-6700 train 5.964384 (lr=2.5244e-05) (hash(x)=147004686)
-6800 val loss 5.9555
-6800 val perplexity 385.8641
-6800 train 5.699266 (lr=2.4979e-05) (hash(x)=133438702)
-6900 val loss 5.9538
-6900 val perplexity 385.2148
-6900 train 5.983283 (lr=2.4711e-05) (hash(x)=157085143)
-7000 val loss 5.9415
-7000 val perplexity 380.5093
-7000 train 5.814411 (lr=2.4442e-05) (hash(x)=139437666)
-7100 val loss 5.9337
-7100 val perplexity 377.5412
-7100 train 5.945026 (lr=2.4171e-05) (hash(x)=159792986)
-7200 val loss 5.9151
-7200 val perplexity 370.6091
-7200 train 5.902672 (lr=2.3899e-05) (hash(x)=144930687)
-7300 val loss 5.9125
-7300 val perplexity 369.6437
-7300 train 5.960479 (lr=2.3625e-05) (hash(x)=156242690)
-7400 val loss 5.9014
-7400 val perplexity 365.5585
-7400 train 5.731031 (lr=2.3349e-05) (hash(x)=148183719)
-7500 val loss 5.8968
-7500 val perplexity 363.8860
-7500 train 5.943295 (lr=2.3073e-05) (hash(x)=152494758)
-7600 val loss 5.8931
-7600 val perplexity 362.5366
-7600 train 5.588400 (lr=2.2795e-05) (hash(x)=142485027)
-7700 val loss 5.8946
-7700 val perplexity 363.0880
-7700 train 5.764569 (lr=2.2515e-05) (hash(x)=147512165)
-7800 val loss 5.8700
-7800 val perplexity 354.2464
-7800 train 5.741778 (lr=2.2235e-05) (hash(x)=160346994)
-7900 val loss 5.8666
-7900 val perplexity 353.0454
-7900 train 5.738772 (lr=2.1954e-05) (hash(x)=144488254)
-8000 val loss 5.8550
-8000 val perplexity 348.9649
-8000 train 5.665582 (lr=2.1671e-05) (hash(x)=147637019)
-8100 val loss 5.8522
-8100 val perplexity 348.0111
-8100 train 5.713085 (lr=2.1388e-05) (hash(x)=147340534)
-8200 val loss 5.8435
-8200 val perplexity 344.9934
-8200 train 5.907599 (lr=2.1105e-05) (hash(x)=151630665)
-8300 val loss 5.8371
-8300 val perplexity 342.7936
-8300 train 5.941170 (lr=2.0820e-05) (hash(x)=149747064)
-8400 val loss 5.8255
-8400 val perplexity 338.8247
-8400 train 6.025146 (lr=2.0536e-05) (hash(x)=154245770)
-8500 val loss 5.8218
-8500 val perplexity 337.5670
-8500 train 5.710631 (lr=2.0250e-05) (hash(x)=152559100)
-8600 val loss 5.8137
-8600 val perplexity 334.8488
-8600 train 6.208880 (lr=1.9965e-05) (hash(x)=181365926)
-8700 val loss 5.8088
-8700 val perplexity 333.2325
-8700 train 5.632130 (lr=1.9679e-05) (hash(x)=154405991)
-8800 val loss 5.7990
-8800 val perplexity 329.9627
-8800 train 5.749090 (lr=1.9393e-05) (hash(x)=153755904)
-8900 val loss 5.7975
-8900 val perplexity 329.4634
-8900 train 5.708569 (lr=1.9107e-05) (hash(x)=152120568)
-9000 val loss 5.7785
-9000 val perplexity 323.2753
-9000 train 5.573270 (lr=1.8821e-05) (hash(x)=142797279)
-9100 val loss 5.7769
-9100 val perplexity 322.7535
-9100 train 5.646819 (lr=1.8535e-05) (hash(x)=143037503)
-9200 val loss 5.7753
-9200 val perplexity 322.2328
-9200 train 5.649633 (lr=1.8250e-05) (hash(x)=113690273)
-9300 val loss 5.7678
-9300 val perplexity 319.8191
-9300 train 5.708163 (lr=1.7964e-05) (hash(x)=158025077)
-9400 val loss 5.7595
-9400 val perplexity 317.1894
-9400 train 5.840967 (lr=1.7680e-05) (hash(x)=158251718)
-9500 val loss 5.7562
-9500 val perplexity 316.1416
-9500 train 5.786424 (lr=1.7395e-05) (hash(x)=154752610)
-9600 val loss 5.7477
-9600 val perplexity 313.4607
-9600 train 5.630883 (lr=1.7112e-05) (hash(x)=146889093)
-9700 val loss 5.7382
-9700 val perplexity 310.4960
-9700 train 5.741565 (lr=1.6829e-05) (hash(x)=156906516)
-9800 val loss 5.7365
-9800 val perplexity 309.9757
-9800 train 5.562198 (lr=1.6546e-05) (hash(x)=153841927)
-9900 val loss 5.7311
-9900 val perplexity 308.3038
-9900 train 5.949374 (lr=1.6265e-05) (hash(x)=163514334)
-10000 val loss 5.7238
-10000 val perplexity 306.0793
-10000 train 5.723799 (lr=1.5985e-05) (hash(x)=144745561)
-10100 val loss 5.7185
-10100 val perplexity 304.4456
-10100 train 5.935614 (lr=1.5705e-05) (hash(x)=161651149)
-10200 val loss 5.7124
-10200 val perplexity 302.5875
-10200 train 5.650381 (lr=1.5427e-05) (hash(x)=143667037)
-10300 val loss 5.7086
-10300 val perplexity 301.4551
-10300 train 5.606673 (lr=1.5151e-05) (hash(x)=151687262)
-10400 val loss 5.7048
-10400 val perplexity 300.2989
-10400 train 5.547319 (lr=1.4875e-05) (hash(x)=153694955)
-10500 val loss 5.7046
-10500 val perplexity 300.2592
-10500 train 5.655263 (lr=1.4601e-05) (hash(x)=150218029)
-10600 val loss 5.6985
-10600 val perplexity 298.4055
-10600 train 5.777823 (lr=1.4329e-05) (hash(x)=155093716)
-10700 val loss 5.6915
-10700 val perplexity 296.3455
-10700 train 5.739979 (lr=1.4058e-05) (hash(x)=151631831)
-10800 val loss 5.6844
-10800 val perplexity 294.2519
-10800 train 5.547766 (lr=1.3789e-05) (hash(x)=136465175)
-10900 val loss 5.6813
-10900 val perplexity 293.3297
-10900 train 5.622696 (lr=1.3521e-05) (hash(x)=154461514)
-11000 val loss 5.6797
-11000 val perplexity 292.8474
-11000 train 5.673146 (lr=1.3256e-05) (hash(x)=141877432)
-11100 val loss 5.6754
-11100 val perplexity 291.5960
-11100 train 5.466080 (lr=1.2992e-05) (hash(x)=139572894)
-11200 val loss 5.6679
-11200 val perplexity 289.4243
-11200 train 5.721173 (lr=1.2731e-05) (hash(x)=155591443)
-11300 val loss 5.6625
-11300 val perplexity 287.8601
-11300 train 5.691182 (lr=1.2472e-05) (hash(x)=153349136)
-11400 val loss 5.6604
-11400 val perplexity 287.2627
-11400 train 5.621408 (lr=1.2215e-05) (hash(x)=148109395)
-11500 val loss 5.6564
-11500 val perplexity 286.1266
-11500 train 5.653296 (lr=1.1960e-05) (hash(x)=153276620)
-11600 val loss 5.6545
-11600 val perplexity 285.5773
-11600 train 5.523118 (lr=1.1708e-05) (hash(x)=151416388)
-11700 val loss 5.6520
-11700 val perplexity 284.8488
-11700 train 5.519225 (lr=1.1458e-05) (hash(x)=143420927)
-11800 val loss 5.6470
-11800 val perplexity 283.4373
-11800 train 5.705198 (lr=1.1210e-05) (hash(x)=165023996)
-11900 val loss 5.6393
-11900 val perplexity 281.2567
-11900 train 5.543955 (lr=1.0966e-05) (hash(x)=144465076)
-12000 val loss 5.6378
-12000 val perplexity 280.8501
-12000 train 5.653812 (lr=1.0724e-05) (hash(x)=155991892)
-12100 val loss 5.6366
-12100 val perplexity 280.5099
-12100 train 5.683947 (lr=1.0485e-05) (hash(x)=150748844)
-12200 val loss 5.6320
-12200 val perplexity 279.2109
-12200 train 5.605118 (lr=1.0249e-05) (hash(x)=149712638)
-12300 val loss 5.6304
-12300 val perplexity 278.7651
-12300 train 5.462433 (lr=1.0016e-05) (hash(x)=140428230)
-12400 val loss 5.6273
-12400 val perplexity 277.9107
-12400 train 5.813768 (lr=9.7854e-06) (hash(x)=163614308)
-12500 val loss 5.6219
-12500 val perplexity 276.4029
-12500 train 5.587658 (lr=9.5583e-06) (hash(x)=153366153)
-12600 val loss 5.6200
-12600 val perplexity 275.8958
-12600 train 5.525670 (lr=9.3345e-06) (hash(x)=146832066)
-12700 val loss 5.6159
-12700 val perplexity 274.7627
-12700 train 5.789717 (lr=9.1139e-06) (hash(x)=154387414)
-12800 val loss 5.6145
-12800 val perplexity 274.3859
-12800 train 5.408749 (lr=8.8967e-06) (hash(x)=139465199)
-12900 val loss 5.6157
-12900 val perplexity 274.7184
-12900 train 5.331150 (lr=8.6829e-06) (hash(x)=143326374)
-13000 val loss 5.6096
-13000 val perplexity 273.0276
-13000 train 5.762860 (lr=8.4726e-06) (hash(x)=158817835)
-13100 val loss 5.6064
-13100 val perplexity 272.1617
-13100 train 5.488267 (lr=8.2658e-06) (hash(x)=142027874)
-13200 val loss 5.6051
-13200 val perplexity 271.8146
-13200 train 5.591190 (lr=8.0626e-06) (hash(x)=148497730)
-13300 val loss 5.6034
-13300 val perplexity 271.3353
-13300 train 5.507898 (lr=7.8632e-06) (hash(x)=148094633)
-13400 val loss 5.5998
-13400 val perplexity 270.3817
-13400 train 5.463295 (lr=7.6674e-06) (hash(x)=150548214)
-13500 val loss 5.5936
-13500 val perplexity 268.7015
-13500 train 5.709482 (lr=7.4756e-06) (hash(x)=152789725)
-13600 val loss 5.5917
-13600 val perplexity 268.1876
-13600 train 5.634488 (lr=7.2875e-06) (hash(x)=153220230)
-13700 val loss 5.5894
-13700 val perplexity 267.5828
-13700 train 5.616460 (lr=7.1035e-06) (hash(x)=147805699)
-13800 val loss 5.5893
-13800 val perplexity 267.5524
-13800 train 5.399392 (lr=6.9234e-06) (hash(x)=146267341)
-13900 val loss 5.5881
-13900 val perplexity 267.2294
-13900 train 5.423314 (lr=6.7474e-06) (hash(x)=139467501)
-14000 val loss 5.5856
-14000 val perplexity 266.5664
-14000 train 5.602534 (lr=6.5756e-06) (hash(x)=152185436)
-14100 val loss 5.5805
-14100 val perplexity 265.2093
-14100 train 5.709170 (lr=6.4079e-06) (hash(x)=176479139)
-14200 val loss 5.5764
-14200 val perplexity 264.1297
-14200 train 5.654222 (lr=6.2444e-06) (hash(x)=151052040)
-14300 val loss 5.5752
-14300 val perplexity 263.8138
-14300 train 5.522494 (lr=6.0852e-06) (hash(x)=149668097)
-14400 val loss 5.5749
-14400 val perplexity 263.7227
-14400 train 5.588047 (lr=5.9304e-06) (hash(x)=151761941)
-14500 val loss 5.5726
-14500 val perplexity 263.1170
-14500 train 5.530627 (lr=5.7800e-06) (hash(x)=149412009)
-14600 val loss 5.5700
-14600 val perplexity 262.4285
-14600 train 5.451919 (lr=5.6340e-06) (hash(x)=145048020)
-14700 val loss 5.5677
-14700 val perplexity 261.8238
-14700 train 5.520936 (lr=5.4925e-06) (hash(x)=141771666)
-14800 val loss 5.5643
-14800 val perplexity 260.9447
-14800 train 5.486890 (lr=5.3555e-06) (hash(x)=148759740)
-14900 val loss 5.5629
-14900 val perplexity 260.5715
-14900 train 5.491292 (lr=5.2231e-06) (hash(x)=146287032)
-15000 val loss 5.5606
-15000 val perplexity 259.9802
-15000 train 5.507795 (lr=5.0954e-06) (hash(x)=143519156)
-15100 val loss 5.5632
-15100 val perplexity 260.6473
-15100 train 5.548056 (lr=4.9723e-06) (hash(x)=151330688)
-15200 val loss 5.5655
-15200 val perplexity 261.2667
-15200 train 5.371873 (lr=4.8539e-06) (hash(x)=140380090)
-15300 val loss 5.5599
-15300 val perplexity 259.7881
-15300 train 5.470203 (lr=4.7403e-06) (hash(x)=152598160)
-15400 val loss 5.5553
-15400 val perplexity 258.5987
-15400 train 5.792345 (lr=4.6314e-06) (hash(x)=164136966)
-15500 val loss 5.5544
-15500 val perplexity 258.3689
-15500 train 5.751496 (lr=4.5274e-06) (hash(x)=158733990)
-15600 val loss 5.5519
-15600 val perplexity 257.7298
-15600 train 5.593985 (lr=4.4282e-06) (hash(x)=149149390)
-15700 val loss 5.5506
-15700 val perplexity 257.3911
-15700 train 5.507520 (lr=4.3339e-06) (hash(x)=146746394)
-15800 val loss 5.5500
-15800 val perplexity 257.2358
-15800 train 5.569361 (lr=4.2446e-06) (hash(x)=159405219)
-15900 val loss 5.5485
-15900 val perplexity 256.8521
-15900 train 5.591457 (lr=4.1601e-06) (hash(x)=154034029)
-16000 val loss 5.5467
-16000 val perplexity 256.3889
-16000 train 5.543062 (lr=4.0807e-06) (hash(x)=153634210)
-16100 val loss 5.5438
-16100 val perplexity 255.6543
-16100 train 5.479768 (lr=4.0063e-06) (hash(x)=155646721)
-16200 val loss 5.5435
-16200 val perplexity 255.5702
-16200 train 5.458189 (lr=3.9368e-06) (hash(x)=142461643)
-16300 val loss 5.5426
-16300 val perplexity 255.3481
-16300 train 5.459714 (lr=3.8725e-06) (hash(x)=141080038)
-16400 val loss 5.5416
-16400 val perplexity 255.0929
-16400 train 5.554860 (lr=3.8132e-06) (hash(x)=153744166)
-16500 val loss 5.5441
-16500 val perplexity 255.7334
-16500 train 5.367388 (lr=3.7590e-06) (hash(x)=149783517)
-16600 val loss 5.5463
-16600 val perplexity 256.2921
-16600 train 5.386395 (lr=3.7099e-06) (hash(x)=142659127)
-16700 val loss 5.5397
-16700 val perplexity 254.5953
-16700 train 5.616637 (lr=3.6659e-06) (hash(x)=149725921)
-16800 val loss 5.5367
-16800 val perplexity 253.8382
-16800 train 5.392825 (lr=3.6271e-06) (hash(x)=140678087)
-16900 val loss 5.5347
-16900 val perplexity 253.3222
-16900 train 5.555767 (lr=3.5934e-06) (hash(x)=146935413)
-17000 val loss 5.5331
-17000 val perplexity 252.9307
-17000 train 5.549460 (lr=3.5649e-06) (hash(x)=144408437)
-17100 val loss 5.5327
-17100 val perplexity 252.8303
-17100 train 5.365889 (lr=3.5415e-06) (hash(x)=149229905)
-17200 val loss 5.5338
-17200 val perplexity 253.1139
-17200 train 5.497445 (lr=3.5234e-06) (hash(x)=150471865)
-17300 val loss 5.5350
-17300 val perplexity 253.3969
-17300 train 5.428278 (lr=3.5104e-06) (hash(x)=155784916)
-17400 val loss 5.5377
-17400 val perplexity 254.0839
-17400 train 5.318845 (lr=3.5026e-06) (hash(x)=148482413)
-17499 val loss 5.5305
-17499 val perplexity 252.2734
+0 train 11.276529 (lr=2.0000e-07) (hash(x)=164406924)
+0 train 11.276529 (lr=1.8750e-07) (hash(x)=164406924)
+100 val loss 9.9069
+100 val perplexity 20067.5703
+100 val loss 9.9126
+100 val perplexity 20183.8613
+100 train 9.956517 (lr=2.0200e-05) (hash(x)=177407419)
+100 train 9.961933 (lr=1.8937e-05) (hash(x)=177407419)
+200 val loss 8.8168
+200 val perplexity 6746.3262
+200 val loss 8.6973
+200 val perplexity 5986.4463
+200 train 8.780914 (lr=4.0000e-05) (hash(x)=144903932)
+200 train 8.659005 (lr=3.7500e-05) (hash(x)=144903932)
+300 val loss 7.8751
+300 val perplexity 2630.9985
+300 train 8.148215 (lr=3.9997e-05) (hash(x)=173839165)
+300 val loss 7.7852
+300 val perplexity 2404.7244
+300 train 8.077199 (lr=3.7497e-05) (hash(x)=173839165)
+400 val loss 7.6065
+400 val perplexity 2011.1913
+400 train 7.687731 (lr=3.9988e-05) (hash(x)=167734596)
+400 val loss 7.5757
+400 val perplexity 1950.2804
+400 train 7.660161 (lr=3.7489e-05) (hash(x)=167734596)
+500 val loss 7.4979
+500 val perplexity 1804.2780
+500 train 7.503215 (lr=3.9973e-05) (hash(x)=153224076)
+500 val loss 7.4857
+500 val perplexity 1782.4413
+500 train 7.490746 (lr=3.7475e-05) (hash(x)=153224076)
+600 val loss 7.4185
+600 val perplexity 1666.4873
+600 train 7.325118 (lr=3.9953e-05) (hash(x)=149619098)
+600 val loss 7.3959
+600 val perplexity 1629.3384
+600 train 7.295888 (lr=3.7456e-05) (hash(x)=149619098)
+700 val loss 7.3749
+700 val perplexity 1595.4503
+700 train 7.299428 (lr=3.9926e-05) (hash(x)=146539909)
+700 val loss 7.3358
+700 val perplexity 1534.2030
+700 train 7.257471 (lr=3.7430e-05) (hash(x)=146539909)
+800 val loss 7.3102
+800 val perplexity 1495.4886
+800 train 7.267064 (lr=3.9893e-05) (hash(x)=153710890)
+800 val loss 7.2643
+800 val perplexity 1428.4465
+800 train 7.219354 (lr=3.7400e-05) (hash(x)=153710890)
+900 val loss 7.2638
+900 val perplexity 1427.6083
+900 train 7.201191 (lr=3.9855e-05) (hash(x)=155873620)
+900 val loss 7.2092
+900 val perplexity 1351.8445
+900 train 7.146883 (lr=3.7364e-05) (hash(x)=155873620)
+1000 val loss 7.2084
+1000 val perplexity 1350.6891
+1000 train 7.128109 (lr=3.9810e-05) (hash(x)=145450636)
+1000 val loss 7.1526
+1000 val perplexity 1277.4160
+1000 train 7.070847 (lr=3.7322e-05) (hash(x)=145450636)
+1100 val loss 7.1457
+1100 val perplexity 1268.6505
+1100 train 7.228844 (lr=3.9760e-05) (hash(x)=154123388)
+1100 val loss 7.0944
+1100 val perplexity 1205.1863
+1100 train 7.183575 (lr=3.7275e-05) (hash(x)=154123388)
+1200 val loss 7.0836
+1200 val perplexity 1192.2507
+1200 train 6.958751 (lr=3.9704e-05) (hash(x)=145249251)
+1200 val loss 7.0348
+1200 val perplexity 1135.4495
+1200 train 6.909027 (lr=3.7223e-05) (hash(x)=145249251)
+1300 val loss 7.0302
+1300 val perplexity 1130.2454
+1300 train 6.915065 (lr=3.9642e-05) (hash(x)=148937127)
+1300 val loss 6.9805
+1300 val perplexity 1075.4382
+1300 train 6.872147 (lr=3.7164e-05) (hash(x)=148937127)
+1400 val loss 6.9896
+1400 val perplexity 1085.2570
+1400 train 7.110299 (lr=3.9574e-05) (hash(x)=150475545)
+1400 val loss 6.9340
+1400 val perplexity 1026.5917
+1400 train 7.049437 (lr=3.7101e-05) (hash(x)=150475545)
+1500 val loss 6.9470
+1500 val perplexity 1040.0145
+1500 train 6.867367 (lr=3.9501e-05) (hash(x)=154653428)
+1500 val loss 6.8934
+1500 val perplexity 985.7771
+1500 train 6.817115 (lr=3.7032e-05) (hash(x)=154653428)
+1600 val loss 6.8915
+1600 val perplexity 983.8972
+1600 train 6.870449 (lr=3.9421e-05) (hash(x)=144483776)
+1600 val loss 6.8501
+1600 val perplexity 944.0123
+1600 train 6.829629 (lr=3.6958e-05) (hash(x)=144483776)
+1700 val loss 6.8386
+1700 val perplexity 933.2001
+1700 train 6.987010 (lr=3.9336e-05) (hash(x)=157395496)
+1700 val loss 6.7947
+1700 val perplexity 893.0623
+1700 train 6.938875 (lr=3.6878e-05) (hash(x)=157395496)
+1800 val loss 6.7901
+1800 val perplexity 889.0110
+1800 train 6.816566 (lr=3.9246e-05) (hash(x)=157916369)
+1800 val loss 6.7504
+1800 val perplexity 854.4164
+1800 train 6.777050 (lr=3.6793e-05) (hash(x)=157916369)
+1900 val loss 6.7361
+1900 val perplexity 842.2986
+1900 train 6.936520 (lr=3.9149e-05) (hash(x)=166073923)
+1900 val loss 6.7043
+1900 val perplexity 815.8821
+1900 train 6.893992 (lr=3.6702e-05) (hash(x)=166073923)
+2000 val loss 6.6925
+2000 val perplexity 806.3372
+2000 train 6.758599 (lr=3.9047e-05) (hash(x)=154856891)
+2000 val loss 6.6664
+2000 val perplexity 785.5396
+2000 train 6.720447 (lr=3.6606e-05) (hash(x)=154856891)
+2100 val loss 6.6506
+2100 val perplexity 773.2477
+2100 train 6.643264 (lr=3.8939e-05) (hash(x)=151925203)
+2100 val loss 6.6219
+2100 val perplexity 751.3624
+2100 train 6.611493 (lr=3.6505e-05) (hash(x)=151925203)
+2200 val loss 6.6021
+2200 val perplexity 736.6467
+2200 train 6.285050 (lr=3.8826e-05) (hash(x)=136191502)
+2200 val loss 6.5854
+2200 val perplexity 724.4288
+2200 train 6.276153 (lr=3.6399e-05) (hash(x)=136191502)
+2300 val loss 6.5581
+2300 val perplexity 704.9544
+2300 train 6.756863 (lr=3.8707e-05) (hash(x)=153273362)
+2300 val loss 6.5412
+2300 val perplexity 693.0952
+2300 train 6.731654 (lr=3.6288e-05) (hash(x)=153273362)
+2400 val loss 6.5256
+2400 val perplexity 682.4135
+2400 train 6.483434 (lr=3.8583e-05) (hash(x)=148021541)
+2400 val loss 6.5070
+2400 val perplexity 669.8347
+2400 train 6.463409 (lr=3.6171e-05) (hash(x)=148021541)
+2500 val loss 6.5038
+2500 val perplexity 667.6447
+2500 train 6.450426 (lr=3.8453e-05) (hash(x)=141356608)
+2500 val loss 6.4819
+2500 val perplexity 653.2144
+2500 train 6.429843 (lr=3.6049e-05) (hash(x)=141356608)
+2600 val loss 6.4649
+2600 val perplexity 642.2280
+2600 train 6.408832 (lr=3.8317e-05) (hash(x)=146005217)
+2600 val loss 6.4501
+2600 val perplexity 632.7565
+2600 train 6.391536 (lr=3.5923e-05) (hash(x)=146005217)
+2700 val loss 6.4427
+2700 val perplexity 628.1240
+2700 train 6.285963 (lr=3.8177e-05) (hash(x)=144511718)
+2700 val loss 6.4316
+2700 val perplexity 621.1712
+2700 train 6.268011 (lr=3.5791e-05) (hash(x)=144511718)
+2800 val loss 6.4141
+2800 val perplexity 610.4131
+2800 train 6.292646 (lr=3.8031e-05) (hash(x)=146019502)
+2800 val loss 6.4099
+2800 val perplexity 607.8426
+2800 train 6.285739 (lr=3.5654e-05) (hash(x)=146019502)
+2900 val loss 6.3908
+2900 val perplexity 596.3517
+2900 train 6.300733 (lr=3.7879e-05) (hash(x)=146496200)
+2900 val loss 6.3863
+2900 val perplexity 593.6297
+2900 train 6.295756 (lr=3.5512e-05) (hash(x)=146496200)
+3000 val loss 6.3779
+3000 val perplexity 588.6973
+3000 train 6.317955 (lr=3.7723e-05) (hash(x)=150127281)
+3000 val loss 6.3716
+3000 val perplexity 585.0152
+3000 train 6.306818 (lr=3.5365e-05) (hash(x)=150127281)
+3100 val loss 6.3459
+3100 val perplexity 570.1234
+3100 train 6.339302 (lr=3.7561e-05) (hash(x)=142022255)
+3100 val loss 6.3427
+3100 val perplexity 568.3057
+3100 train 6.337442 (lr=3.5214e-05) (hash(x)=142022255)
+3200 val loss 6.3191
+3200 val perplexity 555.0615
+3200 train 6.364826 (lr=3.7394e-05) (hash(x)=154120875)
+3200 val loss 6.3214
+3200 val perplexity 556.3721
+3200 train 6.368715 (lr=3.5057e-05) (hash(x)=154120875)
+3300 val loss 6.3082
+3300 val perplexity 549.0667
+3300 train 6.397854 (lr=3.7222e-05) (hash(x)=153999717)
+3300 val loss 6.3112
+3300 val perplexity 550.6960
+3300 train 6.402353 (lr=3.4896e-05) (hash(x)=153999717)
+3400 val loss 6.2924
+3400 val perplexity 540.4279
+3400 train 6.061687 (lr=3.7045e-05) (hash(x)=139694097)
+3400 val loss 6.2955
+3400 val perplexity 542.1262
+3400 train 6.063921 (lr=3.4730e-05) (hash(x)=139694097)
+3500 val loss 6.2640
+3500 val perplexity 525.3035
+3500 train 6.456380 (lr=3.6864e-05) (hash(x)=162992732)
+3500 val loss 6.2708
+3500 val perplexity 528.8883
+3500 train 6.458117 (lr=3.4560e-05) (hash(x)=162992732)
+3600 val loss 6.2387
+3600 val perplexity 512.1668
+3600 train 6.220966 (lr=3.6677e-05) (hash(x)=147574101)
+3600 val loss 6.2449
+3600 val perplexity 515.3562
+3600 train 6.221808 (lr=3.4384e-05) (hash(x)=147574101)
+3700 val loss 6.2285
+3700 val perplexity 506.9707
+3700 train 6.337354 (lr=3.6485e-05) (hash(x)=157763099)
+3700 val loss 6.2352
+3700 val perplexity 510.4246
+3700 train 6.346001 (lr=3.4205e-05) (hash(x)=157763099)
+3800 val loss 6.2041
+3800 val perplexity 494.7653
+3800 train 6.358107 (lr=3.6289e-05) (hash(x)=170800034)
+3800 val loss 6.2118
+3800 val perplexity 498.5732
+3800 train 6.369606 (lr=3.4021e-05) (hash(x)=170800034)
+3900 val loss 6.1864
+3900 val perplexity 486.0738
+3900 train 6.273180 (lr=3.6088e-05) (hash(x)=164984528)
+3900 val loss 6.1991
+3900 val perplexity 492.2966
+3900 train 6.281530 (lr=3.3832e-05) (hash(x)=164984528)
+4000 val loss 6.1776
+4000 val perplexity 481.8339
+4000 train 6.032842 (lr=3.5882e-05) (hash(x)=141743323)
+4000 val loss 6.1853
+4000 val perplexity 485.5659
+4000 train 6.039346 (lr=3.3639e-05) (hash(x)=141743323)
+4100 val loss 6.1726
+4100 val perplexity 479.4245
+4100 train 6.199562 (lr=3.5671e-05) (hash(x)=153392872)
+4100 val loss 6.1791
+4100 val perplexity 482.5476
+4100 train 6.203568 (lr=3.3442e-05) (hash(x)=153392872)
+4200 val loss 6.1605
+4200 val perplexity 473.6565
+4200 train 6.026575 (lr=3.5457e-05) (hash(x)=149074933)
+4200 val loss 6.1702
+4200 val perplexity 478.2826
+4200 train 6.035712 (lr=3.3240e-05) (hash(x)=149074933)
+4300 val loss 6.1363
+4300 val perplexity 462.3177
+4300 train 6.520905 (lr=3.5237e-05) (hash(x)=167823423)
+4300 val loss 6.1484
+4300 val perplexity 467.9566
+4300 train 6.537424 (lr=3.3035e-05) (hash(x)=167823423)
+4400 val loss 6.1232
+4400 val perplexity 456.3228
+4400 train 5.897516 (lr=3.5014e-05) (hash(x)=141203114)
+4400 val loss 6.1266
+4400 val perplexity 457.8788
+4400 train 5.894150 (lr=3.2825e-05) (hash(x)=141203114)
+4500 val loss 6.1109
+4500 val perplexity 450.7238
+4500 train 6.109667 (lr=3.4786e-05) (hash(x)=146284780)
+4500 val loss 6.1169
+4500 val perplexity 453.4737
+4500 train 6.115213 (lr=3.2611e-05) (hash(x)=146284780)
+4600 val loss 6.0856
+4600 val perplexity 439.4913
+4600 train 5.886594 (lr=3.4553e-05) (hash(x)=141126464)
+4600 val loss 6.0963
+4600 val perplexity 444.2163
+4600 train 5.898219 (lr=3.2394e-05) (hash(x)=141126464)
+4700 val loss 6.0757
+4700 val perplexity 435.1674
+4700 train 6.099052 (lr=3.4317e-05) (hash(x)=154751926)
+4700 val loss 6.0886
+4700 val perplexity 440.7993
+4700 train 6.112273 (lr=3.2172e-05) (hash(x)=154751926)
+4800 val loss 6.0661
+4800 val perplexity 430.9826
+4800 train 6.153398 (lr=3.4077e-05) (hash(x)=154793198)
+4800 val loss 6.0750
+4800 val perplexity 434.8571
+4800 train 6.158714 (lr=3.1947e-05) (hash(x)=154793198)
+4900 val loss 6.0632
+4900 val perplexity 429.7419
+4900 train 5.780401 (lr=3.3832e-05) (hash(x)=139406392)
+4900 val loss 6.0728
+4900 val perplexity 433.8838
+4900 train 5.785733 (lr=3.1718e-05) (hash(x)=139406392)
+5000 val loss 6.0692
+5000 val perplexity 432.3254
+5000 train 5.769174 (lr=3.3584e-05) (hash(x)=153548741)
+5000 val loss 6.0797
+5000 val perplexity 436.9094
+5000 train 5.788754 (lr=3.1485e-05) (hash(x)=153548741)
+5100 val loss 6.0468
+5100 val perplexity 422.7672
+5100 train 6.134361 (lr=3.3332e-05) (hash(x)=160488568)
+5100 val loss 6.0504
+5100 val perplexity 424.3024
+5100 train 6.131473 (lr=3.1249e-05) (hash(x)=160488568)
+5200 val loss 6.0239
+5200 val perplexity 413.1979
+5200 train 6.042854 (lr=3.3076e-05) (hash(x)=149645053)
+5200 val loss 6.0349
+5200 val perplexity 417.7651
+5200 train 6.052381 (lr=3.1009e-05) (hash(x)=149645053)
+5300 val loss 6.0072
+5300 val perplexity 406.3470
+5300 train 6.109636 (lr=3.2817e-05) (hash(x)=155820556)
+5300 val loss 6.0128
+5300 val perplexity 408.6330
+5300 train 6.118703 (lr=3.0766e-05) (hash(x)=155820556)
+5400 val loss 6.0016
+5400 val perplexity 404.0920
+5400 train 5.956941 (lr=3.2554e-05) (hash(x)=147538134)
+5400 val loss 6.0048
+5400 val perplexity 405.3886
+5400 train 5.966078 (lr=3.0519e-05) (hash(x)=147538134)
+5500 val loss 5.9913
+5500 val perplexity 399.9270
+5500 val loss 5.9992
+5500 val perplexity 403.1011
+5500 train 6.199021 (lr=3.2287e-05) (hash(x)=166889307)
+5500 train 6.212953 (lr=3.0269e-05) (hash(x)=166889307)
+5600 val loss 5.9801
+5600 val perplexity 395.4617
+5600 val loss 5.9860
+5600 val perplexity 397.8206
+5600 train 5.673012 (lr=3.2017e-05) (hash(x)=139516699)
+5600 train 5.682027 (lr=3.0016e-05) (hash(x)=139516699)
+5700 val loss 5.9813
+5700 val perplexity 395.9561
+5700 val loss 5.9841
+5700 val perplexity 397.0557
+5700 train 5.668679 (lr=3.1744e-05) (hash(x)=140453511)
+5700 train 5.673424 (lr=2.9760e-05) (hash(x)=140453511)
+5800 val loss 5.9834
+5800 val perplexity 396.7742
+5800 val loss 5.9762
+5800 val perplexity 393.9481
+5800 train 5.919316 (lr=2.9501e-05) (hash(x)=162964847)
+5800 train 5.915136 (lr=3.1468e-05) (hash(x)=162964847)
+5900 val loss 5.9686
+5900 val perplexity 390.9478
+5900 val loss 5.9588
+5900 val perplexity 387.1354
+5900 train 5.979049 (lr=2.9239e-05) (hash(x)=150606634)
+5900 train 5.968227 (lr=3.1188e-05) (hash(x)=150606634)
+6000 val loss 5.9452
+6000 val perplexity 381.9059
+6000 val loss 5.9400
+6000 val perplexity 379.9167
+6000 train 6.022529 (lr=2.8974e-05) (hash(x)=149890857)
+6000 train 6.016193 (lr=3.0905e-05) (hash(x)=149890857)
+6100 val loss 5.9463
+6100 val perplexity 382.3314
+6100 val loss 5.9378
+6100 val perplexity 379.1104
+6100 train 6.042576 (lr=2.8706e-05) (hash(x)=173884145)
+6100 train 6.024965 (lr=3.0620e-05) (hash(x)=173884145)
+6200 val loss 5.9322
+6200 val perplexity 376.9831
+6200 train 5.981599 (lr=2.8436e-05) (hash(x)=151987098)
+6200 val loss 5.9273
+6200 val perplexity 375.1512
+6200 train 5.973155 (lr=3.0332e-05) (hash(x)=151987098)
+6300 val loss 5.9248
+6300 val perplexity 374.1998
+6300 train 5.873348 (lr=2.8163e-05) (hash(x)=148853562)
+6300 val loss 5.9169
+6300 val perplexity 371.2656
+6300 train 5.872821 (lr=3.0040e-05) (hash(x)=148853562)
+6400 val loss 5.9075
+6400 val perplexity 367.7746
+6400 train 5.657629 (lr=2.7888e-05) (hash(x)=141530101)
+6400 val loss 5.9059
+6400 val perplexity 367.2035
+6400 train 5.655657 (lr=2.9747e-05) (hash(x)=141530101)
+6500 val loss 5.9027
+6500 val perplexity 366.0108
+6500 train 5.775737 (lr=2.7610e-05) (hash(x)=142297809)
+6500 val loss 5.8991
+6500 val perplexity 364.7047
+6500 train 5.765725 (lr=2.9450e-05) (hash(x)=142297809)
+6600 val loss 5.8898
+6600 val perplexity 361.3448
+6600 train 5.793635 (lr=2.7330e-05) (hash(x)=142447782)
+6600 val loss 5.8855
+6600 val perplexity 359.7716
+6600 train 5.786078 (lr=2.9152e-05) (hash(x)=142447782)
+6700 val loss 5.8867
+6700 val perplexity 360.1985
+6700 train 5.903491 (lr=2.7047e-05) (hash(x)=147004686)
+6700 val loss 5.8801
+6700 val perplexity 357.8412
+6700 train 5.886856 (lr=2.8851e-05) (hash(x)=147004686)
+6800 val loss 5.8747
+6800 val perplexity 355.9338
+6800 train 5.617948 (lr=2.6763e-05) (hash(x)=133438702)
+6800 val loss 5.8695
+6800 val perplexity 354.0645
+6800 train 5.611340 (lr=2.8547e-05) (hash(x)=133438702)
+6900 val loss 5.8742
+6900 val perplexity 355.7494
+6900 train 5.898749 (lr=2.6477e-05) (hash(x)=157085143)
+6900 val loss 5.8642
+6900 val perplexity 352.1968
+6900 train 5.878542 (lr=2.8242e-05) (hash(x)=157085143)
+7000 val loss 5.8634
+7000 val perplexity 351.9237
+7000 train 5.731717 (lr=2.6188e-05) (hash(x)=139437666)
+7000 val loss 5.8582
+7000 val perplexity 350.1105
+7000 train 5.736189 (lr=2.7934e-05) (hash(x)=139437666)
+7100 val loss 5.8561
+7100 val perplexity 349.3535
+7100 train 5.863669 (lr=2.5898e-05) (hash(x)=159792986)
+7100 val loss 5.8547
+7100 val perplexity 348.8631
+7100 train 5.865206 (lr=2.7624e-05) (hash(x)=159792986)
+7200 val loss 5.8377
+7200 val perplexity 343.0039
+7200 train 5.823919 (lr=2.5606e-05) (hash(x)=144930687)
+7200 val loss 5.8343
+7200 val perplexity 341.8415
+7200 train 5.824814 (lr=2.7313e-05) (hash(x)=144930687)
+7300 val loss 5.8308
+7300 val perplexity 340.6451
+7300 train 5.878428 (lr=2.5312e-05) (hash(x)=156242690)
+7300 val loss 5.8277
+7300 val perplexity 339.5932
+7300 train 5.873379 (lr=2.7000e-05) (hash(x)=156242690)
+7400 val loss 5.8255
+7400 val perplexity 338.8294
+7400 train 5.659531 (lr=2.5017e-05) (hash(x)=148183719)
+7400 val loss 5.8215
+7400 val perplexity 337.4912
+7400 train 5.649005 (lr=2.6685e-05) (hash(x)=148183719)
+7500 val loss 5.8227
+7500 val perplexity 337.8752
+7500 train 5.865096 (lr=2.4721e-05) (hash(x)=152494758)
+7500 val loss 5.8188
+7500 val perplexity 336.5761
+7500 train 5.859860 (lr=2.6369e-05) (hash(x)=152494758)
+7600 val loss 5.8145
+7600 val perplexity 335.1399
+7600 train 5.515101 (lr=2.4423e-05) (hash(x)=142485027)
+7600 val loss 5.8094
+7600 val perplexity 333.4160
+7600 train 5.511698 (lr=2.6051e-05) (hash(x)=142485027)
+7700 val loss 5.8187
+7700 val perplexity 336.5345
+7700 train 5.678254 (lr=2.4124e-05) (hash(x)=147512165)
+7700 val loss 5.8137
+7700 val perplexity 334.8448
+7700 train 5.674353 (lr=2.5732e-05) (hash(x)=147512165)
+7800 val loss 5.7926
+7800 val perplexity 327.8691
+7800 train 5.652011 (lr=2.3823e-05) (hash(x)=160346994)
+7800 val loss 5.7872
+7800 val perplexity 326.1078
+7800 train 5.648951 (lr=2.5411e-05) (hash(x)=160346994)
+7900 val loss 5.7913
+7900 val perplexity 327.4312
+7900 train 5.658368 (lr=2.3522e-05) (hash(x)=144488254)
+7900 val loss 5.7874
+7900 val perplexity 326.1503
+7900 train 5.653071 (lr=2.5090e-05) (hash(x)=144488254)
+8000 val loss 5.7809
+8000 val perplexity 324.0535
+8000 train 5.590876 (lr=2.3219e-05) (hash(x)=147637019)
+8000 val loss 5.7754
+8000 val perplexity 322.2737
+8000 train 5.578671 (lr=2.4767e-05) (hash(x)=147637019)
+8100 val loss 5.7795
+8100 val perplexity 323.5949
+8100 train 5.633208 (lr=2.2916e-05) (hash(x)=147340534)
+8100 val loss 5.7713
+8100 val perplexity 320.9568
+8100 train 5.619842 (lr=2.4444e-05) (hash(x)=147340534)
+8200 val loss 5.7721
+8200 val perplexity 321.2093
+8200 train 5.837082 (lr=2.2612e-05) (hash(x)=151630665)
+8200 val loss 5.7666
+8200 val perplexity 319.4461
+8200 train 5.828109 (lr=2.4120e-05) (hash(x)=151630665)
+8300 val loss 5.7654
+8300 val perplexity 319.0818
+8300 train 5.883111 (lr=2.2308e-05) (hash(x)=149747064)
+8300 val loss 5.7570
+8300 val perplexity 316.3929
+8300 train 5.867330 (lr=2.3795e-05) (hash(x)=149747064)
+8400 val loss 5.7532
+8400 val perplexity 315.2084
+8400 train 5.950338 (lr=2.2002e-05) (hash(x)=154245770)
+8400 val loss 5.7465
+8400 val perplexity 313.1074
+8400 train 5.948621 (lr=2.3469e-05) (hash(x)=154245770)
+8500 val loss 5.7487
+8500 val perplexity 313.7896
+8500 train 5.641779 (lr=2.1697e-05) (hash(x)=152559100)
+8500 val loss 5.7415
+8500 val perplexity 311.5444
+8500 train 5.632651 (lr=2.3143e-05) (hash(x)=152559100)
+8600 val loss 5.7416
+8600 val perplexity 311.5613
+8600 train 6.123217 (lr=2.1391e-05) (hash(x)=181365926)
+8600 val loss 5.7348
+8600 val perplexity 309.4497
+8600 train 6.199018 (lr=2.2817e-05) (hash(x)=181365926)
+8700 val loss 5.7373
+8700 val perplexity 310.2185
+8700 train 5.554934 (lr=2.1085e-05) (hash(x)=154405991)
+8700 val loss 5.7295
+8700 val perplexity 307.8279
+8700 train 5.545577 (lr=2.2490e-05) (hash(x)=154405991)
+8800 val loss 5.7307
+8800 val perplexity 308.1836
+8800 train 5.675156 (lr=2.0778e-05) (hash(x)=153755904)
+8800 val loss 5.7258
+8800 val perplexity 306.6717
+8800 train 5.667483 (lr=2.2163e-05) (hash(x)=153755904)
+8900 val loss 5.7295
+8900 val perplexity 307.8115
+8900 train 5.641292 (lr=2.0472e-05) (hash(x)=152120568)
+8900 val loss 5.7224
+8900 val perplexity 305.6432
+8900 train 5.628521 (lr=2.1837e-05) (hash(x)=152120568)
+9000 val loss 5.7100
+9000 val perplexity 301.8787
+9000 train 5.497505 (lr=2.0165e-05) (hash(x)=142797279)
+9000 val loss 5.7040
+9000 val perplexity 300.0584
+9000 train 5.486934 (lr=2.1510e-05) (hash(x)=142797279)
+9100 val loss 5.7074
+9100 val perplexity 301.0725
+9100 train 5.575232 (lr=1.9859e-05) (hash(x)=143037503)
+9100 val loss 5.7013
+9100 val perplexity 299.2558
+9100 train 5.568914 (lr=2.1183e-05) (hash(x)=143037503)
+9200 val loss 5.7056
+9200 val perplexity 300.5474
+9200 train 5.608702 (lr=1.9553e-05) (hash(x)=113690273)
+9200 val loss 5.7053
+9200 val perplexity 300.4619
+9200 train 5.586547 (lr=2.0857e-05) (hash(x)=113690273)
+9300 val loss 5.7022
+9300 val perplexity 299.5243
+9300 train 5.628353 (lr=1.9248e-05) (hash(x)=158025077)
+9300 val loss 5.6996
+9300 val perplexity 298.7454
+9300 train 5.619718 (lr=2.0531e-05) (hash(x)=158025077)
+9400 val loss 5.6929
+9400 val perplexity 296.7406
+9400 train 5.779068 (lr=1.8942e-05) (hash(x)=158251718)
+9400 val loss 5.6887
+9400 val perplexity 295.5107
+9400 train 5.765107 (lr=2.0205e-05) (hash(x)=158251718)
+9500 val loss 5.6903
+9500 val perplexity 295.9885
+9500 train 5.730006 (lr=1.8638e-05) (hash(x)=154752610)
+9500 val loss 5.6844
+9500 val perplexity 294.2540
+9500 train 5.725028 (lr=1.9880e-05) (hash(x)=154752610)
+9600 val loss 5.6897
+9600 val perplexity 295.8063
+9600 train 5.563871 (lr=1.8334e-05) (hash(x)=146889093)
+9600 val loss 5.6851
+9600 val perplexity 294.4477
+9600 train 5.557839 (lr=1.9556e-05) (hash(x)=146889093)
+9700 val loss 5.6745
+9700 val perplexity 291.3521
+9700 train 5.681897 (lr=1.8031e-05) (hash(x)=156906516)
+9700 val loss 5.6738
+9700 val perplexity 291.1449
+9700 train 5.681110 (lr=1.9233e-05) (hash(x)=156906516)
+9800 val loss 5.6750
+9800 val perplexity 291.4800
+9800 train 5.498330 (lr=1.7728e-05) (hash(x)=153841927)
+9800 val loss 5.6701
+9800 val perplexity 290.0769
+9800 train 5.493529 (lr=1.8910e-05) (hash(x)=153841927)
+9900 val loss 5.6684
+9900 val perplexity 289.5805
+9900 train 5.881992 (lr=1.7427e-05) (hash(x)=163514334)
+9900 val loss 5.6649
+9900 val perplexity 288.5633
+9900 train 5.892396 (lr=1.8589e-05) (hash(x)=163514334)
+10000 val loss 5.6602
+10000 val perplexity 287.2001
+10000 train 5.660645 (lr=1.7126e-05) (hash(x)=144745561)
+10000 val loss 5.6588
+10000 val perplexity 286.8079
+10000 train 5.661840 (lr=1.8268e-05) (hash(x)=144745561)
+10100 val loss 5.6585
+10100 val perplexity 286.7325
+10100 train 5.867241 (lr=1.6827e-05) (hash(x)=161651149)
+10100 val loss 5.6526
+10100 val perplexity 285.0405
+10100 train 5.874568 (lr=1.7949e-05) (hash(x)=161651149)
+10200 val loss 5.6528
+10200 val perplexity 285.0913
+10200 train 5.593501 (lr=1.6529e-05) (hash(x)=143667037)
+10200 val loss 5.6487
+10200 val perplexity 283.9085
+10200 train 5.592046 (lr=1.7631e-05) (hash(x)=143667037)
+10300 val loss 5.6470
+10300 val perplexity 283.4483
+10300 train 5.547844 (lr=1.6233e-05) (hash(x)=151687262)
+10300 val loss 5.6406
+10300 val perplexity 281.6277
+10300 train 5.535870 (lr=1.7315e-05) (hash(x)=151687262)
+10400 val loss 5.6433
+10400 val perplexity 282.4055
+10400 train 5.484274 (lr=1.5938e-05) (hash(x)=153694955)
+10400 val loss 5.6398
+10400 val perplexity 281.4023
+10400 train 5.489096 (lr=1.7000e-05) (hash(x)=153694955)
+10500 val loss 5.6450
+10500 val perplexity 282.8637
+10500 train 5.608533 (lr=1.5644e-05) (hash(x)=150218029)
+10500 val loss 5.6402
+10500 val perplexity 281.5293
+10500 train 5.590641 (lr=1.6687e-05) (hash(x)=150218029)
+10600 val loss 5.6364
+10600 val perplexity 280.4643
+10600 train 5.713663 (lr=1.5352e-05) (hash(x)=155093716)
+10600 val loss 5.6319
+10600 val perplexity 279.2037
+10600 train 5.725809 (lr=1.6376e-05) (hash(x)=155093716)
+10700 val loss 5.6334
+10700 val perplexity 279.6174
+10700 train 5.679636 (lr=1.5062e-05) (hash(x)=151631831)
+10700 val loss 5.6294
+10700 val perplexity 278.4958
+10700 train 5.668282 (lr=1.6066e-05) (hash(x)=151631831)
+10800 val loss 5.6237
+10800 val perplexity 276.9008
+10800 train 5.483255 (lr=1.4773e-05) (hash(x)=136465175)
+10800 val loss 5.6205
+10800 val perplexity 276.0296
+10800 train 5.474095 (lr=1.5758e-05) (hash(x)=136465175)
+10900 val loss 5.6199
+10900 val perplexity 275.8506
+10900 train 5.563373 (lr=1.4487e-05) (hash(x)=154461514)
+10900 val loss 5.6162
+10900 val perplexity 274.8297
+10900 train 5.551686 (lr=1.5453e-05) (hash(x)=154461514)
+11000 val loss 5.6188
+11000 val perplexity 275.5578
+11000 train 5.614881 (lr=1.4203e-05) (hash(x)=141877432)
+11000 val loss 5.6164
+11000 val perplexity 274.8996
+11000 train 5.623027 (lr=1.5149e-05) (hash(x)=141877432)
+11100 val loss 5.6162
+11100 val perplexity 274.8346
+11100 train 5.416177 (lr=1.3920e-05) (hash(x)=139572894)
+11100 val loss 5.6148
+11100 val perplexity 274.4604
+11100 train 5.399756 (lr=1.4848e-05) (hash(x)=139572894)
+11200 val loss 5.6093
+11200 val perplexity 272.9595
+11200 train 5.661036 (lr=1.3640e-05) (hash(x)=155591443)
+11200 val loss 5.6064
+11200 val perplexity 272.1530
+11200 train 5.648377 (lr=1.4550e-05) (hash(x)=155591443)
+11300 val loss 5.6051
+11300 val perplexity 271.8068
+11300 train 5.629344 (lr=1.3362e-05) (hash(x)=153349136)
+11300 val loss 5.6002
+11300 val perplexity 270.4747
+11300 train 5.617316 (lr=1.4253e-05) (hash(x)=153349136)
+11400 val loss 5.6023
+11400 val perplexity 271.0404
+11400 train 5.567253 (lr=1.3087e-05) (hash(x)=148109395)
+11400 val loss 5.5999
+11400 val perplexity 270.3969
+11400 train 5.561288 (lr=1.3960e-05) (hash(x)=148109395)
+11500 val loss 5.5961
+11500 val perplexity 269.3818
+11500 train 5.594827 (lr=1.2814e-05) (hash(x)=153276620)
+11500 val loss 5.5965
+11500 val perplexity 269.4873
+11500 train 5.585786 (lr=1.3668e-05) (hash(x)=153276620)
+11600 val loss 5.5967
+11600 val perplexity 269.5294
+11600 train 5.451655 (lr=1.2544e-05) (hash(x)=151416388)
+11600 val loss 5.5949
+11600 val perplexity 269.0572
+11600 train 5.447572 (lr=1.3380e-05) (hash(x)=151416388)
+11700 val loss 5.5911
+11700 val perplexity 268.0205
+11700 train 5.459873 (lr=1.2276e-05) (hash(x)=143420927)
+11700 val loss 5.5915
+11700 val perplexity 268.1421
+11700 train 5.464094 (lr=1.3095e-05) (hash(x)=143420927)
+11800 val loss 5.5852
+11800 val perplexity 266.4524
+11800 train 5.646354 (lr=1.2011e-05) (hash(x)=165023996)
+11800 val loss 5.5853
+11800 val perplexity 266.4740
+11800 train 5.645120 (lr=1.2812e-05) (hash(x)=165023996)
+11900 val loss 5.5815
+11900 val perplexity 265.4713
+11900 train 5.492278 (lr=1.1749e-05) (hash(x)=144465076)
+11900 val loss 5.5815
+11900 val perplexity 265.4736
+11900 train 5.497354 (lr=1.2532e-05) (hash(x)=144465076)
+12000 val loss 5.5786
+12000 val perplexity 264.6908
+12000 train 5.593943 (lr=1.1490e-05) (hash(x)=155991892)
+12000 val loss 5.5800
+12000 val perplexity 265.0674
+12000 train 5.586502 (lr=1.2256e-05) (hash(x)=155991892)
+12100 val loss 5.5768
+12100 val perplexity 264.2198
+12100 train 5.636679 (lr=1.1234e-05) (hash(x)=150748844)
+12100 val loss 5.5766
+12100 val perplexity 264.1783
+12100 train 5.618644 (lr=1.1983e-05) (hash(x)=150748844)
+12200 val loss 5.5723
+12200 val perplexity 263.0429
+12200 train 5.549607 (lr=1.0981e-05) (hash(x)=149712638)
+12200 val loss 5.5750
+12200 val perplexity 263.7563
+12200 train 5.538714 (lr=1.1713e-05) (hash(x)=149712638)
+12300 val loss 5.5712
+12300 val perplexity 262.7422
+12300 train 5.405864 (lr=1.0731e-05) (hash(x)=140428230)
+12300 val loss 5.5733
+12300 val perplexity 263.2964
+12300 train 5.391379 (lr=1.1446e-05) (hash(x)=140428230)
+12400 val loss 5.5672
+12400 val perplexity 261.6985
+12400 train 5.755335 (lr=1.0484e-05) (hash(x)=163614308)
+12400 val loss 5.5704
+12400 val perplexity 262.5314
+12400 train 5.759508 (lr=1.1183e-05) (hash(x)=163614308)
+12500 val loss 5.5616
+12500 val perplexity 260.2372
+12500 train 5.527299 (lr=1.0241e-05) (hash(x)=153366153)
+12500 val loss 5.5650
+12500 val perplexity 261.1380
+12500 train 5.530119 (lr=1.0924e-05) (hash(x)=153366153)
+12600 val loss 5.5603
+12600 val perplexity 259.9073
+12600 train 5.462679 (lr=1.0001e-05) (hash(x)=146832066)
+12600 val loss 5.5609
+12600 val perplexity 260.0463
+12600 train 5.458662 (lr=1.0668e-05) (hash(x)=146832066)
+12700 val loss 5.5565
+12700 val perplexity 258.9038
+12700 train 5.727844 (lr=9.7649e-06) (hash(x)=154387414)
+12700 val loss 5.5582
+12700 val perplexity 259.3439
+12700 train 5.738986 (lr=1.0416e-05) (hash(x)=154387414)
+12800 val loss 5.5547
+12800 val perplexity 258.4451
+12800 train 5.350515 (lr=9.5322e-06) (hash(x)=139465199)
+12800 val loss 5.5582
+12800 val perplexity 259.3517
+12800 train 5.345707 (lr=1.0168e-05) (hash(x)=139465199)
+12900 val loss 5.5572
+12900 val perplexity 259.1073
+12900 train 5.270021 (lr=9.3031e-06) (hash(x)=143326374)
+12900 val loss 5.5619
+12900 val perplexity 260.3083
+12900 train 5.270081 (lr=9.9233e-06) (hash(x)=143326374)
+13000 val loss 5.5495
+13000 val perplexity 257.0962
+13000 train 5.705108 (lr=9.0777e-06) (hash(x)=158817835)
+13000 val loss 5.5532
+13000 val perplexity 258.0558
+13000 train 5.713244 (lr=9.6829e-06) (hash(x)=158817835)
+13100 val loss 5.5475
+13100 val perplexity 256.5825
+13100 train 5.430533 (lr=8.8562e-06) (hash(x)=142027874)
+13100 val loss 5.5516
+13100 val perplexity 257.6517
+13100 train 5.431653 (lr=9.4466e-06) (hash(x)=142027874)
+13200 val loss 5.5475
+13200 val perplexity 256.5872
+13200 train 5.537900 (lr=8.6385e-06) (hash(x)=148497730)
+13200 val loss 5.5520
+13200 val perplexity 257.7479
+13200 train 5.548100 (lr=9.2144e-06) (hash(x)=148497730)
+13300 val loss 5.5442
+13300 val perplexity 255.7459
+13300 train 5.453057 (lr=8.4248e-06) (hash(x)=148094633)
+13300 val loss 5.5486
+13300 val perplexity 256.8882
+13300 train 5.450683 (lr=8.9865e-06) (hash(x)=148094633)
+13400 val loss 5.5415
+13400 val perplexity 255.0583
+13400 train 5.401995 (lr=8.2151e-06) (hash(x)=150548214)
+13400 val loss 5.5463
+13400 val perplexity 256.2863
+13400 train 5.412838 (lr=8.7628e-06) (hash(x)=150548214)
+13500 val loss 5.5343
+13500 val perplexity 253.2366
+13500 train 5.648559 (lr=8.0095e-06) (hash(x)=152789725)
+13500 val loss 5.5386
+13500 val perplexity 254.3236
+13500 train 5.658669 (lr=8.5435e-06) (hash(x)=152789725)
+13600 val loss 5.5324
+13600 val perplexity 252.7610
+13600 train 5.580656 (lr=7.8081e-06) (hash(x)=153220230)
+13600 val loss 5.5382
+13600 val perplexity 254.2139
+13600 train 5.585834 (lr=8.3286e-06) (hash(x)=153220230)
+13700 val loss 5.5300
+13700 val perplexity 252.1347
+13700 train 5.558973 (lr=7.6109e-06) (hash(x)=147805699)
+13700 val loss 5.5359
+13700 val perplexity 253.6477
+13700 train 5.559258 (lr=8.1183e-06) (hash(x)=147805699)
+13800 val loss 5.5312
+13800 val perplexity 252.4363
+13800 train 5.333395 (lr=7.4180e-06) (hash(x)=146267341)
+13800 val loss 5.5364
+13800 val perplexity 253.7737
+13800 train 5.335834 (lr=7.9125e-06) (hash(x)=146267341)
+13900 val loss 5.5294
+13900 val perplexity 251.9943
+13900 train 5.370362 (lr=7.2294e-06) (hash(x)=139467501)
+13900 val loss 5.5355
+13900 val perplexity 253.5304
+13900 train 5.370989 (lr=7.7114e-06) (hash(x)=139467501)
+14000 val loss 5.5268
+14000 val perplexity 251.3447
+14000 train 5.544614 (lr=7.0452e-06) (hash(x)=152185436)
+14000 val loss 5.5326
+14000 val perplexity 252.8019
+14000 train 5.543988 (lr=7.5149e-06) (hash(x)=152185436)
+14100 val loss 5.5214
+14100 val perplexity 249.9880
+14100 train 5.653395 (lr=6.8656e-06) (hash(x)=176479139)
+14100 val loss 5.5266
+14100 val perplexity 251.2776
+14100 train 5.652701 (lr=7.3233e-06) (hash(x)=176479139)
+14200 val loss 5.5177
+14200 val perplexity 249.0710
+14200 train 5.586637 (lr=6.6904e-06) (hash(x)=151052040)
+14200 val loss 5.5232
+14200 val perplexity 250.4466
+14200 train 5.617471 (lr=7.1365e-06) (hash(x)=151052040)
+14300 val loss 5.5162
+14300 val perplexity 248.6834
+14300 train 5.463811 (lr=6.5199e-06) (hash(x)=149668097)
+14300 val loss 5.5228
+14300 val perplexity 250.3304
+14300 train 5.468355 (lr=6.9546e-06) (hash(x)=149668097)
+14400 val loss 5.5139
+14400 val perplexity 248.1140
+14400 train 5.528412 (lr=6.3540e-06) (hash(x)=151761941)
+14400 val loss 5.5212
+14400 val perplexity 249.9304
+14400 train 5.533862 (lr=6.7776e-06) (hash(x)=151761941)
+14500 val loss 5.5126
+14500 val perplexity 247.7880
+14500 train 5.466975 (lr=6.1928e-06) (hash(x)=149412009)
+14500 val loss 5.5209
+14500 val perplexity 249.8611
+14500 train 5.474593 (lr=6.6057e-06) (hash(x)=149412009)
+14600 val loss 5.5100
+14600 val perplexity 247.1472
+14600 train 5.396197 (lr=6.0364e-06) (hash(x)=145048020)
+14600 val loss 5.5190
+14600 val perplexity 249.3952
+14600 train 5.388847 (lr=6.4389e-06) (hash(x)=145048020)
+14700 val loss 5.5088
+14700 val perplexity 246.8573
+14700 train 5.466370 (lr=5.8848e-06) (hash(x)=141771666)
+14700 val loss 5.5155
+14700 val perplexity 248.5044
+14700 train 5.491771 (lr=6.2771e-06) (hash(x)=141771666)
+14800 val loss 5.5046
+14800 val perplexity 245.8187
+14800 train 5.430891 (lr=5.7381e-06) (hash(x)=148759740)
+14800 val loss 5.5129
+14800 val perplexity 247.8759
+14800 train 5.436740 (lr=6.1206e-06) (hash(x)=148759740)
+14900 val loss 5.5054
+14900 val perplexity 246.0100
+14900 train 5.428713 (lr=5.5962e-06) (hash(x)=146287032)
+14900 val loss 5.5117
+14900 val perplexity 247.5706
+14900 train 5.434009 (lr=5.9693e-06) (hash(x)=146287032)
+15000 val loss 5.5023
+15000 val perplexity 245.2603
+15000 train 5.454094 (lr=5.4593e-06) (hash(x)=143519156)
+15000 val loss 5.5109
+15000 val perplexity 247.3839
+15000 train 5.461970 (lr=5.8233e-06) (hash(x)=143519156)
+15100 val loss 5.5041
+15100 val perplexity 245.6988
+15100 train 5.487119 (lr=5.3275e-06) (hash(x)=151330688)
+15100 val loss 5.5128
+15100 val perplexity 247.8524
+15100 train 5.504098 (lr=5.6826e-06) (hash(x)=151330688)
+15200 val loss 5.5066
+15200 val perplexity 246.3017
+15200 train 5.317483 (lr=5.2006e-06) (hash(x)=140380090)
+15200 val loss 5.5160
+15200 val perplexity 248.6277
+15200 train 5.312330 (lr=5.5473e-06) (hash(x)=140380090)
+15300 val loss 5.5005
+15300 val perplexity 244.8256
+15300 train 5.412379 (lr=5.0789e-06) (hash(x)=152598160)
+15300 val loss 5.5097
+15300 val perplexity 247.0870
+15300 train 5.410718 (lr=5.4175e-06) (hash(x)=152598160)
+15400 val loss 5.4954
+15400 val perplexity 243.5679
+15400 train 5.733170 (lr=4.9622e-06) (hash(x)=164136966)
+15400 val loss 5.5048
+15400 val perplexity 245.8654
+15400 train 5.783056 (lr=5.2931e-06) (hash(x)=164136966)
+15500 val loss 5.4956
+15500 val perplexity 243.6188
+15500 train 5.700720 (lr=4.8508e-06) (hash(x)=158733990)
+15500 val loss 5.5042
+15500 val perplexity 245.7172
+15500 train 5.723475 (lr=5.1742e-06) (hash(x)=158733990)
+15600 val loss 5.4928
+15600 val perplexity 242.9356
+15600 train 5.540652 (lr=4.7445e-06) (hash(x)=149149390)
+15600 val loss 5.5022
+15600 val perplexity 245.2186
+15600 train 5.534406 (lr=5.0608e-06) (hash(x)=149149390)
+15700 val loss 5.4914
+15700 val perplexity 242.6066
+15700 train 5.446640 (lr=4.6435e-06) (hash(x)=146746394)
+15700 val loss 5.5010
+15700 val perplexity 244.9401
+15700 train 5.453232 (lr=4.9531e-06) (hash(x)=146746394)
+15800 val loss 5.4919
+15800 val perplexity 242.7099
+15800 train 5.509732 (lr=4.5478e-06) (hash(x)=159405219)
+15800 val loss 5.5007
+15800 val perplexity 244.8553
+15800 train 5.518862 (lr=4.8509e-06) (hash(x)=159405219)
+15900 val loss 5.4900
+15900 val perplexity 242.2480
+15900 train 5.523585 (lr=4.4573e-06) (hash(x)=154034029)
+15900 val loss 5.4997
+15900 val perplexity 244.6285
+15900 train 5.526865 (lr=4.7545e-06) (hash(x)=154034029)
+16000 val loss 5.4874
+16000 val perplexity 241.6343
+16000 train 5.489702 (lr=4.3722e-06) (hash(x)=153634210)
+16000 val loss 5.4971
+16000 val perplexity 243.9921
+16000 train 5.506591 (lr=4.6637e-06) (hash(x)=153634210)
+16100 val loss 5.4855
+16100 val perplexity 241.1760
+16100 train 5.418263 (lr=4.2924e-06) (hash(x)=155646721)
+16100 val loss 5.4950
+16100 val perplexity 243.4680
+16100 train 5.423606 (lr=4.5786e-06) (hash(x)=155646721)
+16200 val loss 5.4849
+16200 val perplexity 241.0224
+16200 train 5.405526 (lr=4.2180e-06) (hash(x)=142461643)
+16200 val loss 5.4950
+16200 val perplexity 243.4673
+16200 train 5.413839 (lr=4.4993e-06) (hash(x)=142461643)
+16300 val loss 5.4845
+16300 val perplexity 240.9184
+16300 train 5.398604 (lr=4.1491e-06) (hash(x)=141080038)
+16300 val loss 5.4945
+16300 val perplexity 243.3531
+16300 train 5.393754 (lr=4.4257e-06) (hash(x)=141080038)
+16400 val loss 5.4839
+16400 val perplexity 240.7880
+16400 train 5.485754 (lr=4.0856e-06) (hash(x)=153744166)
+16400 val loss 5.4943
+16400 val perplexity 243.2899
+16400 train 5.506991 (lr=4.3579e-06) (hash(x)=153744166)
+16500 val loss 5.4868
+16500 val perplexity 241.4812
+16500 train 5.307953 (lr=4.0275e-06) (hash(x)=149783517)
+16500 val loss 5.4972
+16500 val perplexity 243.9996
+16500 train 5.306321 (lr=4.2960e-06) (hash(x)=149783517)
+16600 val loss 5.4881
+16600 val perplexity 241.7923
+16600 train 5.334898 (lr=3.9749e-06) (hash(x)=142659127)
+16600 val loss 5.4990
+16600 val perplexity 244.4431
+16600 train 5.331096 (lr=4.2399e-06) (hash(x)=142659127)
+16700 val loss 5.4805
+16700 val perplexity 239.9655
+16700 train 5.563731 (lr=3.9278e-06) (hash(x)=149725921)
+16700 val loss 5.4919
+16700 val perplexity 242.7237
+16700 train 5.564499 (lr=4.1896e-06) (hash(x)=149725921)
+16800 val loss 5.4784
+16800 val perplexity 239.4544
+16800 train 5.344167 (lr=3.8862e-06) (hash(x)=140678087)
+16800 val loss 5.4894
+16800 val perplexity 242.1178
+16800 train 5.351054 (lr=4.1452e-06) (hash(x)=140678087)
+16900 val loss 5.4780
+16900 val perplexity 239.3585
+16900 train 5.509586 (lr=3.8501e-06) (hash(x)=146935413)
+16900 val loss 5.4879
+16900 val perplexity 241.7598
+16900 train 5.506167 (lr=4.1067e-06) (hash(x)=146935413)
+17000 val loss 5.4756
+17000 val perplexity 238.8020
+17000 train 5.490251 (lr=3.8195e-06) (hash(x)=144408437)
+17000 val loss 5.4863
+17000 val perplexity 241.3699
+17000 train 5.500947 (lr=4.0741e-06) (hash(x)=144408437)
+17100 val loss 5.4754
+17100 val perplexity 238.7511
+17100 train 5.300627 (lr=3.7945e-06) (hash(x)=149229905)
+17100 val loss 5.4860
+17100 val perplexity 241.2910
+17100 train 5.342311 (lr=4.0475e-06) (hash(x)=149229905)
+17200 val loss 5.4754
+17200 val perplexity 238.7410
+17200 train 5.440160 (lr=3.7750e-06) (hash(x)=150471865)
+17200 val loss 5.4862
+17200 val perplexity 241.3465
+17200 train 5.437218 (lr=4.0267e-06) (hash(x)=150471865)
+17300 val loss 5.4771
+17300 val perplexity 239.1510
+17300 train 5.364053 (lr=3.7611e-06) (hash(x)=155784916)
+17300 val loss 5.4887
+17300 val perplexity 241.9519
+17300 train 5.363513 (lr=4.0119e-06) (hash(x)=155784916)
+17400 val loss 5.4797
+17400 val perplexity 239.7847
+17400 train 5.258312 (lr=3.7528e-06) (hash(x)=148482413)
+17400 val loss 5.4919
+17400 val perplexity 242.7172
+17400 train 5.261920 (lr=4.0030e-06) (hash(x)=148482413)
+17499 val loss 5.4726
+17499 val perplexity 238.0666
+17499 val loss 5.4837
+17499 val perplexity 240.7365

attention_kindselective_n_heads4_seed1340/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4923e7ab702d1408072d4b06f276cf73b3ab667510dbbd6f90cda737d2a3559a
+oid sha256:4b94878fbf317ca14cf9f1d8ca2afe3d3c782ec82af0e473faaa020f8ae220f0
 size 92843394

attention_kindselective_n_heads4_seed1340/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:82254200685547f8a942c48ae6e428bb336a452a15fc717fa5d70f603e7ad765
+oid sha256:aa9c531a7dd091b295c0ef72fe3f6cb8df38e9e4e351a622862940383a438670
 size 179406214