Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_30720_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.75e-5_30720_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,528 +1,1054 @@
 max_steps: 17500
+max_steps: 17500
+0 val loss 11.2663
+0 val perplexity 78147.0000
 0 val loss 11.2663
 0 val perplexity 78147.0000
-0 train 11.263341 (lr=1.7500e-07) (hash(x)=150327452)
-100 val loss 9.8666
-100 val perplexity 19276.3164
-100 train 9.875949 (lr=1.7675e-05) (hash(x)=166441190)
-200 val loss 8.5172
-200 val perplexity 5000.0649
-200 train 8.645861 (lr=3.5000e-05) (hash(x)=166780046)
-300 val loss 7.6495
-300 val perplexity 2099.6785
-300 train 7.605138 (lr=3.4997e-05) (hash(x)=159835303)
-400 val loss 7.5080
-400 val perplexity 1822.5958
-400 train 7.283498 (lr=3.4990e-05) (hash(x)=155040610)
-500 val loss 7.4459
-500 val perplexity 1712.7689
-500 train 7.172822 (lr=3.4977e-05) (hash(x)=130190460)
-600 val loss 7.3656
-600 val perplexity 1580.6713
-600 train 7.384192 (lr=3.4958e-05) (hash(x)=155504036)
-700 val loss 7.2947
-700 val perplexity 1472.5343
-700 train 7.147750 (lr=3.4935e-05) (hash(x)=137347213)
-800 val loss 7.2237
-800 val perplexity 1371.5957
-800 train 7.065663 (lr=3.4907e-05) (hash(x)=143823248)
-900 val loss 7.1532
-900 val perplexity 1278.2330
-900 train 7.261928 (lr=3.4873e-05) (hash(x)=156260416)
-1000 val loss 7.0941
-1000 val perplexity 1204.8857
-1000 train 7.049677 (lr=3.4834e-05) (hash(x)=143734685)
-1100 val loss 7.0353
-1100 val perplexity 1136.0641
-1100 train 6.952003 (lr=3.4790e-05) (hash(x)=160013925)
-1200 val loss 6.9993
-1200 val perplexity 1095.8428
-1200 train 6.722279 (lr=3.4741e-05) (hash(x)=150678249)
-1300 val loss 6.9463
-1300 val perplexity 1039.2848
-1300 train 6.712331 (lr=3.4687e-05) (hash(x)=149073315)
-1400 val loss 6.8822
-1400 val perplexity 974.7924
-1400 train 7.177571 (lr=3.4628e-05) (hash(x)=175802021)
-1500 val loss 6.8204
-1500 val perplexity 916.3141
-1500 train 7.088338 (lr=3.4563e-05) (hash(x)=171034639)
-1600 val loss 6.7715
-1600 val perplexity 872.6337
-1600 train 6.987903 (lr=3.4494e-05) (hash(x)=158681215)
-1700 val loss 6.7281
-1700 val perplexity 835.5875
-1700 train 6.675895 (lr=3.4419e-05) (hash(x)=152116061)
-1800 val loss 6.6849
-1800 val perplexity 800.2023
-1800 train 6.617714 (lr=3.4340e-05) (hash(x)=146108145)
-1900 val loss 6.6472
-1900 val perplexity 770.6526
-1900 train 6.465954 (lr=3.4255e-05) (hash(x)=147598108)
-2000 val loss 6.6229
-2000 val perplexity 752.1500
-2000 train 6.451444 (lr=3.4166e-05) (hash(x)=154996086)
-2100 val loss 6.5878
-2100 val perplexity 726.2009
-2100 train 6.356413 (lr=3.4072e-05) (hash(x)=153396183)
-2200 val loss 6.5320
-2200 val perplexity 686.7681
-2200 train 6.529510 (lr=3.3973e-05) (hash(x)=153885445)
-2300 val loss 6.4981
-2300 val perplexity 663.9014
-2300 train 6.507496 (lr=3.3869e-05) (hash(x)=159666385)
-2400 val loss 6.4592
-2400 val perplexity 638.5427
-2400 train 6.455539 (lr=3.3760e-05) (hash(x)=142353087)
-2500 val loss 6.4283
-2500 val perplexity 619.1046
-2500 train 6.443000 (lr=3.3646e-05) (hash(x)=146491718)
-2600 val loss 6.4042
-2600 val perplexity 604.3793
-2600 train 6.326907 (lr=3.3528e-05) (hash(x)=150750353)
-2700 val loss 6.3784
-2700 val perplexity 588.9938
-2700 train 6.114101 (lr=3.3405e-05) (hash(x)=129849193)
-2800 val loss 6.3725
-2800 val perplexity 585.5407
-2800 train 6.196213 (lr=3.3277e-05) (hash(x)=152767913)
-2900 val loss 6.3522
-2900 val perplexity 573.7309
-2900 train 6.126045 (lr=3.3144e-05) (hash(x)=146531140)
-3000 val loss 6.3223
-3000 val perplexity 556.8661
-3000 train 6.413922 (lr=3.3008e-05) (hash(x)=151562048)
-3100 val loss 6.2940
-3100 val perplexity 541.3404
-3100 train 6.252115 (lr=3.2866e-05) (hash(x)=146001424)
-3200 val loss 6.2814
-3200 val perplexity 534.5200
-3200 train 6.352508 (lr=3.2720e-05) (hash(x)=166486165)
-3300 val loss 6.2526
-3300 val perplexity 519.3692
-3300 train 6.191504 (lr=3.2570e-05) (hash(x)=150866680)
-3400 val loss 6.2387
-3400 val perplexity 512.1734
-3400 train 6.131408 (lr=3.2415e-05) (hash(x)=143900419)
-3500 val loss 6.2234
-3500 val perplexity 504.4341
-3500 train 6.034927 (lr=3.2256e-05) (hash(x)=148845794)
-3600 val loss 6.2103
-3600 val perplexity 497.8275
-3600 train 6.038193 (lr=3.2092e-05) (hash(x)=145667796)
-3700 val loss 6.2000
-3700 val perplexity 492.7471
-3700 train 6.219586 (lr=3.1924e-05) (hash(x)=163563851)
-3800 val loss 6.1898
-3800 val perplexity 487.7324
-3800 train 6.139465 (lr=3.1753e-05) (hash(x)=147488689)
-3900 val loss 6.1621
-3900 val perplexity 474.4142
-3900 train 6.174405 (lr=3.1577e-05) (hash(x)=148186608)
-4000 val loss 6.1436
-4000 val perplexity 465.7114
-4000 train 6.032261 (lr=3.1397e-05) (hash(x)=142970187)
-4100 val loss 6.1282
-4100 val perplexity 458.5933
-4100 train 6.276835 (lr=3.1212e-05) (hash(x)=141584883)
-4200 val loss 6.1200
-4200 val perplexity 454.8668
-4200 train 5.921480 (lr=3.1024e-05) (hash(x)=145664585)
-4300 val loss 6.1068
-4300 val perplexity 448.8869
-4300 train 5.991696 (lr=3.0833e-05) (hash(x)=143736499)
-4400 val loss 6.1066
-4400 val perplexity 448.7936
-4400 train 5.929280 (lr=3.0637e-05) (hash(x)=151883322)
-4500 val loss 6.0970
-4500 val perplexity 444.5335
-4500 train 5.972202 (lr=3.0437e-05) (hash(x)=153904871)
-4600 val loss 6.0794
-4600 val perplexity 436.7792
-4600 train 6.184064 (lr=3.0234e-05) (hash(x)=154893521)
-4700 val loss 6.0611
-4700 val perplexity 428.8307
-4700 train 6.102214 (lr=3.0027e-05) (hash(x)=152323949)
-4800 val loss 6.0513
-4800 val perplexity 424.6822
-4800 train 5.999310 (lr=2.9817e-05) (hash(x)=154104619)
-4900 val loss 6.0326
-4900 val perplexity 416.8012
-4900 train 6.160482 (lr=2.9603e-05) (hash(x)=146311426)
-5000 val loss 6.0271
-5000 val perplexity 414.5144
-5000 train 5.998255 (lr=2.9386e-05) (hash(x)=156741847)
-5100 val loss 6.0212
-5100 val perplexity 412.0829
-5100 train 5.801046 (lr=2.9166e-05) (hash(x)=142086346)
-5200 val loss 6.0182
-5200 val perplexity 410.8245
-5200 train 5.785246 (lr=2.8942e-05) (hash(x)=150265428)
-5300 val loss 6.0053
-5300 val perplexity 405.5529
-5300 train 5.873408 (lr=2.8715e-05) (hash(x)=151339108)
-5400 val loss 5.9988
-5400 val perplexity 402.9337
-5400 train 5.956289 (lr=2.8484e-05) (hash(x)=154654372)
-5500 val loss 5.9793
-5500 val perplexity 395.1808
-5500 train 5.988954 (lr=2.8251e-05) (hash(x)=150575051)
-5600 val loss 5.9670
-5600 val perplexity 390.3333
-5600 train 5.884884 (lr=2.8015e-05) (hash(x)=140396423)
-5700 val loss 5.9546
-5700 val perplexity 385.5175
-5700 train 5.811714 (lr=2.7776e-05) (hash(x)=144678758)
-5800 val loss 5.9474
-5800 val perplexity 382.7544
-5800 train 6.006756 (lr=2.7534e-05) (hash(x)=151992743)
-5900 val loss 5.9399
-5900 val perplexity 379.9125
-5900 train 5.736372 (lr=2.7290e-05) (hash(x)=144396927)
-6000 val loss 5.9352
-6000 val perplexity 378.1267
-6000 train 5.888235 (lr=2.7042e-05) (hash(x)=165478625)
-6100 val loss 5.9382
-6100 val perplexity 379.2473
-6100 train 5.665012 (lr=2.6792e-05) (hash(x)=147088621)
-6200 val loss 5.9269
-6200 val perplexity 374.9813
-6200 train 5.690819 (lr=2.6540e-05) (hash(x)=140794994)
-6300 val loss 5.9086
-6300 val perplexity 368.1940
-6300 train 5.854196 (lr=2.6285e-05) (hash(x)=134780906)
-6400 val loss 5.8961
-6400 val perplexity 363.6166
-6400 train 5.876956 (lr=2.6028e-05) (hash(x)=149023655)
-6500 val loss 5.8875
-6500 val perplexity 360.5128
-6500 train 5.736791 (lr=2.5769e-05) (hash(x)=147497796)
-6600 val loss 5.8871
-6600 val perplexity 360.3513
-6600 train 5.739757 (lr=2.5508e-05) (hash(x)=152902689)
-6700 val loss 5.8851
-6700 val perplexity 359.6428
-6700 train 5.922760 (lr=2.5244e-05) (hash(x)=153846046)
-6800 val loss 5.8640
-6800 val perplexity 352.1464
-6800 train 5.991194 (lr=2.4979e-05) (hash(x)=158512738)
-6900 val loss 5.8555
-6900 val perplexity 349.1653
-6900 train 6.398537 (lr=2.4711e-05) (hash(x)=156849968)
-7000 val loss 5.8474
-7000 val perplexity 346.3384
-7000 train 5.704376 (lr=2.4442e-05) (hash(x)=142395855)
-7100 val loss 5.8435
-7100 val perplexity 344.9890
-7100 train 5.685997 (lr=2.4171e-05) (hash(x)=147114884)
-7200 val loss 5.8335
-7200 val perplexity 341.5437
-7200 train 5.867506 (lr=2.3899e-05) (hash(x)=156979839)
-7300 val loss 5.8267
-7300 val perplexity 339.2277
-7300 train 5.626989 (lr=2.3625e-05) (hash(x)=145584373)
-7400 val loss 5.8235
-7400 val perplexity 338.1602
-7400 train 5.572762 (lr=2.3349e-05) (hash(x)=141508204)
-7500 val loss 5.8126
-7500 val perplexity 334.4917
-7500 train 5.851742 (lr=2.3073e-05) (hash(x)=148803965)
-7600 val loss 5.8067
-7600 val perplexity 332.5083
-7600 train 5.863437 (lr=2.2795e-05) (hash(x)=151019676)
-7700 val loss 5.7939
-7700 val perplexity 328.2856
-7700 train 5.906473 (lr=2.2515e-05) (hash(x)=143155750)
-7800 val loss 5.7892
-7800 val perplexity 326.7418
-7800 train 5.820032 (lr=2.2235e-05) (hash(x)=152569653)
-7900 val loss 5.7830
-7900 val perplexity 324.7243
-7900 train 5.654320 (lr=2.1954e-05) (hash(x)=143519455)
-8000 val loss 5.7751
-8000 val perplexity 322.1766
-8000 train 5.862270 (lr=2.1671e-05) (hash(x)=161180944)
-8100 val loss 5.7742
-8100 val perplexity 321.8991
-8100 train 5.804288 (lr=2.1388e-05) (hash(x)=154107345)
-8200 val loss 5.7724
-8200 val perplexity 321.2982
-8200 train 5.814382 (lr=2.1105e-05) (hash(x)=152486517)
-8300 val loss 5.7592
-8300 val perplexity 317.1099
-8300 train 5.759204 (lr=2.0820e-05) (hash(x)=156167749)
-8400 val loss 5.7493
-8400 val perplexity 313.9706
-8400 train 5.835824 (lr=2.0536e-05) (hash(x)=149155006)
-8500 val loss 5.7415
-8500 val perplexity 311.5284
-8500 train 5.687334 (lr=2.0250e-05) (hash(x)=147844390)
-8600 val loss 5.7388
-8600 val perplexity 310.6940
-8600 train 5.997749 (lr=1.9965e-05) (hash(x)=165753320)
-8700 val loss 5.7355
-8700 val perplexity 309.6590
-8700 train 5.514201 (lr=1.9679e-05) (hash(x)=146079979)
-8800 val loss 5.7332
-8800 val perplexity 308.9646
-8800 train 5.915848 (lr=1.9393e-05) (hash(x)=172259509)
-8900 val loss 5.7253
-8900 val perplexity 306.5404
-8900 train 5.405478 (lr=1.9107e-05) (hash(x)=145148314)
-9000 val loss 5.7233
-9000 val perplexity 305.9171
-9000 train 5.642859 (lr=1.8821e-05) (hash(x)=144250633)
-9100 val loss 5.7168
-9100 val perplexity 303.9372
-9100 train 5.809632 (lr=1.8535e-05) (hash(x)=157219797)
-9200 val loss 5.7049
-9200 val perplexity 300.3360
-9200 train 5.752951 (lr=1.8250e-05) (hash(x)=142743778)
-9300 val loss 5.7006
-9300 val perplexity 299.0528
-9300 train 5.633637 (lr=1.7964e-05) (hash(x)=139669771)
-9400 val loss 5.6915
-9400 val perplexity 296.3345
-9400 train 5.733429 (lr=1.7680e-05) (hash(x)=145916843)
-9500 val loss 5.6955
-9500 val perplexity 297.5302
-9500 train 5.614336 (lr=1.7395e-05) (hash(x)=150196125)
-9600 val loss 5.6859
-9600 val perplexity 294.6796
-9600 train 6.000877 (lr=1.7112e-05) (hash(x)=160041419)
-9700 val loss 5.6857
-9700 val perplexity 294.6254
-9700 train 5.472782 (lr=1.6829e-05) (hash(x)=139931627)
-9800 val loss 5.6803
-9800 val perplexity 293.0335
-9800 train 5.666329 (lr=1.6546e-05) (hash(x)=150370792)
-9900 val loss 5.6722
-9900 val perplexity 290.6800
-9900 train 5.500357 (lr=1.6265e-05) (hash(x)=153014886)
-10000 val loss 5.6696
-10000 val perplexity 289.9293
-10000 train 5.582622 (lr=1.5985e-05) (hash(x)=141915265)
-10100 val loss 5.6658
-10100 val perplexity 288.8095
-10100 train 6.173605 (lr=1.5705e-05) (hash(x)=197770952)
-10200 val loss 5.6572
-10200 val perplexity 286.3348
-10200 train 5.623529 (lr=1.5427e-05) (hash(x)=156806005)
-10300 val loss 5.6543
-10300 val perplexity 285.5200
-10300 train 5.462598 (lr=1.5151e-05) (hash(x)=145378216)
-10400 val loss 5.6495
-10400 val perplexity 284.1413
-10400 train 5.563924 (lr=1.4875e-05) (hash(x)=145125407)
-10500 val loss 5.6463
-10500 val perplexity 283.2400
-10500 train 5.467100 (lr=1.4601e-05) (hash(x)=147553745)
-10600 val loss 5.6426
-10600 val perplexity 282.1878
-10600 train 5.293966 (lr=1.4329e-05) (hash(x)=134894889)
-10700 val loss 5.6358
-10700 val perplexity 280.2725
-10700 train 5.560827 (lr=1.4058e-05) (hash(x)=151041040)
-10800 val loss 5.6313
-10800 val perplexity 279.0345
-10800 train 5.559671 (lr=1.3789e-05) (hash(x)=145685829)
-10900 val loss 5.6262
-10900 val perplexity 277.6062
-10900 train 5.715626 (lr=1.3521e-05) (hash(x)=156000716)
-11000 val loss 5.6236
-11000 val perplexity 276.8808
-11000 train 5.636567 (lr=1.3256e-05) (hash(x)=149858163)
-11100 val loss 5.6234
-11100 val perplexity 276.8307
-11100 train 5.906702 (lr=1.2992e-05) (hash(x)=164873575)
-11200 val loss 5.6220
-11200 val perplexity 276.4303
-11200 train 5.503514 (lr=1.2731e-05) (hash(x)=156370069)
-11300 val loss 5.6210
-11300 val perplexity 276.1702
-11300 train 5.706107 (lr=1.2472e-05) (hash(x)=160356892)
-11400 val loss 5.6073
-11400 val perplexity 272.3991
-11400 train 5.678671 (lr=1.2215e-05) (hash(x)=158955430)
-11500 val loss 5.6066
-11500 val perplexity 272.2245
-11500 train 5.573491 (lr=1.1960e-05) (hash(x)=146598377)
-11600 val loss 5.6235
-11600 val perplexity 276.8658
-11600 train 5.762615 (lr=1.1708e-05) (hash(x)=156045795)
-11700 val loss 5.6025
-11700 val perplexity 271.1102
-11700 train 5.456719 (lr=1.1458e-05) (hash(x)=143963334)
-11800 val loss 5.6006
-11800 val perplexity 270.5925
-11800 train 5.440306 (lr=1.1210e-05) (hash(x)=145831579)
-11900 val loss 5.5957
-11900 val perplexity 269.2682
-11900 train 5.755351 (lr=1.0966e-05) (hash(x)=165154141)
-12000 val loss 5.5908
-12000 val perplexity 267.9550
-12000 train 5.674036 (lr=1.0724e-05) (hash(x)=160373794)
-12100 val loss 5.5874
-12100 val perplexity 267.0450
-12100 train 5.375375 (lr=1.0485e-05) (hash(x)=141476346)
-12200 val loss 5.5856
-12200 val perplexity 266.5506
-12200 train 5.835909 (lr=1.0249e-05) (hash(x)=164508058)
-12300 val loss 5.5835
-12300 val perplexity 265.9999
-12300 train 5.569179 (lr=1.0016e-05) (hash(x)=140407664)
-12400 val loss 5.5811
-12400 val perplexity 265.3701
-12400 train 5.437617 (lr=9.7854e-06) (hash(x)=149526189)
-12500 val loss 5.5757
-12500 val perplexity 263.9212
-12500 train 5.430196 (lr=9.5583e-06) (hash(x)=142060916)
-12600 val loss 5.5706
-12600 val perplexity 262.5867
-12600 train 5.744236 (lr=9.3345e-06) (hash(x)=168650848)
-12700 val loss 5.5701
-12700 val perplexity 262.4474
-12700 train 5.596745 (lr=9.1139e-06) (hash(x)=154353018)
-12800 val loss 5.5697
-12800 val perplexity 262.3454
-12800 train 5.737992 (lr=8.8967e-06) (hash(x)=156115265)
-12900 val loss 5.5665
-12900 val perplexity 261.5076
-12900 train 5.494372 (lr=8.6829e-06) (hash(x)=143378175)
-13000 val loss 5.5639
-13000 val perplexity 260.8358
-13000 train 5.656729 (lr=8.4726e-06) (hash(x)=166124864)
-13100 val loss 5.5626
-13100 val perplexity 260.4978
-13100 train 5.440381 (lr=8.2658e-06) (hash(x)=146615385)
-13200 val loss 5.5623
-13200 val perplexity 260.4130
-13200 train 5.458254 (lr=8.0626e-06) (hash(x)=149236047)
-13300 val loss 5.5570
-13300 val perplexity 259.0388
-13300 train 5.490791 (lr=7.8632e-06) (hash(x)=151346972)
-13400 val loss 5.5544
-13400 val perplexity 258.3721
-13400 train 5.652813 (lr=7.6674e-06) (hash(x)=159167093)
-13500 val loss 5.5534
-13500 val perplexity 258.1016
-13500 train 5.282621 (lr=7.4756e-06) (hash(x)=135054650)
-13600 val loss 5.5507
-13600 val perplexity 257.4178
-13600 train 5.435532 (lr=7.2875e-06) (hash(x)=151033927)
-13700 val loss 5.5507
-13700 val perplexity 257.4250
-13700 train 5.252520 (lr=7.1035e-06) (hash(x)=141119172)
-13800 val loss 5.5498
-13800 val perplexity 257.1873
-13800 train 5.263982 (lr=6.9234e-06) (hash(x)=144197766)
-13900 val loss 5.5452
-13900 val perplexity 256.0157
-13900 train 5.482715 (lr=6.7474e-06) (hash(x)=145148158)
-14000 val loss 5.5415
-14000 val perplexity 255.0668
-14000 train 6.022576 (lr=6.5756e-06) (hash(x)=179068751)
-14100 val loss 5.5413
-14100 val perplexity 255.0131
-14100 train 5.412605 (lr=6.4079e-06) (hash(x)=144988373)
-14200 val loss 5.5398
-14200 val perplexity 254.6327
-14200 train 5.489853 (lr=6.2444e-06) (hash(x)=133379696)
-14300 val loss 5.5397
-14300 val perplexity 254.6027
-14300 train 5.420756 (lr=6.0852e-06) (hash(x)=149883422)
-14400 val loss 5.5397
-14400 val perplexity 254.5917
-14400 train 5.406084 (lr=5.9304e-06) (hash(x)=145709198)
-14500 val loss 5.5349
-14500 val perplexity 253.3946
-14500 train 5.546863 (lr=5.7800e-06) (hash(x)=148016579)
-14600 val loss 5.5333
-14600 val perplexity 252.9894
-14600 train 5.513137 (lr=5.6340e-06) (hash(x)=150218222)
-14700 val loss 5.5329
-14700 val perplexity 252.8771
-14700 train 5.468463 (lr=5.4925e-06) (hash(x)=142258737)
-14800 val loss 5.5297
-14800 val perplexity 252.0703
-14800 train 5.427268 (lr=5.3555e-06) (hash(x)=147628003)
-14900 val loss 5.5314
-14900 val perplexity 252.5087
-14900 train 5.382735 (lr=5.2231e-06) (hash(x)=149144036)
-15000 val loss 5.5304
-15000 val perplexity 252.2545
-15000 train 5.463471 (lr=5.0954e-06) (hash(x)=145925409)
-15100 val loss 5.5261
-15100 val perplexity 251.1536
-15100 train 5.413812 (lr=4.9723e-06) (hash(x)=144555751)
-15200 val loss 5.5241
-15200 val perplexity 250.6498
-15200 train 5.348018 (lr=4.8539e-06) (hash(x)=143448852)
-15300 val loss 5.5229
-15300 val perplexity 250.3561
-15300 train 5.524452 (lr=4.7403e-06) (hash(x)=156347253)
-15400 val loss 5.5216
-15400 val perplexity 250.0448
-15400 train 5.474526 (lr=4.6314e-06) (hash(x)=153010179)
-15500 val loss 5.5222
-15500 val perplexity 250.1826
-15500 train 5.299636 (lr=4.5274e-06) (hash(x)=144066445)
-15600 val loss 5.5216
-15600 val perplexity 250.0413
-15600 train 5.255478 (lr=4.4282e-06) (hash(x)=146107722)
-15700 val loss 5.5178
-15700 val perplexity 249.0807
-15700 train 5.667055 (lr=4.3339e-06) (hash(x)=152533799)
-15800 val loss 5.5155
-15800 val perplexity 248.5203
-15800 train 5.569731 (lr=4.2446e-06) (hash(x)=151756366)
-15900 val loss 5.5149
-15900 val perplexity 248.3722
-15900 train 5.498937 (lr=4.1601e-06) (hash(x)=151735039)
-16000 val loss 5.5144
-16000 val perplexity 248.2384
-16000 train 5.541584 (lr=4.0807e-06) (hash(x)=154633226)
-16100 val loss 5.5148
-16100 val perplexity 248.3285
-16100 train 5.358697 (lr=4.0063e-06) (hash(x)=158094333)
-16200 val loss 5.5145
-16200 val perplexity 248.2621
-16200 train 5.270772 (lr=3.9368e-06) (hash(x)=142772927)
-16300 val loss 5.5136
-16300 val perplexity 248.0458
-16300 train 5.414096 (lr=3.8725e-06) (hash(x)=148042896)
-16400 val loss 5.5103
-16400 val perplexity 247.2169
-16400 train 5.995365 (lr=3.8132e-06) (hash(x)=170306399)
-16500 val loss 5.5097
-16500 val perplexity 247.0674
-16500 train 5.531100 (lr=3.7590e-06) (hash(x)=141049320)
-16600 val loss 5.5105
-16600 val perplexity 247.2748
-16600 train 5.366843 (lr=3.7099e-06) (hash(x)=150512136)
-16700 val loss 5.5079
-16700 val perplexity 246.6345
-16700 train 5.512495 (lr=3.6659e-06) (hash(x)=151843645)
-16800 val loss 5.5054
-16800 val perplexity 246.0190
-16800 train 5.574283 (lr=3.6271e-06) (hash(x)=140073659)
-16900 val loss 5.5040
-16900 val perplexity 245.6649
-16900 train 5.649993 (lr=3.5934e-06) (hash(x)=171157212)
-17000 val loss 5.5037
-17000 val perplexity 245.6022
-17000 train 5.439902 (lr=3.5649e-06) (hash(x)=144553276)
-17100 val loss 5.5033
-17100 val perplexity 245.5049
-17100 train 5.497724 (lr=3.5415e-06) (hash(x)=157717565)
-17200 val loss 5.5036
-17200 val perplexity 245.5721
-17200 train 5.361936 (lr=3.5234e-06) (hash(x)=149843860)
-17300 val loss 5.5012
-17300 val perplexity 244.9887
-17300 train 5.560598 (lr=3.5104e-06) (hash(x)=162171979)
-17400 val loss 5.5006
-17400 val perplexity 244.8329
-17400 train 5.472299 (lr=3.5026e-06) (hash(x)=148400702)
-17499 val loss 5.4991
-17499 val perplexity 244.4820
+0 train 11.263341 (lr=1.8750e-07) (hash(x)=150327452)
+0 train 11.263341 (lr=2.0000e-07) (hash(x)=150327452)
+100 val loss 9.8595
+100 val perplexity 19138.5859
+100 val loss 9.8359
+100 val perplexity 18693.2441
+100 train 9.868124 (lr=1.8937e-05) (hash(x)=166441190)
+100 train 9.845812 (lr=2.0200e-05) (hash(x)=166441190)
+200 val loss 8.5181
+200 val perplexity 5004.7261
+200 val loss 8.6538
+200 val perplexity 5731.6953
+200 train 8.651850 (lr=4.0000e-05) (hash(x)=166780046)
+200 train 8.781153 (lr=3.7500e-05) (hash(x)=166780046)
+300 val loss 7.7057
+300 val perplexity 2221.0286
+300 val loss 7.6969
+300 val perplexity 2201.4924
+300 train 7.656827 (lr=3.9997e-05) (hash(x)=159835303)
+300 train 7.646784 (lr=3.7497e-05) (hash(x)=159835303)
+400 val loss 7.5543
+400 val perplexity 1909.0188
+400 train 7.335799 (lr=3.9988e-05) (hash(x)=155040610)
+400 val loss 7.5167
+400 val perplexity 1838.4224
+400 train 7.293813 (lr=3.7489e-05) (hash(x)=155040610)
+500 val loss 7.4826
+500 val perplexity 1776.7881
+500 train 7.209811 (lr=3.9973e-05) (hash(x)=130190460)
+500 val loss 7.4408
+500 val perplexity 1704.1182
+500 train 7.158998 (lr=3.7475e-05) (hash(x)=130190460)
+600 val loss 7.4172
+600 val perplexity 1664.2971
+600 train 7.436842 (lr=3.9953e-05) (hash(x)=155504036)
+600 val loss 7.3542
+600 val perplexity 1562.7886
+600 train 7.372247 (lr=3.7456e-05) (hash(x)=155504036)
+700 val loss 7.3468
+700 val perplexity 1551.1505
+700 train 7.196675 (lr=3.9926e-05) (hash(x)=137347213)
+700 val loss 7.2823
+700 val perplexity 1454.3055
+700 train 7.140363 (lr=3.7430e-05) (hash(x)=137347213)
+800 val loss 7.2940
+800 val perplexity 1471.5103
+800 train 7.142735 (lr=3.9893e-05) (hash(x)=143823248)
+800 val loss 7.2112
+800 val perplexity 1354.4867
+800 train 7.056271 (lr=3.7400e-05) (hash(x)=143823248)
+900 val loss 7.2423
+900 val perplexity 1397.3634
+900 train 7.346307 (lr=3.9855e-05) (hash(x)=156260416)
+900 val loss 7.1369
+900 val perplexity 1257.5696
+900 train 7.245272 (lr=3.7364e-05) (hash(x)=156260416)
+1000 val loss 7.1949
+1000 val perplexity 1332.5796
+1000 train 7.155627 (lr=3.9810e-05) (hash(x)=143734685)
+1000 val loss 7.0832
+1000 val perplexity 1191.7954
+1000 train 7.037784 (lr=3.7322e-05) (hash(x)=143734685)
+1100 val loss 7.1500
+1100 val perplexity 1274.1158
+1100 train 7.072702 (lr=3.9760e-05) (hash(x)=160013925)
+1100 val loss 7.0295
+1100 val perplexity 1129.5018
+1100 train 6.946764 (lr=3.7275e-05) (hash(x)=160013925)
+1200 val loss 7.1062
+1200 val perplexity 1219.4874
+1200 train 6.836085 (lr=3.9704e-05) (hash(x)=150678249)
+1200 val loss 6.9870
+1200 val perplexity 1082.4991
+1200 train 6.713125 (lr=3.7223e-05) (hash(x)=150678249)
+1300 val loss 7.0673
+1300 val perplexity 1173.0056
+1300 train 6.833193 (lr=3.9642e-05) (hash(x)=149073315)
+1300 val loss 6.9427
+1300 val perplexity 1035.5397
+1300 train 6.705812 (lr=3.7164e-05) (hash(x)=149073315)
+1400 val loss 7.0057
+1400 val perplexity 1102.8519
+1400 train 7.326665 (lr=3.9574e-05) (hash(x)=175802021)
+1400 val loss 6.8794
+1400 val perplexity 972.0729
+1400 train 7.181681 (lr=3.7101e-05) (hash(x)=175802021)
+1500 val loss 6.9568
+1500 val perplexity 1050.2328
+1500 train 7.220825 (lr=3.9501e-05) (hash(x)=171034639)
+1500 val loss 6.8245
+1500 val perplexity 920.0974
+1500 train 7.093277 (lr=3.7032e-05) (hash(x)=171034639)
+1600 val loss 6.9040
+1600 val perplexity 996.2525
+1600 train 7.110297 (lr=3.9421e-05) (hash(x)=158681215)
+1600 val loss 6.7638
+1600 val perplexity 865.9183
+1600 train 6.971634 (lr=3.6958e-05) (hash(x)=158681215)
+1700 val loss 6.8712
+1700 val perplexity 964.1384
+1700 train 6.815831 (lr=3.9336e-05) (hash(x)=152116061)
+1700 val loss 6.7173
+1700 val perplexity 826.6052
+1700 train 6.662481 (lr=3.6878e-05) (hash(x)=152116061)
+1800 val loss 6.8333
+1800 val perplexity 928.2429
+1800 train 6.769077 (lr=3.9246e-05) (hash(x)=146108145)
+1800 val loss 6.6703
+1800 val perplexity 788.5938
+1800 train 6.601736 (lr=3.6793e-05) (hash(x)=146108145)
+1900 val loss 6.7823
+1900 val perplexity 882.0527
+1900 train 6.609385 (lr=3.9149e-05) (hash(x)=147598108)
+1900 val loss 6.6343
+1900 val perplexity 760.7567
+1900 train 6.457673 (lr=3.6702e-05) (hash(x)=147598108)
+2000 val loss 6.7541
+2000 val perplexity 857.5994
+2000 train 6.593875 (lr=3.9047e-05) (hash(x)=154996086)
+2000 val loss 6.6140
+2000 val perplexity 745.4609
+2000 train 6.439816 (lr=3.6606e-05) (hash(x)=154996086)
+2100 val loss 6.7144
+2100 val perplexity 824.1536
+2100 train 6.490150 (lr=3.8939e-05) (hash(x)=153396183)
+2100 val loss 6.5849
+2100 val perplexity 724.1075
+2100 train 6.354748 (lr=3.6505e-05) (hash(x)=153396183)
+2200 val loss 6.6532
+2200 val perplexity 775.2857
+2200 train 6.646795 (lr=3.8826e-05) (hash(x)=153885445)
+2200 val loss 6.5269
+2200 val perplexity 683.2792
+2200 train 6.532281 (lr=3.6399e-05) (hash(x)=153885445)
+2300 val loss 6.6119
+2300 val perplexity 743.8623
+2300 train 6.614337 (lr=3.8707e-05) (hash(x)=159666385)
+2300 val loss 6.5068
+2300 val perplexity 669.6613
+2300 train 6.508514 (lr=3.6288e-05) (hash(x)=159666385)
+2400 val loss 6.5667
+2400 val perplexity 711.0148
+2400 train 6.558314 (lr=3.8583e-05) (hash(x)=142353087)
+2400 val loss 6.4607
+2400 val perplexity 639.5293
+2400 train 6.453071 (lr=3.6171e-05) (hash(x)=142353087)
+2500 val loss 6.5303
+2500 val perplexity 685.6010
+2500 train 6.551023 (lr=3.8453e-05) (hash(x)=146491718)
+2500 val loss 6.4310
+2500 val perplexity 620.7972
+2500 train 6.445704 (lr=3.6049e-05) (hash(x)=146491718)
+2600 val loss 6.4981
+2600 val perplexity 663.8701
+2600 train 6.420842 (lr=3.8317e-05) (hash(x)=150750353)
+2600 val loss 6.4089
+2600 val perplexity 607.2394
+2600 train 6.333244 (lr=3.5923e-05) (hash(x)=150750353)
+2700 val loss 6.4712
+2700 val perplexity 646.2736
+2700 train 6.197730 (lr=3.8177e-05) (hash(x)=129849193)
+2700 val loss 6.3935
+2700 val perplexity 597.9317
+2700 train 6.113802 (lr=3.5791e-05) (hash(x)=129849193)
+2800 val loss 6.4585
+2800 val perplexity 638.0903
+2800 train 6.284863 (lr=3.8031e-05) (hash(x)=152767913)
+2800 val loss 6.3859
+2800 val perplexity 593.3999
+2800 train 6.207295 (lr=3.5654e-05) (hash(x)=152767913)
+2900 val loss 6.4376
+2900 val perplexity 624.8848
+2900 train 6.206501 (lr=3.7879e-05) (hash(x)=146531140)
+2900 val loss 6.3696
+2900 val perplexity 583.8331
+2900 train 6.140223 (lr=3.5512e-05) (hash(x)=146531140)
+3000 val loss 6.4011
+3000 val perplexity 602.4949
+3000 train 6.483201 (lr=3.7723e-05) (hash(x)=151562048)
+3000 val loss 6.3378
+3000 val perplexity 565.5319
+3000 train 6.426512 (lr=3.5365e-05) (hash(x)=151562048)
+3100 val loss 6.3668
+3100 val perplexity 582.2114
+3100 train 6.322643 (lr=3.7561e-05) (hash(x)=146001424)
+3100 val loss 6.3160
+3100 val perplexity 553.3704
+3100 train 6.270426 (lr=3.5214e-05) (hash(x)=146001424)
+3200 val loss 6.3596
+3200 val perplexity 577.9986
+3200 train 6.421289 (lr=3.7394e-05) (hash(x)=166486165)
+3200 val loss 6.3009
+3200 val perplexity 545.0633
+3200 train 6.371137 (lr=3.5057e-05) (hash(x)=166486165)
+3300 val loss 6.3258
+3300 val perplexity 558.8276
+3300 train 6.260159 (lr=3.7222e-05) (hash(x)=150866680)
+3300 val loss 6.2683
+3300 val perplexity 527.5996
+3300 train 6.203961 (lr=3.4896e-05) (hash(x)=150866680)
+3400 val loss 6.3106
+3400 val perplexity 550.3920
+3400 train 6.202477 (lr=3.7045e-05) (hash(x)=143900419)
+3400 val loss 6.2639
+3400 val perplexity 525.2636
+3400 train 6.162298 (lr=3.4730e-05) (hash(x)=143900419)
+3500 val loss 6.2926
+3500 val perplexity 540.5385
+3500 train 6.108777 (lr=3.6864e-05) (hash(x)=148845794)
+3500 val loss 6.2436
+3500 val perplexity 514.7196
+3500 train 6.060290 (lr=3.4560e-05) (hash(x)=148845794)
+3600 val loss 6.2743
+3600 val perplexity 530.7532
+3600 train 6.098810 (lr=3.6677e-05) (hash(x)=145667796)
+3600 val loss 6.2313
+3600 val perplexity 508.4273
+3600 train 6.051695 (lr=3.4384e-05) (hash(x)=145667796)
+3700 val loss 6.2667
+3700 val perplexity 526.7236
+3700 train 6.283004 (lr=3.6485e-05) (hash(x)=163563851)
+3700 val loss 6.2186
+3700 val perplexity 502.0200
+3700 train 6.234773 (lr=3.4205e-05) (hash(x)=163563851)
+3800 val loss 6.2501
+3800 val perplexity 518.0671
+3800 train 6.205328 (lr=3.6289e-05) (hash(x)=147488689)
+3800 val loss 6.2043
+3800 val perplexity 494.8778
+3800 train 6.156865 (lr=3.4021e-05) (hash(x)=147488689)
+3900 val loss 6.2277
+3900 val perplexity 506.5876
+3900 train 6.239246 (lr=3.6088e-05) (hash(x)=148186608)
+3900 val loss 6.1811
+3900 val perplexity 483.5410
+3900 train 6.190732 (lr=3.3832e-05) (hash(x)=148186608)
+4000 val loss 6.2055
+4000 val perplexity 495.4650
+4000 train 6.086553 (lr=3.5882e-05) (hash(x)=142970187)
+4000 val loss 6.1604
+4000 val perplexity 473.5967
+4000 train 6.046245 (lr=3.3639e-05) (hash(x)=142970187)
+4100 val loss 6.1877
+4100 val perplexity 486.7348
+4100 train 6.322111 (lr=3.5671e-05) (hash(x)=141584883)
+4100 val loss 6.1480
+4100 val perplexity 467.8040
+4100 train 6.294571 (lr=3.3442e-05) (hash(x)=141584883)
+4200 val loss 6.1772
+4200 val perplexity 481.6568
+4200 train 5.979122 (lr=3.5457e-05) (hash(x)=145664585)
+4200 val loss 6.1330
+4200 val perplexity 460.8101
+4200 train 5.931549 (lr=3.3240e-05) (hash(x)=145664585)
+4300 val loss 6.1621
+4300 val perplexity 474.4248
+4300 train 6.036442 (lr=3.5237e-05) (hash(x)=143736499)
+4300 val loss 6.1214
+4300 val perplexity 455.5243
+4300 train 5.990649 (lr=3.3035e-05) (hash(x)=143736499)
+4400 val loss 6.1566
+4400 val perplexity 471.8342
+4400 train 5.982193 (lr=3.5014e-05) (hash(x)=151883322)
+4400 val loss 6.1144
+4400 val perplexity 452.3284
+4400 train 5.935634 (lr=3.2825e-05) (hash(x)=151883322)
+4500 val loss 6.1477
+4500 val perplexity 467.6193
+4500 train 6.027667 (lr=3.4786e-05) (hash(x)=153904871)
+4500 val loss 6.1101
+4500 val perplexity 450.3830
+4500 train 5.990096 (lr=3.2611e-05) (hash(x)=153904871)
+4600 val loss 6.1353
+4600 val perplexity 461.8768
+4600 train 6.231039 (lr=3.4553e-05) (hash(x)=154893521)
+4600 val loss 6.0971
+4600 val perplexity 444.5587
+4600 train 6.197966 (lr=3.2394e-05) (hash(x)=154893521)
+4700 val loss 6.1131
+4700 val perplexity 451.7544
+4700 train 6.154427 (lr=3.4317e-05) (hash(x)=152323949)
+4700 val loss 6.0800
+4700 val perplexity 437.0204
+4700 train 6.121953 (lr=3.2172e-05) (hash(x)=152323949)
+4800 val loss 6.0982
+4800 val perplexity 445.0459
+4800 train 6.050216 (lr=3.4077e-05) (hash(x)=154104619)
+4800 val loss 6.0606
+4800 val perplexity 428.6257
+4800 train 6.015411 (lr=3.1947e-05) (hash(x)=154104619)
+4900 val loss 6.0874
+4900 val perplexity 440.2722
+4900 train 6.218273 (lr=3.3832e-05) (hash(x)=146311426)
+4900 val loss 6.0524
+4900 val perplexity 425.1423
+4900 train 6.163857 (lr=3.1718e-05) (hash(x)=146311426)
+5000 val loss 6.0780
+5000 val perplexity 436.1702
+5000 train 6.054106 (lr=3.3584e-05) (hash(x)=156741847)
+5000 val loss 6.0432
+5000 val perplexity 421.2397
+5000 train 6.020451 (lr=3.1485e-05) (hash(x)=156741847)
+5100 val loss 6.0687
+5100 val perplexity 432.1014
+5100 train 5.851731 (lr=3.3332e-05) (hash(x)=142086346)
+5100 val loss 6.0378
+5100 val perplexity 418.9780
+5100 train 5.812495 (lr=3.1249e-05) (hash(x)=142086346)
+5200 val loss 6.0645
+5200 val perplexity 430.2945
+5200 train 5.838616 (lr=3.3076e-05) (hash(x)=150265428)
+5200 val loss 6.0325
+5200 val perplexity 416.7517
+5200 train 5.801290 (lr=3.1009e-05) (hash(x)=150265428)
+5300 val loss 6.0519
+5300 val perplexity 424.9258
+5300 train 5.920601 (lr=3.2817e-05) (hash(x)=151339108)
+5300 val loss 6.0259
+5300 val perplexity 414.0215
+5300 train 5.896479 (lr=3.0766e-05) (hash(x)=151339108)
+5400 val loss 6.0438
+5400 val perplexity 421.4719
+5400 train 6.020870 (lr=3.2554e-05) (hash(x)=154654372)
+5400 val loss 6.0132
+5400 val perplexity 408.7832
+5400 train 5.971983 (lr=3.0519e-05) (hash(x)=154654372)
+5500 val loss 6.0302
+5500 val perplexity 415.8119
+5500 train 6.037389 (lr=3.2287e-05) (hash(x)=150575051)
+5500 val loss 5.9965
+5500 val perplexity 402.0139
+5500 train 6.008774 (lr=3.0269e-05) (hash(x)=150575051)
+5600 val loss 6.0248
+5600 val perplexity 413.5508
+5600 train 5.938440 (lr=3.2017e-05) (hash(x)=140396423)
+5600 val loss 5.9856
+5600 val perplexity 397.6622
+5600 train 5.897523 (lr=3.0016e-05) (hash(x)=140396423)
+5700 val loss 6.0045
+5700 val perplexity 405.2670
+5700 train 5.853642 (lr=3.1744e-05) (hash(x)=144678758)
+5700 val loss 5.9772
+5700 val perplexity 394.3266
+5700 train 5.828709 (lr=2.9760e-05) (hash(x)=144678758)
+5800 val loss 5.9977
+5800 val perplexity 402.4905
+5800 train 6.057874 (lr=3.1468e-05) (hash(x)=151992743)
+5800 val loss 5.9700
+5800 val perplexity 391.5067
+5800 train 6.027828 (lr=2.9501e-05) (hash(x)=151992743)
+5900 val loss 5.9894
+5900 val perplexity 399.1632
+5900 train 5.782817 (lr=3.1188e-05) (hash(x)=144396927)
+5900 val loss 5.9604
+5900 val perplexity 387.7543
+5900 train 5.755085 (lr=2.9239e-05) (hash(x)=144396927)
+6000 val loss 5.9792
+6000 val perplexity 395.1375
+6000 train 5.933915 (lr=3.0905e-05) (hash(x)=165478625)
+6000 val loss 5.9519
+6000 val perplexity 384.4901
+6000 train 5.896648 (lr=2.8974e-05) (hash(x)=165478625)
+6100 val loss 5.9793
+6100 val perplexity 395.1680
+6100 train 5.708070 (lr=3.0620e-05) (hash(x)=147088621)
+6100 val loss 5.9592
+6100 val perplexity 387.3049
+6100 train 5.687679 (lr=2.8706e-05) (hash(x)=147088621)
+6200 val loss 5.9699
+6200 val perplexity 391.4770
+6200 train 5.730008 (lr=3.0332e-05) (hash(x)=140794994)
+6200 val loss 5.9432
+6200 val perplexity 381.1576
+6200 train 5.709146 (lr=2.8436e-05) (hash(x)=140794994)
+6300 val loss 5.9605
+6300 val perplexity 387.8083
+6300 train 5.902456 (lr=3.0040e-05) (hash(x)=134780906)
+6300 val loss 5.9288
+6300 val perplexity 375.7183
+6300 train 5.868412 (lr=2.8163e-05) (hash(x)=134780906)
+6400 val loss 5.9421
+6400 val perplexity 380.7487
+6400 train 5.926373 (lr=2.9747e-05) (hash(x)=149023655)
+6400 val loss 5.9124
+6400 val perplexity 369.5739
+6400 train 5.901180 (lr=2.7888e-05) (hash(x)=149023655)
+6500 val loss 5.9334
+6500 val perplexity 377.4468
+6500 train 5.781084 (lr=2.9450e-05) (hash(x)=147497796)
+6500 val loss 5.9058
+6500 val perplexity 367.1653
+6500 train 5.750584 (lr=2.7610e-05) (hash(x)=147497796)
+6600 val loss 5.9301
+6600 val perplexity 376.1914
+6600 train 5.772248 (lr=2.9152e-05) (hash(x)=152902689)
+6600 val loss 5.9013
+6600 val perplexity 365.4952
+6600 train 5.752711 (lr=2.7330e-05) (hash(x)=152902689)
+6700 val loss 5.9384
+6700 val perplexity 379.3110
+6700 train 5.978984 (lr=2.8851e-05) (hash(x)=153846046)
+6700 val loss 5.9061
+6700 val perplexity 367.2744
+6700 train 5.948668 (lr=2.7047e-05) (hash(x)=153846046)
+6800 val loss 5.9072
+6800 val perplexity 367.6755
+6800 train 6.038209 (lr=2.8547e-05) (hash(x)=158512738)
+6800 val loss 5.8796
+6800 val perplexity 357.6506
+6800 train 6.010082 (lr=2.6763e-05) (hash(x)=158512738)
+6900 val loss 5.9006
+6900 val perplexity 365.2498
+6900 train 6.420359 (lr=2.8242e-05) (hash(x)=156849968)
+6900 val loss 5.8750
+6900 val perplexity 356.0296
+6900 train 6.411304 (lr=2.6477e-05) (hash(x)=156849968)
+7000 val loss 5.8930
+7000 val perplexity 362.4887
+7000 train 5.751428 (lr=2.7934e-05) (hash(x)=142395855)
+7000 val loss 5.8643
+7000 val perplexity 352.2307
+7000 train 5.722020 (lr=2.6188e-05) (hash(x)=142395855)
+7100 val loss 5.8898
+7100 val perplexity 361.3358
+7100 train 5.744161 (lr=2.7624e-05) (hash(x)=147114884)
+7100 val loss 5.8620
+7100 val perplexity 351.4352
+7100 train 5.708766 (lr=2.5898e-05) (hash(x)=147114884)
+7200 val loss 5.8776
+7200 val perplexity 356.9494
+7200 train 5.911930 (lr=2.7313e-05) (hash(x)=156979839)
+7200 val loss 5.8497
+7200 val perplexity 347.1398
+7200 train 5.874716 (lr=2.5606e-05) (hash(x)=156979839)
+7300 val loss 5.8737
+7300 val perplexity 355.5622
+7300 train 5.665551 (lr=2.7000e-05) (hash(x)=145584373)
+7300 val loss 5.8484
+7300 val perplexity 346.6742
+7300 train 5.648200 (lr=2.5312e-05) (hash(x)=145584373)
+7400 val loss 5.8650
+7400 val perplexity 352.4915
+7400 train 5.609486 (lr=2.6685e-05) (hash(x)=141508204)
+7400 val loss 5.8448
+7400 val perplexity 345.4498
+7400 train 5.591662 (lr=2.5017e-05) (hash(x)=141508204)
+7500 val loss 5.8531
+7500 val perplexity 348.3254
+7500 train 5.891374 (lr=2.6369e-05) (hash(x)=148803965)
+7500 val loss 5.8303
+7500 val perplexity 340.4442
+7500 train 5.863412 (lr=2.4721e-05) (hash(x)=148803965)
+7600 val loss 5.8480
+7600 val perplexity 346.5572
+7600 train 5.902753 (lr=2.6051e-05) (hash(x)=151019676)
+7600 val loss 5.8243
+7600 val perplexity 338.4339
+7600 train 5.884450 (lr=2.4423e-05) (hash(x)=151019676)
+7700 val loss 5.8308
+7700 val perplexity 340.6423
+7700 train 5.933746 (lr=2.5732e-05) (hash(x)=143155750)
+7700 val loss 5.8114
+7700 val perplexity 334.0776
+7700 train 5.916495 (lr=2.4124e-05) (hash(x)=143155750)
+7800 val loss 5.8294
+7800 val perplexity 340.1667
+7800 train 5.860056 (lr=2.5411e-05) (hash(x)=152569653)
+7800 val loss 5.8093
+7800 val perplexity 333.3885
+7800 train 5.832789 (lr=2.3823e-05) (hash(x)=152569653)
+7900 val loss 5.8226
+7900 val perplexity 337.8345
+7900 train 5.699649 (lr=2.5090e-05) (hash(x)=143519455)
+7900 val loss 5.8013
+7900 val perplexity 330.7151
+7900 train 5.672275 (lr=2.3522e-05) (hash(x)=143519455)
+8000 val loss 5.8151
+8000 val perplexity 335.3357
+8000 train 5.913954 (lr=2.4767e-05) (hash(x)=161180944)
+8000 val loss 5.7937
+8000 val perplexity 328.2259
+8000 train 5.883524 (lr=2.3219e-05) (hash(x)=161180944)
+8100 val loss 5.8179
+8100 val perplexity 336.2599
+8100 train 5.844440 (lr=2.4444e-05) (hash(x)=154107345)
+8100 val loss 5.7921
+8100 val perplexity 327.6842
+8100 train 5.819099 (lr=2.2916e-05) (hash(x)=154107345)
+8200 val loss 5.8122
+8200 val perplexity 334.3447
+8200 train 5.851220 (lr=2.4120e-05) (hash(x)=152486517)
+8200 val loss 5.7916
+8200 val perplexity 327.5419
+8200 train 5.825037 (lr=2.2612e-05) (hash(x)=152486517)
+8300 val loss 5.7987
+8300 val perplexity 329.8811
+8300 train 5.804880 (lr=2.3795e-05) (hash(x)=156167749)
+8300 val loss 5.7805
+8300 val perplexity 323.9186
+8300 train 5.781146 (lr=2.2308e-05) (hash(x)=156167749)
+8400 val loss 5.7949
+8400 val perplexity 328.6116
+8400 train 5.884000 (lr=2.3469e-05) (hash(x)=149155006)
+8400 val loss 5.7753
+8400 val perplexity 322.2279
+8400 train 5.864207 (lr=2.2002e-05) (hash(x)=149155006)
+8500 val loss 5.7817
+8500 val perplexity 324.3089
+8500 train 5.722820 (lr=2.3143e-05) (hash(x)=147844390)
+8500 val loss 5.7626
+8500 val perplexity 318.1658
+8500 train 5.712843 (lr=2.1697e-05) (hash(x)=147844390)
+8600 val loss 5.7772
+8600 val perplexity 322.8451
+8600 train 6.027016 (lr=2.2817e-05) (hash(x)=165753320)
+8600 val loss 5.7581
+8600 val perplexity 316.7603
+8600 train 6.013570 (lr=2.1391e-05) (hash(x)=165753320)
+8700 val loss 5.7711
+8700 val perplexity 320.8813
+8700 train 5.550282 (lr=2.2490e-05) (hash(x)=146079979)
+8700 val loss 5.7549
+8700 val perplexity 315.7386
+8700 train 5.528018 (lr=2.1085e-05) (hash(x)=146079979)
+8800 val loss 5.7729
+8800 val perplexity 321.4843
+8800 train 5.959742 (lr=2.2163e-05) (hash(x)=172259509)
+8800 val loss 5.7516
+8800 val perplexity 314.7053
+8800 train 5.941445 (lr=2.0778e-05) (hash(x)=172259509)
+8900 val loss 5.7657
+8900 val perplexity 319.1769
+8900 train 5.439955 (lr=2.1837e-05) (hash(x)=145148314)
+8900 val loss 5.7465
+8900 val perplexity 313.0956
+8900 train 5.426799 (lr=2.0472e-05) (hash(x)=145148314)
+9000 val loss 5.7599
+9000 val perplexity 317.3039
+9000 train 5.682996 (lr=2.1510e-05) (hash(x)=144250633)
+9000 val loss 5.7424
+9000 val perplexity 311.8057
+9000 train 5.664256 (lr=2.0165e-05) (hash(x)=144250633)
+9100 val loss 5.7545
+9100 val perplexity 315.6111
+9100 train 5.857194 (lr=2.1183e-05) (hash(x)=157219797)
+9100 val loss 5.7381
+9100 val perplexity 310.4626
+9100 train 5.840177 (lr=1.9859e-05) (hash(x)=157219797)
+9200 val loss 5.7413
+9200 val perplexity 311.4770
+9200 train 5.797914 (lr=2.0857e-05) (hash(x)=142743778)
+9200 val loss 5.7290
+9200 val perplexity 307.6659
+9200 train 5.779316 (lr=1.9553e-05) (hash(x)=142743778)
+9300 val loss 5.7385
+9300 val perplexity 310.6078
+9300 train 5.666136 (lr=2.0531e-05) (hash(x)=139669771)
+9300 val loss 5.7255
+9300 val perplexity 306.5905
+9300 train 5.659318 (lr=1.9248e-05) (hash(x)=139669771)
+9400 val loss 5.7292
+9400 val perplexity 307.7287
+9400 train 5.776366 (lr=2.0205e-05) (hash(x)=145916843)
+9400 val loss 5.7158
+9400 val perplexity 303.6291
+9400 train 5.751498 (lr=1.8942e-05) (hash(x)=145916843)
+9500 val loss 5.7304
+9500 val perplexity 308.0824
+9500 train 5.645638 (lr=1.9880e-05) (hash(x)=150196125)
+9500 val loss 5.7119
+9500 val perplexity 302.4328
+9500 train 5.636480 (lr=1.8638e-05) (hash(x)=150196125)
+9600 val loss 5.7219
+9600 val perplexity 305.4899
+9600 train 6.033727 (lr=1.9556e-05) (hash(x)=160041419)
+9600 val loss 5.7099
+9600 val perplexity 301.8288
+9600 train 6.023571 (lr=1.8334e-05) (hash(x)=160041419)
+9700 val loss 5.7220
+9700 val perplexity 305.5088
+9700 train 5.503003 (lr=1.9233e-05) (hash(x)=139931627)
+9700 val loss 5.7082
+9700 val perplexity 301.3137
+9700 train 5.491812 (lr=1.8031e-05) (hash(x)=139931627)
+9800 val loss 5.7155
+9800 val perplexity 303.5482
+9800 train 5.696881 (lr=1.8910e-05) (hash(x)=150370792)
+9800 val loss 5.7005
+9800 val perplexity 299.0040
+9800 train 5.680792 (lr=1.7728e-05) (hash(x)=150370792)
+9900 val loss 5.7093
+9900 val perplexity 301.6725
+9900 train 5.542067 (lr=1.8589e-05) (hash(x)=153014886)
+9900 val loss 5.6953
+9900 val perplexity 297.4720
+9900 train 5.523115 (lr=1.7427e-05) (hash(x)=153014886)
+10000 val loss 5.7047
+10000 val perplexity 300.2856
+10000 train 5.607134 (lr=1.8268e-05) (hash(x)=141915265)
+10000 val loss 5.6927
+10000 val perplexity 296.6991
+10000 train 5.594075 (lr=1.7126e-05) (hash(x)=141915265)
+10100 val loss 5.6994
+10100 val perplexity 298.7018
+10100 train 6.204209 (lr=1.7949e-05) (hash(x)=197770952)
+10100 val loss 5.6883
+10100 val perplexity 295.3954
+10100 train 6.189380 (lr=1.6827e-05) (hash(x)=197770952)
+10200 val loss 5.6916
+10200 val perplexity 296.3653
+10200 train 5.666861 (lr=1.7631e-05) (hash(x)=156806005)
+10200 val loss 5.6819
+10200 val perplexity 293.5147
+10200 train 5.647396 (lr=1.6529e-05) (hash(x)=156806005)
+10300 val loss 5.6888
+10300 val perplexity 295.5269
+10300 train 5.500148 (lr=1.7315e-05) (hash(x)=145378216)
+10300 val loss 5.6795
+10300 val perplexity 292.8058
+10300 train 5.484988 (lr=1.6233e-05) (hash(x)=145378216)
+10400 val loss 5.6817
+10400 val perplexity 293.4449
+10400 train 5.593282 (lr=1.7000e-05) (hash(x)=145125407)
+10400 val loss 5.6708
+10400 val perplexity 290.2617
+10400 train 5.582484 (lr=1.5938e-05) (hash(x)=145125407)
+10500 val loss 5.6802
+10500 val perplexity 293.0004
+10500 train 5.493242 (lr=1.6687e-05) (hash(x)=147553745)
+10500 val loss 5.6710
+10500 val perplexity 290.3318
+10500 train 5.494811 (lr=1.5644e-05) (hash(x)=147553745)
+10600 val loss 5.6777
+10600 val perplexity 292.2679
+10600 train 5.317801 (lr=1.6376e-05) (hash(x)=134894889)
+10600 val loss 5.6662
+10600 val perplexity 288.9396
+10600 train 5.318324 (lr=1.5352e-05) (hash(x)=134894889)
+10700 val loss 5.6693
+10700 val perplexity 289.8371
+10700 train 5.593366 (lr=1.6066e-05) (hash(x)=151041040)
+10700 val loss 5.6609
+10700 val perplexity 287.4037
+10700 train 5.582285 (lr=1.5062e-05) (hash(x)=151041040)
+10800 val loss 5.6634
+10800 val perplexity 288.1177
+10800 train 5.595318 (lr=1.5758e-05) (hash(x)=145685829)
+10800 val loss 5.6543
+10800 val perplexity 285.5227
+10800 train 5.583758 (lr=1.4773e-05) (hash(x)=145685829)
+10900 val loss 5.6585
+10900 val perplexity 286.7199
+10900 train 5.739454 (lr=1.5453e-05) (hash(x)=156000716)
+10900 val loss 5.6492
+10900 val perplexity 284.0699
+10900 train 5.734696 (lr=1.4487e-05) (hash(x)=156000716)
+11000 val loss 5.6547
+11000 val perplexity 285.6243
+11000 train 5.665429 (lr=1.5149e-05) (hash(x)=149858163)
+11000 val loss 5.6467
+11000 val perplexity 283.3628
+11000 train 5.654157 (lr=1.4203e-05) (hash(x)=149858163)
+11100 val loss 5.6523
+11100 val perplexity 284.9481
+11100 train 5.932511 (lr=1.4848e-05) (hash(x)=164873575)
+11100 val loss 5.6453
+11100 val perplexity 282.9635
+11100 train 5.932080 (lr=1.3920e-05) (hash(x)=164873575)
+11200 val loss 5.6532
+11200 val perplexity 285.2096
+11200 train 5.536577 (lr=1.4550e-05) (hash(x)=156370069)
+11200 val loss 5.6454
+11200 val perplexity 282.9827
+11200 train 5.519896 (lr=1.3640e-05) (hash(x)=156370069)
+11300 val loss 5.6495
+11300 val perplexity 284.1511
+11300 train 5.738060 (lr=1.4253e-05) (hash(x)=160356892)
+11300 val loss 5.6433
+11300 val perplexity 282.3891
+11300 train 5.722073 (lr=1.3362e-05) (hash(x)=160356892)
+11400 val loss 5.6395
+11400 val perplexity 281.3151
+11400 train 5.707570 (lr=1.3960e-05) (hash(x)=158955430)
+11400 val loss 5.6331
+11400 val perplexity 279.5309
+11400 train 5.699232 (lr=1.3087e-05) (hash(x)=158955430)
+11500 val loss 5.6344
+11500 val perplexity 279.8896
+11500 train 5.614134 (lr=1.3668e-05) (hash(x)=146598377)
+11500 val loss 5.6283
+11500 val perplexity 278.2024
+11500 train 5.589320 (lr=1.2814e-05) (hash(x)=146598377)
+11600 val loss 5.6628
+11600 val perplexity 287.9396
+11600 train 5.808537 (lr=1.3380e-05) (hash(x)=156045795)
+11600 val loss 5.6480
+11600 val perplexity 283.7278
+11600 train 5.784801 (lr=1.2544e-05) (hash(x)=156045795)
+11700 val loss 5.6311
+11700 val perplexity 278.9812
+11700 train 5.494103 (lr=1.3095e-05) (hash(x)=143963334)
+11700 val loss 5.6257
+11700 val perplexity 277.4742
+11700 train 5.477342 (lr=1.2276e-05) (hash(x)=143963334)
+11800 val loss 5.6293
+11800 val perplexity 278.4727
+11800 train 5.468873 (lr=1.2812e-05) (hash(x)=145831579)
+11800 val loss 5.6245
+11800 val perplexity 277.1250
+11800 train 5.454970 (lr=1.2011e-05) (hash(x)=145831579)
+11900 val loss 5.6252
+11900 val perplexity 277.3227
+11900 train 5.793966 (lr=1.2532e-05) (hash(x)=165154141)
+11900 val loss 5.6200
+11900 val perplexity 275.9032
+11900 train 5.778847 (lr=1.1749e-05) (hash(x)=165154141)
+12000 val loss 5.6199
+12000 val perplexity 275.8593
+12000 train 5.701312 (lr=1.2256e-05) (hash(x)=160373794)
+12000 val loss 5.6135
+12000 val perplexity 274.1089
+12000 train 5.691423 (lr=1.1490e-05) (hash(x)=160373794)
+12100 val loss 5.6162
+12100 val perplexity 274.8550
+12100 train 5.406276 (lr=1.1983e-05) (hash(x)=141476346)
+12100 val loss 5.6107
+12100 val perplexity 273.3370
+12100 train 5.404443 (lr=1.1234e-05) (hash(x)=141476346)
+12200 val loss 5.6134
+12200 val perplexity 274.0763
+12200 train 5.868688 (lr=1.1713e-05) (hash(x)=164508058)
+12200 val loss 5.6091
+12200 val perplexity 272.9045
+12200 train 5.864333 (lr=1.0981e-05) (hash(x)=164508058)
+12300 val loss 5.6108
+12300 val perplexity 273.3618
+12300 train 5.597710 (lr=1.1446e-05) (hash(x)=140407664)
+12300 val loss 5.6059
+12300 val perplexity 272.0320
+12300 train 5.588737 (lr=1.0731e-05) (hash(x)=140407664)
+12400 val loss 5.6079
+12400 val perplexity 272.5839
+12400 train 5.461676 (lr=1.1183e-05) (hash(x)=149526189)
+12400 val loss 5.6054
+12400 val perplexity 271.8886
+12400 train 5.460255 (lr=1.0484e-05) (hash(x)=149526189)
+12500 val loss 5.6030
+12500 val perplexity 271.2485
+12500 train 5.449765 (lr=1.0924e-05) (hash(x)=142060916)
+12500 val loss 5.6003
+12500 val perplexity 270.4955
+12500 train 5.456053 (lr=1.0241e-05) (hash(x)=142060916)
+12600 val loss 5.5984
+12600 val perplexity 269.9859
+12600 train 5.778718 (lr=1.0668e-05) (hash(x)=168650848)
+12600 val loss 5.5952
+12600 val perplexity 269.1323
+12600 train 5.773306 (lr=1.0001e-05) (hash(x)=168650848)
+12700 val loss 5.5969
+12700 val perplexity 269.5833
+12700 train 5.618319 (lr=1.0416e-05) (hash(x)=154353018)
+12700 val loss 5.5934
+12700 val perplexity 268.6388
+12700 train 5.614925 (lr=9.7649e-06) (hash(x)=154353018)
+12800 val loss 5.5980
+12800 val perplexity 269.8842
+12800 train 5.762591 (lr=1.0168e-05) (hash(x)=156115265)
+12800 val loss 5.5941
+12800 val perplexity 268.8290
+12900 val loss 5.5944
+12900 val perplexity 268.9202
+12800 train 5.749104 (lr=9.5322e-06) (hash(x)=156115265)
+12900 train 5.519612 (lr=9.9233e-06) (hash(x)=143378175)
+13000 val loss 5.5921
+13000 val perplexity 268.3089
+12900 val loss 5.5926
+12900 val perplexity 268.4366
+13000 train 5.694238 (lr=9.6829e-06) (hash(x)=166124864)
+12900 train 5.517610 (lr=9.3031e-06) (hash(x)=143378175)
+13000 val loss 5.5889
+13000 val perplexity 267.4538
+13000 train 5.684518 (lr=9.0777e-06) (hash(x)=166124864)
+13100 val loss 5.5895
+13100 val perplexity 267.5899
+13100 train 5.472170 (lr=9.4466e-06) (hash(x)=146615385)
+13200 val loss 5.5914
+13200 val perplexity 268.1238
+13200 train 5.491530 (lr=9.2144e-06) (hash(x)=149236047)
+13100 val loss 5.5882
+13100 val perplexity 267.2526
+13100 train 5.465880 (lr=8.8562e-06) (hash(x)=146615385)
+13300 val loss 5.5855
+13300 val perplexity 266.5218
+13300 train 5.513978 (lr=8.9865e-06) (hash(x)=151346972)
+13200 val loss 5.5862
+13200 val perplexity 266.7275
+13200 train 5.484173 (lr=8.6385e-06) (hash(x)=149236047)
+13400 val loss 5.5827
+13400 val perplexity 265.7754
+13400 train 5.675163 (lr=8.7628e-06) (hash(x)=159167093)
+13300 val loss 5.5833
+13300 val perplexity 265.9412
+13300 train 5.509476 (lr=8.4248e-06) (hash(x)=151346972)
+13500 val loss 5.5806
+13500 val perplexity 265.2330
+13500 train 5.302473 (lr=8.5435e-06) (hash(x)=135054650)
+13400 val loss 5.5798
+13400 val perplexity 265.0265
+13400 train 5.674298 (lr=8.2151e-06) (hash(x)=159167093)
+13600 val loss 5.5801
+13600 val perplexity 265.0899
+13600 train 5.463277 (lr=8.3286e-06) (hash(x)=151033927)
+13500 val loss 5.5785
+13500 val perplexity 264.6842
+13500 train 5.312832 (lr=8.0095e-06) (hash(x)=135054650)
+13700 val loss 5.5792
+13700 val perplexity 264.8670
+13700 train 5.272966 (lr=8.1183e-06) (hash(x)=141119172)
+13600 val loss 5.5776
+13600 val perplexity 264.4442
+13600 train 5.465922 (lr=7.8081e-06) (hash(x)=151033927)
+13800 val loss 5.5764
+13800 val perplexity 264.1136
+13800 train 5.289057 (lr=7.9125e-06) (hash(x)=144197766)
+13700 val loss 5.5762
+13700 val perplexity 264.0586
+13700 train 5.273306 (lr=7.6109e-06) (hash(x)=141119172)
+13900 val loss 5.5724
+13900 val perplexity 263.0594
+13900 train 5.510831 (lr=7.7114e-06) (hash(x)=145148158)
+13800 val loss 5.5757
+13800 val perplexity 263.9417
+13800 train 5.286690 (lr=7.4180e-06) (hash(x)=144197766)
+14000 val loss 5.5687
+14000 val perplexity 262.0901
+14000 train 6.069955 (lr=7.5149e-06) (hash(x)=179068751)
+13900 val loss 5.5698
+13900 val perplexity 262.3841
+13900 train 5.509115 (lr=7.2294e-06) (hash(x)=145148158)
+14100 val loss 5.5677
+14100 val perplexity 261.8275
+14100 train 5.435109 (lr=7.3233e-06) (hash(x)=144988373)
+14000 val loss 5.5673
+14000 val perplexity 261.7274
+14000 train 6.063221 (lr=7.0452e-06) (hash(x)=179068751)
+14200 val loss 5.5658
+14200 val perplexity 261.3357
+14200 train 5.512167 (lr=7.1365e-06) (hash(x)=133379696)
+14100 val loss 5.5657
+14100 val perplexity 261.2974
+14100 train 5.440147 (lr=6.8656e-06) (hash(x)=144988373)
+14300 val loss 5.5669
+14300 val perplexity 261.6292
+14300 train 5.442722 (lr=6.9546e-06) (hash(x)=149883422)
+14200 val loss 5.5650
+14200 val perplexity 261.1154
+14200 train 5.502961 (lr=6.6904e-06) (hash(x)=133379696)
+14400 val loss 5.5665
+14400 val perplexity 261.5081
+14400 train 5.428222 (lr=6.7776e-06) (hash(x)=145709198)
+14300 val loss 5.5660
+14300 val perplexity 261.3931
+14300 train 5.445783 (lr=6.5199e-06) (hash(x)=149883422)
+14500 val loss 5.5604
+14500 val perplexity 259.9273
+14500 train 5.575936 (lr=6.6057e-06) (hash(x)=148016579)
+14400 val loss 5.5656
+14400 val perplexity 261.2873
+14400 train 5.421428 (lr=6.3540e-06) (hash(x)=145709198)
+14600 val loss 5.5589
+14600 val perplexity 259.5357
+14600 train 5.529923 (lr=6.4389e-06) (hash(x)=150218222)
+14500 val loss 5.5601
+14500 val perplexity 259.8371
+14500 train 5.578542 (lr=6.1928e-06) (hash(x)=148016579)
+14700 val loss 5.5592
+14700 val perplexity 259.6161
+14700 train 5.484089 (lr=6.2771e-06) (hash(x)=142258737)
+14600 val loss 5.5581
+14600 val perplexity 259.3359
+14600 train 5.534735 (lr=6.0364e-06) (hash(x)=150218222)
+14800 val loss 5.5551
+14800 val perplexity 258.5652
+14800 train 5.453934 (lr=6.1206e-06) (hash(x)=147628003)
+14700 val loss 5.5577
+14700 val perplexity 259.2366
+14700 train 5.493864 (lr=5.8848e-06) (hash(x)=142258737)
+14900 val loss 5.5566
+14900 val perplexity 258.9344
+14900 train 5.411365 (lr=5.9693e-06) (hash(x)=149144036)
+14800 val loss 5.5555
+14800 val perplexity 258.6584
+14800 train 5.448818 (lr=5.7381e-06) (hash(x)=147628003)
+15000 val loss 5.5561
+15000 val perplexity 258.8178
+15000 train 5.489982 (lr=5.8233e-06) (hash(x)=145925409)
+14900 val loss 5.5569
+14900 val perplexity 259.0222
+14900 train 5.407426 (lr=5.5962e-06) (hash(x)=149144036)
+15100 val loss 5.5518
+15100 val perplexity 257.7099
+15100 train 5.436214 (lr=5.6826e-06) (hash(x)=144555751)
+15000 val loss 5.5554
+15000 val perplexity 258.6341
+15000 train 5.492520 (lr=5.4593e-06) (hash(x)=145925409)
+15200 val loss 5.5504
+15200 val perplexity 257.3345
+15200 train 5.364831 (lr=5.5473e-06) (hash(x)=143448852)
+15100 val loss 5.5514
+15100 val perplexity 257.6073
+15100 train 5.432653 (lr=5.3275e-06) (hash(x)=144555751)
+15300 val loss 5.5487
+15300 val perplexity 256.9110
+15300 train 5.554673 (lr=5.4175e-06) (hash(x)=156347253)
+15200 val loss 5.5495
+15200 val perplexity 257.0970
+15200 train 5.369192 (lr=5.2006e-06) (hash(x)=143448852)
+15400 val loss 5.5467
+15400 val perplexity 256.3846
+15400 train 5.494298 (lr=5.2931e-06) (hash(x)=153010179)
+15300 val loss 5.5487
+15300 val perplexity 256.9073
+15300 train 5.551206 (lr=5.0789e-06) (hash(x)=156347253)
+15500 val loss 5.5477
+15500 val perplexity 256.6517
+15500 train 5.325893 (lr=5.1742e-06) (hash(x)=144066445)
+15400 val loss 5.5468
+15400 val perplexity 256.4174
+15400 train 5.497810 (lr=4.9622e-06) (hash(x)=153010179)
+15600 val loss 5.5476
+15600 val perplexity 256.6144
+15600 train 5.283917 (lr=5.0608e-06) (hash(x)=146107722)
+15500 val loss 5.5471
+15500 val perplexity 256.4917
+15500 train 5.329374 (lr=4.8508e-06) (hash(x)=144066445)
+15700 val loss 5.5433
+15700 val perplexity 255.5193
+15700 train 5.703694 (lr=4.9531e-06) (hash(x)=152533799)
+15600 val loss 5.5457
+15600 val perplexity 256.1370
+15600 train 5.281954 (lr=4.7445e-06) (hash(x)=146107722)
+15800 val loss 5.5401
+15800 val perplexity 254.7081
+15800 train 5.594624 (lr=4.8509e-06) (hash(x)=151756366)
+15700 val loss 5.5431
+15700 val perplexity 255.4666
+15700 train 5.690786 (lr=4.6435e-06) (hash(x)=152533799)
+15900 val loss 5.5398
+15900 val perplexity 254.6376
+15900 train 5.522333 (lr=4.7545e-06) (hash(x)=151735039)
+15800 val loss 5.5402
+15800 val perplexity 254.7284
+15800 train 5.592268 (lr=4.5478e-06) (hash(x)=151756366)
+16000 val loss 5.5389
+16000 val perplexity 254.4079
+16000 train 5.564443 (lr=4.6637e-06) (hash(x)=154633226)
+15900 val loss 5.5406
+15900 val perplexity 254.8195
+15900 train 5.530737 (lr=4.4573e-06) (hash(x)=151735039)
+16100 val loss 5.5407
+16100 val perplexity 254.8598
+16100 train 5.373457 (lr=4.5786e-06) (hash(x)=158094333)
+16000 val loss 5.5398
+16000 val perplexity 254.6348
+16000 train 5.566914 (lr=4.3722e-06) (hash(x)=154633226)
+16200 val loss 5.5395
+16200 val perplexity 254.5566
+16200 train 5.301944 (lr=4.4993e-06) (hash(x)=142772927)
+16100 val loss 5.5413
+16100 val perplexity 255.0077
+16100 train 5.377645 (lr=4.2924e-06) (hash(x)=158094333)
+16300 val loss 5.5382
+16300 val perplexity 254.2214
+16300 train 5.442211 (lr=4.4257e-06) (hash(x)=148042896)
+16200 val loss 5.5397
+16200 val perplexity 254.6087
+16200 train 5.298206 (lr=4.2180e-06) (hash(x)=142772927)
+16400 val loss 5.5343
+16400 val perplexity 253.2240
+16400 train 6.030413 (lr=4.3579e-06) (hash(x)=170306399)
+16300 val loss 5.5383
+16300 val perplexity 254.2502
+16300 train 5.440192 (lr=4.1491e-06) (hash(x)=148042896)
+16500 val loss 5.5344
+16500 val perplexity 253.2674
+16500 train 5.555060 (lr=4.2960e-06) (hash(x)=141049320)
+16400 val loss 5.5351
+16400 val perplexity 253.4445
+16400 train 6.030590 (lr=4.0856e-06) (hash(x)=170306399)
+16600 val loss 5.5348
+16600 val perplexity 253.3616
+16600 train 5.383412 (lr=4.2399e-06) (hash(x)=150512136)
+16500 val loss 5.5352
+16500 val perplexity 253.4678
+16500 train 5.552647 (lr=4.0275e-06) (hash(x)=141049320)
+16700 val loss 5.5323
+16700 val perplexity 252.7309
+16700 train 5.544337 (lr=4.1896e-06) (hash(x)=151843645)
+16600 val loss 5.5355
+16600 val perplexity 253.5231
+16600 train 5.393153 (lr=3.9749e-06) (hash(x)=150512136)
+16800 val loss 5.5298
+16800 val perplexity 252.1039
+16800 train 5.617894 (lr=4.1452e-06) (hash(x)=140073659)
+16700 val loss 5.5328
+16700 val perplexity 252.8530
+16700 train 5.542039 (lr=3.9278e-06) (hash(x)=151843645)
+16900 val loss 5.5283
+16900 val perplexity 251.7074
+16900 train 5.680115 (lr=4.1067e-06) (hash(x)=171157212)
+16800 val loss 5.5304
+16800 val perplexity 252.2500
+16800 train 5.605481 (lr=3.8862e-06) (hash(x)=140073659)
+17000 val loss 5.5268
+17000 val perplexity 251.3435
+17000 train 5.455764 (lr=4.0741e-06) (hash(x)=144553276)
+16900 val loss 5.5295
+16900 val perplexity 252.0072
+16900 train 5.689977 (lr=3.8501e-06) (hash(x)=171157212)
+17100 val loss 5.5280
+17100 val perplexity 251.6405
+17100 train 5.518610 (lr=4.0475e-06) (hash(x)=157717565)
+17000 val loss 5.5295
+17000 val perplexity 252.0275
+17000 train 5.469572 (lr=3.8195e-06) (hash(x)=144553276)
+17200 val loss 5.5286
+17200 val perplexity 251.8003
+17200 train 5.387378 (lr=4.0267e-06) (hash(x)=149843860)
+17100 val loss 5.5289
+17100 val perplexity 251.8564
+17100 train 5.516634 (lr=3.7945e-06) (hash(x)=157717565)
+17300 val loss 5.5249
+17300 val perplexity 250.8722
+17300 train 5.581902 (lr=4.0119e-06) (hash(x)=162171979)
+17200 val loss 5.5297
+17200 val perplexity 252.0703
+17200 train 5.394426 (lr=3.7750e-06) (hash(x)=149843860)
+17400 val loss 5.5241
+17400 val perplexity 250.6583
+17400 train 5.499109 (lr=4.0030e-06) (hash(x)=148400702)
+17300 val loss 5.5271
+17300 val perplexity 251.4217
+17300 train 5.585364 (lr=3.7611e-06) (hash(x)=162171979)
+17499 val loss 5.5230
+17499 val perplexity 250.3805
+17400 val loss 5.5252
+17400 val perplexity 250.9298
+17400 train 5.501467 (lr=3.7528e-06) (hash(x)=148400702)

attention_kindselective_n_heads4_seed1338/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9bccf2461b0c494a3687ecba37da35128d7decbe154d840de37be045dd40b15
+oid sha256:278408312592a175d7ef97e31580042adc4983160ade6d84eefec00db3f627b6
 size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bbe1f41dc0665df8af9fb548e6cf38e4843c70c08313c5c880d130411c57a236
+oid sha256:98acd634505624795929b0bce783d7c16bc675ab95a8831af11c40182685208e
 size 179406214