Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_30720_4_1339", "n_embd": 256}

attention_kindselective_n_heads4_seed1339/log2.txt

@@ -1,528 +1,1056 @@
 max_steps: 17500
+max_steps: 17500
+0 val loss 11.2810
+0 val perplexity 79298.4922
 0 val loss 11.2810
 0 val perplexity 79298.4922
-0 train 11.295413 (lr=1.3750e-07) (hash(x)=150724848)
-100 val loss 9.9603
-100 val perplexity 21170.0430
-100 train 10.007094 (lr=1.3888e-05) (hash(x)=149217061)
-200 val loss 8.9723
-200 val perplexity 7882.0664
-200 train 8.990173 (lr=2.7500e-05) (hash(x)=149910534)
-300 val loss 7.9449
-300 val perplexity 2821.2769
-300 train 7.869526 (lr=2.7498e-05) (hash(x)=142185643)
-400 val loss 7.6747
-400 val perplexity 2153.2439
-400 train 7.723728 (lr=2.7492e-05) (hash(x)=148123706)
-500 val loss 7.5919
-500 val perplexity 1982.0444
-500 train 7.469253 (lr=2.7482e-05) (hash(x)=149952383)
-600 val loss 7.5425
-600 val perplexity 1886.4757
-600 train 7.369303 (lr=2.7467e-05) (hash(x)=146678221)
-700 val loss 7.4943
-700 val perplexity 1797.8137
-700 train 7.524321 (lr=2.7449e-05) (hash(x)=156180736)
-800 val loss 7.4513
-800 val perplexity 1722.1606
-800 train 7.442773 (lr=2.7427e-05) (hash(x)=151700982)
-900 val loss 7.4230
-900 val perplexity 1673.9784
-900 train 7.216110 (lr=2.7400e-05) (hash(x)=147288467)
-1000 val loss 7.3971
-1000 val perplexity 1631.3278
-1000 train 7.439397 (lr=2.7370e-05) (hash(x)=156182087)
-1100 val loss 7.3783
-1100 val perplexity 1600.9114
-1100 train 7.255154 (lr=2.7335e-05) (hash(x)=147861550)
-1200 val loss 7.3593
-1200 val perplexity 1570.6859
-1200 train 7.232437 (lr=2.7297e-05) (hash(x)=149318660)
-1300 val loss 7.3489
-1300 val perplexity 1554.4735
-1300 train 7.209687 (lr=2.7254e-05) (hash(x)=151631103)
-1400 val loss 7.3331
-1400 val perplexity 1530.1715
-1400 train 7.381249 (lr=2.7207e-05) (hash(x)=150482428)
-1500 val loss 7.3048
-1500 val perplexity 1487.3849
-1500 train 7.156554 (lr=2.7157e-05) (hash(x)=144458617)
-1600 val loss 7.2813
-1600 val perplexity 1452.9476
-1600 train 7.155491 (lr=2.7102e-05) (hash(x)=143268605)
-1700 val loss 7.2596
-1700 val perplexity 1421.6182
-1700 train 7.191982 (lr=2.7044e-05) (hash(x)=150817011)
-1800 val loss 7.2414
-1800 val perplexity 1396.0886
-1800 train 7.228529 (lr=2.6981e-05) (hash(x)=152322423)
-1900 val loss 7.2235
-1900 val perplexity 1371.2922
-1900 train 7.044053 (lr=2.6915e-05) (hash(x)=143574126)
-2000 val loss 7.2147
-2000 val perplexity 1359.2687
-2000 train 6.985788 (lr=2.6845e-05) (hash(x)=147904298)
-2100 val loss 7.1918
-2100 val perplexity 1328.4425
-2100 train 6.953863 (lr=2.6771e-05) (hash(x)=144719400)
-2200 val loss 7.1646
-2200 val perplexity 1292.8639
-2200 train 7.402001 (lr=2.6693e-05) (hash(x)=154343147)
-2300 val loss 7.1374
-2300 val perplexity 1258.2095
-2300 train 7.186423 (lr=2.6611e-05) (hash(x)=154372635)
-2400 val loss 7.1253
-2400 val perplexity 1242.9806
-2400 train 6.955888 (lr=2.6525e-05) (hash(x)=141843115)
-2500 val loss 7.0979
-2500 val perplexity 1209.3766
-2500 train 6.993516 (lr=2.6436e-05) (hash(x)=149417679)
-2600 val loss 7.0803
-2600 val perplexity 1188.2710
-2600 train 6.944737 (lr=2.6343e-05) (hash(x)=145279030)
-2700 val loss 7.0618
-2700 val perplexity 1166.5103
-2700 train 6.953462 (lr=2.6246e-05) (hash(x)=146363911)
-2800 val loss 7.0477
-2800 val perplexity 1150.1772
-2800 train 7.019940 (lr=2.6146e-05) (hash(x)=152507639)
-2900 val loss 7.0327
-2900 val perplexity 1133.1031
-2900 train 6.825722 (lr=2.6042e-05) (hash(x)=139296191)
-3000 val loss 7.0194
-3000 val perplexity 1118.1536
-3000 train 7.043332 (lr=2.5934e-05) (hash(x)=148473774)
-3100 val loss 6.9894
-3100 val perplexity 1085.0210
-3100 train 7.095091 (lr=2.5823e-05) (hash(x)=154130527)
-3200 val loss 6.9654
-3200 val perplexity 1059.3645
-3200 train 6.898325 (lr=2.5709e-05) (hash(x)=151117002)
-3300 val loss 6.9591
-3300 val perplexity 1052.6724
-3300 train 7.079136 (lr=2.5590e-05) (hash(x)=151035383)
-3400 val loss 6.9313
-3400 val perplexity 1023.8008
-3400 train 7.011525 (lr=2.5469e-05) (hash(x)=150614749)
-3500 val loss 6.9130
-3500 val perplexity 1005.3008
-3500 train 6.961452 (lr=2.5344e-05) (hash(x)=151431391)
-3600 val loss 6.8958
-3600 val perplexity 988.1005
-3600 train 6.819849 (lr=2.5215e-05) (hash(x)=151140797)
-3700 val loss 6.8788
-3700 val perplexity 971.4802
-3700 train 7.120714 (lr=2.5083e-05) (hash(x)=154608340)
-3800 val loss 6.8610
-3800 val perplexity 954.3665
-3800 train 6.883213 (lr=2.4948e-05) (hash(x)=169641309)
-3900 val loss 6.8405
-3900 val perplexity 934.9172
-3900 train 6.791928 (lr=2.4810e-05) (hash(x)=150592479)
-4000 val loss 6.8152
-4000 val perplexity 911.6139
-4000 train 6.790205 (lr=2.4669e-05) (hash(x)=155495317)
-4100 val loss 6.8076
-4100 val perplexity 904.6551
-4100 train 7.009809 (lr=2.4524e-05) (hash(x)=170323970)
-4200 val loss 6.7861
-4200 val perplexity 885.4940
-4200 train 6.587790 (lr=2.4376e-05) (hash(x)=140854360)
-4300 val loss 6.7558
-4300 val perplexity 859.0294
-4300 train 6.750789 (lr=2.4226e-05) (hash(x)=148926644)
-4400 val loss 6.7324
-4400 val perplexity 839.1576
-4400 train 6.774224 (lr=2.4072e-05) (hash(x)=149997871)
-4500 val loss 6.7066
-4500 val perplexity 817.7622
-4500 train 6.658325 (lr=2.3915e-05) (hash(x)=149219289)
-4600 val loss 6.6926
-4600 val perplexity 806.4287
-4600 train 6.489277 (lr=2.3755e-05) (hash(x)=139836197)
-4700 val loss 6.6692
-4700 val perplexity 787.7410
-4700 train 6.632764 (lr=2.3593e-05) (hash(x)=150722883)
-4800 val loss 6.6471
-4800 val perplexity 770.5707
-4800 train 6.862972 (lr=2.3428e-05) (hash(x)=166864648)
-4900 val loss 6.6300
-4900 val perplexity 757.4689
-4900 train 6.746885 (lr=2.3260e-05) (hash(x)=157746743)
-5000 val loss 6.6030
-5000 val perplexity 737.3123
-5000 train 6.564852 (lr=2.3089e-05) (hash(x)=153032257)
-5100 val loss 6.5809
-5100 val perplexity 721.2192
-5100 train 6.662528 (lr=2.2916e-05) (hash(x)=151862371)
-5200 val loss 6.5598
-5200 val perplexity 706.0972
-5200 train 6.881927 (lr=2.2740e-05) (hash(x)=168862995)
-5300 val loss 6.5497
-5300 val perplexity 699.0560
-5300 train 6.526187 (lr=2.2561e-05) (hash(x)=148243567)
-5400 val loss 6.5310
-5400 val perplexity 686.0756
-5400 train 6.454970 (lr=2.2381e-05) (hash(x)=145381742)
-5500 val loss 6.5144
-5500 val perplexity 674.7809
-5500 train 6.669298 (lr=2.2197e-05) (hash(x)=155602174)
-5600 val loss 6.4878
-5600 val perplexity 657.1042
-5600 train 6.578669 (lr=2.2012e-05) (hash(x)=160637672)
-5700 val loss 6.4728
-5700 val perplexity 647.2852
-5700 train 6.409344 (lr=2.1824e-05) (hash(x)=152101131)
-5800 val loss 6.4536
-5800 val perplexity 634.9938
-5800 train 6.406909 (lr=2.1634e-05) (hash(x)=152327089)
-5900 val loss 6.4456
-5900 val perplexity 629.9174
-5900 train 6.379840 (lr=2.1442e-05) (hash(x)=151094543)
-6000 val loss 6.4294
-6000 val perplexity 619.8093
-6000 train 6.527921 (lr=2.1248e-05) (hash(x)=160233315)
-6100 val loss 6.4108
-6100 val perplexity 608.3984
-6100 train 6.511137 (lr=2.1051e-05) (hash(x)=158295337)
-6200 val loss 6.3918
-6200 val perplexity 596.9457
-6200 train 6.349186 (lr=2.0853e-05) (hash(x)=151269817)
-6300 val loss 6.3760
-6300 val perplexity 587.5455
-6300 train 6.490969 (lr=2.0653e-05) (hash(x)=156912756)
-6400 val loss 6.3643
-6400 val perplexity 580.7366
-6400 train 6.304334 (lr=2.0451e-05) (hash(x)=125779637)
-6500 val loss 6.3546
-6500 val perplexity 575.1245
-6500 train 6.241210 (lr=2.0247e-05) (hash(x)=148157661)
-6600 val loss 6.3439
-6600 val perplexity 569.0137
-6600 train 6.392389 (lr=2.0042e-05) (hash(x)=152820049)
-6700 val loss 6.3313
-6700 val perplexity 561.9103
-6700 train 6.277595 (lr=1.9835e-05) (hash(x)=155818717)
-6800 val loss 6.3142
-6800 val perplexity 552.3473
-6800 train 6.357372 (lr=1.9626e-05) (hash(x)=158833881)
-6900 val loss 6.3115
-6900 val perplexity 550.8728
-6900 train 6.280612 (lr=1.9416e-05) (hash(x)=164984909)
-7000 val loss 6.2944
-7000 val perplexity 541.5046
-7000 train 6.183699 (lr=1.9205e-05) (hash(x)=151172493)
-7100 val loss 6.2913
-7100 val perplexity 539.8703
-7100 train 6.301157 (lr=1.8992e-05) (hash(x)=154023633)
-7200 val loss 6.2841
-7200 val perplexity 535.9922
-7200 train 6.126408 (lr=1.8778e-05) (hash(x)=139351250)
-7300 val loss 6.2701
-7300 val perplexity 528.5300
-7300 train 6.216894 (lr=1.8562e-05) (hash(x)=158719458)
-7400 val loss 6.2545
-7400 val perplexity 520.3541
-7400 train 6.342453 (lr=1.8346e-05) (hash(x)=162250810)
-7500 val loss 6.2451
-7500 val perplexity 515.4594
-7500 train 6.107119 (lr=1.8128e-05) (hash(x)=142827557)
-7600 val loss 6.2375
-7600 val perplexity 511.5827
-7600 train 6.134127 (lr=1.7910e-05) (hash(x)=147830801)
-7700 val loss 6.2256
-7700 val perplexity 505.5025
-7700 train 6.331251 (lr=1.7691e-05) (hash(x)=165233314)
-7800 val loss 6.2179
-7800 val perplexity 501.6711
-7800 train 6.216796 (lr=1.7470e-05) (hash(x)=160654717)
-7900 val loss 6.2136
-7900 val perplexity 499.4775
-7900 train 6.051061 (lr=1.7249e-05) (hash(x)=140712586)
-8000 val loss 6.1968
-8000 val perplexity 491.1817
-8000 train 6.355695 (lr=1.7028e-05) (hash(x)=150309061)
-8100 val loss 6.1933
-8100 val perplexity 489.4791
-8100 train 6.204559 (lr=1.6805e-05) (hash(x)=158805466)
-8200 val loss 6.1826
-8200 val perplexity 484.2360
-8200 train 6.047708 (lr=1.6582e-05) (hash(x)=142814078)
-8300 val loss 6.1745
-8300 val perplexity 480.3483
-8300 train 6.138761 (lr=1.6359e-05) (hash(x)=148340417)
-8400 val loss 6.1701
-8400 val perplexity 478.2222
-8400 train 6.109375 (lr=1.6135e-05) (hash(x)=157132639)
-8500 val loss 6.1692
-8500 val perplexity 477.8114
-8500 train 6.108513 (lr=1.5911e-05) (hash(x)=152994240)
-8600 val loss 6.1531
-8600 val perplexity 470.1633
-8600 train 6.107152 (lr=1.5687e-05) (hash(x)=152210154)
-8700 val loss 6.1455
-8700 val perplexity 466.6085
-8700 train 6.205480 (lr=1.5462e-05) (hash(x)=149631247)
-8800 val loss 6.1372
-8800 val perplexity 462.7359
-8800 train 5.981178 (lr=1.5237e-05) (hash(x)=145617784)
-8900 val loss 6.1299
-8900 val perplexity 459.3880
-8900 train 6.226513 (lr=1.5013e-05) (hash(x)=155984970)
-9000 val loss 6.1254
-9000 val perplexity 457.3501
-9000 train 5.854463 (lr=1.4788e-05) (hash(x)=143433013)
-9100 val loss 6.1202
-9100 val perplexity 454.9649
-9100 train 6.005787 (lr=1.4563e-05) (hash(x)=152507533)
-9200 val loss 6.1142
-9200 val perplexity 452.2314
-9200 train 6.255028 (lr=1.4339e-05) (hash(x)=162400650)
-9300 val loss 6.1044
-9300 val perplexity 447.8143
-9300 train 6.153483 (lr=1.4115e-05) (hash(x)=148419068)
-9400 val loss 6.0967
-9400 val perplexity 444.3860
-9400 train 6.057858 (lr=1.3891e-05) (hash(x)=145025169)
-9500 val loss 6.0875
-9500 val perplexity 440.3331
-9500 train 6.061573 (lr=1.3668e-05) (hash(x)=145104235)
-9600 val loss 6.0901
-9600 val perplexity 441.4541
-9600 train 6.020576 (lr=1.3445e-05) (hash(x)=150499797)
-9700 val loss 6.0780
-9700 val perplexity 436.1382
-9700 train 5.881082 (lr=1.3222e-05) (hash(x)=139690022)
-9800 val loss 6.0699
-9800 val perplexity 432.6385
-9800 train 6.058245 (lr=1.3001e-05) (hash(x)=145645279)
-9900 val loss 6.0611
-9900 val perplexity 428.8531
-9900 train 5.971967 (lr=1.2780e-05) (hash(x)=150103432)
-10000 val loss 6.0610
-10000 val perplexity 428.7884
-10000 train 6.213492 (lr=1.2559e-05) (hash(x)=162309473)
-10100 val loss 6.0507
-10100 val perplexity 424.4024
-10100 train 5.937528 (lr=1.2340e-05) (hash(x)=146586772)
-10200 val loss 6.0450
-10200 val perplexity 422.0151
-10200 train 5.940793 (lr=1.2122e-05) (hash(x)=149256313)
-10300 val loss 6.0396
-10300 val perplexity 419.7351
-10300 train 6.018402 (lr=1.1904e-05) (hash(x)=151455083)
-10400 val loss 6.0374
-10400 val perplexity 418.8122
-10400 train 5.842232 (lr=1.1688e-05) (hash(x)=150210444)
-10500 val loss 6.0397
-10500 val perplexity 419.7683
-10500 train 5.804628 (lr=1.1472e-05) (hash(x)=152144428)
-10600 val loss 6.0296
-10600 val perplexity 415.5560
-10600 train 6.099782 (lr=1.1258e-05) (hash(x)=168628935)
-10700 val loss 6.0206
-10700 val perplexity 411.8316
-10700 train 5.880192 (lr=1.1045e-05) (hash(x)=164667785)
-10800 val loss 6.0117
-10800 val perplexity 408.1864
-10800 train 5.955902 (lr=1.0834e-05) (hash(x)=143621491)
-10900 val loss 6.0106
-10900 val perplexity 407.7130
-10900 train 5.899141 (lr=1.0624e-05) (hash(x)=143914826)
-11000 val loss 6.0058
-11000 val perplexity 405.7754
-11000 train 5.781573 (lr=1.0415e-05) (hash(x)=137663435)
-11100 val loss 6.0036
-11100 val perplexity 404.8896
-11100 train 5.873570 (lr=1.0208e-05) (hash(x)=149054007)
-11200 val loss 6.0013
-11200 val perplexity 403.9420
-11200 train 6.005395 (lr=1.0003e-05) (hash(x)=153473924)
-11300 val loss 6.0030
-11300 val perplexity 404.6429
-11300 train 5.808824 (lr=9.7991e-06) (hash(x)=158563275)
-11400 val loss 6.0014
-11400 val perplexity 403.9809
-11400 train 6.135341 (lr=9.5972e-06) (hash(x)=161016667)
-11500 val loss 5.9971
-11500 val perplexity 402.2647
-11500 train 6.046785 (lr=9.3970e-06) (hash(x)=157344822)
-11600 val loss 5.9819
-11600 val perplexity 396.1853
-11600 train 5.841331 (lr=9.1988e-06) (hash(x)=139977713)
-11700 val loss 5.9752
-11700 val perplexity 393.5630
-11700 train 5.986742 (lr=9.0025e-06) (hash(x)=156511738)
-11800 val loss 5.9721
-11800 val perplexity 392.3307
-11800 train 6.460408 (lr=8.8082e-06) (hash(x)=185372558)
-11900 val loss 5.9709
-11900 val perplexity 391.8408
-11900 train 5.801854 (lr=8.6160e-06) (hash(x)=138085470)
-12000 val loss 5.9692
-12000 val perplexity 391.1958
-12000 train 5.824976 (lr=8.4260e-06) (hash(x)=142928235)
-12100 val loss 5.9703
-12100 val perplexity 391.6365
-12100 train 5.825260 (lr=8.2381e-06) (hash(x)=151960396)
-12200 val loss 5.9584
-12200 val perplexity 386.9728
-12200 train 6.033947 (lr=8.0526e-06) (hash(x)=156794523)
-12300 val loss 5.9553
-12300 val perplexity 385.8007
-12300 train 5.767831 (lr=7.8694e-06) (hash(x)=138082358)
-12400 val loss 5.9486
-12400 val perplexity 383.2321
-12400 train 5.852021 (lr=7.6885e-06) (hash(x)=155255899)
-12500 val loss 5.9469
-12500 val perplexity 382.5752
-12500 train 5.925366 (lr=7.5101e-06) (hash(x)=150490528)
-12600 val loss 5.9437
-12600 val perplexity 381.3263
-12600 train 6.066210 (lr=7.3342e-06) (hash(x)=154831027)
-12700 val loss 5.9425
-12700 val perplexity 380.8722
-12700 train 5.882764 (lr=7.1609e-06) (hash(x)=147341052)
-12800 val loss 5.9403
-12800 val perplexity 380.0535
-12800 train 5.977871 (lr=6.9903e-06) (hash(x)=159333795)
-12900 val loss 5.9416
-12900 val perplexity 380.5322
-12900 train 5.784962 (lr=6.8223e-06) (hash(x)=143896867)
-13000 val loss 5.9327
-13000 val perplexity 377.1534
-13000 train 5.921026 (lr=6.6570e-06) (hash(x)=147161537)
-13100 val loss 5.9279
-13100 val perplexity 375.3620
-13100 train 6.057076 (lr=6.4945e-06) (hash(x)=147081918)
-13200 val loss 5.9254
-13200 val perplexity 374.4168
-13200 train 5.689006 (lr=6.3349e-06) (hash(x)=145079287)
-13300 val loss 5.9268
-13300 val perplexity 374.9709
-13300 train 5.729674 (lr=6.1782e-06) (hash(x)=142515041)
-13400 val loss 5.9236
-13400 val perplexity 373.7578
-13400 train 5.724186 (lr=6.0244e-06) (hash(x)=144081985)
-13500 val loss 5.9207
-13500 val perplexity 372.6601
-13500 train 5.990724 (lr=5.8737e-06) (hash(x)=149957292)
-13600 val loss 5.9153
-13600 val perplexity 370.6725
-13600 train 5.946911 (lr=5.7259e-06) (hash(x)=157123033)
-13700 val loss 5.9115
-13700 val perplexity 369.2667
-13700 train 5.899968 (lr=5.5813e-06) (hash(x)=148277247)
-13800 val loss 5.9118
-13800 val perplexity 369.3729
-13800 train 6.018199 (lr=5.4398e-06) (hash(x)=142769907)
-13900 val loss 5.9075
-13900 val perplexity 367.7686
-13900 train 5.817693 (lr=5.3016e-06) (hash(x)=144932004)
-14000 val loss 5.9045
-14000 val perplexity 366.6660
-14000 train 5.907049 (lr=5.1665e-06) (hash(x)=149532265)
-14100 val loss 5.9036
-14100 val perplexity 366.3682
-14100 train 5.748715 (lr=5.0348e-06) (hash(x)=149301374)
-14200 val loss 5.9043
-14200 val perplexity 366.5968
-14200 train 5.656896 (lr=4.9063e-06) (hash(x)=142871075)
-14300 val loss 5.8980
-14300 val perplexity 364.3106
-14300 train 6.041042 (lr=4.7813e-06) (hash(x)=166744960)
-14400 val loss 5.8939
-14400 val perplexity 362.8291
-14400 train 5.875367 (lr=4.6596e-06) (hash(x)=145950920)
-14500 val loss 5.8929
-14500 val perplexity 362.4689
-14500 train 5.894318 (lr=4.5414e-06) (hash(x)=141089155)
-14600 val loss 5.8899
-14600 val perplexity 361.3864
-14600 train 6.058316 (lr=4.4267e-06) (hash(x)=174516275)
-14700 val loss 5.8880
-14700 val perplexity 360.7004
-14700 train 5.669492 (lr=4.3155e-06) (hash(x)=137896065)
-14800 val loss 5.8868
-14800 val perplexity 360.2480
-14800 train 5.918042 (lr=4.2079e-06) (hash(x)=154396556)
-14900 val loss 5.8871
-14900 val perplexity 360.3737
-14900 train 5.747381 (lr=4.1039e-06) (hash(x)=149322394)
-15000 val loss 5.8871
-15000 val perplexity 360.3573
-15000 train 5.802734 (lr=4.0035e-06) (hash(x)=147410513)
-15100 val loss 5.8844
-15100 val perplexity 359.3901
-15100 train 5.803135 (lr=3.9068e-06) (hash(x)=145245634)
-15200 val loss 5.8800
-15200 val perplexity 357.8166
-15200 train 5.965070 (lr=3.8138e-06) (hash(x)=148341108)
-15300 val loss 5.8774
-15300 val perplexity 356.8876
-15300 train 5.697327 (lr=3.7245e-06) (hash(x)=140162437)
-15400 val loss 5.8762
-15400 val perplexity 356.4378
-15400 train 6.069530 (lr=3.6390e-06) (hash(x)=142166081)
-15500 val loss 5.8757
-15500 val perplexity 356.2906
-15500 train 5.752399 (lr=3.5572e-06) (hash(x)=141042196)
-15600 val loss 5.8742
-15600 val perplexity 355.7265
-15600 train 5.975054 (lr=3.4793e-06) (hash(x)=148672701)
-15700 val loss 5.8742
-15700 val perplexity 355.7508
-15700 train 5.772561 (lr=3.4052e-06) (hash(x)=149672385)
-15800 val loss 5.8731
-15800 val perplexity 355.3330
-15800 train 5.898124 (lr=3.3350e-06) (hash(x)=152379562)
-15900 val loss 5.8693
-15900 val perplexity 354.0105
-15900 train 5.903119 (lr=3.2687e-06) (hash(x)=154674265)
-16000 val loss 5.8670
-16000 val perplexity 353.1752
-16000 train 5.897068 (lr=3.2063e-06) (hash(x)=150368078)
-16100 val loss 5.8651
-16100 val perplexity 352.5308
-16100 train 5.962111 (lr=3.1478e-06) (hash(x)=160390561)
-16200 val loss 5.8642
-16200 val perplexity 352.1844
-16200 train 6.208712 (lr=3.0932e-06) (hash(x)=155218378)
-16300 val loss 5.8629
-16300 val perplexity 351.7262
-16300 train 6.052020 (lr=3.0427e-06) (hash(x)=160891077)
-16400 val loss 5.8618
-16400 val perplexity 351.3526
-16400 train 5.771412 (lr=2.9961e-06) (hash(x)=142789253)
-16500 val loss 5.8614
-16500 val perplexity 351.1985
-16500 train 5.603426 (lr=2.9535e-06) (hash(x)=141497841)
-16600 val loss 5.8614
-16600 val perplexity 351.2000
-16600 train 5.843324 (lr=2.9149e-06) (hash(x)=157947018)
-16700 val loss 5.8569
-16700 val perplexity 349.6255
-16700 train 5.805718 (lr=2.8804e-06) (hash(x)=146039309)
-16800 val loss 5.8567
-16800 val perplexity 349.5811
-16800 train 5.820101 (lr=2.8498e-06) (hash(x)=156381445)
-16900 val loss 5.8545
-16900 val perplexity 348.8135
-16900 train 5.816824 (lr=2.8234e-06) (hash(x)=146537354)
-17000 val loss 5.8537
-17000 val perplexity 348.5178
-17000 train 5.735730 (lr=2.8010e-06) (hash(x)=135939444)
-17100 val loss 5.8534
-17100 val perplexity 348.4101
-17100 train 5.909929 (lr=2.7826e-06) (hash(x)=147819652)
-17200 val loss 5.8540
-17200 val perplexity 348.6358
-17200 train 5.964046 (lr=2.7684e-06) (hash(x)=155307909)
-17300 val loss 5.8497
-17300 val perplexity 347.1329
-17300 train 5.801245 (lr=2.7582e-06) (hash(x)=151879548)
-17400 val loss 5.8487
-17400 val perplexity 346.7682
-17400 train 5.754910 (lr=2.7520e-06) (hash(x)=143641792)
-17499 val loss 5.8466
-17499 val perplexity 346.0586
+0 train 11.295413 (lr=1.8750e-07) (hash(x)=150724848)
+0 train 11.295413 (lr=2.0000e-07) (hash(x)=150724848)
+100 val loss 9.8920
+100 val perplexity 19770.6621
+100 train 9.940850 (lr=1.8937e-05) (hash(x)=149217061)
+100 val loss 9.8892
+100 val perplexity 19716.9434
+100 train 9.938035 (lr=2.0200e-05) (hash(x)=149217061)
+200 val loss 8.7035
+200 val perplexity 6023.8257
+200 train 8.722006 (lr=3.7500e-05) (hash(x)=149910534)
+200 val loss 8.5860
+200 val perplexity 5356.3530
+200 train 8.602024 (lr=4.0000e-05) (hash(x)=149910534)
+300 val loss 7.9287
+300 val perplexity 2775.8965
+300 train 7.848921 (lr=3.7497e-05) (hash(x)=142185643)
+300 val loss 7.7731
+300 val perplexity 2375.8701
+300 train 7.680683 (lr=3.9997e-05) (hash(x)=142185643)
+400 val loss 7.6983
+400 val perplexity 2204.5137
+400 train 7.749911 (lr=3.7489e-05) (hash(x)=148123706)
+400 val loss 7.6116
+400 val perplexity 2021.4500
+400 train 7.655191 (lr=3.9988e-05) (hash(x)=148123706)
+500 val loss 7.6214
+500 val perplexity 2041.4889
+500 train 7.503228 (lr=3.7475e-05) (hash(x)=149952383)
+500 val loss 7.5217
+500 val perplexity 1847.7963
+500 train 7.393625 (lr=3.9973e-05) (hash(x)=149952383)
+600 val loss 7.5742
+600 val perplexity 1947.2408
+600 train 7.401229 (lr=3.7456e-05) (hash(x)=146678221)
+600 val loss 7.4720
+600 val perplexity 1758.0408
+600 train 7.277638 (lr=3.9953e-05) (hash(x)=146678221)
+700 val loss 7.5268
+700 val perplexity 1857.0763
+700 train 7.560060 (lr=3.7430e-05) (hash(x)=156180736)
+700 val loss 7.3901
+700 val perplexity 1619.8132
+700 train 7.425849 (lr=3.9926e-05) (hash(x)=156180736)
+800 val loss 7.4827
+800 val perplexity 1776.9448
+800 train 7.479188 (lr=3.7400e-05) (hash(x)=151700982)
+800 val loss 7.3359
+800 val perplexity 1534.3604
+800 train 7.325125 (lr=3.9893e-05) (hash(x)=151700982)
+900 val loss 7.4518
+900 val perplexity 1723.0034
+900 train 7.249286 (lr=3.7364e-05) (hash(x)=147288467)
+900 val loss 7.2879
+900 val perplexity 1462.5081
+900 train 7.073882 (lr=3.9855e-05) (hash(x)=147288467)
+1000 val loss 7.4266
+1000 val perplexity 1680.1207
+1000 train 7.467032 (lr=3.7322e-05) (hash(x)=156182087)
+1000 val loss 7.2358
+1000 val perplexity 1388.1892
+1000 train 7.278591 (lr=3.9810e-05) (hash(x)=156182087)
+1100 val loss 7.4096
+1100 val perplexity 1651.7021
+1100 train 7.287074 (lr=3.7275e-05) (hash(x)=147861550)
+1100 val loss 7.1888
+1100 val perplexity 1324.5430
+1100 train 7.058900 (lr=3.9760e-05) (hash(x)=147861550)
+1200 val loss 7.3875
+1200 val perplexity 1615.6384
+1200 train 7.260944 (lr=3.7223e-05) (hash(x)=149318660)
+1200 val loss 7.1559
+1200 val perplexity 1281.6619
+1200 train 7.013008 (lr=3.9704e-05) (hash(x)=149318660)
+1300 val loss 7.3797
+1300 val perplexity 1603.1160
+1300 train 7.235526 (lr=3.7164e-05) (hash(x)=151631103)
+1300 val loss 7.1329
+1300 val perplexity 1252.5103
+1300 train 6.970014 (lr=3.9642e-05) (hash(x)=151631103)
+1400 val loss 7.3629
+1400 val perplexity 1576.4673
+1400 train 7.414872 (lr=3.7101e-05) (hash(x)=150482428)
+1400 val loss 7.0961
+1400 val perplexity 1207.2863
+1400 train 7.131195 (lr=3.9574e-05) (hash(x)=150482428)
+1500 val loss 7.3347
+1500 val perplexity 1532.5674
+1500 train 7.194934 (lr=3.7032e-05) (hash(x)=144458617)
+1500 val loss 7.0334
+1500 val perplexity 1133.9139
+1500 train 6.878065 (lr=3.9501e-05) (hash(x)=144458617)
+1600 val loss 7.3127
+1600 val perplexity 1499.2800
+1600 train 7.187172 (lr=3.6958e-05) (hash(x)=143268605)
+1600 val loss 6.9786
+1600 val perplexity 1073.4591
+1600 train 6.860932 (lr=3.9421e-05) (hash(x)=143268605)
+1700 val loss 7.2833
+1700 val perplexity 1455.7445
+1700 train 7.218442 (lr=3.6878e-05) (hash(x)=150817011)
+1700 val loss 6.9268
+1700 val perplexity 1019.2236
+1700 train 6.862553 (lr=3.9336e-05) (hash(x)=150817011)
+1800 val loss 7.2652
+1800 val perplexity 1429.6526
+1800 train 7.249866 (lr=3.6793e-05) (hash(x)=152322423)
+1800 val loss 6.8768
+1800 val perplexity 969.4940
+1800 train 6.858002 (lr=3.9246e-05) (hash(x)=152322423)
+1900 val loss 7.2479
+1900 val perplexity 1405.1208
+1900 train 7.074307 (lr=3.6702e-05) (hash(x)=143574126)
+1900 val loss 6.8296
+1900 val perplexity 924.8268
+1900 train 6.618206 (lr=3.9149e-05) (hash(x)=143574126)
+2000 val loss 7.2321
+2000 val perplexity 1383.1353
+2000 train 7.012033 (lr=3.6606e-05) (hash(x)=147904298)
+2000 val loss 6.7887
+2000 val perplexity 887.7144
+2000 train 6.533892 (lr=3.9047e-05) (hash(x)=147904298)
+2100 val loss 7.2056
+2100 val perplexity 1346.9266
+2100 train 6.963788 (lr=3.6505e-05) (hash(x)=144719400)
+2100 val loss 6.7451
+2100 val perplexity 849.8469
+2100 train 6.479913 (lr=3.8939e-05) (hash(x)=144719400)
+2200 val loss 7.1795
+2200 val perplexity 1312.2140
+2200 train 7.416286 (lr=3.6399e-05) (hash(x)=154343147)
+2200 val loss 6.6894
+2200 val perplexity 803.8664
+2200 train 6.935144 (lr=3.8826e-05) (hash(x)=154343147)
+2300 val loss 7.1358
+2300 val perplexity 1256.1078
+2300 train 7.187479 (lr=3.6288e-05) (hash(x)=154372635)
+2300 val loss 6.6321
+2300 val perplexity 759.0529
+2300 train 6.670909 (lr=3.8707e-05) (hash(x)=154372635)
+2400 val loss 7.1059
+2400 val perplexity 1219.1160
+2400 train 6.941741 (lr=3.6171e-05) (hash(x)=141843115)
+2400 val loss 6.5962
+2400 val perplexity 732.2883
+2400 train 6.420857 (lr=3.8583e-05) (hash(x)=141843115)
+2500 val loss 7.0782
+2500 val perplexity 1185.8218
+2500 train 6.975012 (lr=3.6049e-05) (hash(x)=149417679)
+2500 val loss 6.5524
+2500 val perplexity 700.9512
+2500 train 6.445707 (lr=3.8453e-05) (hash(x)=149417679)
+2600 val loss 7.0639
+2600 val perplexity 1168.9380
+2600 train 6.928061 (lr=3.5923e-05) (hash(x)=145279030)
+2600 val loss 6.5181
+2600 val perplexity 677.3092
+2600 train 6.392206 (lr=3.8317e-05) (hash(x)=145279030)
+2700 val loss 7.0370
+2700 val perplexity 1137.9823
+2700 train 6.938586 (lr=3.5791e-05) (hash(x)=146363911)
+2700 val loss 6.4870
+2700 val perplexity 656.5254
+2700 train 6.406978 (lr=3.8177e-05) (hash(x)=146363911)
+2800 val loss 7.0138
+2800 val perplexity 1111.8834
+2800 train 6.982116 (lr=3.5654e-05) (hash(x)=152507639)
+2800 val loss 6.4852
+2800 val perplexity 655.3987
+2800 train 6.433135 (lr=3.8031e-05) (hash(x)=152507639)
+2900 val loss 6.9887
+2900 val perplexity 1084.3630
+2900 train 6.780720 (lr=3.5512e-05) (hash(x)=139296191)
+2900 val loss 6.4438
+2900 val perplexity 628.7961
+2900 train 6.226076 (lr=3.7879e-05) (hash(x)=139296191)
+3000 val loss 6.9635
+3000 val perplexity 1057.3408
+3000 train 6.976216 (lr=3.5365e-05) (hash(x)=148473774)
+3000 val loss 6.4230
+3000 val perplexity 615.8414
+3000 train 6.430891 (lr=3.7723e-05) (hash(x)=148473774)
+3100 val loss 6.9340
+3100 val perplexity 1026.5642
+3100 train 7.038289 (lr=3.5214e-05) (hash(x)=154130527)
+3100 val loss 6.3778
+3100 val perplexity 588.6588
+3100 train 6.464539 (lr=3.7561e-05) (hash(x)=154130527)
+3200 val loss 6.8995
+3200 val perplexity 991.7739
+3200 train 6.837024 (lr=3.5057e-05) (hash(x)=151117002)
+3200 val loss 6.3583
+3200 val perplexity 577.2902
+3200 train 6.312740 (lr=3.7394e-05) (hash(x)=151117002)
+3300 val loss 6.8909
+3300 val perplexity 983.2383
+3300 train 7.017479 (lr=3.4896e-05) (hash(x)=151035383)
+3300 val loss 6.3573
+3300 val perplexity 576.6871
+3300 train 6.524789 (lr=3.7222e-05) (hash(x)=151035383)
+3400 val loss 6.8506
+3400 val perplexity 944.4594
+3400 train 6.938754 (lr=3.4730e-05) (hash(x)=150614749)
+3400 val loss 6.3145
+3400 val perplexity 552.5436
+3400 train 6.451954 (lr=3.7045e-05) (hash(x)=150614749)
+3500 val loss 6.8218
+3500 val perplexity 917.6775
+3500 train 6.866987 (lr=3.4560e-05) (hash(x)=151431391)
+3500 val loss 6.2900
+3500 val perplexity 539.1597
+3500 train 6.274175 (lr=3.6864e-05) (hash(x)=151431391)
+3600 val loss 6.7947
+3600 val perplexity 893.1364
+3600 train 6.717680 (lr=3.4384e-05) (hash(x)=151140797)
+3600 val loss 6.2696
+3600 val perplexity 528.2466
+3600 train 6.207747 (lr=3.6677e-05) (hash(x)=151140797)
+3700 val loss 6.7724
+3700 val perplexity 873.3726
+3700 train 6.998176 (lr=3.4205e-05) (hash(x)=154608340)
+3700 val loss 6.2443
+3700 val perplexity 515.0886
+3700 train 6.412389 (lr=3.6485e-05) (hash(x)=154608340)
+3800 val loss 6.7447
+3800 val perplexity 849.5219
+3800 train 6.773592 (lr=3.4021e-05) (hash(x)=169641309)
+3800 val loss 6.2278
+3800 val perplexity 506.6567
+3800 train 6.265309 (lr=3.6289e-05) (hash(x)=169641309)
+3900 val loss 6.7102
+3900 val perplexity 820.7698
+3900 train 6.665777 (lr=3.3832e-05) (hash(x)=150592479)
+3900 val loss 6.2083
+3900 val perplexity 496.8782
+3900 train 6.183831 (lr=3.6088e-05) (hash(x)=150592479)
+4000 val loss 6.6884
+4000 val perplexity 803.0477
+4000 train 6.668771 (lr=3.3639e-05) (hash(x)=155495317)
+4000 val loss 6.1923
+4000 val perplexity 488.9843
+4000 train 6.184135 (lr=3.5882e-05) (hash(x)=155495317)
+4100 val loss 6.6582
+4100 val perplexity 779.1608
+4100 train 6.861329 (lr=3.3442e-05) (hash(x)=170323970)
+4100 val loss 6.1827
+4100 val perplexity 484.3083
+4100 train 6.371322 (lr=3.5671e-05) (hash(x)=170323970)
+4200 val loss 6.6526
+4200 val perplexity 774.7971
+4200 train 6.458316 (lr=3.3240e-05) (hash(x)=140854360)
+4200 val loss 6.1780
+4200 val perplexity 482.0302
+4200 train 6.021700 (lr=3.5457e-05) (hash(x)=140854360)
+4300 val loss 6.6006
+4300 val perplexity 735.5699
+4300 train 6.599814 (lr=3.3035e-05) (hash(x)=148926644)
+4300 val loss 6.1474
+4300 val perplexity 467.4936
+4300 train 6.165276 (lr=3.5237e-05) (hash(x)=148926644)
+4400 val loss 6.5684
+4400 val perplexity 712.2296
+4400 train 6.605387 (lr=3.2825e-05) (hash(x)=149997871)
+4400 val loss 6.1302
+4400 val perplexity 459.5324
+4400 train 6.172115 (lr=3.5014e-05) (hash(x)=149997871)
+4500 val loss 6.5417
+4500 val perplexity 693.4786
+4500 train 6.497378 (lr=3.2611e-05) (hash(x)=149219289)
+4500 val loss 6.1173
+4500 val perplexity 453.6170
+4500 train 6.054480 (lr=3.4786e-05) (hash(x)=149219289)
+4600 val loss 6.5138
+4600 val perplexity 674.4045
+4600 train 6.310131 (lr=3.2394e-05) (hash(x)=139836197)
+4600 val loss 6.0995
+4600 val perplexity 445.6441
+4600 train 5.907494 (lr=3.4553e-05) (hash(x)=139836197)
+4700 val loss 6.4972
+4700 val perplexity 663.3148
+4700 train 6.451049 (lr=3.2172e-05) (hash(x)=150722883)
+4700 val loss 6.0911
+4700 val perplexity 441.9263
+4700 train 6.046262 (lr=3.4317e-05) (hash(x)=150722883)
+4800 val loss 6.4677
+4800 val perplexity 644.0198
+4800 train 6.666210 (lr=3.1947e-05) (hash(x)=166864648)
+4800 val loss 6.0826
+4800 val perplexity 438.1559
+4800 train 6.288001 (lr=3.4077e-05) (hash(x)=166864648)
+4900 val loss 6.4483
+4900 val perplexity 631.6287
+4900 train 6.556377 (lr=3.1718e-05) (hash(x)=157746743)
+4900 val loss 6.0648
+4900 val perplexity 430.4201
+4900 train 6.170938 (lr=3.3832e-05) (hash(x)=157746743)
+5000 val loss 6.4205
+5000 val perplexity 614.2855
+5000 train 6.371796 (lr=3.1485e-05) (hash(x)=153032257)
+5000 val loss 6.0495
+5000 val perplexity 423.8893
+5000 train 5.990033 (lr=3.3584e-05) (hash(x)=153032257)
+5100 val loss 6.4025
+5100 val perplexity 603.3539
+5100 train 6.482961 (lr=3.1249e-05) (hash(x)=151862371)
+5100 val loss 6.0405
+5100 val perplexity 420.1125
+5100 train 6.099496 (lr=3.3332e-05) (hash(x)=151862371)
+5200 val loss 6.3816
+5200 val perplexity 590.8616
+5200 train 6.701185 (lr=3.1009e-05) (hash(x)=168862995)
+5200 val loss 6.0248
+5200 val perplexity 413.5569
+5200 train 6.326960 (lr=3.3076e-05) (hash(x)=168862995)
+5300 val loss 6.3669
+5300 val perplexity 582.2534
+5300 train 6.339127 (lr=3.0766e-05) (hash(x)=148243567)
+5300 val loss 6.0171
+5300 val perplexity 410.3859
+5300 train 5.976335 (lr=3.2817e-05) (hash(x)=148243567)
+5400 val loss 6.3465
+5400 val perplexity 570.4892
+5400 train 6.265824 (lr=3.0519e-05) (hash(x)=145381742)
+5400 val loss 6.0088
+5400 val perplexity 407.0121
+5400 train 5.949846 (lr=3.2554e-05) (hash(x)=145381742)
+5500 val loss 6.3343
+5500 val perplexity 563.5786
+5500 train 6.492792 (lr=3.0269e-05) (hash(x)=155602174)
+5500 val loss 5.9979
+5500 val perplexity 402.5925
+5500 train 6.146212 (lr=3.2287e-05) (hash(x)=155602174)
+5600 val loss 6.3026
+5600 val perplexity 545.9857
+5600 train 6.393070 (lr=3.0016e-05) (hash(x)=160637672)
+5600 val loss 5.9769
+5600 val perplexity 394.2003
+5600 train 6.044743 (lr=3.2017e-05) (hash(x)=160637672)
+5700 val loss 6.2884
+5700 val perplexity 538.2720
+5700 train 6.213086 (lr=2.9760e-05) (hash(x)=152101131)
+5700 val loss 5.9691
+5700 val perplexity 391.1516
+5700 train 5.917782 (lr=3.1744e-05) (hash(x)=152101131)
+5800 val loss 6.2701
+5800 val perplexity 528.5106
+5800 train 6.232066 (lr=2.9501e-05) (hash(x)=152327089)
+5800 val loss 5.9564
+5800 val perplexity 386.2349
+5800 train 5.918398 (lr=3.1468e-05) (hash(x)=152327089)
+5900 val loss 6.2712
+5900 val perplexity 529.1312
+5900 train 6.214243 (lr=2.9239e-05) (hash(x)=151094543)
+5900 val loss 5.9514
+5900 val perplexity 384.3059
+5900 train 5.916299 (lr=3.1188e-05) (hash(x)=151094543)
+6000 val loss 6.2522
+6000 val perplexity 519.1394
+6000 train 6.358566 (lr=2.8974e-05) (hash(x)=160233315)
+6000 val loss 5.9423
+6000 val perplexity 380.8282
+6000 train 6.028115 (lr=3.0905e-05) (hash(x)=160233315)
+6100 val loss 6.2356
+6100 val perplexity 510.5935
+6100 train 6.335859 (lr=2.8706e-05) (hash(x)=158295337)
+6100 val loss 5.9316
+6100 val perplexity 376.7630
+6100 train 6.034167 (lr=3.0620e-05) (hash(x)=158295337)
+6200 val loss 6.2173
+6200 val perplexity 501.3618
+6200 train 6.177991 (lr=2.8436e-05) (hash(x)=151269817)
+6200 val loss 5.9158
+6200 val perplexity 370.8564
+6200 train 5.867013 (lr=3.0332e-05) (hash(x)=151269817)
+6300 val loss 6.1963
+6300 val perplexity 490.9506
+6300 train 6.339870 (lr=2.8163e-05) (hash(x)=156912756)
+6300 val loss 5.9049
+6300 val perplexity 366.8421
+6300 train 6.066171 (lr=3.0040e-05) (hash(x)=156912756)
+6400 val loss 6.1899
+6400 val perplexity 487.8059
+6400 train 6.127525 (lr=2.7888e-05) (hash(x)=125779637)
+6400 val loss 5.8971
+6400 val perplexity 363.9658
+6400 train 5.816467 (lr=2.9747e-05) (hash(x)=125779637)
+6500 val loss 6.1775
+6500 val perplexity 481.8019
+6500 train 6.068864 (lr=2.7610e-05) (hash(x)=148157661)
+6500 val loss 5.8918
+6500 val perplexity 362.0443
+6500 train 5.793604 (lr=2.9450e-05) (hash(x)=148157661)
+6600 val loss 6.1667
+6600 val perplexity 476.6075
+6600 train 6.197901 (lr=2.7330e-05) (hash(x)=152820049)
+6600 val loss 5.8860
+6600 val perplexity 359.9784
+6600 train 5.888762 (lr=2.9152e-05) (hash(x)=152820049)
+6700 val loss 6.1553
+6700 val perplexity 471.1977
+6700 train 6.099213 (lr=2.7047e-05) (hash(x)=155818717)
+6700 val loss 5.8753
+6700 val perplexity 356.1459
+6700 train 5.810640 (lr=2.8851e-05) (hash(x)=155818717)
+6800 val loss 6.1344
+6800 val perplexity 461.4436
+6800 train 6.172260 (lr=2.6763e-05) (hash(x)=158833881)
+6800 val loss 5.8571
+6800 val perplexity 349.6964
+6800 train 5.852377 (lr=2.8547e-05) (hash(x)=158833881)
+6900 val loss 6.1326
+6900 val perplexity 460.6177
+6900 train 6.066556 (lr=2.6477e-05) (hash(x)=164984909)
+6900 val loss 5.8510
+6900 val perplexity 347.5731
+6900 train 5.733030 (lr=2.8242e-05) (hash(x)=164984909)
+7000 val loss 6.1168
+7000 val perplexity 453.4012
+7000 train 6.016676 (lr=2.6188e-05) (hash(x)=151172493)
+7000 val loss 5.8428
+7000 val perplexity 344.7496
+7000 train 5.743183 (lr=2.7934e-05) (hash(x)=151172493)
+7100 val loss 6.1157
+7100 val perplexity 452.9291
+7100 train 6.155096 (lr=2.5898e-05) (hash(x)=154023633)
+7100 val loss 5.8395
+7100 val perplexity 343.6040
+7100 train 5.904043 (lr=2.7624e-05) (hash(x)=154023633)
+7200 val loss 6.0983
+7200 val perplexity 445.1168
+7200 train 5.955569 (lr=2.5606e-05) (hash(x)=139351250)
+7200 val loss 5.8291
+7200 val perplexity 340.0587
+7200 train 5.676873 (lr=2.7313e-05) (hash(x)=139351250)
+7300 val loss 6.0909
+7300 val perplexity 441.8268
+7300 train 6.039002 (lr=2.5312e-05) (hash(x)=158719458)
+7300 val loss 5.8239
+7300 val perplexity 338.2815
+7300 train 5.789658 (lr=2.7000e-05) (hash(x)=158719458)
+7400 val loss 6.0725
+7400 val perplexity 433.7473
+7400 train 6.140794 (lr=2.5017e-05) (hash(x)=162250810)
+7400 val loss 5.8076
+7400 val perplexity 332.8254
+7400 train 5.877717 (lr=2.6685e-05) (hash(x)=162250810)
+7500 val loss 6.0607
+7500 val perplexity 428.6643
+7500 train 5.937808 (lr=2.4721e-05) (hash(x)=142827557)
+7500 val loss 5.7964
+7500 val perplexity 329.1051
+7500 train 5.685374 (lr=2.6369e-05) (hash(x)=142827557)
+7600 val loss 6.0534
+7600 val perplexity 425.5662
+7600 train 5.953993 (lr=2.4423e-05) (hash(x)=147830801)
+7600 val loss 5.7925
+7600 val perplexity 327.8361
+7600 train 5.709580 (lr=2.6051e-05) (hash(x)=147830801)
+7700 val loss 6.0466
+7700 val perplexity 422.6890
+7700 train 6.135414 (lr=2.4124e-05) (hash(x)=165233314)
+7700 val loss 5.7853
+7700 val perplexity 325.4878
+7700 train 5.887728 (lr=2.5732e-05) (hash(x)=165233314)
+7800 val loss 6.0379
+7800 val perplexity 418.9930
+7800 train 6.028669 (lr=2.3823e-05) (hash(x)=160654717)
+7800 val loss 5.7776
+7800 val perplexity 322.9735
+7800 train 5.765421 (lr=2.5411e-05) (hash(x)=160654717)
+7900 val loss 6.0338
+7900 val perplexity 417.2910
+7900 train 5.882531 (lr=2.3522e-05) (hash(x)=140712586)
+7900 val loss 5.7761
+7900 val perplexity 322.4857
+7900 train 5.653329 (lr=2.5090e-05) (hash(x)=140712586)
+8000 val loss 6.0164
+8000 val perplexity 410.0987
+8000 train 6.180608 (lr=2.3219e-05) (hash(x)=150309061)
+8000 val loss 5.7610
+8000 val perplexity 317.6568
+8000 train 5.930829 (lr=2.4767e-05) (hash(x)=150309061)
+8100 val loss 6.0151
+8100 val perplexity 409.5820
+8100 train 6.010431 (lr=2.2916e-05) (hash(x)=158805466)
+8100 val loss 5.7605
+8100 val perplexity 317.4916
+8100 train 5.754591 (lr=2.4444e-05) (hash(x)=158805466)
+8200 val loss 5.9996
+8200 val perplexity 403.2486
+8200 train 5.881342 (lr=2.2612e-05) (hash(x)=142814078)
+8200 val loss 5.7481
+8200 val perplexity 313.6090
+8200 train 5.645060 (lr=2.4120e-05) (hash(x)=142814078)
+8300 val loss 5.9939
+8300 val perplexity 400.9659
+8300 train 5.953493 (lr=2.2308e-05) (hash(x)=148340417)
+8300 val loss 5.7425
+8300 val perplexity 311.8304
+8300 train 5.710927 (lr=2.3795e-05) (hash(x)=148340417)
+8400 val loss 5.9912
+8400 val perplexity 399.8812
+8400 train 5.927708 (lr=2.2002e-05) (hash(x)=157132639)
+8400 val loss 5.7362
+8400 val perplexity 309.8906
+8400 train 5.664308 (lr=2.3469e-05) (hash(x)=157132639)
+8500 val loss 5.9836
+8500 val perplexity 396.8719
+8500 train 5.911011 (lr=2.1697e-05) (hash(x)=152994240)
+8500 val loss 5.7317
+8500 val perplexity 308.4876
+8500 train 5.661629 (lr=2.3143e-05) (hash(x)=152994240)
+8600 val loss 5.9692
+8600 val perplexity 391.2051
+8600 train 5.927732 (lr=2.1391e-05) (hash(x)=152210154)
+8600 val loss 5.7182
+8600 val perplexity 304.3698
+8600 train 5.687449 (lr=2.2817e-05) (hash(x)=152210154)
+8700 val loss 5.9625
+8700 val perplexity 388.5811
+8700 train 6.040589 (lr=2.1085e-05) (hash(x)=149631247)
+8700 val loss 5.7136
+8700 val perplexity 302.9607
+8700 train 5.818038 (lr=2.2490e-05) (hash(x)=149631247)
+8800 val loss 5.9546
+8800 val perplexity 385.5366
+8800 train 5.811369 (lr=2.0778e-05) (hash(x)=145617784)
+8800 val loss 5.7087
+8800 val perplexity 301.4867
+8800 train 5.592268 (lr=2.2163e-05) (hash(x)=145617784)
+8900 val loss 5.9494
+8900 val perplexity 383.5336
+8900 train 6.043489 (lr=2.0472e-05) (hash(x)=155984970)
+8900 val loss 5.7001
+8900 val perplexity 298.8987
+8900 train 5.810061 (lr=2.1837e-05) (hash(x)=155984970)
+9000 val loss 5.9442
+9000 val perplexity 381.5475
+9000 train 5.677837 (lr=2.0165e-05) (hash(x)=143433013)
+9000 val loss 5.6995
+9000 val perplexity 298.7101
+9000 train 5.443996 (lr=2.1510e-05) (hash(x)=143433013)
+9100 val loss 5.9373
+9100 val perplexity 378.9221
+9100 train 5.827633 (lr=1.9859e-05) (hash(x)=152507533)
+9100 val loss 5.6936
+9100 val perplexity 296.9637
+9100 train 5.598932 (lr=2.1183e-05) (hash(x)=152507533)
+9200 val loss 5.9295
+9200 val perplexity 375.9717
+9200 train 6.067265 (lr=1.9553e-05) (hash(x)=162400650)
+9200 val loss 5.6854
+9200 val perplexity 294.5436
+9200 train 5.809351 (lr=2.0857e-05) (hash(x)=162400650)
+9300 val loss 5.9234
+9300 val perplexity 373.6941
+9300 train 5.969053 (lr=1.9248e-05) (hash(x)=148419068)
+9300 val loss 5.6822
+9300 val perplexity 293.5955
+9300 train 5.727042 (lr=2.0531e-05) (hash(x)=148419068)
+9400 val loss 5.9160
+9400 val perplexity 370.9360
+9400 train 5.881715 (lr=1.8942e-05) (hash(x)=145025169)
+9400 val loss 5.6712
+9400 val perplexity 290.3748
+9400 train 5.642601 (lr=2.0205e-05) (hash(x)=145025169)
+9500 val loss 5.9054
+9500 val perplexity 367.0152
+9500 train 5.877307 (lr=1.8638e-05) (hash(x)=145104235)
+9500 val loss 5.6654
+9500 val perplexity 288.7047
+9500 train 5.641552 (lr=1.9880e-05) (hash(x)=145104235)
+9600 val loss 5.9024
+9600 val perplexity 365.9068
+9600 train 5.836945 (lr=1.8334e-05) (hash(x)=150499797)
+9600 val loss 5.6634
+9600 val perplexity 288.1191
+9600 train 5.599575 (lr=1.9556e-05) (hash(x)=150499797)
+9700 val loss 5.8969
+9700 val perplexity 363.9092
+9700 train 5.695178 (lr=1.8031e-05) (hash(x)=139690022)
+9700 val loss 5.6585
+9700 val perplexity 286.7106
+9700 train 5.472332 (lr=1.9233e-05) (hash(x)=139690022)
+9800 val loss 5.8880
+9800 val perplexity 360.6846
+9800 train 5.884799 (lr=1.7728e-05) (hash(x)=145645279)
+9800 val loss 5.6541
+9800 val perplexity 285.4582
+9800 train 5.667852 (lr=1.8910e-05) (hash(x)=145645279)
+9900 val loss 5.8780
+9900 val perplexity 357.1094
+9900 train 5.797610 (lr=1.7427e-05) (hash(x)=150103432)
+9900 val loss 5.6451
+9900 val perplexity 282.8907
+9900 train 5.576068 (lr=1.8589e-05) (hash(x)=150103432)
+10000 val loss 5.8753
+10000 val perplexity 356.1355
+10000 train 6.028806 (lr=1.7126e-05) (hash(x)=162309473)
+10000 val loss 5.6387
+10000 val perplexity 281.0946
+10000 train 5.791598 (lr=1.8268e-05) (hash(x)=162309473)
+10100 val loss 5.8695
+10100 val perplexity 354.0836
+10100 train 5.753899 (lr=1.6827e-05) (hash(x)=146586772)
+10100 val loss 5.6350
+10100 val perplexity 280.0534
+10100 train 5.533384 (lr=1.7949e-05) (hash(x)=146586772)
+10200 val loss 5.8623
+10200 val perplexity 351.5154
+10200 train 5.765655 (lr=1.6529e-05) (hash(x)=149256313)
+10200 val loss 5.6319
+10200 val perplexity 279.1962
+10200 train 5.533719 (lr=1.7631e-05) (hash(x)=149256313)
+10300 val loss 5.8580
+10300 val perplexity 350.0123
+10300 train 5.819844 (lr=1.6233e-05) (hash(x)=151455083)
+10300 val loss 5.6270
+10300 val perplexity 277.8158
+10300 train 5.576078 (lr=1.7315e-05) (hash(x)=151455083)
+10400 val loss 5.8561
+10400 val perplexity 349.3732
+10400 train 5.640420 (lr=1.5938e-05) (hash(x)=150210444)
+10400 val loss 5.6245
+10400 val perplexity 277.1461
+10400 train 5.374650 (lr=1.7000e-05) (hash(x)=150210444)
+10500 val loss 5.8610
+10500 val perplexity 351.0689
+10500 train 5.633465 (lr=1.5644e-05) (hash(x)=152144428)
+10500 val loss 5.6257
+10500 val perplexity 277.4727
+10500 train 5.393909 (lr=1.6687e-05) (hash(x)=152144428)
+10600 val loss 5.8443
+10600 val perplexity 345.2592
+10600 train 5.903284 (lr=1.5352e-05) (hash(x)=168628935)
+10600 val loss 5.6154
+10600 val perplexity 274.6116
+10600 train 5.629866 (lr=1.6376e-05) (hash(x)=168628935)
+10700 val loss 5.8357
+10700 val perplexity 342.3191
+10700 train 5.653769 (lr=1.5062e-05) (hash(x)=164667785)
+10700 val loss 5.6089
+10700 val perplexity 272.8553
+10700 train 5.377524 (lr=1.6066e-05) (hash(x)=164667785)
+10800 val loss 5.8284
+10800 val perplexity 339.8024
+10800 train 5.784979 (lr=1.4773e-05) (hash(x)=143621491)
+10800 val loss 5.6017
+10800 val perplexity 270.8960
+10800 train 5.585045 (lr=1.5758e-05) (hash(x)=143621491)
+10900 val loss 5.8282
+10900 val perplexity 339.7538
+10900 train 5.716103 (lr=1.4487e-05) (hash(x)=143914826)
+10900 val loss 5.5983
+10900 val perplexity 269.9630
+10900 train 5.495556 (lr=1.5453e-05) (hash(x)=143914826)
+11000 val loss 5.8235
+11000 val perplexity 338.1537
+11000 train 5.596159 (lr=1.4203e-05) (hash(x)=137663435)
+11000 val loss 5.5962
+11000 val perplexity 269.4052
+11000 train 5.388721 (lr=1.5149e-05) (hash(x)=137663435)
+11100 val loss 5.8186
+11100 val perplexity 336.5119
+11100 train 5.695492 (lr=1.3920e-05) (hash(x)=149054007)
+11100 val loss 5.5939
+11100 val perplexity 268.7805
+11100 train 5.476272 (lr=1.4848e-05) (hash(x)=149054007)
+11200 val loss 5.8212
+11200 val perplexity 337.3617
+11200 train 5.827100 (lr=1.3640e-05) (hash(x)=153473924)
+11200 val loss 5.5936
+11200 val perplexity 268.7033
+11200 train 5.615970 (lr=1.4550e-05) (hash(x)=153473924)
+11300 val loss 5.8178
+11300 val perplexity 336.2411
+11300 train 5.608507 (lr=1.3362e-05) (hash(x)=158563275)
+11300 val loss 5.5922
+11300 val perplexity 268.3133
+11300 train 5.359133 (lr=1.4253e-05) (hash(x)=158563275)
+11400 val loss 5.8136
+11400 val perplexity 334.8136
+11400 train 5.946541 (lr=1.3087e-05) (hash(x)=161016667)
+11400 val loss 5.5860
+11400 val perplexity 266.6766
+11400 train 5.716009 (lr=1.3960e-05) (hash(x)=161016667)
+11500 val loss 5.8071
+11500 val perplexity 332.6627
+11500 train 5.853693 (lr=1.2814e-05) (hash(x)=157344822)
+11500 val loss 5.5861
+11500 val perplexity 266.6995
+11500 train 5.636026 (lr=1.3668e-05) (hash(x)=157344822)
+11600 val loss 5.7953
+11600 val perplexity 328.7356
+11600 train 5.679436 (lr=1.2544e-05) (hash(x)=139977713)
+11600 val loss 5.5719
+11600 val perplexity 262.9367
+11600 train 5.476897 (lr=1.3380e-05) (hash(x)=139977713)
+11700 val loss 5.7915
+11700 val perplexity 327.4971
+11700 train 5.782972 (lr=1.2276e-05) (hash(x)=156511738)
+11700 val loss 5.5674
+11700 val perplexity 261.7418
+11700 train 5.542856 (lr=1.3095e-05) (hash(x)=156511738)
+11800 val loss 5.7884
+11800 val perplexity 326.4805
+11800 train 6.230186 (lr=1.2011e-05) (hash(x)=185372558)
+11800 val loss 5.5674
+11800 val perplexity 261.7533
+11800 train 5.948181 (lr=1.2812e-05) (hash(x)=185372558)
+11900 val loss 5.7832
+11900 val perplexity 324.7906
+11900 train 5.621591 (lr=1.1749e-05) (hash(x)=138085470)
+11900 val loss 5.5627
+11900 val perplexity 260.5332
+11900 train 5.431148 (lr=1.2532e-05) (hash(x)=138085470)
+12000 val loss 5.7834
+12000 val perplexity 324.8578
+12000 train 5.635805 (lr=1.1490e-05) (hash(x)=142928235)
+12000 val loss 5.5626
+12000 val perplexity 260.5064
+12000 train 5.406928 (lr=1.2256e-05) (hash(x)=142928235)
+12100 val loss 5.7828
+12100 val perplexity 324.6824
+12100 train 5.632250 (lr=1.1234e-05) (hash(x)=151960396)
+12100 val loss 5.5622
+12100 val perplexity 260.4002
+12100 train 5.407411 (lr=1.1983e-05) (hash(x)=151960396)
+12200 val loss 5.7718
+12200 val perplexity 321.1224
+12200 train 5.828484 (lr=1.0981e-05) (hash(x)=156794523)
+12200 val loss 5.5532
+12200 val perplexity 258.0563
+12200 train 5.590004 (lr=1.1713e-05) (hash(x)=156794523)
+12300 val loss 5.7668
+12300 val perplexity 319.5281
+12300 train 5.579047 (lr=1.0731e-05) (hash(x)=138082358)
+12300 val loss 5.5463
+12300 val perplexity 256.2923
+12300 train 5.355970 (lr=1.1446e-05) (hash(x)=138082358)
+12400 val loss 5.7622
+12400 val perplexity 318.0525
+12400 train 5.666190 (lr=1.0484e-05) (hash(x)=155255899)
+12400 val loss 5.5444
+12400 val perplexity 255.8121
+12400 train 5.450579 (lr=1.1183e-05) (hash(x)=155255899)
+12500 val loss 5.7579
+12500 val perplexity 316.6952
+12500 train 5.723560 (lr=1.0241e-05) (hash(x)=150490528)
+12500 val loss 5.5400
+12500 val perplexity 254.6694
+12500 train 5.493838 (lr=1.0924e-05) (hash(x)=150490528)
+12600 val loss 5.7566
+12600 val perplexity 316.2633
+12600 train 5.901272 (lr=1.0001e-05) (hash(x)=154831027)
+12600 val loss 5.5394
+12600 val perplexity 254.5151
+12600 train 5.718703 (lr=1.0668e-05) (hash(x)=154831027)
+12700 val loss 5.7545
+12700 val perplexity 315.5970
+12700 train 5.704981 (lr=9.7649e-06) (hash(x)=147341052)
+12700 val loss 5.5385
+12700 val perplexity 254.3070
+12700 train 5.490286 (lr=1.0416e-05) (hash(x)=147341052)
+12800 val loss 5.7534
+12800 val perplexity 315.2712
+12800 train 5.785439 (lr=9.5322e-06) (hash(x)=159333795)
+12800 val loss 5.5359
+12800 val perplexity 253.6327
+12800 train 5.563532 (lr=1.0168e-05) (hash(x)=159333795)
+12900 val loss 5.7550
+12900 val perplexity 315.7724
+12900 train 5.598507 (lr=9.3031e-06) (hash(x)=143896867)
+12900 val loss 5.5382
+12900 val perplexity 254.2111
+12900 train 5.371953 (lr=9.9233e-06) (hash(x)=143896867)
+13000 val loss 5.7436
+13000 val perplexity 312.1739
+13000 train 5.751026 (lr=9.0777e-06) (hash(x)=147161537)
+13000 val loss 5.5275
+13000 val perplexity 251.5187
+13000 train 5.549591 (lr=9.6829e-06) (hash(x)=147161537)
+13100 val loss 5.7422
+13100 val perplexity 311.7380
+13100 train 5.870570 (lr=8.8562e-06) (hash(x)=147081918)
+13100 val loss 5.5249
+13100 val perplexity 250.8569
+13100 train 5.663218 (lr=9.4466e-06) (hash(x)=147081918)
+13200 val loss 5.7385
+13200 val perplexity 310.5829
+13200 train 5.503904 (lr=8.6385e-06) (hash(x)=145079287)
+13200 val loss 5.5241
+13200 val perplexity 250.6500
+13200 train 5.294759 (lr=9.2144e-06) (hash(x)=145079287)
+13300 val loss 5.7375
+13300 val perplexity 310.2796
+13300 train 5.542725 (lr=8.4248e-06) (hash(x)=142515041)
+13300 val loss 5.5217
+13300 val perplexity 250.0569
+13300 train 5.322534 (lr=8.9865e-06) (hash(x)=142515041)
+13400 val loss 5.7343
+13400 val perplexity 309.3112
+13400 train 5.524352 (lr=8.2151e-06) (hash(x)=144081985)
+13400 val loss 5.5212
+13400 val perplexity 249.9334
+13400 train 5.311021 (lr=8.7628e-06) (hash(x)=144081985)
+13500 val loss 5.7319
+13500 val perplexity 308.5533
+13500 train 5.817440 (lr=8.0095e-06) (hash(x)=149957292)
+13500 val loss 5.5183
+13500 val perplexity 249.2086
+13500 train 5.610835 (lr=8.5435e-06) (hash(x)=149957292)
+13600 val loss 5.7283
+13600 val perplexity 307.4504
+13600 train 5.761745 (lr=7.8081e-06) (hash(x)=157123033)
+13600 val loss 5.5120
+13600 val perplexity 247.6491
+13600 train 5.547890 (lr=8.3286e-06) (hash(x)=157123033)
+13700 val loss 5.7234
+13700 val perplexity 305.9554
+13700 train 5.703632 (lr=7.6109e-06) (hash(x)=148277247)
+13700 val loss 5.5094
+13700 val perplexity 247.0001
+13700 train 5.489895 (lr=8.1183e-06) (hash(x)=148277247)
+13800 val loss 5.7274
+13800 val perplexity 307.1811
+13800 train 5.850340 (lr=7.4180e-06) (hash(x)=142769907)
+13800 val loss 5.5152
+13800 val perplexity 248.4417
+13800 train 5.671909 (lr=7.9125e-06) (hash(x)=142769907)
+13900 val loss 5.7188
+13900 val perplexity 304.5252
+13900 train 5.651237 (lr=7.2294e-06) (hash(x)=144932004)
+13900 val loss 5.5049
+13900 val perplexity 245.9041
+13900 train 5.479329 (lr=7.7114e-06) (hash(x)=144932004)
+14000 val loss 5.7157
+14000 val perplexity 303.5841
+14000 train 5.726858 (lr=7.0452e-06) (hash(x)=149532265)
+14000 val loss 5.5043
+14000 val perplexity 245.7397
+14000 train 5.515040 (lr=7.5149e-06) (hash(x)=149532265)
+14100 val loss 5.7150
+14100 val perplexity 303.3718
+14100 train 5.543785 (lr=6.8656e-06) (hash(x)=149301374)
+14100 val loss 5.5016
+14100 val perplexity 245.0862
+14100 train 5.308578 (lr=7.3233e-06) (hash(x)=149301374)
+14200 val loss 5.7140
+14200 val perplexity 303.0953
+14200 train 5.461953 (lr=6.6904e-06) (hash(x)=142871075)
+14200 val loss 5.5020
+14200 val perplexity 245.1914
+14200 train 5.235135 (lr=7.1365e-06) (hash(x)=142871075)
+14300 val loss 5.7094
+14300 val perplexity 301.6765
+14300 train 5.865572 (lr=6.5199e-06) (hash(x)=166744960)
+14300 val loss 5.4964
+14300 val perplexity 243.8035
+14300 train 5.635840 (lr=6.9546e-06) (hash(x)=166744960)
+14400 val loss 5.7051
+14400 val perplexity 300.4096
+14400 train 5.674760 (lr=6.3540e-06) (hash(x)=145950920)
+14400 val loss 5.4931
+14400 val perplexity 243.0033
+14400 train 5.448152 (lr=6.7776e-06) (hash(x)=145950920)
+14500 val loss 5.7030
+14500 val perplexity 299.7748
+14500 train 5.690929 (lr=6.1928e-06) (hash(x)=141089155)
+14500 val loss 5.4908
+14500 val perplexity 242.4427
+14500 train 5.497694 (lr=6.6057e-06) (hash(x)=141089155)
+14600 val loss 5.7002
+14600 val perplexity 298.9186
+14600 train 5.852167 (lr=6.0364e-06) (hash(x)=174516275)
+14600 val loss 5.4892
+14600 val perplexity 242.0709
+14600 train 5.642091 (lr=6.4389e-06) (hash(x)=174516275)
+14700 val loss 5.6985
+14700 val perplexity 298.4080
+14700 train 5.502239 (lr=5.8848e-06) (hash(x)=137896065)
+14700 val loss 5.4878
+14700 val perplexity 241.7357
+14700 train 5.310824 (lr=6.2771e-06) (hash(x)=137896065)
+14800 val loss 5.6982
+14800 val perplexity 298.3280
+14800 train 5.728738 (lr=5.7381e-06) (hash(x)=154396556)
+14800 val loss 5.4868
+14800 val perplexity 241.4946
+14800 train 5.521058 (lr=6.1206e-06) (hash(x)=154396556)
+14900 val loss 5.6981
+14900 val perplexity 298.3020
+14900 train 5.544933 (lr=5.5962e-06) (hash(x)=149322394)
+14900 val loss 5.4872
+14900 val perplexity 241.5863
+14900 train 5.323401 (lr=5.9693e-06) (hash(x)=149322394)
+15000 val loss 5.6981
+15000 val perplexity 298.3115
+15000 train 5.610722 (lr=5.4593e-06) (hash(x)=147410513)
+15000 val loss 5.4872
+15000 val perplexity 241.5842
+15000 train 5.404994 (lr=5.8233e-06) (hash(x)=147410513)
+15100 val loss 5.6947
+15100 val perplexity 297.2931
+15100 train 5.604572 (lr=5.3275e-06) (hash(x)=145245634)
+15100 val loss 5.4839
+15100 val perplexity 240.7724
+15100 train 5.395303 (lr=5.6826e-06) (hash(x)=145245634)
+15200 val loss 5.6903
+15200 val perplexity 295.9903
+15200 train 5.774277 (lr=5.2006e-06) (hash(x)=148341108)
+15200 val loss 5.4795
+15200 val perplexity 239.7191
+15200 train 5.560233 (lr=5.5473e-06) (hash(x)=148341108)
+15300 val loss 5.6883
+15300 val perplexity 295.3842
+15300 train 5.497147 (lr=5.0789e-06) (hash(x)=140162437)
+15300 val loss 5.4766
+15300 val perplexity 239.0419
+15300 train 5.292723 (lr=5.4175e-06) (hash(x)=140162437)
+15400 val loss 5.6865
+15400 val perplexity 294.8504
+15400 train 5.889404 (lr=4.9622e-06) (hash(x)=142166081)
+15400 val loss 5.4757
+15400 val perplexity 238.8262
+15400 train 5.706107 (lr=5.2931e-06) (hash(x)=142166081)
+15500 val loss 5.6855
+15500 val perplexity 294.5707
+15500 train 5.590559 (lr=4.8508e-06) (hash(x)=141042196)
+15500 val loss 5.4749
+15500 val perplexity 238.6198
+15500 train 5.418190 (lr=5.1742e-06) (hash(x)=141042196)
+15600 val loss 5.6839
+15600 val perplexity 294.1076
+15600 train 5.804760 (lr=4.7445e-06) (hash(x)=148672701)
+15600 val loss 5.4752
+15600 val perplexity 238.6886
+15600 train 5.604007 (lr=5.0608e-06) (hash(x)=148672701)
+15700 val loss 5.6840
+15700 val perplexity 294.1362
+15700 train 5.586970 (lr=4.6435e-06) (hash(x)=149672385)
+15700 val loss 5.4744
+15700 val perplexity 238.5025
+15700 train 5.381526 (lr=4.9531e-06) (hash(x)=149672385)
+15800 val loss 5.6840
+15800 val perplexity 294.1187
+15800 train 5.705602 (lr=4.5478e-06) (hash(x)=152379562)
+15800 val loss 5.4739
+15800 val perplexity 238.3925
+15800 train 5.498224 (lr=4.8509e-06) (hash(x)=152379562)
+15900 val loss 5.6794
+15900 val perplexity 292.7766
+15900 train 5.719800 (lr=4.4573e-06) (hash(x)=154674265)
+15900 val loss 5.4694
+15900 val perplexity 237.3285
+15900 train 5.510931 (lr=4.7545e-06) (hash(x)=154674265)
+16000 val loss 5.6778
+16000 val perplexity 292.3029
+16000 train 5.719006 (lr=4.3722e-06) (hash(x)=150368078)
+16000 val loss 5.4672
+16000 val perplexity 236.8024
+16000 train 5.519518 (lr=4.6637e-06) (hash(x)=150368078)
+16100 val loss 5.6746
+16100 val perplexity 291.3651
+16100 train 5.784335 (lr=4.2924e-06) (hash(x)=160390561)
+16100 val loss 5.4654
+16100 val perplexity 236.3686
+16100 train 5.580384 (lr=4.5786e-06) (hash(x)=160390561)
+16200 val loss 5.6741
+16200 val perplexity 291.2354
+16200 train 6.027804 (lr=4.2180e-06) (hash(x)=155218378)
+16200 val loss 5.4652
+16200 val perplexity 236.3124
+16200 train 5.828511 (lr=4.4993e-06) (hash(x)=155218378)
+16300 val loss 5.6723
+16300 val perplexity 290.6990
+16300 train 5.863964 (lr=4.1491e-06) (hash(x)=160891077)
+16300 val loss 5.4638
+16300 val perplexity 235.9971
+16300 train 5.650701 (lr=4.4257e-06) (hash(x)=160891077)
+16400 val loss 5.6722
+16400 val perplexity 290.6846
+16400 train 5.577696 (lr=4.0856e-06) (hash(x)=142789253)
+16400 val loss 5.4630
+16400 val perplexity 235.7978
+16400 train 5.364003 (lr=4.3579e-06) (hash(x)=142789253)
+16500 val loss 5.6721
+16500 val perplexity 290.6511
+16500 train 5.416285 (lr=4.0275e-06) (hash(x)=141497841)
+16500 val loss 5.4626
+16500 val perplexity 235.7060
+16500 train 5.210999 (lr=4.2960e-06) (hash(x)=141497841)
+16600 val loss 5.6708
+16600 val perplexity 290.2732
+16600 train 5.635384 (lr=3.9749e-06) (hash(x)=157947018)
+16600 val loss 5.4621
+16600 val perplexity 235.5873
+16600 train 5.423563 (lr=4.2399e-06) (hash(x)=157947018)
+16700 val loss 5.6669
+16700 val perplexity 289.1373
+16700 train 5.618238 (lr=3.9278e-06) (hash(x)=146039309)
+16700 val loss 5.4583
+16700 val perplexity 234.6897
+16700 train 5.432078 (lr=4.1896e-06) (hash(x)=146039309)
+16800 val loss 5.6668
+16800 val perplexity 289.1097
+16800 train 5.634438 (lr=3.8862e-06) (hash(x)=156381445)
+16800 val loss 5.4569
+16800 val perplexity 234.3673
+16800 train 5.423639 (lr=4.1452e-06) (hash(x)=156381445)
+16900 val loss 5.6643
+16900 val perplexity 288.3940
+16900 train 5.624663 (lr=3.8501e-06) (hash(x)=146537354)
+16900 val loss 5.4553
+16900 val perplexity 233.9896
+16900 train 5.410178 (lr=4.1067e-06) (hash(x)=146537354)
+17000 val loss 5.6638
+17000 val perplexity 288.2406
+17000 train 5.582037 (lr=3.8195e-06) (hash(x)=135939444)
+17000 val loss 5.4545
+17000 val perplexity 233.8000
+17000 train 5.407039 (lr=4.0741e-06) (hash(x)=135939444)
+17100 val loss 5.6635
+17100 val perplexity 288.1500
+17100 train 5.736537 (lr=3.7945e-06) (hash(x)=147819652)
+17100 val loss 5.4544
+17100 val perplexity 233.7901
+17100 train 5.548837 (lr=4.0475e-06) (hash(x)=147819652)
+17200 val loss 5.6636
+17200 val perplexity 288.1846
+17200 train 5.764005 (lr=3.7750e-06) (hash(x)=155307909)
+17200 val loss 5.4544
+17200 val perplexity 233.7908
+17200 train 5.525397 (lr=4.0267e-06) (hash(x)=155307909)
+17300 val loss 5.6608
+17300 val perplexity 287.3757
+17300 train 5.615438 (lr=3.7611e-06) (hash(x)=151879548)
+17300 val loss 5.4510
+17300 val perplexity 233.0012
+17300 train 5.396317 (lr=4.0119e-06) (hash(x)=151879548)
+17400 val loss 5.6585
+17400 val perplexity 286.7057
+17400 train 5.578130 (lr=3.7528e-06) (hash(x)=143641792)
+17400 val loss 5.4501
+17400 val perplexity 232.7875
+17400 train 5.374997 (lr=4.0030e-06) (hash(x)=143641792)
+17499 val loss 5.6564
+17499 val perplexity 286.1226
+17499 val loss 5.4482
+17499 val perplexity 232.3446

attention_kindselective_n_heads4_seed1339/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a8f3c5900182fece7b2ad89221ee3a19873525cfb62a9be93a17deb36b902e1
+oid sha256:edd56ce72d060006a978c84f12922673f5fd5d7cce4ef197c2dc55828e5da6b6
 size 92843394

attention_kindselective_n_heads4_seed1339/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3bea0b6ffe448da1642694f3629ed45aa464f2db4f9c4a8bd07ebe572c330412
+oid sha256:d2fef469299677eb41c2a4b0f379db456edb3603c976938ab2de09e0956c072b
 size 179406214