Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1343/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_30720_4_1343", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_30720_4_1343", "n_embd": 256}

attention_kindselective_n_heads4_seed1343/log2.txt

@@ -1,528 +1,1055 @@
 max_steps: 17500
 0 val loss 11.3001
 0 val perplexity 80829.3594
-0 train 11.304919 (lr=1.7500e-07) (hash(x)=153418542)
-100 val loss 9.8579
-100 val perplexity 19109.1309
-100 train 9.824990 (lr=1.7675e-05) (hash(x)=154406429)
-200 val loss 8.8331
-200 val perplexity 6857.3115
-200 train 8.746761 (lr=3.5000e-05) (hash(x)=143722001)
-300 val loss 7.9027
-300 val perplexity 2704.6018
-300 train 7.843596 (lr=3.4997e-05) (hash(x)=152421341)
-400 val loss 7.5710
-400 val perplexity 1941.1111
-400 train 7.456883 (lr=3.4990e-05) (hash(x)=152989689)
-500 val loss 7.4553
-500 val perplexity 1729.0040
-500 train 7.379359 (lr=3.4977e-05) (hash(x)=145113544)
-600 val loss 7.3587
-600 val perplexity 1569.8698
-600 train 7.348912 (lr=3.4958e-05) (hash(x)=150071018)
-700 val loss 7.2927
-700 val perplexity 1469.5853
-700 train 7.403689 (lr=3.4935e-05) (hash(x)=156594182)
-800 val loss 7.2374
-800 val perplexity 1390.4237
-800 train 7.242920 (lr=3.4907e-05) (hash(x)=153358238)
-900 val loss 7.1837
-900 val perplexity 1317.7135
-900 train 7.002797 (lr=3.4873e-05) (hash(x)=146617394)
-1000 val loss 7.1280
-1000 val perplexity 1246.3452
-1000 train 7.063962 (lr=3.4834e-05) (hash(x)=148983354)
-1100 val loss 7.0634
-1100 val perplexity 1168.3807
-1100 train 7.084983 (lr=3.4790e-05) (hash(x)=146313352)
-1200 val loss 7.0047
-1200 val perplexity 1101.7477
-1200 train 6.932942 (lr=3.4741e-05) (hash(x)=150770333)
-1300 val loss 6.9237
-1300 val perplexity 1016.0912
-1300 train 6.875015 (lr=3.4687e-05) (hash(x)=153237933)
-1400 val loss 6.8673
-1400 val perplexity 960.3375
-1400 train 6.700178 (lr=3.4628e-05) (hash(x)=145246201)
-1500 val loss 6.8174
-1500 val perplexity 913.5900
-1500 train 6.794487 (lr=3.4563e-05) (hash(x)=154002540)
-1600 val loss 6.7607
-1600 val perplexity 863.2115
-1600 train 6.541095 (lr=3.4494e-05) (hash(x)=142635842)
-1700 val loss 6.7104
-1700 val perplexity 820.9233
-1700 train 6.758193 (lr=3.4419e-05) (hash(x)=151450555)
-1800 val loss 6.6554
-1800 val perplexity 776.9322
-1800 train 6.578032 (lr=3.4340e-05) (hash(x)=148299832)
-1900 val loss 6.6127
-1900 val perplexity 744.5060
-1900 train 6.748245 (lr=3.4255e-05) (hash(x)=155487476)
-2000 val loss 6.5803
-2000 val perplexity 720.7286
-2000 train 6.563023 (lr=3.4166e-05) (hash(x)=152887721)
-2100 val loss 6.5454
-2100 val perplexity 696.0361
-2100 train 6.236790 (lr=3.4072e-05) (hash(x)=143430603)
-2200 val loss 6.5212
-2200 val perplexity 679.4134
-2200 train 6.524914 (lr=3.3973e-05) (hash(x)=153721194)
-2300 val loss 6.4924
-2300 val perplexity 660.1232
-2300 train 6.651157 (lr=3.3869e-05) (hash(x)=148702429)
-2400 val loss 6.4638
-2400 val perplexity 641.5140
-2400 train 6.519583 (lr=3.3760e-05) (hash(x)=153047184)
-2500 val loss 6.4378
-2500 val perplexity 625.0129
-2500 train 6.324356 (lr=3.3646e-05) (hash(x)=149720123)
-2600 val loss 6.4174
-2600 val perplexity 612.4166
-2600 train 6.436531 (lr=3.3528e-05) (hash(x)=152466045)
-2700 val loss 6.4221
-2700 val perplexity 615.3074
-2700 train 6.333328 (lr=3.3405e-05) (hash(x)=155796305)
-2800 val loss 6.3789
-2800 val perplexity 589.3065
-2800 train 6.213628 (lr=3.3277e-05) (hash(x)=151991552)
-2900 val loss 6.3549
-2900 val perplexity 575.2861
-2900 train 6.934335 (lr=3.3144e-05) (hash(x)=166537989)
-3000 val loss 6.3344
-3000 val perplexity 563.6105
-3000 train 6.332823 (lr=3.3008e-05) (hash(x)=147892594)
-3100 val loss 6.3253
-3100 val perplexity 558.5463
-3100 train 6.284938 (lr=3.2866e-05) (hash(x)=149536927)
-3200 val loss 6.3026
-3200 val perplexity 545.9706
-3200 train 6.015903 (lr=3.2720e-05) (hash(x)=135678663)
-3300 val loss 6.2866
-3300 val perplexity 537.3096
-3300 train 6.366162 (lr=3.2570e-05) (hash(x)=148523646)
-3400 val loss 6.2729
-3400 val perplexity 530.0279
-3400 train 6.336071 (lr=3.2415e-05) (hash(x)=150281778)
-3500 val loss 6.2572
-3500 val perplexity 521.7344
-3500 train 6.212903 (lr=3.2256e-05) (hash(x)=151527953)
-3600 val loss 6.2510
-3600 val perplexity 518.5555
-3600 train 6.464981 (lr=3.2092e-05) (hash(x)=141303739)
-3700 val loss 6.2310
-3700 val perplexity 508.2712
-3700 train 6.281719 (lr=3.1924e-05) (hash(x)=146778597)
-3800 val loss 6.2172
-3800 val perplexity 501.2825
-3800 train 6.171426 (lr=3.1753e-05) (hash(x)=146902797)
-3900 val loss 6.2040
-3900 val perplexity 494.7110
-3900 train 6.130551 (lr=3.1577e-05) (hash(x)=142022555)
-4000 val loss 6.1886
-4000 val perplexity 487.1476
-4000 train 6.098490 (lr=3.1397e-05) (hash(x)=151590309)
-4100 val loss 6.1778
-4100 val perplexity 481.9368
-4100 train 6.023282 (lr=3.1212e-05) (hash(x)=144127100)
-4200 val loss 6.1688
-4200 val perplexity 477.6228
-4200 train 6.234536 (lr=3.1024e-05) (hash(x)=149526774)
-4300 val loss 6.1632
-4300 val perplexity 474.9631
-4300 train 6.102820 (lr=3.0833e-05) (hash(x)=150924362)
-4400 val loss 6.1428
-4400 val perplexity 465.3742
-4400 train 6.281572 (lr=3.0637e-05) (hash(x)=155547282)
-4500 val loss 6.1375
-4500 val perplexity 462.8767
-4500 train 6.125348 (lr=3.0437e-05) (hash(x)=152479203)
-4600 val loss 6.1183
-4600 val perplexity 454.0975
-4600 train 6.030381 (lr=3.0234e-05) (hash(x)=146739760)
-4700 val loss 6.1079
-4700 val perplexity 449.4118
-4700 train 6.328187 (lr=3.0027e-05) (hash(x)=159977388)
-4800 val loss 6.0983
-4800 val perplexity 445.0985
-4800 train 5.991615 (lr=2.9817e-05) (hash(x)=142959739)
-4900 val loss 6.0819
-4900 val perplexity 437.8497
-4900 train 6.032333 (lr=2.9603e-05) (hash(x)=152253034)
-5000 val loss 6.0800
-5000 val perplexity 437.0294
-5000 train 6.051497 (lr=2.9386e-05) (hash(x)=140379747)
-5100 val loss 6.0608
-5100 val perplexity 428.7304
-5100 train 6.091932 (lr=2.9166e-05) (hash(x)=151826392)
-5200 val loss 6.0533
-5200 val perplexity 425.5295
-5200 train 6.021602 (lr=2.8942e-05) (hash(x)=152869594)
-5300 val loss 6.0463
-5300 val perplexity 422.5276
-5300 train 5.831899 (lr=2.8715e-05) (hash(x)=139093709)
-5400 val loss 6.0359
-5400 val perplexity 418.1806
-5400 train 5.915012 (lr=2.8484e-05) (hash(x)=145908738)
-5500 val loss 6.0321
-5500 val perplexity 416.5761
-5500 train 5.895004 (lr=2.8251e-05) (hash(x)=137913426)
-5600 val loss 6.0108
-5600 val perplexity 407.8055
-5600 train 5.881226 (lr=2.8015e-05) (hash(x)=140770390)
-5700 val loss 6.0147
-5700 val perplexity 409.4116
-5700 train 5.930855 (lr=2.7776e-05) (hash(x)=143728578)
-5800 val loss 5.9952
-5800 val perplexity 401.5169
-5800 train 6.058795 (lr=2.7534e-05) (hash(x)=154469787)
-5900 val loss 5.9872
-5900 val perplexity 398.3034
-5900 train 5.950858 (lr=2.7290e-05) (hash(x)=139316559)
-6000 val loss 5.9789
-6000 val perplexity 395.0224
-6000 train 5.991045 (lr=2.7042e-05) (hash(x)=152058488)
-6100 val loss 5.9701
-6100 val perplexity 391.5618
-6100 train 5.985957 (lr=2.6792e-05) (hash(x)=152958689)
-6200 val loss 5.9554
-6200 val perplexity 385.8266
-6200 train 5.911655 (lr=2.6540e-05) (hash(x)=150887864)
-6300 val loss 5.9482
-6300 val perplexity 383.0494
-6300 train 5.901646 (lr=2.6285e-05) (hash(x)=156290469)
-6400 val loss 5.9369
-6400 val perplexity 378.7653
-6400 train 6.258915 (lr=2.6028e-05) (hash(x)=159160159)
-6500 val loss 5.9340
-6500 val perplexity 377.6659
-6500 train 5.852345 (lr=2.5769e-05) (hash(x)=151239121)
-6600 val loss 5.9260
-6600 val perplexity 374.6520
-6600 train 5.947871 (lr=2.5508e-05) (hash(x)=151964101)
-6700 val loss 5.9197
-6700 val perplexity 372.3041
-6700 train 5.845406 (lr=2.5244e-05) (hash(x)=141291229)
-6800 val loss 5.9132
-6800 val perplexity 369.8967
-6800 train 5.825259 (lr=2.4979e-05) (hash(x)=160416800)
-6900 val loss 5.9017
-6900 val perplexity 365.6624
-6900 train 5.810899 (lr=2.4711e-05) (hash(x)=143382691)
-7000 val loss 5.8909
-7000 val perplexity 361.7261
-7000 train 5.795605 (lr=2.4442e-05) (hash(x)=147606572)
-7100 val loss 5.8815
-7100 val perplexity 358.3473
-7100 train 5.707331 (lr=2.4171e-05) (hash(x)=145901538)
-7200 val loss 5.8815
-7200 val perplexity 358.3601
-7200 train 5.719882 (lr=2.3899e-05) (hash(x)=139099153)
-7300 val loss 5.8714
-7300 val perplexity 354.7415
-7300 train 5.836294 (lr=2.3625e-05) (hash(x)=140434004)
-7400 val loss 5.8686
-7400 val perplexity 353.7577
-7400 train 5.989874 (lr=2.3349e-05) (hash(x)=155496146)
-7500 val loss 5.8615
-7500 val perplexity 351.2433
-7500 train 5.792509 (lr=2.3073e-05) (hash(x)=146940054)
-7600 val loss 5.8472
-7600 val perplexity 346.2521
-7600 train 6.075488 (lr=2.2795e-05) (hash(x)=165785723)
-7700 val loss 5.8365
-7700 val perplexity 342.5613
-7700 train 5.990919 (lr=2.2515e-05) (hash(x)=156339008)
-7800 val loss 5.8346
-7800 val perplexity 341.9387
-7800 train 5.831989 (lr=2.2235e-05) (hash(x)=148593556)
-7900 val loss 5.8383
-7900 val perplexity 343.1989
-7900 train 5.761374 (lr=2.1954e-05) (hash(x)=148079742)
-8000 val loss 5.8212
-8000 val perplexity 337.3739
-8000 train 5.754209 (lr=2.1671e-05) (hash(x)=159706718)
-8100 val loss 5.8161
-8100 val perplexity 335.6689
-8100 train 5.803094 (lr=2.1388e-05) (hash(x)=160203225)
-8200 val loss 5.8042
-8200 val perplexity 331.6880
-8200 train 5.745097 (lr=2.1105e-05) (hash(x)=146176536)
-8300 val loss 5.7950
-8300 val perplexity 328.6607
-8300 train 5.881962 (lr=2.0820e-05) (hash(x)=158883824)
-8400 val loss 5.7912
-8400 val perplexity 327.4128
-8400 train 5.611819 (lr=2.0536e-05) (hash(x)=137102572)
-8500 val loss 5.7839
-8500 val perplexity 325.0120
-8500 train 5.818129 (lr=2.0250e-05) (hash(x)=152812797)
-8600 val loss 5.7820
-8600 val perplexity 324.4048
-8600 train 5.625070 (lr=1.9965e-05) (hash(x)=148854244)
-8700 val loss 5.7930
-8700 val perplexity 328.0061
-8700 train 5.611191 (lr=1.9679e-05) (hash(x)=135590023)
-8800 val loss 5.7679
-8800 val perplexity 319.8712
-8800 train 5.704375 (lr=1.9393e-05) (hash(x)=144913509)
-8900 val loss 5.7618
-8900 val perplexity 317.9144
-8900 train 5.900129 (lr=1.9107e-05) (hash(x)=148045669)
-9000 val loss 5.7572
-9000 val perplexity 316.4612
-9000 train 5.788160 (lr=1.8821e-05) (hash(x)=146661746)
-9100 val loss 5.7500
-9100 val perplexity 314.1996
-9100 train 5.697944 (lr=1.8535e-05) (hash(x)=145354881)
-9200 val loss 5.7463
-9200 val perplexity 313.0410
-9200 train 5.611643 (lr=1.8250e-05) (hash(x)=150939310)
-9300 val loss 5.7398
-9300 val perplexity 311.0170
-9300 train 5.541914 (lr=1.7964e-05) (hash(x)=162258309)
-9400 val loss 5.7344
-9400 val perplexity 309.3314
-9400 train 5.737438 (lr=1.7680e-05) (hash(x)=154701830)
-9500 val loss 5.7263
-9500 val perplexity 306.8179
-9500 train 5.623687 (lr=1.7395e-05) (hash(x)=142241404)
-9600 val loss 5.7201
-9600 val perplexity 304.9395
-9600 train 5.693145 (lr=1.7112e-05) (hash(x)=148439858)
-9700 val loss 5.7193
-9700 val perplexity 304.7034
-9700 train 5.717032 (lr=1.6829e-05) (hash(x)=152204410)
-9800 val loss 5.7127
-9800 val perplexity 302.6905
-9800 train 5.571279 (lr=1.6546e-05) (hash(x)=141294630)
-9900 val loss 5.7124
-9900 val perplexity 302.6070
-9900 train 5.687763 (lr=1.6265e-05) (hash(x)=147390033)
-10000 val loss 5.7070
-10000 val perplexity 300.9729
-10000 train 5.728780 (lr=1.5985e-05) (hash(x)=144131604)
-10100 val loss 5.6945
-10100 val perplexity 297.2338
-10100 train 5.905474 (lr=1.5705e-05) (hash(x)=158814818)
-10200 val loss 5.6932
-10200 val perplexity 296.8331
-10200 train 5.781689 (lr=1.5427e-05) (hash(x)=153491388)
-10300 val loss 5.6900
-10300 val perplexity 295.8885
-10300 train 5.668776 (lr=1.5151e-05) (hash(x)=149600707)
-10400 val loss 5.6850
-10400 val perplexity 294.4254
-10400 train 5.614613 (lr=1.4875e-05) (hash(x)=152277064)
-10500 val loss 5.6834
-10500 val perplexity 293.9567
-10500 train 5.584078 (lr=1.4601e-05) (hash(x)=150447091)
-10600 val loss 5.6768
-10600 val perplexity 292.0200
-10600 train 5.693406 (lr=1.4329e-05) (hash(x)=151606822)
-10700 val loss 5.6694
-10700 val perplexity 289.8598
-10700 train 5.642289 (lr=1.4058e-05) (hash(x)=158730240)
-10800 val loss 5.6663
-10800 val perplexity 288.9665
-10800 train 5.702177 (lr=1.3789e-05) (hash(x)=163095518)
-10900 val loss 5.6634
-10900 val perplexity 288.1227
-10900 train 5.426070 (lr=1.3521e-05) (hash(x)=145260395)
-11000 val loss 5.6586
-11000 val perplexity 286.7513
-11000 train 5.655396 (lr=1.3256e-05) (hash(x)=155970264)
-11100 val loss 5.6565
-11100 val perplexity 286.1584
-11100 train 5.748341 (lr=1.2992e-05) (hash(x)=159871097)
-11200 val loss 5.6514
-11200 val perplexity 284.6913
-11200 train 5.730143 (lr=1.2731e-05) (hash(x)=161135244)
-11300 val loss 5.6451
-11300 val perplexity 282.8916
-11300 train 5.687905 (lr=1.2472e-05) (hash(x)=150649336)
-11400 val loss 5.6426
-11400 val perplexity 282.1888
-11400 train 5.608679 (lr=1.2215e-05) (hash(x)=152230013)
-11500 val loss 5.6394
-11500 val perplexity 281.3017
-11500 train 5.663232 (lr=1.1960e-05) (hash(x)=163300613)
-11600 val loss 5.6377
-11600 val perplexity 280.8187
-11600 train 5.610705 (lr=1.1708e-05) (hash(x)=151203838)
-11700 val loss 5.6325
-11700 val perplexity 279.3708
-11700 train 5.657187 (lr=1.1458e-05) (hash(x)=155255111)
-11800 val loss 5.6326
-11800 val perplexity 279.3909
-11800 train 5.592705 (lr=1.1210e-05) (hash(x)=139380668)
-11900 val loss 5.6238
-11900 val perplexity 276.9400
-11900 train 5.564804 (lr=1.0966e-05) (hash(x)=144033405)
-12000 val loss 5.6209
-12000 val perplexity 276.1438
-12000 train 5.544782 (lr=1.0724e-05) (hash(x)=148689574)
-12100 val loss 5.6226
-12100 val perplexity 276.5996
-12100 train 5.551520 (lr=1.0485e-05) (hash(x)=143272239)
-12200 val loss 5.6176
-12200 val perplexity 275.2220
-12200 train 5.656269 (lr=1.0249e-05) (hash(x)=157671609)
-12300 val loss 5.6146
-12300 val perplexity 274.3976
-12300 train 5.639105 (lr=1.0016e-05) (hash(x)=150363247)
-12400 val loss 5.6096
-12400 val perplexity 273.0280
-12400 train 5.499437 (lr=9.7854e-06) (hash(x)=146857787)
-12500 val loss 5.6083
-12500 val perplexity 272.6883
-12500 train 5.576447 (lr=9.5583e-06) (hash(x)=153327320)
-12600 val loss 5.6047
-12600 val perplexity 271.6973
-12600 train 5.337243 (lr=9.3345e-06) (hash(x)=143612983)
-12700 val loss 5.6017
-12700 val perplexity 270.8927
-12700 train 5.535681 (lr=9.1139e-06) (hash(x)=150739132)
-12800 val loss 5.6011
-12800 val perplexity 270.7128
-12800 train 5.455906 (lr=8.8967e-06) (hash(x)=147986204)
-12900 val loss 5.5987
-12900 val perplexity 270.0814
-12900 train 5.773186 (lr=8.6829e-06) (hash(x)=155218833)
-13000 val loss 5.5935
-13000 val perplexity 268.6810
-13000 train 5.588228 (lr=8.4726e-06) (hash(x)=148867960)
-13100 val loss 5.5914
-13100 val perplexity 268.1018
-13100 train 5.612458 (lr=8.2658e-06) (hash(x)=160101478)
-13200 val loss 5.5920
-13200 val perplexity 268.2831
-13200 train 5.452489 (lr=8.0626e-06) (hash(x)=151080097)
-13300 val loss 5.5899
-13300 val perplexity 267.7130
-13300 train 5.557879 (lr=7.8632e-06) (hash(x)=150403039)
-13400 val loss 5.5862
-13400 val perplexity 266.7191
-13400 train 5.541445 (lr=7.6674e-06) (hash(x)=129728571)
-13500 val loss 5.5814
-13500 val perplexity 265.4492
-13500 train 5.501922 (lr=7.4756e-06) (hash(x)=148988625)
-13600 val loss 5.5807
-13600 val perplexity 265.2461
-13600 train 5.370127 (lr=7.2875e-06) (hash(x)=149790861)
-13700 val loss 5.5796
-13700 val perplexity 264.9609
-13700 train 5.498874 (lr=7.1035e-06) (hash(x)=153505062)
-13800 val loss 5.5790
-13800 val perplexity 264.7946
-13800 train 5.486200 (lr=6.9234e-06) (hash(x)=143845463)
-13900 val loss 5.5776
-13900 val perplexity 264.4264
-13900 train 5.383967 (lr=6.7474e-06) (hash(x)=143378076)
-14000 val loss 5.5723
-14000 val perplexity 263.0425
-14000 train 5.532637 (lr=6.5756e-06) (hash(x)=148681755)
-14100 val loss 5.5704
-14100 val perplexity 262.5340
-14100 train 5.456738 (lr=6.4079e-06) (hash(x)=145530465)
-14200 val loss 5.5713
-14200 val perplexity 262.7664
-14200 train 5.505947 (lr=6.2444e-06) (hash(x)=162139713)
-14300 val loss 5.5674
-14300 val perplexity 261.7432
-14300 train 5.574866 (lr=6.0852e-06) (hash(x)=141997596)
-14400 val loss 5.5670
-14400 val perplexity 261.6435
-14400 train 5.542020 (lr=5.9304e-06) (hash(x)=146156313)
-14500 val loss 5.5668
-14500 val perplexity 261.5967
-14500 train 5.344202 (lr=5.7800e-06) (hash(x)=145298178)
-14600 val loss 5.5618
-14600 val perplexity 260.2844
-14600 train 5.648545 (lr=5.6340e-06) (hash(x)=148232610)
-14700 val loss 5.5608
-14700 val perplexity 260.0356
-14700 train 5.436423 (lr=5.4925e-06) (hash(x)=151079115)
-14800 val loss 5.5602
-14800 val perplexity 259.8626
-14800 train 5.553704 (lr=5.3555e-06) (hash(x)=151340983)
-14900 val loss 5.5582
-14900 val perplexity 259.3505
-14900 train 5.774446 (lr=5.2231e-06) (hash(x)=150213803)
-15000 val loss 5.5574
-15000 val perplexity 259.1565
-15000 train 5.464648 (lr=5.0954e-06) (hash(x)=157652334)
-15100 val loss 5.5565
-15100 val perplexity 258.9096
-15100 train 5.274495 (lr=4.9723e-06) (hash(x)=145078577)
-15200 val loss 5.5530
-15200 val perplexity 258.0162
-15200 train 5.499384 (lr=4.8539e-06) (hash(x)=155898006)
-15300 val loss 5.5518
-15300 val perplexity 257.6959
-15300 train 5.588014 (lr=4.7403e-06) (hash(x)=149346671)
-15400 val loss 5.5496
-15400 val perplexity 257.1445
-15400 train 5.391021 (lr=4.6314e-06) (hash(x)=145708113)
-15500 val loss 5.5477
-15500 val perplexity 256.6455
-15500 train 5.739224 (lr=4.5274e-06) (hash(x)=165102970)
-15600 val loss 5.5488
-15600 val perplexity 256.9364
-15600 train 5.333233 (lr=4.4282e-06) (hash(x)=139106517)
-15700 val loss 5.5474
-15700 val perplexity 256.5752
-15700 train 5.435939 (lr=4.3339e-06) (hash(x)=141680838)
-15800 val loss 5.5448
-15800 val perplexity 255.9047
-15800 train 5.447368 (lr=4.2446e-06) (hash(x)=136152310)
-15900 val loss 5.5426
-15900 val perplexity 255.3536
-15900 train 5.602579 (lr=4.1601e-06) (hash(x)=151104640)
-16000 val loss 5.5418
-16000 val perplexity 255.1399
-16000 train 5.491660 (lr=4.0807e-06) (hash(x)=153333281)
-16100 val loss 5.5407
-16100 val perplexity 254.8457
-16100 train 5.496932 (lr=4.0063e-06) (hash(x)=145628059)
-16200 val loss 5.5414
-16200 val perplexity 255.0395
-16200 train 5.268761 (lr=3.9368e-06) (hash(x)=137220041)
-16300 val loss 5.5398
-16300 val perplexity 254.6356
-16300 train 5.710855 (lr=3.8725e-06) (hash(x)=161215791)
-16400 val loss 5.5392
-16400 val perplexity 254.4846
-16400 train 5.249424 (lr=3.8132e-06) (hash(x)=136194680)
-16500 val loss 5.5397
-16500 val perplexity 254.6127
-16500 train 5.334385 (lr=3.7590e-06) (hash(x)=152209568)
-16600 val loss 5.5366
-16600 val perplexity 253.8228
-16600 train 5.652360 (lr=3.7099e-06) (hash(x)=157345721)
-16700 val loss 5.5351
-16700 val perplexity 253.4332
-16700 train 5.666698 (lr=3.6659e-06) (hash(x)=146883642)
-16800 val loss 5.5349
-16800 val perplexity 253.3878
-16800 train 5.509064 (lr=3.6271e-06) (hash(x)=153037972)
-16900 val loss 5.5340
-16900 val perplexity 253.1516
-16900 train 5.297623 (lr=3.5934e-06) (hash(x)=150691367)
-17000 val loss 5.5341
-17000 val perplexity 253.1721
-17000 train 5.288759 (lr=3.5649e-06) (hash(x)=148975206)
-17100 val loss 5.5351
-17100 val perplexity 253.4207
-17100 train 5.278723 (lr=3.5415e-06) (hash(x)=142026221)
-17200 val loss 5.5315
-17200 val perplexity 252.5322
-17200 train 5.597274 (lr=3.5234e-06) (hash(x)=138211880)
-17300 val loss 5.5293
-17300 val perplexity 251.9741
-17300 train 5.385768 (lr=3.5104e-06) (hash(x)=143366666)
-17400 val loss 5.5290
-17400 val perplexity 251.8806
-17400 train 5.530194 (lr=3.5026e-06) (hash(x)=150791766)
-17499 val loss 5.5286
-17499 val perplexity 251.7872
+0 val loss 11.3001
+0 val perplexity 80829.3594
+0 train 11.304919 (lr=1.8750e-07) (hash(x)=153418542)
+0 train 11.304919 (lr=2.0000e-07) (hash(x)=153418542)
+100 val loss 9.8329
+100 val perplexity 18636.0176
+100 train 9.800076 (lr=1.8937e-05) (hash(x)=154406429)
+100 val loss 9.8212
+100 val perplexity 18420.7812
+100 train 9.789283 (lr=2.0200e-05) (hash(x)=154406429)
+200 val loss 8.5308
+200 val perplexity 5068.6626
+200 train 8.423743 (lr=3.7500e-05) (hash(x)=143722001)
+200 val loss 8.4768
+200 val perplexity 4802.1499
+200 train 8.366443 (lr=4.0000e-05) (hash(x)=143722001)
+300 val loss 7.8027
+300 val perplexity 2447.1411
+300 train 7.741059 (lr=3.7497e-05) (hash(x)=152421341)
+300 val loss 7.7235
+300 val perplexity 2260.9478
+300 train 7.657517 (lr=3.9997e-05) (hash(x)=152421341)
+400 val loss 7.6001
+400 val perplexity 1998.4463
+400 val loss 7.5567
+400 val perplexity 1913.5374
+400 train 7.495296 (lr=3.7489e-05) (hash(x)=152989689)
+400 train 7.447391 (lr=3.9988e-05) (hash(x)=152989689)
+500 val loss 7.5282
+500 val perplexity 1859.7047
+500 val loss 7.4923
+500 val perplexity 1794.2245
+500 train 7.458594 (lr=3.7475e-05) (hash(x)=145113544)
+500 train 7.418945 (lr=3.9973e-05) (hash(x)=145113544)
+600 val loss 7.4701
+600 val perplexity 1754.8171
+600 val loss 7.4354
+600 val perplexity 1694.8523
+600 train 7.463784 (lr=3.7456e-05) (hash(x)=150071018)
+600 train 7.434020 (lr=3.9953e-05) (hash(x)=150071018)
+700 val loss 7.3884
+700 val perplexity 1617.1136
+700 val loss 7.4224
+700 val perplexity 1673.0927
+700 train 7.480190 (lr=3.9926e-05) (hash(x)=156594182)
+700 train 7.505778 (lr=3.7430e-05) (hash(x)=156594182)
+800 val loss 7.3583
+800 val perplexity 1569.1648
+800 train 7.357888 (lr=3.9893e-05) (hash(x)=153358238)
+800 val loss 7.3882
+800 val perplexity 1616.8392
+800 train 7.388269 (lr=3.7400e-05) (hash(x)=153358238)
+900 val loss 7.3160
+900 val perplexity 1504.1028
+900 train 7.138016 (lr=3.9855e-05) (hash(x)=146617394)
+900 val loss 7.3554
+900 val perplexity 1564.5654
+900 train 7.184053 (lr=3.7364e-05) (hash(x)=146617394)
+1000 val loss 7.2891
+1000 val perplexity 1464.2344
+1000 train 7.218844 (lr=3.9810e-05) (hash(x)=148983354)
+1000 val loss 7.3291
+1000 val perplexity 1524.0352
+1000 train 7.271225 (lr=3.7322e-05) (hash(x)=148983354)
+1100 val loss 7.2489
+1100 val perplexity 1406.5225
+1100 train 7.274782 (lr=3.9760e-05) (hash(x)=146313352)
+1100 val loss 7.2985
+1100 val perplexity 1478.0883
+1100 train 7.326784 (lr=3.7275e-05) (hash(x)=146313352)
+1200 val loss 7.2332
+1200 val perplexity 1384.6940
+1200 train 7.190815 (lr=3.9704e-05) (hash(x)=150770333)
+1200 val loss 7.2798
+1200 val perplexity 1450.7379
+1200 train 7.232755 (lr=3.7223e-05) (hash(x)=150770333)
+1300 val loss 7.1657
+1300 val perplexity 1294.2185
+1300 train 7.110763 (lr=3.9642e-05) (hash(x)=153237933)
+1300 val loss 7.2388
+1300 val perplexity 1392.4480
+1300 train 7.181512 (lr=3.7164e-05) (hash(x)=153237933)
+1400 val loss 7.1323
+1400 val perplexity 1251.7812
+1400 train 6.970374 (lr=3.9574e-05) (hash(x)=145246201)
+1400 val loss 7.2173
+1400 val perplexity 1362.7434
+1400 train 7.056479 (lr=3.7101e-05) (hash(x)=145246201)
+1500 val loss 7.0966
+1500 val perplexity 1207.8966
+1500 train 7.092261 (lr=3.9501e-05) (hash(x)=154002540)
+1500 val loss 7.1916
+1500 val perplexity 1328.2701
+1500 train 7.191994 (lr=3.7032e-05) (hash(x)=154002540)
+1600 val loss 7.0737
+1600 val perplexity 1180.4943
+1600 train 6.847246 (lr=3.9421e-05) (hash(x)=142635842)
+1600 val loss 7.1695
+1600 val perplexity 1299.2076
+1600 train 6.949299 (lr=3.6958e-05) (hash(x)=142635842)
+1700 val loss 7.0351
+1700 val perplexity 1135.7549
+1700 train 7.076663 (lr=3.9336e-05) (hash(x)=151450555)
+1700 val loss 7.1469
+1700 val perplexity 1270.1534
+1700 train 7.183446 (lr=3.6878e-05) (hash(x)=151450555)
+1800 val loss 7.0089
+1800 val perplexity 1106.4053
+1800 train 6.922809 (lr=3.9246e-05) (hash(x)=148299832)
+1800 val loss 7.1197
+1800 val perplexity 1236.1178
+1800 train 7.035375 (lr=3.6793e-05) (hash(x)=148299832)
+1900 val loss 6.9638
+1900 val perplexity 1057.6011
+1900 train 7.122456 (lr=3.9149e-05) (hash(x)=155487476)
+1900 val loss 7.0849
+1900 val perplexity 1193.7941
+1900 train 7.249062 (lr=3.6702e-05) (hash(x)=155487476)
+2000 val loss 6.9417
+2000 val perplexity 1034.5106
+2000 train 6.923683 (lr=3.9047e-05) (hash(x)=152887721)
+2000 val loss 7.0502
+2000 val perplexity 1153.0652
+2000 train 7.028882 (lr=3.6606e-05) (hash(x)=152887721)
+2100 val loss 6.9170
+2100 val perplexity 1009.2787
+2100 train 6.620973 (lr=3.8939e-05) (hash(x)=143430603)
+2100 val loss 7.0140
+2100 val perplexity 1112.0854
+2100 train 6.726614 (lr=3.6505e-05) (hash(x)=143430603)
+2200 val loss 6.8884
+2200 val perplexity 980.8248
+2200 train 6.878071 (lr=3.8826e-05) (hash(x)=153721194)
+2200 val loss 6.9844
+2200 val perplexity 1079.6273
+2200 train 6.971331 (lr=3.6399e-05) (hash(x)=153721194)
+2300 val loss 6.8506
+2300 val perplexity 944.4535
+2300 train 6.979174 (lr=3.8707e-05) (hash(x)=148702429)
+2300 val loss 6.9476
+2300 val perplexity 1040.6342
+2300 train 7.065608 (lr=3.6288e-05) (hash(x)=148702429)
+2400 val loss 6.8137
+2400 val perplexity 910.2053
+2400 train 6.860815 (lr=3.8583e-05) (hash(x)=153047184)
+2400 val loss 6.9253
+2400 val perplexity 1017.6667
+2400 train 6.966145 (lr=3.6171e-05) (hash(x)=153047184)
+2500 val loss 6.7714
+2500 val perplexity 872.5063
+2500 train 6.653331 (lr=3.8453e-05) (hash(x)=149720123)
+2500 val loss 6.8891
+2500 val perplexity 981.5322
+2500 train 6.767345 (lr=3.6049e-05) (hash(x)=149720123)
+2600 val loss 6.7305
+2600 val perplexity 837.5394
+2600 train 6.760979 (lr=3.8317e-05) (hash(x)=152466045)
+2600 val loss 6.8655
+2600 val perplexity 958.6515
+2600 train 6.900772 (lr=3.5923e-05) (hash(x)=152466045)
+2700 val loss 6.7068
+2700 val perplexity 817.9697
+2700 train 6.640062 (lr=3.8177e-05) (hash(x)=155796305)
+2700 val loss 6.8521
+2700 val perplexity 945.9030
+2700 train 6.800490 (lr=3.5791e-05) (hash(x)=155796305)
+2800 val loss 6.6646
+2800 val perplexity 784.1478
+2800 train 6.503381 (lr=3.8031e-05) (hash(x)=151991552)
+2800 val loss 6.8246
+2800 val perplexity 920.1984
+2800 train 6.668451 (lr=3.5654e-05) (hash(x)=151991552)
+2900 val loss 6.6274
+2900 val perplexity 755.5084
+2900 train 7.209426 (lr=3.7879e-05) (hash(x)=166537989)
+2900 val loss 6.7901
+2900 val perplexity 888.9970
+2900 train 7.444793 (lr=3.5512e-05) (hash(x)=166537989)
+3000 val loss 6.5884
+3000 val perplexity 726.5895
+3000 train 6.592463 (lr=3.7723e-05) (hash(x)=147892594)
+3000 val loss 6.7554
+3000 val perplexity 858.6501
+3000 train 6.760037 (lr=3.5365e-05) (hash(x)=147892594)
+3100 val loss 6.5645
+3100 val perplexity 709.4786
+3100 train 6.522549 (lr=3.7561e-05) (hash(x)=149536927)
+3100 val loss 6.7237
+3100 val perplexity 831.8831
+3100 train 6.678670 (lr=3.5214e-05) (hash(x)=149536927)
+3200 val loss 6.5361
+3200 val perplexity 689.5991
+3200 train 6.249228 (lr=3.7394e-05) (hash(x)=135678663)
+3200 val loss 6.6918
+3200 val perplexity 805.7526
+3200 train 6.404359 (lr=3.5057e-05) (hash(x)=135678663)
+3300 val loss 6.5172
+3300 val perplexity 676.7065
+3300 train 6.580379 (lr=3.7222e-05) (hash(x)=148523646)
+3300 val loss 6.6596
+3300 val perplexity 780.2591
+3300 train 6.714272 (lr=3.4896e-05) (hash(x)=148523646)
+3400 val loss 6.4957
+3400 val perplexity 662.2582
+3400 train 6.568460 (lr=3.7045e-05) (hash(x)=150281778)
+3400 val loss 6.6333
+3400 val perplexity 760.0051
+3400 train 6.704147 (lr=3.4730e-05) (hash(x)=150281778)
+3500 val loss 6.4701
+3500 val perplexity 645.5488
+3500 train 6.417324 (lr=3.6864e-05) (hash(x)=151527953)
+3500 val loss 6.6012
+3500 val perplexity 735.9909
+3500 train 6.537759 (lr=3.4560e-05) (hash(x)=151527953)
+3600 val loss 6.4607
+3600 val perplexity 639.5080
+3600 train 6.673931 (lr=3.6677e-05) (hash(x)=141303739)
+3600 val loss 6.5851
+3600 val perplexity 724.2502
+3600 train 6.815054 (lr=3.4384e-05) (hash(x)=141303739)
+3700 val loss 6.4387
+3700 val perplexity 625.5988
+3700 train 6.471379 (lr=3.6485e-05) (hash(x)=146778597)
+3700 val loss 6.5494
+3700 val perplexity 698.8573
+3700 train 6.569457 (lr=3.4205e-05) (hash(x)=146778597)
+3800 val loss 6.4031
+3800 val perplexity 603.6975
+3800 train 6.358904 (lr=3.6289e-05) (hash(x)=146902797)
+3800 val loss 6.5228
+3800 val perplexity 680.4645
+3800 train 6.484237 (lr=3.4021e-05) (hash(x)=146902797)
+3900 val loss 6.3836
+3900 val perplexity 592.0266
+3900 train 6.312788 (lr=3.6088e-05) (hash(x)=142022555)
+3900 val loss 6.4976
+3900 val perplexity 663.5311
+3900 train 6.423472 (lr=3.3832e-05) (hash(x)=142022555)
+4000 val loss 6.3624
+4000 val perplexity 579.6483
+4000 train 6.290960 (lr=3.5882e-05) (hash(x)=151590309)
+4000 val loss 6.4783
+4000 val perplexity 650.8406
+4000 train 6.398770 (lr=3.3639e-05) (hash(x)=151590309)
+4100 val loss 6.3443
+4100 val perplexity 569.2514
+4100 train 6.188325 (lr=3.5671e-05) (hash(x)=144127100)
+4100 val loss 6.4617
+4100 val perplexity 640.1420
+4100 train 6.302608 (lr=3.3442e-05) (hash(x)=144127100)
+4200 val loss 6.3344
+4200 val perplexity 563.6245
+4200 train 6.400013 (lr=3.5457e-05) (hash(x)=149526774)
+4200 val loss 6.4405
+4200 val perplexity 626.7184
+4200 train 6.507928 (lr=3.3240e-05) (hash(x)=149526774)
+4300 val loss 6.3215
+4300 val perplexity 556.4191
+4300 train 6.272267 (lr=3.5237e-05) (hash(x)=150924362)
+4300 val loss 6.4347
+4300 val perplexity 623.0805
+4300 train 6.396029 (lr=3.3035e-05) (hash(x)=150924362)
+4400 val loss 6.2921
+4400 val perplexity 540.2718
+4400 train 6.439682 (lr=3.5014e-05) (hash(x)=155547282)
+4400 val loss 6.3944
+4400 val perplexity 598.5039
+4400 train 6.551724 (lr=3.2825e-05) (hash(x)=155547282)
+4500 val loss 6.2809
+4500 val perplexity 534.2578
+4500 train 6.262727 (lr=3.4786e-05) (hash(x)=152479203)
+4500 val loss 6.3806
+4500 val perplexity 590.2866
+4500 train 6.360958 (lr=3.2611e-05) (hash(x)=152479203)
+4600 val loss 6.2638
+4600 val perplexity 525.2311
+4600 train 6.177735 (lr=3.4553e-05) (hash(x)=146739760)
+4600 val loss 6.3688
+4600 val perplexity 583.3686
+4600 train 6.277540 (lr=3.2394e-05) (hash(x)=146739760)
+4700 val loss 6.2521
+4700 val perplexity 519.1068
+4700 train 6.473020 (lr=3.4317e-05) (hash(x)=159977388)
+4700 val loss 6.3566
+4700 val perplexity 576.2929
+4700 train 6.578290 (lr=3.2172e-05) (hash(x)=159977388)
+4800 val loss 6.2422
+4800 val perplexity 513.9703
+4800 train 6.129420 (lr=3.4077e-05) (hash(x)=142959739)
+4800 val loss 6.3445
+4800 val perplexity 569.3329
+4800 train 6.230520 (lr=3.1947e-05) (hash(x)=142959739)
+4900 val loss 6.2251
+4900 val perplexity 505.2736
+4900 train 6.180996 (lr=3.3832e-05) (hash(x)=152253034)
+4900 val loss 6.3244
+4900 val perplexity 558.0325
+4900 train 6.286006 (lr=3.1718e-05) (hash(x)=152253034)
+5000 val loss 6.2122
+5000 val perplexity 498.8215
+5000 train 6.184723 (lr=3.3584e-05) (hash(x)=140379747)
+5000 val loss 6.3155
+5000 val perplexity 553.0589
+5000 train 6.280877 (lr=3.1485e-05) (hash(x)=140379747)
+5100 val loss 6.1928
+5100 val perplexity 489.2184
+5100 train 6.213193 (lr=3.3332e-05) (hash(x)=151826392)
+5100 val loss 6.2938
+5100 val perplexity 541.1830
+5100 train 6.315104 (lr=3.1249e-05) (hash(x)=151826392)
+5200 val loss 6.1897
+5200 val perplexity 487.7077
+5200 train 6.150966 (lr=3.3076e-05) (hash(x)=152869594)
+5200 val loss 6.2842
+5200 val perplexity 536.0088
+5200 train 6.236502 (lr=3.1009e-05) (hash(x)=152869594)
+5300 val loss 6.1714
+5300 val perplexity 478.8367
+5300 train 5.955120 (lr=3.2817e-05) (hash(x)=139093709)
+5300 val loss 6.2693
+5300 val perplexity 528.1216
+5300 train 6.061146 (lr=3.0766e-05) (hash(x)=139093709)
+5400 val loss 6.1644
+5400 val perplexity 475.5219
+5400 train 6.037286 (lr=3.2554e-05) (hash(x)=145908738)
+5400 val loss 6.2641
+5400 val perplexity 525.3493
+5400 train 6.129711 (lr=3.0519e-05) (hash(x)=145908738)
+5500 val loss 6.1606
+5500 val perplexity 473.6938
+5500 train 6.016478 (lr=3.2287e-05) (hash(x)=137913426)
+5500 val loss 6.2502
+5500 val perplexity 518.0980
+5500 train 6.108302 (lr=3.0269e-05) (hash(x)=137913426)
+5600 val loss 6.1410
+5600 val perplexity 464.5248
+5600 train 5.993292 (lr=3.2017e-05) (hash(x)=140770390)
+5600 val loss 6.2283
+5600 val perplexity 506.8769
+5600 train 6.066168 (lr=3.0016e-05) (hash(x)=140770390)
+5700 val loss 6.1385
+5700 val perplexity 463.3590
+5700 train 6.046831 (lr=3.1744e-05) (hash(x)=143728578)
+5700 val loss 6.2293
+5700 val perplexity 507.3886
+5700 train 6.146498 (lr=2.9760e-05) (hash(x)=143728578)
+5800 val loss 6.1174
+5800 val perplexity 453.7044
+5800 train 6.182424 (lr=3.1468e-05) (hash(x)=154469787)
+5800 val loss 6.2102
+5800 val perplexity 497.8142
+5800 train 6.285611 (lr=2.9501e-05) (hash(x)=154469787)
+5900 val loss 6.1137
+5900 val perplexity 452.0091
+5900 train 6.078083 (lr=3.1188e-05) (hash(x)=139316559)
+5900 val loss 6.2014
+5900 val perplexity 493.4557
+5900 train 6.150805 (lr=2.9239e-05) (hash(x)=139316559)
+6000 val loss 6.1008
+6000 val perplexity 446.2053
+6000 train 6.113969 (lr=3.0905e-05) (hash(x)=152058488)
+6000 val loss 6.1958
+6000 val perplexity 490.6758
+6000 train 6.203081 (lr=2.8974e-05) (hash(x)=152058488)
+6100 val loss 6.0900
+6100 val perplexity 441.4070
+6100 train 6.101003 (lr=3.0620e-05) (hash(x)=152958689)
+6100 val loss 6.1783
+6100 val perplexity 482.1736
+6100 train 6.180849 (lr=2.8706e-05) (hash(x)=152958689)
+6200 val loss 6.0709
+6200 val perplexity 433.0719
+6200 train 6.035614 (lr=3.0332e-05) (hash(x)=150887864)
+6200 val loss 6.1637
+6200 val perplexity 475.1640
+6200 train 6.128726 (lr=2.8436e-05) (hash(x)=150887864)
+6300 val loss 6.0649
+6300 val perplexity 430.4847
+6300 train 6.011372 (lr=3.0040e-05) (hash(x)=156290469)
+6300 val loss 6.1603
+6300 val perplexity 473.5673
+6300 train 6.107643 (lr=2.8163e-05) (hash(x)=156290469)
+6400 val loss 6.0562
+6400 val perplexity 426.7660
+6400 train 6.360909 (lr=2.9747e-05) (hash(x)=159160159)
+6400 val loss 6.1479
+6400 val perplexity 467.7326
+6400 train 6.456499 (lr=2.7888e-05) (hash(x)=159160159)
+6500 val loss 6.0513
+6500 val perplexity 424.6838
+6500 train 5.965971 (lr=2.9450e-05) (hash(x)=151239121)
+6500 val loss 6.1376
+6500 val perplexity 462.9326
+6500 train 6.059669 (lr=2.7610e-05) (hash(x)=151239121)
+6600 val loss 6.0396
+6600 val perplexity 419.7285
+6600 train 6.053043 (lr=2.9152e-05) (hash(x)=151964101)
+6600 val loss 6.1316
+6600 val perplexity 460.1588
+6600 train 6.140378 (lr=2.7330e-05) (hash(x)=151964101)
+6700 val loss 6.0341
+6700 val perplexity 417.4256
+6700 train 5.943383 (lr=2.8851e-05) (hash(x)=141291229)
+6700 val loss 6.1208
+6700 val perplexity 455.2326
+6700 train 6.024530 (lr=2.7047e-05) (hash(x)=141291229)
+6800 val loss 6.0232
+6800 val perplexity 412.8877
+6800 train 5.946563 (lr=2.8547e-05) (hash(x)=160416800)
+6800 val loss 6.1158
+6800 val perplexity 452.9381
+6800 train 6.040329 (lr=2.6763e-05) (hash(x)=160416800)
+6900 val loss 6.0162
+6900 val perplexity 410.0209
+6900 train 5.912896 (lr=2.8242e-05) (hash(x)=143382691)
+6900 val loss 6.1049
+6900 val perplexity 448.0463
+6900 train 6.002789 (lr=2.6477e-05) (hash(x)=143382691)
+7000 val loss 5.9973
+7000 val perplexity 402.3608
+7000 train 5.902618 (lr=2.7934e-05) (hash(x)=147606572)
+7000 val loss 6.0921
+7000 val perplexity 442.3517
+7000 train 6.005036 (lr=2.6188e-05) (hash(x)=147606572)
+7100 val loss 5.9907
+7100 val perplexity 399.6799
+7100 train 5.814173 (lr=2.7624e-05) (hash(x)=145901538)
+7100 val loss 6.0824
+7100 val perplexity 438.0784
+7100 train 5.912265 (lr=2.5898e-05) (hash(x)=145901538)
+7200 val loss 5.9901
+7200 val perplexity 399.4538
+7200 train 5.821192 (lr=2.7313e-05) (hash(x)=139099153)
+7200 val loss 6.0794
+7200 val perplexity 436.7787
+7200 train 5.910559 (lr=2.5606e-05) (hash(x)=139099153)
+7300 val loss 5.9797
+7300 val perplexity 395.3340
+7300 train 5.948170 (lr=2.7000e-05) (hash(x)=140434004)
+7300 val loss 6.0718
+7300 val perplexity 433.4543
+7300 train 6.038386 (lr=2.5312e-05) (hash(x)=140434004)
+7400 val loss 5.9791
+7400 val perplexity 395.1002
+7400 train 6.095177 (lr=2.6685e-05) (hash(x)=155496146)
+7400 val loss 6.0672
+7400 val perplexity 431.4613
+7400 train 6.180016 (lr=2.5017e-05) (hash(x)=155496146)
+7500 val loss 5.9664
+7500 val perplexity 390.1005
+7500 train 5.892523 (lr=2.6369e-05) (hash(x)=146940054)
+7500 val loss 6.0597
+7500 val perplexity 428.2500
+7500 train 5.979121 (lr=2.4721e-05) (hash(x)=146940054)
+7600 val loss 5.9557
+7600 val perplexity 385.9278
+7600 train 6.182656 (lr=2.6051e-05) (hash(x)=165785723)
+7600 val loss 6.0449
+7600 val perplexity 421.9463
+7600 train 6.269816 (lr=2.4423e-05) (hash(x)=165785723)
+7700 val loss 5.9467
+7700 val perplexity 382.4761
+7700 train 6.093819 (lr=2.5732e-05) (hash(x)=156339008)
+7700 val loss 6.0365
+7700 val perplexity 418.4242
+7700 train 6.181146 (lr=2.4124e-05) (hash(x)=156339008)
+7800 val loss 5.9486
+7800 val perplexity 383.2071
+7800 train 5.945467 (lr=2.5411e-05) (hash(x)=148593556)
+7800 val loss 6.0319
+7800 val perplexity 416.4887
+7800 train 6.030032 (lr=2.3823e-05) (hash(x)=148593556)
+7900 val loss 5.9564
+7900 val perplexity 386.2014
+7900 train 5.891070 (lr=2.5090e-05) (hash(x)=148079742)
+7900 val loss 6.0443
+7900 val perplexity 421.7001
+7900 train 5.989381 (lr=2.3522e-05) (hash(x)=148079742)
+8000 val loss 5.9298
+8000 val perplexity 376.0685
+8000 train 5.867480 (lr=2.4767e-05) (hash(x)=159706718)
+8000 val loss 6.0159
+8000 val perplexity 409.8966
+8000 train 5.955351 (lr=2.3219e-05) (hash(x)=159706718)
+8100 val loss 5.9287
+8100 val perplexity 375.6778
+8100 train 5.920499 (lr=2.4444e-05) (hash(x)=160203225)
+8100 val loss 6.0136
+8100 val perplexity 408.9505
+8100 train 6.008880 (lr=2.2916e-05) (hash(x)=160203225)
+8200 val loss 5.9155
+8200 val perplexity 370.7519
+8200 train 5.866872 (lr=2.4120e-05) (hash(x)=146176536)
+8200 val loss 5.9986
+8200 val perplexity 402.8794
+8200 train 5.960113 (lr=2.2612e-05) (hash(x)=146176536)
+8300 val loss 5.9061
+8300 val perplexity 367.2567
+8300 train 5.982905 (lr=2.3795e-05) (hash(x)=158883824)
+8300 val loss 5.9935
+8300 val perplexity 400.8128
+8300 train 6.061623 (lr=2.2308e-05) (hash(x)=158883824)
+8400 val loss 5.8996
+8400 val perplexity 364.8946
+8400 train 5.716026 (lr=2.3469e-05) (hash(x)=137102572)
+8400 val loss 5.9840
+8400 val perplexity 397.0131
+8400 train 5.799762 (lr=2.2002e-05) (hash(x)=137102572)
+8500 val loss 5.8970
+8500 val perplexity 363.9478
+8500 train 5.928124 (lr=2.3143e-05) (hash(x)=152812797)
+8500 val loss 5.9767
+8500 val perplexity 394.1475
+8500 train 6.004527 (lr=2.1697e-05) (hash(x)=152812797)
+8600 val loss 5.8957
+8600 val perplexity 363.4700
+8600 train 5.742877 (lr=2.2817e-05) (hash(x)=148854244)
+8600 val loss 5.9755
+8600 val perplexity 393.6647
+8600 train 5.820330 (lr=2.1391e-05) (hash(x)=148854244)
+8700 val loss 5.9036
+8700 val perplexity 366.3378
+8700 train 5.714509 (lr=2.2490e-05) (hash(x)=135590023)
+8700 val loss 5.9854
+8700 val perplexity 397.5729
+8700 train 5.797290 (lr=2.1085e-05) (hash(x)=135590023)
+8800 val loss 5.8742
+8800 val perplexity 355.7392
+8800 train 5.809571 (lr=2.2163e-05) (hash(x)=144913509)
+8800 val loss 5.9599
+8800 val perplexity 387.5530
+8800 train 5.892747 (lr=2.0778e-05) (hash(x)=144913509)
+8900 val loss 5.8714
+8900 val perplexity 354.7586
+8900 train 5.992274 (lr=2.1837e-05) (hash(x)=148045669)
+8900 val loss 5.9579
+8900 val perplexity 386.7786
+8900 train 6.079168 (lr=2.0472e-05) (hash(x)=148045669)
+9000 val loss 5.8665
+9000 val perplexity 353.0065
+9000 train 5.906809 (lr=2.1510e-05) (hash(x)=146661746)
+9000 val loss 5.9502
+9000 val perplexity 383.8190
+9000 train 5.979196 (lr=2.0165e-05) (hash(x)=146661746)
+9100 val loss 5.8603
+9100 val perplexity 350.8411
+9100 train 5.797707 (lr=2.1183e-05) (hash(x)=145354881)
+9100 val loss 5.9399
+9100 val perplexity 379.8830
+9100 train 5.876511 (lr=1.9859e-05) (hash(x)=145354881)
+9200 val loss 5.8547
+9200 val perplexity 348.8842
+9200 train 5.729348 (lr=2.0857e-05) (hash(x)=150939310)
+9200 val loss 5.9363
+9200 val perplexity 378.5494
+9200 train 5.809578 (lr=1.9553e-05) (hash(x)=150939310)
+9300 val loss 5.8502
+9300 val perplexity 347.3205
+9300 train 5.692621 (lr=2.0531e-05) (hash(x)=162258309)
+9300 val loss 5.9312
+9300 val perplexity 376.6234
+9300 train 5.781178 (lr=1.9248e-05) (hash(x)=162258309)
+9400 val loss 5.8477
+9400 val perplexity 346.4448
+9400 train 5.856801 (lr=2.0205e-05) (hash(x)=154701830)
+9400 val loss 5.9250
+9400 val perplexity 374.2867
+9400 train 5.932467 (lr=1.8942e-05) (hash(x)=154701830)
+9500 val loss 5.8364
+9500 val perplexity 342.5458
+9500 train 5.730147 (lr=1.9880e-05) (hash(x)=142241404)
+9500 val loss 5.9229
+9500 val perplexity 373.5072
+9500 train 5.808607 (lr=1.8638e-05) (hash(x)=142241404)
+9600 val loss 5.8332
+9600 val perplexity 341.4551
+9600 train 5.795203 (lr=1.9556e-05) (hash(x)=148439858)
+9600 val loss 5.9120
+9600 val perplexity 369.4564
+9600 train 5.880486 (lr=1.8334e-05) (hash(x)=148439858)
+9700 val loss 5.8316
+9700 val perplexity 340.8906
+9700 train 5.828146 (lr=1.9233e-05) (hash(x)=152204410)
+9700 val loss 5.9098
+9700 val perplexity 368.6377
+9700 train 5.907160 (lr=1.8031e-05) (hash(x)=152204410)
+9800 val loss 5.8245
+9800 val perplexity 338.5030
+9800 train 5.668409 (lr=1.8910e-05) (hash(x)=141294630)
+9800 val loss 5.9046
+9800 val perplexity 366.7115
+9800 train 5.741839 (lr=1.7728e-05) (hash(x)=141294630)
+9900 val loss 5.8172
+9900 val perplexity 336.0203
+9900 train 5.789067 (lr=1.8589e-05) (hash(x)=147390033)
+9900 val loss 5.9026
+9900 val perplexity 365.9911
+9900 train 5.876742 (lr=1.7427e-05) (hash(x)=147390033)
+10000 val loss 5.8138
+10000 val perplexity 334.8972
+10000 train 5.811567 (lr=1.8268e-05) (hash(x)=144131604)
+10000 val loss 5.8944
+10000 val perplexity 363.0030
+10000 train 5.886058 (lr=1.7126e-05) (hash(x)=144131604)
+10100 val loss 5.8044
+10100 val perplexity 331.7603
+10100 train 6.006137 (lr=1.7949e-05) (hash(x)=158814818)
+10100 val loss 5.8887
+10100 val perplexity 360.9442
+10100 train 6.090841 (lr=1.6827e-05) (hash(x)=158814818)
+10200 val loss 5.8000
+10200 val perplexity 330.2987
+10200 train 5.880665 (lr=1.7631e-05) (hash(x)=153491388)
+10200 val loss 5.8807
+10200 val perplexity 358.0732
+10200 train 5.957265 (lr=1.6529e-05) (hash(x)=153491388)
+10300 val loss 5.8002
+10300 val perplexity 330.3508
+10300 train 5.778177 (lr=1.7315e-05) (hash(x)=149600707)
+10300 val loss 5.8789
+10300 val perplexity 357.4311
+10300 train 5.849335 (lr=1.6233e-05) (hash(x)=149600707)
+10400 val loss 5.7900
+10400 val perplexity 327.0261
+10400 train 5.710597 (lr=1.7000e-05) (hash(x)=152277064)
+10400 val loss 5.8727
+10400 val perplexity 355.2067
+10400 train 5.788152 (lr=1.5938e-05) (hash(x)=152277064)
+10500 val loss 5.7888
+10500 val perplexity 326.6337
+10500 train 5.694821 (lr=1.6687e-05) (hash(x)=150447091)
+10500 val loss 5.8737
+10500 val perplexity 355.5547
+10500 train 5.782475 (lr=1.5644e-05) (hash(x)=150447091)
+10600 val loss 5.7777
+10600 val perplexity 323.0083
+10600 train 5.801088 (lr=1.6376e-05) (hash(x)=151606822)
+10600 val loss 5.8607
+10600 val perplexity 350.9606
+10600 train 5.884093 (lr=1.5352e-05) (hash(x)=151606822)
+10700 val loss 5.7708
+10700 val perplexity 320.8090
+10700 train 5.758115 (lr=1.6066e-05) (hash(x)=158730240)
+10700 val loss 5.8516
+10700 val perplexity 347.7813
+10700 train 5.837807 (lr=1.5062e-05) (hash(x)=158730240)
+10800 val loss 5.7688
+10800 val perplexity 320.1381
+10800 train 5.818008 (lr=1.5758e-05) (hash(x)=163095518)
+10800 val loss 5.8491
+10800 val perplexity 346.9075
+10800 train 5.900390 (lr=1.4773e-05) (hash(x)=163095518)
+10900 val loss 5.7664
+10900 val perplexity 319.3910
+10900 train 5.526252 (lr=1.5453e-05) (hash(x)=145260395)
+10900 val loss 5.8431
+10900 val perplexity 344.8545
+10900 train 5.609083 (lr=1.4487e-05) (hash(x)=145260395)
+11000 val loss 5.7589
+11000 val perplexity 316.9950
+11000 train 5.760559 (lr=1.5149e-05) (hash(x)=155970264)
+11000 val loss 5.8394
+11000 val perplexity 343.5878
+11000 train 5.845952 (lr=1.4203e-05) (hash(x)=155970264)
+11100 val loss 5.7567
+11100 val perplexity 316.3170
+11100 train 5.850685 (lr=1.4848e-05) (hash(x)=159871097)
+11100 val loss 5.8376
+11100 val perplexity 342.9427
+11100 train 5.931939 (lr=1.3920e-05) (hash(x)=159871097)
+11200 val loss 5.7492
+11200 val perplexity 313.9388
+11200 train 5.841630 (lr=1.4550e-05) (hash(x)=161135244)
+11200 val loss 5.8295
+11200 val perplexity 340.1743
+11200 train 5.919682 (lr=1.3640e-05) (hash(x)=161135244)
+11300 val loss 5.7410
+11300 val perplexity 311.3792
+11300 train 5.780535 (lr=1.4253e-05) (hash(x)=150649336)
+11300 val loss 5.8223
+11300 val perplexity 337.7465
+11300 train 5.851455 (lr=1.3362e-05) (hash(x)=150649336)
+11400 val loss 5.7409
+11400 val perplexity 311.3582
+11400 train 5.710909 (lr=1.3960e-05) (hash(x)=152230013)
+11400 val loss 5.8216
+11400 val perplexity 337.5094
+11400 train 5.791006 (lr=1.3087e-05) (hash(x)=152230013)
+11500 val loss 5.7378
+11500 val perplexity 310.3662
+11500 train 5.758396 (lr=1.3668e-05) (hash(x)=163300613)
+11500 val loss 5.8162
+11500 val perplexity 335.6881
+11500 train 5.859279 (lr=1.2814e-05) (hash(x)=163300613)
+11600 val loss 5.7372
+11600 val perplexity 310.1859
+11600 train 5.718981 (lr=1.3380e-05) (hash(x)=151203838)
+11600 val loss 5.8142
+11600 val perplexity 335.0154
+11600 train 5.800889 (lr=1.2544e-05) (hash(x)=151203838)
+11700 val loss 5.7318
+11700 val perplexity 308.5238
+11700 train 5.752216 (lr=1.3095e-05) (hash(x)=155255111)
+11700 val loss 5.8111
+11700 val perplexity 333.9854
+11700 train 5.829021 (lr=1.2276e-05) (hash(x)=155255111)
+11800 val loss 5.7282
+11800 val perplexity 307.4052
+11800 train 5.689683 (lr=1.2812e-05) (hash(x)=139380668)
+11800 val loss 5.8103
+11800 val perplexity 333.7100
+11800 train 5.770501 (lr=1.2011e-05) (hash(x)=139380668)
+11900 val loss 5.7189
+11900 val perplexity 304.5619
+11900 train 5.653548 (lr=1.2532e-05) (hash(x)=144033405)
+11900 val loss 5.8001
+11900 val perplexity 330.3267
+11900 train 5.729938 (lr=1.1749e-05) (hash(x)=144033405)
+12000 val loss 5.7152
+12000 val perplexity 303.4598
+12000 train 5.636559 (lr=1.2256e-05) (hash(x)=148689574)
+12000 val loss 5.7968
+12000 val perplexity 329.2376
+12000 train 5.719021 (lr=1.1490e-05) (hash(x)=148689574)
+12100 val loss 5.7144
+12100 val perplexity 303.2094
+12100 train 5.659702 (lr=1.1983e-05) (hash(x)=143272239)
+12100 val loss 5.7941
+12100 val perplexity 328.3486
+12100 train 5.742701 (lr=1.1234e-05) (hash(x)=143272239)
+12200 val loss 5.7108
+12200 val perplexity 302.1156
+12200 train 5.794576 (lr=1.1713e-05) (hash(x)=157671609)
+12200 val loss 5.7898
+12200 val perplexity 326.9385
+12200 train 5.877266 (lr=1.0981e-05) (hash(x)=157671609)
+12300 val loss 5.7066
+12300 val perplexity 300.8392
+12300 train 5.723989 (lr=1.1446e-05) (hash(x)=150363247)
+12300 val loss 5.7873
+12300 val perplexity 326.1414
+12300 train 5.806932 (lr=1.0731e-05) (hash(x)=150363247)
+12400 val loss 5.7022
+12400 val perplexity 299.5182
+12400 train 5.592573 (lr=1.1183e-05) (hash(x)=146857787)
+12400 val loss 5.7808
+12400 val perplexity 324.0274
+12400 train 5.672683 (lr=1.0484e-05) (hash(x)=146857787)
+12500 val loss 5.7012
+12500 val perplexity 299.2310
+12500 train 5.667606 (lr=1.0924e-05) (hash(x)=153327320)
+12500 val loss 5.7830
+12500 val perplexity 324.7299
+12500 train 5.742136 (lr=1.0241e-05) (hash(x)=153327320)
+12600 val loss 5.6965
+12600 val perplexity 297.8202
+12600 train 5.419716 (lr=1.0668e-05) (hash(x)=143612983)
+12600 val loss 5.7757
+12600 val perplexity 322.3680
+12600 train 5.494945 (lr=1.0001e-05) (hash(x)=143612983)
+12700 val loss 5.6943
+12700 val perplexity 297.1594
+12700 train 5.625340 (lr=1.0416e-05) (hash(x)=150739132)
+12700 val loss 5.7727
+12700 val perplexity 321.4064
+12700 train 5.713191 (lr=9.7649e-06) (hash(x)=150739132)
+12800 val loss 5.6909
+12800 val perplexity 296.1632
+12800 train 5.540470 (lr=1.0168e-05) (hash(x)=147986204)
+12800 val loss 5.7704
+12800 val perplexity 320.6530
+12800 train 5.618282 (lr=9.5322e-06) (hash(x)=147986204)
+12900 val loss 5.6897
+12900 val perplexity 295.8014
+12900 train 5.856245 (lr=9.9233e-06) (hash(x)=155218833)
+12900 val loss 5.7692
+12900 val perplexity 320.2925
+12900 train 5.924139 (lr=9.3031e-06) (hash(x)=155218833)
+13000 val loss 5.6853
+13000 val perplexity 294.5142
+13000 train 5.679413 (lr=9.6829e-06) (hash(x)=148867960)
+13000 val loss 5.7643
+13000 val perplexity 318.7263
+13000 train 5.762329 (lr=9.0777e-06) (hash(x)=148867960)
+13100 val loss 5.6814
+13100 val perplexity 293.3666
+13100 train 5.713234 (lr=9.4466e-06) (hash(x)=160101478)
+13100 val loss 5.7600
+13100 val perplexity 317.3558
+13100 train 5.790013 (lr=8.8562e-06) (hash(x)=160101478)
+13200 val loss 5.6827
+13200 val perplexity 293.7467
+13200 train 5.541912 (lr=9.2144e-06) (hash(x)=151080097)
+13200 val loss 5.7604
+13200 val perplexity 317.4737
+13200 train 5.619044 (lr=8.6385e-06) (hash(x)=151080097)
+13300 val loss 5.6789
+13300 val perplexity 292.6209
+13300 train 5.679009 (lr=8.9865e-06) (hash(x)=150403039)
+13300 val loss 5.7578
+13300 val perplexity 316.6603
+13300 train 5.761934 (lr=8.4248e-06) (hash(x)=150403039)
+13400 val loss 5.6757
+13400 val perplexity 291.6800
+13400 train 5.628583 (lr=8.7628e-06) (hash(x)=129728571)
+13400 val loss 5.7552
+13400 val perplexity 315.8320
+13400 train 5.708203 (lr=8.2151e-06) (hash(x)=129728571)
+13500 val loss 5.6703
+13500 val perplexity 290.1295
+13500 train 5.593028 (lr=8.5435e-06) (hash(x)=148988625)
+13500 val loss 5.7486
+13500 val perplexity 313.7387
+13500 train 5.668513 (lr=8.0095e-06) (hash(x)=148988625)
+13600 val loss 5.6693
+13600 val perplexity 289.8436
+13600 train 5.457337 (lr=8.3286e-06) (hash(x)=149790861)
+13600 val loss 5.7477
+13600 val perplexity 313.4661
+13600 train 5.534057 (lr=7.8081e-06) (hash(x)=149790861)
+13700 val loss 5.6673
+13700 val perplexity 289.2669
+13700 train 5.596827 (lr=8.1183e-06) (hash(x)=153505062)
+13700 val loss 5.7464
+13700 val perplexity 313.0768
+13700 train 5.678047 (lr=7.6109e-06) (hash(x)=153505062)
+13800 val loss 5.6684
+13800 val perplexity 289.5752
+13800 train 5.570222 (lr=7.9125e-06) (hash(x)=143845463)
+13800 val loss 5.7460
+13800 val perplexity 312.9442
+13800 train 5.656930 (lr=7.4180e-06) (hash(x)=143845463)
+13900 val loss 5.6655
+13900 val perplexity 288.7445
+13900 train 5.464289 (lr=7.7114e-06) (hash(x)=143378076)
+13900 val loss 5.7445
+13900 val perplexity 312.4745
+13900 train 5.542448 (lr=7.2294e-06) (hash(x)=143378076)
+14000 val loss 5.6604
+14000 val perplexity 287.2565
+14000 train 5.620647 (lr=7.5149e-06) (hash(x)=148681755)
+14000 val loss 5.7382
+14000 val perplexity 310.4907
+14000 train 5.702466 (lr=7.0452e-06) (hash(x)=148681755)
+14100 val loss 5.6578
+14100 val perplexity 286.5279
+14100 train 5.536157 (lr=7.3233e-06) (hash(x)=145530465)
+14100 val loss 5.7370
+14100 val perplexity 310.1180
+14100 train 5.622849 (lr=6.8656e-06) (hash(x)=145530465)
+14200 val loss 5.6592
+14200 val perplexity 286.9243
+14200 train 5.594131 (lr=7.1365e-06) (hash(x)=162139713)
+14200 val loss 5.7349
+14200 val perplexity 309.4690
+14200 train 5.676126 (lr=6.6904e-06) (hash(x)=162139713)
+14300 val loss 5.6543
+14300 val perplexity 285.5057
+14300 train 5.672934 (lr=6.9546e-06) (hash(x)=141997596)
+14300 val loss 5.7317
+14300 val perplexity 308.5032
+14300 train 5.741300 (lr=6.5199e-06) (hash(x)=141997596)
+14400 val loss 5.6544
+14400 val perplexity 285.5538
+14400 train 5.624477 (lr=6.7776e-06) (hash(x)=146156313)
+14400 val loss 5.7329
+14400 val perplexity 308.8662
+14400 train 5.704103 (lr=6.3540e-06) (hash(x)=146156313)
+14500 val loss 5.6525
+14500 val perplexity 285.0071
+14500 train 5.428591 (lr=6.6057e-06) (hash(x)=145298178)
+14500 val loss 5.7310
+14500 val perplexity 308.2753
+14500 train 5.500850 (lr=6.1928e-06) (hash(x)=145298178)
+14600 val loss 5.6484
+14600 val perplexity 283.8466
+14600 train 5.744937 (lr=6.4389e-06) (hash(x)=148232610)
+14600 val loss 5.7246
+14600 val perplexity 306.3022
+14600 train 5.808825 (lr=6.0364e-06) (hash(x)=148232610)
+14700 val loss 5.6463
+14700 val perplexity 283.2440
+14700 train 5.532464 (lr=6.2771e-06) (hash(x)=151079115)
+14700 val loss 5.7234
+14700 val perplexity 305.9421
+14700 train 5.605141 (lr=5.8848e-06) (hash(x)=151079115)
+14800 val loss 5.6452
+14800 val perplexity 282.9186
+14800 train 5.631949 (lr=6.1206e-06) (hash(x)=151340983)
+14800 val loss 5.7220
+14800 val perplexity 305.5163
+14800 train 5.697337 (lr=5.7381e-06) (hash(x)=151340983)
+14900 val loss 5.6443
+14900 val perplexity 282.6823
+14900 train 5.854811 (lr=5.9693e-06) (hash(x)=150213803)
+14900 val loss 5.7197
+14900 val perplexity 304.8096
+14900 train 5.932733 (lr=5.5962e-06) (hash(x)=150213803)
+15000 val loss 5.6443
+15000 val perplexity 282.6677
+15000 train 5.559547 (lr=5.8233e-06) (hash(x)=157652334)
+15000 val loss 5.7195
+15000 val perplexity 304.7498
+15000 train 5.633491 (lr=5.4593e-06) (hash(x)=157652334)
+15100 val loss 5.6418
+15100 val perplexity 281.9667
+15100 train 5.371411 (lr=5.6826e-06) (hash(x)=145078577)
+15100 val loss 5.7171
+15100 val perplexity 304.0203
+15100 train 5.451268 (lr=5.3275e-06) (hash(x)=145078577)
+15200 val loss 5.6389
+15200 val perplexity 281.1660
+15200 train 5.578742 (lr=5.5473e-06) (hash(x)=155898006)
+15200 val loss 5.7147
+15200 val perplexity 303.3001
+15200 train 5.655529 (lr=5.2006e-06) (hash(x)=155898006)
+15300 val loss 5.6362
+15300 val perplexity 280.3892
+15300 train 5.665936 (lr=5.4175e-06) (hash(x)=149346671)
+15300 val loss 5.7129
+15300 val perplexity 302.7553
+15300 train 5.740127 (lr=5.0789e-06) (hash(x)=149346671)
+15400 val loss 5.6351
+15400 val perplexity 280.0972
+15400 train 5.487409 (lr=5.2931e-06) (hash(x)=145708113)
+15400 val loss 5.7106
+15400 val perplexity 302.0578
+15400 train 5.557944 (lr=4.9622e-06) (hash(x)=145708113)
+15500 val loss 5.6321
+15500 val perplexity 279.2454
+15500 train 5.809040 (lr=5.1742e-06) (hash(x)=165102970)
+15500 val loss 5.7091
+15500 val perplexity 301.5966
+15500 train 5.890470 (lr=4.8508e-06) (hash(x)=165102970)
+15600 val loss 5.6338
+15600 val perplexity 279.7115
+15600 train 5.420284 (lr=5.0608e-06) (hash(x)=139106517)
+15600 val loss 5.7103
+15600 val perplexity 301.9613
+15600 train 5.507070 (lr=4.7445e-06) (hash(x)=139106517)
+15700 val loss 5.6319
+15700 val perplexity 279.2039
+15700 train 5.497750 (lr=4.9531e-06) (hash(x)=141680838)
+15700 val loss 5.7078
+15700 val perplexity 301.2041
+15700 train 5.570164 (lr=4.6435e-06) (hash(x)=141680838)
+15800 val loss 5.6292
+15800 val perplexity 278.4494
+15800 train 5.530750 (lr=4.8509e-06) (hash(x)=136152310)
+15800 val loss 5.7050
+15800 val perplexity 300.3506
+15800 train 5.599582 (lr=4.5478e-06) (hash(x)=136152310)
+15900 val loss 5.6275
+15900 val perplexity 277.9763
+15900 train 5.688832 (lr=4.7545e-06) (hash(x)=151104640)
+15900 val loss 5.7034
+15900 val perplexity 299.8883
+15900 train 5.773046 (lr=4.4573e-06) (hash(x)=151104640)
+16000 val loss 5.6271
+16000 val perplexity 277.8607
+16000 train 5.584705 (lr=4.6637e-06) (hash(x)=153333281)
+16000 val loss 5.7023
+16000 val perplexity 299.5677
+16000 train 5.651556 (lr=4.3722e-06) (hash(x)=153333281)
+16100 val loss 5.6251
+16100 val perplexity 277.2950
+16100 train 5.577593 (lr=4.5786e-06) (hash(x)=145628059)
+16100 val loss 5.7011
+16100 val perplexity 299.1999
+16100 train 5.658373 (lr=4.2924e-06) (hash(x)=145628059)
+16200 val loss 5.6259
+16200 val perplexity 277.5304
+16200 train 5.351377 (lr=4.4993e-06) (hash(x)=137220041)
+16200 val loss 5.7009
+16200 val perplexity 299.1331
+16200 train 5.426072 (lr=4.2180e-06) (hash(x)=137220041)
+16300 val loss 5.6244
+16300 val perplexity 277.0934
+16300 train 5.800812 (lr=4.4257e-06) (hash(x)=161215791)
+16300 val loss 5.6993
+16300 val perplexity 298.6519
+16300 train 5.872560 (lr=4.1491e-06) (hash(x)=161215791)
+16400 val loss 5.6233
+16400 val perplexity 276.8119
+16400 train 5.331463 (lr=4.3579e-06) (hash(x)=136194680)
+16400 val loss 5.6981
+16400 val perplexity 298.3057
+16400 train 5.408731 (lr=4.0856e-06) (hash(x)=136194680)
+16500 val loss 5.6232
+16500 val perplexity 276.7720
+16500 train 5.408642 (lr=4.2960e-06) (hash(x)=152209568)
+16500 val loss 5.6986
+16500 val perplexity 298.4346
+16500 train 5.489052 (lr=4.0275e-06) (hash(x)=152209568)
+16600 val loss 5.6196
+16600 val perplexity 275.7748
+16600 train 5.738178 (lr=4.2399e-06) (hash(x)=157345721)
+16600 val loss 5.6951
+16600 val perplexity 297.4131
+16600 train 5.814222 (lr=3.9749e-06) (hash(x)=157345721)
+16700 val loss 5.6179
+16700 val perplexity 275.3081
+16700 train 5.744716 (lr=4.1896e-06) (hash(x)=146883642)
+16700 val loss 5.6931
+16700 val perplexity 296.8066
+16700 train 5.804691 (lr=3.9278e-06) (hash(x)=146883642)
+16800 val loss 5.6177
+16800 val perplexity 275.2513
+16800 train 5.589443 (lr=4.1452e-06) (hash(x)=153037972)
+16800 val loss 5.6927
+16800 val perplexity 296.6937
+16800 train 5.661279 (lr=3.8862e-06) (hash(x)=153037972)
+16900 val loss 5.6175
+16900 val perplexity 275.1880
+16900 train 5.389691 (lr=4.1067e-06) (hash(x)=150691367)
+16900 val loss 5.6925
+16900 val perplexity 296.6332
+16900 train 5.463097 (lr=3.8501e-06) (hash(x)=150691367)
+17000 val loss 5.6172
+17000 val perplexity 275.1071
+17000 train 5.362617 (lr=4.0741e-06) (hash(x)=148975206)
+17000 val loss 5.6918
+17000 val perplexity 296.4331
+17000 train 5.442601 (lr=3.8195e-06) (hash(x)=148975206)
+17100 val loss 5.6178
+17100 val perplexity 275.2888
+17100 train 5.347479 (lr=4.0475e-06) (hash(x)=142026221)
+17100 val loss 5.6924
+17100 val perplexity 296.6107
+17100 train 5.421805 (lr=3.7945e-06) (hash(x)=142026221)
+17200 val loss 5.6141
+17200 val perplexity 274.2535
+17200 train 5.675111 (lr=4.0267e-06) (hash(x)=138211880)
+17200 val loss 5.6894
+17200 val perplexity 295.7159
+17200 train 5.752285 (lr=3.7750e-06) (hash(x)=138211880)
+17300 val loss 5.6114
+17300 val perplexity 273.5252
+17300 train 5.478740 (lr=4.0119e-06) (hash(x)=143366666)
+17300 val loss 5.6868
+17300 val perplexity 294.9388
+17300 train 5.559046 (lr=3.7611e-06) (hash(x)=143366666)
+17400 val loss 5.6115
+17400 val perplexity 273.5565
+17400 train 5.612268 (lr=4.0030e-06) (hash(x)=150791766)
+17400 val loss 5.6855
+17400 val perplexity 294.5702
+17400 train 5.678813 (lr=3.7528e-06) (hash(x)=150791766)
+17499 val loss 5.6105
+17499 val perplexity 273.2749
+17499 val loss 5.6855
+17499 val perplexity 294.5559

attention_kindselective_n_heads4_seed1343/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:27081619b463bbaad238902544341ba0a158e6c08674f088292609d0df8be2db
+oid sha256:ded6ca4e3e7b8f45ca68a41610d3de86c7a4efd76f4baa78b390042695cc51c8
 size 92843394

attention_kindselective_n_heads4_seed1343/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:73b852ab286f52955da2ba1e0c51cb905db4b5700fae874d7533bc7e4896228d
+oid sha256:edace29feb1d342ff12f12491dc047a9f4a8fe182db893063f951aee9383726c
 size 179406214