Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_30720_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_30720_4_1341", "n_embd": 256}

attention_kindselective_n_heads4_seed1341/log2.txt

@@ -1,528 +1,1055 @@
 max_steps: 17500
 0 val loss 11.3107
 0 val perplexity 81693.5234
-0 train 11.308768 (lr=1.7500e-07) (hash(x)=145079536)
-100 val loss 9.9497
-100 val perplexity 20946.6250
-100 train 9.888896 (lr=1.7675e-05) (hash(x)=154745873)
-200 val loss 8.9449
-200 val perplexity 7668.6709
-200 train 8.987842 (lr=3.5000e-05) (hash(x)=155800595)
-300 val loss 7.9903
-300 val perplexity 2952.0789
-300 train 7.955874 (lr=3.4997e-05) (hash(x)=148595389)
-400 val loss 7.7101
-400 val perplexity 2230.8018
-400 train 7.625350 (lr=3.4990e-05) (hash(x)=145606733)
-500 val loss 7.5636
-500 val perplexity 1926.8578
-500 train 7.398027 (lr=3.4977e-05) (hash(x)=138221231)
-600 val loss 7.4980
-600 val perplexity 1804.5145
-600 train 7.503819 (lr=3.4958e-05) (hash(x)=150367139)
-700 val loss 7.4583
-700 val perplexity 1734.2299
-700 train 7.608730 (lr=3.4935e-05) (hash(x)=155579314)
-800 val loss 7.4133
-800 val perplexity 1657.8750
-800 train 7.490392 (lr=3.4907e-05) (hash(x)=155747374)
-900 val loss 7.3716
-900 val perplexity 1590.1399
-900 train 7.577220 (lr=3.4873e-05) (hash(x)=159334575)
-1000 val loss 7.3410
-1000 val perplexity 1542.3295
-1000 train 7.224279 (lr=3.4834e-05) (hash(x)=140604760)
-1100 val loss 7.3157
-1100 val perplexity 1503.7556
-1100 train 7.157549 (lr=3.4790e-05) (hash(x)=146208052)
-1200 val loss 7.2921
-1200 val perplexity 1468.6102
-1200 train 7.204362 (lr=3.4741e-05) (hash(x)=148404734)
-1300 val loss 7.2524
-1300 val perplexity 1411.5232
-1300 train 7.193460 (lr=3.4687e-05) (hash(x)=155681970)
-1400 val loss 7.2227
-1400 val perplexity 1370.2190
-1400 train 7.172364 (lr=3.4628e-05) (hash(x)=148115934)
-1500 val loss 7.1729
-1500 val perplexity 1303.5782
-1500 train 7.317149 (lr=3.4563e-05) (hash(x)=157074034)
-1600 val loss 7.1336
-1600 val perplexity 1253.3771
-1600 train 6.900460 (lr=3.4494e-05) (hash(x)=137464699)
-1700 val loss 7.0965
-1700 val perplexity 1207.7498
-1700 train 7.269404 (lr=3.4419e-05) (hash(x)=166955614)
-1800 val loss 7.0607
-1800 val perplexity 1165.2827
-1800 train 6.887536 (lr=3.4340e-05) (hash(x)=143886042)
-1900 val loss 7.0308
-1900 val perplexity 1130.8890
-1900 train 7.645374 (lr=3.4255e-05) (hash(x)=193610391)
-2000 val loss 7.0013
-2000 val perplexity 1098.0428
-2000 train 7.174060 (lr=3.4166e-05) (hash(x)=163799796)
-2100 val loss 6.9508
-2100 val perplexity 1043.9821
-2100 train 6.924165 (lr=3.4072e-05) (hash(x)=154107339)
-2200 val loss 6.9160
-2200 val perplexity 1008.2556
-2200 train 6.909832 (lr=3.3973e-05) (hash(x)=144592844)
-2300 val loss 6.8809
-2300 val perplexity 973.4561
-2300 train 6.959703 (lr=3.3869e-05) (hash(x)=154535861)
-2400 val loss 6.8430
-2400 val perplexity 937.2637
-2400 train 7.566927 (lr=3.3760e-05) (hash(x)=204706354)
-2500 val loss 6.8175
-2500 val perplexity 913.6815
-2500 train 6.620405 (lr=3.3646e-05) (hash(x)=143783202)
-2600 val loss 6.7831
-2600 val perplexity 882.8004
-2600 train 6.690417 (lr=3.3528e-05) (hash(x)=150862210)
-2700 val loss 6.7496
-2700 val perplexity 853.6903
-2700 train 6.799547 (lr=3.3405e-05) (hash(x)=150753426)
-2800 val loss 6.7300
-2800 val perplexity 837.1569
-2800 train 6.731043 (lr=3.3277e-05) (hash(x)=147766811)
-2900 val loss 6.6856
-2900 val perplexity 800.8203
-2900 train 6.624177 (lr=3.3144e-05) (hash(x)=151177814)
-3000 val loss 6.6480
-3000 val perplexity 771.2670
-3000 train 6.508345 (lr=3.3008e-05) (hash(x)=135925327)
-3100 val loss 6.6213
-3100 val perplexity 750.9255
-3100 train 6.264478 (lr=3.2866e-05) (hash(x)=124842353)
-3200 val loss 6.5985
-3200 val perplexity 733.9844
-3200 train 6.701370 (lr=3.2720e-05) (hash(x)=160440642)
-3300 val loss 6.5726
-3300 val perplexity 715.2368
-3300 train 6.485851 (lr=3.2570e-05) (hash(x)=143017131)
-3400 val loss 6.5430
-3400 val perplexity 694.3995
-3400 train 6.860902 (lr=3.2415e-05) (hash(x)=148143474)
-3500 val loss 6.5207
-3500 val perplexity 679.0399
-3500 train 6.339900 (lr=3.2256e-05) (hash(x)=148732639)
-3600 val loss 6.4931
-3600 val perplexity 660.5637
-3600 train 6.487075 (lr=3.2092e-05) (hash(x)=146333803)
-3700 val loss 6.4768
-3700 val perplexity 649.8641
-3700 train 6.284898 (lr=3.1924e-05) (hash(x)=143611445)
-3800 val loss 6.4673
-3800 val perplexity 643.7672
-3800 train 6.257189 (lr=3.1753e-05) (hash(x)=141272115)
-3900 val loss 6.4417
-3900 val perplexity 627.4771
-3900 train 6.371986 (lr=3.1577e-05) (hash(x)=149161848)
-4000 val loss 6.4124
-4000 val perplexity 609.3493
-4000 train 6.491913 (lr=3.1397e-05) (hash(x)=152036955)
-4100 val loss 6.4030
-4100 val perplexity 603.6633
-4100 train 6.459525 (lr=3.1212e-05) (hash(x)=153260968)
-4200 val loss 6.3843
-4200 val perplexity 592.4824
-4200 train 6.265935 (lr=3.1024e-05) (hash(x)=147629478)
-4300 val loss 6.3717
-4300 val perplexity 585.0626
-4300 train 6.359217 (lr=3.0833e-05) (hash(x)=161701080)
-4400 val loss 6.3599
-4400 val perplexity 578.1844
-4400 train 6.402924 (lr=3.0637e-05) (hash(x)=155290524)
-4500 val loss 6.3511
-4500 val perplexity 573.1031
-4500 train 6.127981 (lr=3.0437e-05) (hash(x)=144726593)
-4600 val loss 6.3341
-4600 val perplexity 563.4740
-4600 train 6.280225 (lr=3.0234e-05) (hash(x)=149517202)
-4700 val loss 6.3134
-4700 val perplexity 551.9003
-4700 train 6.312892 (lr=3.0027e-05) (hash(x)=139403008)
-4800 val loss 6.2895
-4800 val perplexity 538.9004
-4800 train 6.371090 (lr=2.9817e-05) (hash(x)=153387238)
-4900 val loss 6.2852
-4900 val perplexity 536.5527
-4900 train 6.030615 (lr=2.9603e-05) (hash(x)=139881569)
-5000 val loss 6.2632
-5000 val perplexity 524.9213
-5000 train 6.252951 (lr=2.9386e-05) (hash(x)=152602580)
-5100 val loss 6.2518
-5100 val perplexity 518.9592
-5100 train 6.300901 (lr=2.9166e-05) (hash(x)=150598460)
-5200 val loss 6.2453
-5200 val perplexity 515.5968
-5200 train 6.123269 (lr=2.8942e-05) (hash(x)=147660496)
-5300 val loss 6.2314
-5300 val perplexity 508.4545
-5300 train 5.967817 (lr=2.8715e-05) (hash(x)=147693848)
-5400 val loss 6.2224
-5400 val perplexity 503.9259
-5400 train 5.967390 (lr=2.8484e-05) (hash(x)=136048145)
-5500 val loss 6.2030
-5500 val perplexity 494.2373
-5500 train 6.205599 (lr=2.8251e-05) (hash(x)=140313123)
-5600 val loss 6.1946
-5600 val perplexity 490.0963
-5600 train 6.213743 (lr=2.8015e-05) (hash(x)=156997993)
-5700 val loss 6.1764
-5700 val perplexity 481.2511
-5700 train 6.185669 (lr=2.7776e-05) (hash(x)=143513880)
-5800 val loss 6.1682
-5800 val perplexity 477.3484
-5800 train 6.282696 (lr=2.7534e-05) (hash(x)=153736628)
-5900 val loss 6.1591
-5900 val perplexity 472.9898
-5900 train 6.298066 (lr=2.7290e-05) (hash(x)=166385417)
-6000 val loss 6.1455
-6000 val perplexity 466.6283
-6000 train 6.385063 (lr=2.7042e-05) (hash(x)=153216517)
-6100 val loss 6.1470
-6100 val perplexity 467.2961
-6100 train 6.026530 (lr=2.6792e-05) (hash(x)=145387545)
-6200 val loss 6.1355
-6200 val perplexity 461.9876
-6200 train 6.258276 (lr=2.6540e-05) (hash(x)=156306460)
-6300 val loss 6.1155
-6300 val perplexity 452.8189
-6300 train 6.044140 (lr=2.6285e-05) (hash(x)=148610447)
-6400 val loss 6.1131
-6400 val perplexity 451.7383
-6400 train 6.328663 (lr=2.6028e-05) (hash(x)=158343535)
-6500 val loss 6.0930
-6500 val perplexity 442.7654
-6500 train 6.260117 (lr=2.5769e-05) (hash(x)=157742557)
-6600 val loss 6.0867
-6600 val perplexity 439.9755
-6600 train 5.990105 (lr=2.5508e-05) (hash(x)=150457308)
-6700 val loss 6.0869
-6700 val perplexity 440.0525
-6700 train 5.952641 (lr=2.5244e-05) (hash(x)=142426205)
-6800 val loss 6.0792
-6800 val perplexity 436.6671
-6800 train 5.866668 (lr=2.4979e-05) (hash(x)=151522525)
-6900 val loss 6.0638
-6900 val perplexity 430.0030
-6900 train 6.014320 (lr=2.4711e-05) (hash(x)=152029630)
-7000 val loss 6.0503
-7000 val perplexity 424.2452
-7000 train 5.893929 (lr=2.4442e-05) (hash(x)=142978954)
-7100 val loss 6.0438
-7100 val perplexity 421.4832
-7100 train 5.922884 (lr=2.4171e-05) (hash(x)=148064359)
-7200 val loss 6.0364
-7200 val perplexity 418.3951
-7200 train 5.940358 (lr=2.3899e-05) (hash(x)=155771845)
-7300 val loss 6.0284
-7300 val perplexity 415.0425
-7300 train 5.838033 (lr=2.3625e-05) (hash(x)=141639844)
-7400 val loss 6.0295
-7400 val perplexity 415.5189
-7400 train 6.080736 (lr=2.3349e-05) (hash(x)=153466021)
-7500 val loss 6.0143
-7500 val perplexity 409.2429
-7500 train 6.005939 (lr=2.3073e-05) (hash(x)=148843587)
-7600 val loss 5.9990
-7600 val perplexity 403.0125
-7600 train 6.114269 (lr=2.2795e-05) (hash(x)=152053933)
-7700 val loss 5.9965
-7700 val perplexity 402.0277
-7700 train 5.850546 (lr=2.2515e-05) (hash(x)=141564003)
-7800 val loss 5.9888
-7800 val perplexity 398.9208
-7800 train 5.864420 (lr=2.2235e-05) (hash(x)=148617843)
-7900 val loss 5.9869
-7900 val perplexity 398.1961
-7900 train 5.957241 (lr=2.1954e-05) (hash(x)=146417668)
-8000 val loss 5.9815
-8000 val perplexity 396.0533
-8000 train 5.777200 (lr=2.1671e-05) (hash(x)=150165488)
-8100 val loss 5.9701
-8100 val perplexity 391.5258
-8100 train 5.962206 (lr=2.1388e-05) (hash(x)=144818855)
-8200 val loss 5.9619
-8200 val perplexity 388.3450
-8200 train 6.043481 (lr=2.1105e-05) (hash(x)=143074954)
-8300 val loss 5.9542
-8300 val perplexity 385.3598
-8300 train 5.830782 (lr=2.0820e-05) (hash(x)=145172494)
-8400 val loss 5.9498
-8400 val perplexity 383.6585
-8400 train 5.986149 (lr=2.0536e-05) (hash(x)=152284393)
-8500 val loss 5.9401
-8500 val perplexity 379.9685
-8500 train 5.797303 (lr=2.0250e-05) (hash(x)=147044705)
-8600 val loss 5.9401
-8600 val perplexity 379.9708
-8600 train 5.818129 (lr=1.9965e-05) (hash(x)=147762608)
-8700 val loss 5.9321
-8700 val perplexity 376.9359
-8700 train 5.937027 (lr=1.9679e-05) (hash(x)=141994567)
-8800 val loss 5.9250
-8800 val perplexity 374.2906
-8800 train 6.014191 (lr=1.9393e-05) (hash(x)=152552896)
-8900 val loss 5.9151
-8900 val perplexity 370.5843
-8900 train 5.756435 (lr=1.9107e-05) (hash(x)=139714454)
-9000 val loss 5.9092
-9000 val perplexity 368.4096
-9000 train 5.939328 (lr=1.8821e-05) (hash(x)=152274419)
-9100 val loss 5.9043
-9100 val perplexity 366.5936
-9100 train 5.997716 (lr=1.8535e-05) (hash(x)=160536885)
-9200 val loss 5.9018
-9200 val perplexity 365.6812
-9200 train 5.709610 (lr=1.8250e-05) (hash(x)=138386899)
-9300 val loss 5.8949
-9300 val perplexity 363.1737
-9300 train 5.940402 (lr=1.7964e-05) (hash(x)=149957457)
-9400 val loss 5.8850
-9400 val perplexity 359.5974
-9400 train 5.901648 (lr=1.7680e-05) (hash(x)=161671548)
-9500 val loss 5.8853
-9500 val perplexity 359.7241
-9500 train 6.005171 (lr=1.7395e-05) (hash(x)=171857400)
-9600 val loss 5.8742
-9600 val perplexity 355.7437
-9600 train 5.633665 (lr=1.7112e-05) (hash(x)=135378471)
-9700 val loss 5.8697
-9700 val perplexity 354.1471
-9700 train 5.806231 (lr=1.6829e-05) (hash(x)=149514591)
-9800 val loss 5.8668
-9800 val perplexity 353.1118
-9800 train 5.848464 (lr=1.6546e-05) (hash(x)=152679072)
-9900 val loss 5.8576
-9900 val perplexity 349.8673
-9900 train 5.807221 (lr=1.6265e-05) (hash(x)=150415193)
-10000 val loss 5.8518
-10000 val perplexity 347.8661
-10000 train 5.696980 (lr=1.5985e-05) (hash(x)=146910087)
-10100 val loss 5.8481
-10100 val perplexity 346.5745
-10100 train 5.764876 (lr=1.5705e-05) (hash(x)=152091050)
-10200 val loss 5.8494
-10200 val perplexity 347.0356
-10200 train 5.692203 (lr=1.5427e-05) (hash(x)=143101306)
-10300 val loss 5.8395
-10300 val perplexity 343.5908
-10300 train 5.852290 (lr=1.5151e-05) (hash(x)=150803985)
-10400 val loss 5.8389
-10400 val perplexity 343.4077
-10400 train 5.813169 (lr=1.4875e-05) (hash(x)=148671575)
-10500 val loss 5.8290
-10500 val perplexity 340.0123
-10500 train 5.629170 (lr=1.4601e-05) (hash(x)=146312794)
-10600 val loss 5.8302
-10600 val perplexity 340.4258
-10600 train 5.658768 (lr=1.4329e-05) (hash(x)=137878985)
-10700 val loss 5.8253
-10700 val perplexity 338.7693
-10700 train 5.635949 (lr=1.4058e-05) (hash(x)=148897486)
-10800 val loss 5.8196
-10800 val perplexity 336.8351
-10800 train 5.928453 (lr=1.3789e-05) (hash(x)=147219580)
-10900 val loss 5.8238
-10900 val perplexity 338.2571
-10900 train 5.850612 (lr=1.3521e-05) (hash(x)=152048929)
-11000 val loss 5.8069
-11000 val perplexity 332.5901
-11000 train 5.701795 (lr=1.3256e-05) (hash(x)=146417066)
-11100 val loss 5.8023
-11100 val perplexity 331.0467
-11100 train 5.661292 (lr=1.2992e-05) (hash(x)=148784436)
-11200 val loss 5.7998
-11200 val perplexity 330.2259
-11200 train 5.766848 (lr=1.2731e-05) (hash(x)=144721870)
-11300 val loss 5.7966
-11300 val perplexity 329.1706
-11300 train 5.718678 (lr=1.2472e-05) (hash(x)=149442981)
-11400 val loss 5.7939
-11400 val perplexity 328.2987
-11400 train 5.687094 (lr=1.2215e-05) (hash(x)=154481228)
-11500 val loss 5.7893
-11500 val perplexity 326.7811
-11500 train 5.484087 (lr=1.1960e-05) (hash(x)=115164581)
-11600 val loss 5.7824
-11600 val perplexity 324.5251
-11600 train 6.010474 (lr=1.1708e-05) (hash(x)=155048082)
-11700 val loss 5.7800
-11700 val perplexity 323.7676
-11700 train 5.845362 (lr=1.1458e-05) (hash(x)=152123770)
-11800 val loss 5.7762
-11800 val perplexity 322.5318
-11800 train 5.716846 (lr=1.1210e-05) (hash(x)=152705857)
-11900 val loss 5.7743
-11900 val perplexity 321.9169
-11900 train 5.496723 (lr=1.0966e-05) (hash(x)=137392362)
-12000 val loss 5.7721
-12000 val perplexity 321.2125
-12000 train 5.578417 (lr=1.0724e-05) (hash(x)=143492562)
-12100 val loss 5.7669
-12100 val perplexity 319.5511
-12100 train 5.743323 (lr=1.0485e-05) (hash(x)=141825431)
-12200 val loss 5.7618
-12200 val perplexity 317.9296
-12200 train 5.665538 (lr=1.0249e-05) (hash(x)=141774370)
-12300 val loss 5.7583
-12300 val perplexity 316.8130
-12300 train 5.782146 (lr=1.0016e-05) (hash(x)=152995173)
-12400 val loss 5.7583
-12400 val perplexity 316.8082
-12400 train 5.578952 (lr=9.7854e-06) (hash(x)=144703471)
-12500 val loss 5.7548
-12500 val perplexity 315.6921
-12500 train 5.653757 (lr=9.5583e-06) (hash(x)=144866179)
-12600 val loss 5.7517
-12600 val perplexity 314.7222
-12600 train 5.762004 (lr=9.3345e-06) (hash(x)=157019663)
-12700 val loss 5.7504
-12700 val perplexity 314.3056
-12700 train 5.716350 (lr=9.1139e-06) (hash(x)=152872020)
-12800 val loss 5.7459
-12800 val perplexity 312.9010
-12800 train 5.782084 (lr=8.8967e-06) (hash(x)=158975636)
-12900 val loss 5.7390
-12900 val perplexity 310.7508
-12900 train 5.793307 (lr=8.6829e-06) (hash(x)=143495701)
-13000 val loss 5.7369
-13000 val perplexity 310.0933
-13000 train 5.660654 (lr=8.4726e-06) (hash(x)=151940005)
-13100 val loss 5.7339
-13100 val perplexity 309.1845
-13100 train 6.017331 (lr=8.2658e-06) (hash(x)=155920713)
-13200 val loss 5.7332
-13200 val perplexity 308.9682
-13200 train 5.909652 (lr=8.0626e-06) (hash(x)=152145878)
-13300 val loss 5.7318
-13300 val perplexity 308.5327
-13300 train 5.530159 (lr=7.8632e-06) (hash(x)=134141709)
-13400 val loss 5.7328
-13400 val perplexity 308.8322
-13400 train 5.650486 (lr=7.6674e-06) (hash(x)=146096299)
-13500 val loss 5.7307
-13500 val perplexity 308.1761
-13500 train 5.490761 (lr=7.4756e-06) (hash(x)=144011431)
-13600 val loss 5.7228
-13600 val perplexity 305.7547
-13600 train 5.965028 (lr=7.2875e-06) (hash(x)=163950932)
-13700 val loss 5.7204
-13700 val perplexity 305.0365
-13700 train 5.863823 (lr=7.1035e-06) (hash(x)=156652406)
-13800 val loss 5.7158
-13800 val perplexity 303.6303
-13800 train 5.633546 (lr=6.9234e-06) (hash(x)=145753166)
-13900 val loss 5.7157
-13900 val perplexity 303.5893
-13900 train 5.580726 (lr=6.7474e-06) (hash(x)=147065551)
-14000 val loss 5.7123
-14000 val perplexity 302.5801
-14000 train 6.179626 (lr=6.5756e-06) (hash(x)=179077542)
-14100 val loss 5.7122
-14100 val perplexity 302.5332
-14100 train 5.790305 (lr=6.4079e-06) (hash(x)=171962802)
-14200 val loss 5.7115
-14200 val perplexity 302.3179
-14200 train 5.780046 (lr=6.2444e-06) (hash(x)=153299913)
-14300 val loss 5.7120
-14300 val perplexity 302.4830
-14300 train 5.646414 (lr=6.0852e-06) (hash(x)=154750565)
-14400 val loss 5.7095
-14400 val perplexity 301.7082
-14400 train 5.990693 (lr=5.9304e-06) (hash(x)=154536694)
-14500 val loss 5.7045
-14500 val perplexity 300.2213
-14500 train 5.758692 (lr=5.7800e-06) (hash(x)=151236383)
-14600 val loss 5.7014
-14600 val perplexity 299.2985
-14600 train 5.712729 (lr=5.6340e-06) (hash(x)=152584272)
-14700 val loss 5.6987
-14700 val perplexity 298.4829
-14700 train 5.690105 (lr=5.4925e-06) (hash(x)=175301826)
-14800 val loss 5.6993
-14800 val perplexity 298.6620
-14800 train 5.619073 (lr=5.3555e-06) (hash(x)=148729408)
-14900 val loss 5.6973
-14900 val perplexity 298.0557
-14900 train 5.883558 (lr=5.2231e-06) (hash(x)=150978283)
-15000 val loss 5.6978
-15000 val perplexity 298.2142
-15000 train 5.641515 (lr=5.0954e-06) (hash(x)=153621389)
-15100 val loss 5.6975
-15100 val perplexity 298.1349
-15100 train 5.619658 (lr=4.9723e-06) (hash(x)=150653793)
-15200 val loss 5.6946
-15200 val perplexity 297.2447
-15200 train 5.870111 (lr=4.8539e-06) (hash(x)=140802934)
-15300 val loss 5.6893
-15300 val perplexity 295.6989
-15300 train 5.667443 (lr=4.7403e-06) (hash(x)=152007556)
-15400 val loss 5.6884
-15400 val perplexity 295.4150
-15400 train 5.644412 (lr=4.6314e-06) (hash(x)=149057446)
-15500 val loss 5.6864
-15500 val perplexity 294.8278
-15500 train 5.606894 (lr=4.5274e-06) (hash(x)=147521774)
-15600 val loss 5.6852
-15600 val perplexity 294.4681
-15600 train 5.655397 (lr=4.4282e-06) (hash(x)=148944405)
-15700 val loss 5.6842
-15700 val perplexity 294.1700
-15700 train 5.634020 (lr=4.3339e-06) (hash(x)=148472532)
-15800 val loss 5.6840
-15800 val perplexity 294.1218
-15800 train 5.457642 (lr=4.2446e-06) (hash(x)=149048780)
-15900 val loss 5.6851
-15900 val perplexity 294.4477
-15900 train 5.646353 (lr=4.1601e-06) (hash(x)=150175778)
-16000 val loss 5.6857
-16000 val perplexity 294.6167
-16000 train 5.474238 (lr=4.0807e-06) (hash(x)=145943831)
-16100 val loss 5.6786
-16100 val perplexity 292.5426
-16100 train 5.846851 (lr=4.0063e-06) (hash(x)=134314842)
-16200 val loss 5.6767
-16200 val perplexity 291.9774
-16200 train 5.697859 (lr=3.9368e-06) (hash(x)=153755135)
-16300 val loss 5.6770
-16300 val perplexity 292.0574
-16300 train 5.805291 (lr=3.8725e-06) (hash(x)=146916346)
-16400 val loss 5.6778
-16400 val perplexity 292.3182
-16400 train 5.589097 (lr=3.8132e-06) (hash(x)=155023643)
-16500 val loss 5.6745
-16500 val perplexity 291.3519
-16500 train 5.755394 (lr=3.7590e-06) (hash(x)=148910191)
-16600 val loss 5.6737
-16600 val perplexity 291.1219
-16600 train 5.594974 (lr=3.7099e-06) (hash(x)=150107746)
-16700 val loss 5.6710
-16700 val perplexity 290.3261
-16700 train 5.806321 (lr=3.6659e-06) (hash(x)=159943976)
-16800 val loss 5.6696
-16800 val perplexity 289.9181
-16800 train 5.669618 (lr=3.6271e-06) (hash(x)=160255226)
-16900 val loss 5.6680
-16900 val perplexity 289.4692
-16900 train 5.656839 (lr=3.5934e-06) (hash(x)=157121826)
-17000 val loss 5.6676
-17000 val perplexity 289.3403
-17000 train 5.800177 (lr=3.5649e-06) (hash(x)=159298029)
-17100 val loss 5.6665
-17100 val perplexity 289.0216
-17100 train 5.624098 (lr=3.5415e-06) (hash(x)=149355903)
-17200 val loss 5.6673
-17200 val perplexity 289.2592
-17200 train 5.485609 (lr=3.5234e-06) (hash(x)=144467909)
-17300 val loss 5.6645
-17300 val perplexity 288.4494
-17300 train 5.834759 (lr=3.5104e-06) (hash(x)=157150350)
-17400 val loss 5.6622
-17400 val perplexity 287.7772
-17400 train 5.710876 (lr=3.5026e-06) (hash(x)=154825212)
-17499 val loss 5.6616
-17499 val perplexity 287.6055
+0 val loss 11.3107
+0 val perplexity 81693.5234
+0 train 11.308768 (lr=1.8750e-07) (hash(x)=145079536)
+0 train 11.308768 (lr=2.0000e-07) (hash(x)=145079536)
+100 val loss 9.9475
+100 val perplexity 20900.9492
+100 val loss 9.9305
+100 val perplexity 20547.3184
+100 train 9.887659 (lr=1.8937e-05) (hash(x)=154745873)
+100 train 9.869424 (lr=2.0200e-05) (hash(x)=154745873)
+200 val loss 8.8930
+200 val perplexity 7280.7363
+200 val loss 8.6350
+200 val perplexity 5625.3657
+200 train 8.938697 (lr=3.7500e-05) (hash(x)=155800595)
+200 train 8.700185 (lr=4.0000e-05) (hash(x)=155800595)
+300 val loss 7.9772
+300 val perplexity 2913.7043
+300 val loss 7.7205
+300 val perplexity 2253.9800
+300 train 7.942238 (lr=3.7497e-05) (hash(x)=148595389)
+300 train 7.677109 (lr=3.9997e-05) (hash(x)=148595389)
+400 val loss 7.7660
+400 val perplexity 2359.0081
+400 train 7.692175 (lr=3.7489e-05) (hash(x)=145606733)
+400 val loss 7.5253
+400 val perplexity 1854.3280
+400 train 7.429271 (lr=3.9988e-05) (hash(x)=145606733)
+500 val loss 7.6252
+500 val perplexity 2049.1321
+500 train 7.473835 (lr=3.7475e-05) (hash(x)=138221231)
+500 val loss 7.4571
+500 val perplexity 1732.1571
+500 train 7.274553 (lr=3.9973e-05) (hash(x)=138221231)
+600 val loss 7.5417
+600 val perplexity 1885.0631
+600 train 7.552938 (lr=3.7456e-05) (hash(x)=150367139)
+600 val loss 7.4091
+600 val perplexity 1651.0148
+600 train 7.414547 (lr=3.9953e-05) (hash(x)=150367139)
+700 val loss 7.4944
+700 val perplexity 1797.9723
+700 train 7.645608 (lr=3.7430e-05) (hash(x)=155579314)
+700 val loss 7.3478
+700 val perplexity 1552.7489
+700 train 7.501362 (lr=3.9926e-05) (hash(x)=155579314)
+800 val loss 7.4339
+800 val perplexity 1692.3730
+800 train 7.515994 (lr=3.7400e-05) (hash(x)=155747374)
+800 val loss 7.2731
+800 val perplexity 1441.0717
+800 train 7.349138 (lr=3.9893e-05) (hash(x)=155747374)
+900 val loss 7.3800
+900 val perplexity 1603.6160
+900 train 7.586455 (lr=3.7364e-05) (hash(x)=159334575)
+900 val loss 7.2117
+900 val perplexity 1355.2490
+900 train 7.430864 (lr=3.9855e-05) (hash(x)=159334575)
+1000 val loss 7.3251
+1000 val perplexity 1517.9445
+1000 train 7.211405 (lr=3.7322e-05) (hash(x)=140604760)
+1000 val loss 7.1531
+1000 val perplexity 1278.1234
+1000 train 7.017516 (lr=3.9810e-05) (hash(x)=140604760)
+1100 val loss 7.2738
+1100 val perplexity 1441.9576
+1100 train 7.113899 (lr=3.7275e-05) (hash(x)=146208052)
+1100 val loss 7.0989
+1100 val perplexity 1210.6454
+1100 train 6.925892 (lr=3.9760e-05) (hash(x)=146208052)
+1200 val loss 7.2267
+1200 val perplexity 1375.7426
+1200 train 7.126267 (lr=3.7223e-05) (hash(x)=148404734)
+1200 val loss 7.0482
+1200 val perplexity 1150.7867
+1200 train 6.938736 (lr=3.9704e-05) (hash(x)=148404734)
+1300 val loss 7.1736
+1300 val perplexity 1304.5807
+1300 train 7.107864 (lr=3.7164e-05) (hash(x)=155681970)
+1300 val loss 7.0146
+1300 val perplexity 1112.7230
+1300 train 6.945279 (lr=3.9642e-05) (hash(x)=155681970)
+1400 val loss 7.1235
+1400 val perplexity 1240.8369
+1400 train 7.073852 (lr=3.7101e-05) (hash(x)=148115934)
+1400 val loss 6.9556
+1400 val perplexity 1049.0596
+1400 train 6.901715 (lr=3.9574e-05) (hash(x)=148115934)
+1500 val loss 7.0624
+1500 val perplexity 1167.1885
+1500 train 7.218080 (lr=3.7032e-05) (hash(x)=157074034)
+1500 val loss 6.8954
+1500 val perplexity 987.7095
+1500 train 7.059405 (lr=3.9501e-05) (hash(x)=157074034)
+1600 val loss 6.9965
+1600 val perplexity 1092.7928
+1600 train 6.758173 (lr=3.6958e-05) (hash(x)=137464699)
+1600 val loss 6.8454
+1600 val perplexity 939.5462
+1600 train 6.603693 (lr=3.9421e-05) (hash(x)=137464699)
+1700 val loss 6.9363
+1700 val perplexity 1028.9852
+1700 train 7.090509 (lr=3.6878e-05) (hash(x)=166955614)
+1700 val loss 6.7900
+1700 val perplexity 888.9067
+1700 train 6.936810 (lr=3.9336e-05) (hash(x)=166955614)
+1800 val loss 6.8773
+1800 val perplexity 970.0369
+1800 train 6.705127 (lr=3.6793e-05) (hash(x)=143886042)
+1800 val loss 6.7448
+1800 val perplexity 849.5964
+1800 train 6.570076 (lr=3.9246e-05) (hash(x)=143886042)
+1900 val loss 6.8304
+1900 val perplexity 925.5649
+1900 train 7.393573 (lr=3.6702e-05) (hash(x)=193610391)
+1900 val loss 6.6976
+1900 val perplexity 810.4745
+1900 train 7.261598 (lr=3.9149e-05) (hash(x)=193610391)
+2000 val loss 6.7689
+2000 val perplexity 870.3190
+2000 train 6.956485 (lr=3.6606e-05) (hash(x)=163799796)
+2000 val loss 6.6535
+2000 val perplexity 775.5157
+2000 train 6.857941 (lr=3.9047e-05) (hash(x)=163799796)
+2100 val loss 6.7018
+2100 val perplexity 813.8666
+2100 train 6.689584 (lr=3.6505e-05) (hash(x)=154107339)
+2100 val loss 6.6103
+2100 val perplexity 742.7250
+2100 train 6.602917 (lr=3.8939e-05) (hash(x)=154107339)
+2200 val loss 6.6605
+2200 val perplexity 780.9216
+2200 train 6.669657 (lr=3.6399e-05) (hash(x)=144592844)
+2200 val loss 6.5823
+2200 val perplexity 722.1735
+2200 train 6.591764 (lr=3.8826e-05) (hash(x)=144592844)
+2300 val loss 6.6079
+2300 val perplexity 740.9298
+2300 train 6.679429 (lr=3.6288e-05) (hash(x)=154535861)
+2300 val loss 6.5377
+2300 val perplexity 690.7051
+2300 train 6.601418 (lr=3.8707e-05) (hash(x)=154535861)
+2400 val loss 6.5631
+2400 val perplexity 708.4941
+2400 train 7.329075 (lr=3.6171e-05) (hash(x)=204706354)
+2400 val loss 6.5092
+2400 val perplexity 671.2576
+2400 train 7.266478 (lr=3.8583e-05) (hash(x)=204706354)
+2500 val loss 6.5323
+2500 val perplexity 687.0010
+2500 train 6.326653 (lr=3.6049e-05) (hash(x)=143783202)
+2500 val loss 6.4879
+2500 val perplexity 657.1377
+2500 train 6.280975 (lr=3.8453e-05) (hash(x)=143783202)
+2600 val loss 6.4941
+2600 val perplexity 661.2271
+2600 train 6.404017 (lr=3.5923e-05) (hash(x)=150862210)
+2600 val loss 6.4599
+2600 val perplexity 638.9831
+2600 train 6.371375 (lr=3.8317e-05) (hash(x)=150862210)
+2700 val loss 6.4670
+2700 val perplexity 643.5612
+2700 train 6.509830 (lr=3.5791e-05) (hash(x)=150753426)
+2700 val loss 6.4312
+2700 val perplexity 620.9432
+2700 train 6.481739 (lr=3.8177e-05) (hash(x)=150753426)
+2800 val loss 6.4464
+2800 val perplexity 630.4233
+2800 train 6.465562 (lr=3.5654e-05) (hash(x)=147766811)
+2800 val loss 6.4168
+2800 val perplexity 612.0283
+2800 train 6.451035 (lr=3.8031e-05) (hash(x)=147766811)
+2900 val loss 6.3979
+2900 val perplexity 600.5582
+2900 train 6.329579 (lr=3.5512e-05) (hash(x)=151177814)
+2900 val loss 6.3885
+2900 val perplexity 594.9806
+2900 train 6.327680 (lr=3.7879e-05) (hash(x)=151177814)
+3000 val loss 6.3747
+3000 val perplexity 586.7963
+3000 train 6.239610 (lr=3.5365e-05) (hash(x)=135925327)
+3000 val loss 6.3632
+3000 val perplexity 580.0998
+3000 train 6.232740 (lr=3.7723e-05) (hash(x)=135925327)
+3100 val loss 6.3508
+3100 val perplexity 572.9315
+3100 train 5.987338 (lr=3.5214e-05) (hash(x)=124842353)
+3100 val loss 6.3534
+3100 val perplexity 574.4473
+3100 train 5.985197 (lr=3.7561e-05) (hash(x)=124842353)
+3200 val loss 6.3339
+3200 val perplexity 563.3701
+3200 train 6.424920 (lr=3.5057e-05) (hash(x)=160440642)
+3200 val loss 6.3508
+3200 val perplexity 572.9645
+3200 train 6.449918 (lr=3.7394e-05) (hash(x)=160440642)
+3300 val loss 6.3022
+3300 val perplexity 545.7853
+3300 train 6.219065 (lr=3.4896e-05) (hash(x)=143017131)
+3300 val loss 6.3124
+3300 val perplexity 551.3576
+3300 train 6.227836 (lr=3.7222e-05) (hash(x)=143017131)
+3400 val loss 6.2772
+3400 val perplexity 532.2734
+3400 train 6.647445 (lr=3.4730e-05) (hash(x)=148143474)
+3400 val loss 6.2929
+3400 val perplexity 540.7349
+3400 train 6.647270 (lr=3.7045e-05) (hash(x)=148143474)
+3500 val loss 6.2591
+3500 val perplexity 522.7372
+3500 train 6.077838 (lr=3.4560e-05) (hash(x)=148732639)
+3500 val loss 6.2809
+3500 val perplexity 534.2861
+3500 train 6.102720 (lr=3.6864e-05) (hash(x)=148732639)
+3600 val loss 6.2419
+3600 val perplexity 513.8419
+3600 train 6.244680 (lr=3.4384e-05) (hash(x)=146333803)
+3600 val loss 6.2661
+3600 val perplexity 526.4097
+3600 train 6.271680 (lr=3.6677e-05) (hash(x)=146333803)
+3700 val loss 6.2261
+3700 val perplexity 505.7911
+3700 train 6.029370 (lr=3.4205e-05) (hash(x)=143611445)
+3700 val loss 6.2551
+3700 val perplexity 520.6425
+3700 train 6.060405 (lr=3.6485e-05) (hash(x)=143611445)
+3800 val loss 6.2239
+3800 val perplexity 504.6608
+3800 train 6.008307 (lr=3.4021e-05) (hash(x)=141272115)
+3800 val loss 6.2480
+3800 val perplexity 516.9984
+3800 train 6.028848 (lr=3.6289e-05) (hash(x)=141272115)
+3900 val loss 6.2016
+3900 val perplexity 493.5485
+3900 train 6.137794 (lr=3.3832e-05) (hash(x)=149161848)
+3900 val loss 6.2293
+3900 val perplexity 507.4062
+3900 train 6.166859 (lr=3.6088e-05) (hash(x)=149161848)
+4000 val loss 6.1724
+4000 val perplexity 479.3447
+4000 train 6.246626 (lr=3.3639e-05) (hash(x)=152036955)
+4000 val loss 6.1951
+4000 val perplexity 490.3334
+4000 train 6.271713 (lr=3.5882e-05) (hash(x)=152036955)
+4100 val loss 6.1561
+4100 val perplexity 471.5683
+4100 train 6.201271 (lr=3.3442e-05) (hash(x)=153260968)
+4100 val loss 6.1798
+4100 val perplexity 482.8908
+4100 train 6.228312 (lr=3.5671e-05) (hash(x)=153260968)
+4200 val loss 6.1402
+4200 val perplexity 464.1679
+4200 train 6.029848 (lr=3.3240e-05) (hash(x)=147629478)
+4200 val loss 6.1666
+4200 val perplexity 476.5579
+4200 train 6.052325 (lr=3.5457e-05) (hash(x)=147629478)
+4300 val loss 6.1300
+4300 val perplexity 459.4489
+4300 train 6.085530 (lr=3.3035e-05) (hash(x)=161701080)
+4300 val loss 6.1532
+4300 val perplexity 470.1998
+4300 train 6.117069 (lr=3.5237e-05) (hash(x)=161701080)
+4400 val loss 6.1198
+4400 val perplexity 454.7519
+4400 train 6.167680 (lr=3.2825e-05) (hash(x)=155290524)
+4400 val loss 6.1466
+4400 val perplexity 467.1368
+4400 train 6.194079 (lr=3.5014e-05) (hash(x)=155290524)
+4500 val loss 6.1049
+4500 val perplexity 448.0491
+4500 train 5.891583 (lr=3.2611e-05) (hash(x)=144726593)
+4500 val loss 6.1316
+4500 val perplexity 460.1760
+4500 train 5.918100 (lr=3.4786e-05) (hash(x)=144726593)
+4600 val loss 6.0951
+4600 val perplexity 443.6972
+4600 train 6.051011 (lr=3.2394e-05) (hash(x)=149517202)
+4600 val loss 6.1236
+4600 val perplexity 456.5213
+4600 train 6.087997 (lr=3.4553e-05) (hash(x)=149517202)
+4700 val loss 6.0744
+4700 val perplexity 434.5847
+4700 train 6.067421 (lr=3.2172e-05) (hash(x)=139403008)
+4700 val loss 6.1027
+4700 val perplexity 447.0701
+4700 train 6.095179 (lr=3.4317e-05) (hash(x)=139403008)
+4800 val loss 6.0584
+4800 val perplexity 427.6721
+4800 train 6.180338 (lr=3.1947e-05) (hash(x)=153387238)
+4800 val loss 6.0834
+4800 val perplexity 438.5213
+4800 train 6.183041 (lr=3.4077e-05) (hash(x)=153387238)
+4900 val loss 6.0543
+4900 val perplexity 425.9605
+4900 train 5.802900 (lr=3.1718e-05) (hash(x)=139881569)
+4900 val loss 6.0811
+4900 val perplexity 437.5304
+4900 train 5.832566 (lr=3.3832e-05) (hash(x)=139881569)
+5000 val loss 6.0322
+5000 val perplexity 416.6431
+5000 train 6.041107 (lr=3.1485e-05) (hash(x)=152602580)
+5000 val loss 6.0648
+5000 val perplexity 430.4250
+5000 train 6.070035 (lr=3.3584e-05) (hash(x)=152602580)
+5100 val loss 6.0235
+5100 val perplexity 413.0090
+5100 train 6.071440 (lr=3.1249e-05) (hash(x)=150598460)
+5100 val loss 6.0549
+5100 val perplexity 426.1752
+5100 train 6.101313 (lr=3.3332e-05) (hash(x)=150598460)
+5200 val loss 6.0116
+5200 val perplexity 408.1312
+5200 train 5.894236 (lr=3.1009e-05) (hash(x)=147660496)
+5200 val loss 6.0430
+5200 val perplexity 421.1597
+5200 train 5.921767 (lr=3.3076e-05) (hash(x)=147660496)
+5300 val loss 6.0084
+5300 val perplexity 406.8307
+5300 train 5.754591 (lr=3.0766e-05) (hash(x)=147693848)
+5300 val loss 6.0379
+5300 val perplexity 419.0294
+5300 train 5.791131 (lr=3.2817e-05) (hash(x)=147693848)
+5400 val loss 6.0008
+5400 val perplexity 403.7434
+5400 train 5.763398 (lr=3.0519e-05) (hash(x)=136048145)
+5400 val loss 6.0402
+5400 val perplexity 419.9767
+5400 train 5.802145 (lr=3.2554e-05) (hash(x)=136048145)
+5500 val loss 5.9803
+5500 val perplexity 395.5671
+5500 train 6.009804 (lr=3.0269e-05) (hash(x)=140313123)
+5500 val loss 6.0176
+5500 val perplexity 410.5718
+5500 train 6.038659 (lr=3.2287e-05) (hash(x)=140313123)
+5600 val loss 5.9709
+5600 val perplexity 391.8647
+5600 train 6.000503 (lr=3.0016e-05) (hash(x)=156997993)
+5600 val loss 6.0068
+5600 val perplexity 406.1997
+5600 train 6.036944 (lr=3.2017e-05) (hash(x)=156997993)
+5700 val loss 5.9577
+5700 val perplexity 386.7048
+5700 train 5.974874 (lr=2.9760e-05) (hash(x)=143513880)
+5700 val loss 5.9976
+5700 val perplexity 402.4466
+5700 train 6.007860 (lr=3.1744e-05) (hash(x)=143513880)
+5800 val loss 5.9488
+5800 val perplexity 383.3037
+5800 train 6.059963 (lr=2.9501e-05) (hash(x)=153736628)
+5800 val loss 5.9862
+5800 val perplexity 397.9112
+5800 train 6.105621 (lr=3.1468e-05) (hash(x)=153736628)
+5900 val loss 5.9390
+5900 val perplexity 379.5628
+5900 train 6.063349 (lr=2.9239e-05) (hash(x)=166385417)
+5900 val loss 5.9854
+5900 val perplexity 397.5974
+5900 train 6.117510 (lr=3.1188e-05) (hash(x)=166385417)
+6000 val loss 5.9255
+6000 val perplexity 374.4518
+6000 train 6.170571 (lr=2.8974e-05) (hash(x)=153216517)
+6000 val loss 5.9711
+6000 val perplexity 391.9264
+6000 train 6.213697 (lr=3.0905e-05) (hash(x)=153216517)
+6100 val loss 5.9246
+6100 val perplexity 374.1172
+6100 train 5.816231 (lr=2.8706e-05) (hash(x)=145387545)
+6100 val loss 5.9652
+6100 val perplexity 389.6248
+6100 train 5.857091 (lr=3.0620e-05) (hash(x)=145387545)
+6200 val loss 5.9235
+6200 val perplexity 373.7118
+6200 train 6.051315 (lr=2.8436e-05) (hash(x)=156306460)
+6200 val loss 5.9607
+6200 val perplexity 387.8851
+6200 train 6.091648 (lr=3.0332e-05) (hash(x)=156306460)
+6300 val loss 5.9015
+6300 val perplexity 365.5843
+6300 train 5.842206 (lr=2.8163e-05) (hash(x)=148610447)
+6300 val loss 5.9386
+6300 val perplexity 379.4168
+6300 train 5.881337 (lr=3.0040e-05) (hash(x)=148610447)
+6400 val loss 5.8869
+6400 val perplexity 360.2831
+6400 train 6.113601 (lr=2.7888e-05) (hash(x)=158343535)
+6400 val loss 5.9284
+6400 val perplexity 375.5528
+6400 train 6.155274 (lr=2.9747e-05) (hash(x)=158343535)
+6500 val loss 5.8778
+6500 val perplexity 357.0185
+6500 train 6.037257 (lr=2.7610e-05) (hash(x)=157742557)
+6500 val loss 5.9186
+6500 val perplexity 371.8734
+6500 train 6.085427 (lr=2.9450e-05) (hash(x)=157742557)
+6600 val loss 5.8700
+6600 val perplexity 354.2516
+6600 train 5.763630 (lr=2.7330e-05) (hash(x)=150457308)
+6600 val loss 5.9109
+6600 val perplexity 369.0442
+6600 train 5.798040 (lr=2.9152e-05) (hash(x)=150457308)
+6700 val loss 5.8640
+6700 val perplexity 352.1355
+6700 train 5.719018 (lr=2.7047e-05) (hash(x)=142426205)
+6700 val loss 5.9085
+6700 val perplexity 368.1502
+6700 train 5.757698 (lr=2.8851e-05) (hash(x)=142426205)
+6800 val loss 5.8662
+6800 val perplexity 352.8958
+6800 train 5.642277 (lr=2.6763e-05) (hash(x)=151522525)
+6800 val loss 5.9020
+6800 val perplexity 365.7698
+6800 train 5.676707 (lr=2.8547e-05) (hash(x)=151522525)
+6900 val loss 5.8505
+6900 val perplexity 347.3940
+6900 train 5.823310 (lr=2.6477e-05) (hash(x)=152029630)
+6900 val loss 5.8897
+6900 val perplexity 361.2955
+6900 train 5.848513 (lr=2.8242e-05) (hash(x)=152029630)
+7000 val loss 5.8346
+7000 val perplexity 341.9149
+7000 train 5.679857 (lr=2.6188e-05) (hash(x)=142978954)
+7000 val loss 5.8743
+7000 val perplexity 355.7711
+7000 train 5.724010 (lr=2.7934e-05) (hash(x)=142978954)
+7100 val loss 5.8294
+7100 val perplexity 340.1654
+7100 train 5.713923 (lr=2.5898e-05) (hash(x)=148064359)
+7100 val loss 5.8667
+7100 val perplexity 353.0793
+7100 train 5.747740 (lr=2.7624e-05) (hash(x)=148064359)
+7200 val loss 5.8227
+7200 val perplexity 337.8991
+7200 train 5.718215 (lr=2.5606e-05) (hash(x)=155771845)
+7200 val loss 5.8588
+7200 val perplexity 350.3069
+7200 train 5.752675 (lr=2.7313e-05) (hash(x)=155771845)
+7300 val loss 5.8232
+7300 val perplexity 338.0643
+7300 train 5.622985 (lr=2.5312e-05) (hash(x)=141639844)
+7300 val loss 5.8545
+7300 val perplexity 348.7894
+7300 train 5.646307 (lr=2.7000e-05) (hash(x)=141639844)
+7400 val loss 5.8208
+7400 val perplexity 337.2388
+7400 train 5.871749 (lr=2.5017e-05) (hash(x)=153466021)
+7400 val loss 5.8575
+7400 val perplexity 349.8495
+7400 train 5.910269 (lr=2.6685e-05) (hash(x)=153466021)
+7500 val loss 5.8062
+7500 val perplexity 332.3530
+7500 train 5.788906 (lr=2.4721e-05) (hash(x)=148843587)
+7500 val loss 5.8403
+7500 val perplexity 343.8840
+7500 train 5.822583 (lr=2.6369e-05) (hash(x)=148843587)
+7600 val loss 5.7932
+7600 val perplexity 328.0590
+7600 train 5.915468 (lr=2.4423e-05) (hash(x)=152053933)
+7600 val loss 5.8307
+7600 val perplexity 340.6115
+7600 train 5.955600 (lr=2.6051e-05) (hash(x)=152053933)
+7700 val loss 5.7872
+7700 val perplexity 326.0890
+7700 train 5.670220 (lr=2.4124e-05) (hash(x)=141564003)
+7700 val loss 5.8225
+7700 val perplexity 337.8119
+7700 train 5.695722 (lr=2.5732e-05) (hash(x)=141564003)
+7800 val loss 5.7776
+7800 val perplexity 322.9746
+7800 train 5.652206 (lr=2.3823e-05) (hash(x)=148617843)
+7800 val loss 5.8148
+7800 val perplexity 335.2256
+7800 train 5.692866 (lr=2.5411e-05) (hash(x)=148617843)
+7900 val loss 5.7755
+7900 val perplexity 322.3173
+7900 train 5.765408 (lr=2.3522e-05) (hash(x)=146417668)
+7900 val loss 5.8111
+7900 val perplexity 333.9737
+7900 train 5.790226 (lr=2.5090e-05) (hash(x)=146417668)
+8000 val loss 5.7700
+8000 val perplexity 320.5481
+8000 train 5.556377 (lr=2.3219e-05) (hash(x)=150165488)
+8000 val loss 5.8064
+8000 val perplexity 332.4193
+8000 train 5.587837 (lr=2.4767e-05) (hash(x)=150165488)
+8100 val loss 5.7600
+8100 val perplexity 317.3409
+8100 train 5.759561 (lr=2.2916e-05) (hash(x)=144818855)
+8100 val loss 5.7962
+8100 val perplexity 329.0320
+8100 train 5.799084 (lr=2.4444e-05) (hash(x)=144818855)
+8200 val loss 5.7594
+8200 val perplexity 317.1577
+8200 train 5.855401 (lr=2.2612e-05) (hash(x)=143074954)
+8200 val loss 5.7938
+8200 val perplexity 328.2616
+8200 train 5.898982 (lr=2.4120e-05) (hash(x)=143074954)
+8300 val loss 5.7483
+8300 val perplexity 313.6542
+8300 train 5.614585 (lr=2.2308e-05) (hash(x)=145172494)
+8300 val loss 5.7821
+8300 val perplexity 324.4247
+8300 train 5.645600 (lr=2.3795e-05) (hash(x)=145172494)
+8400 val loss 5.7403
+8400 val perplexity 311.1508
+8400 train 5.787636 (lr=2.2002e-05) (hash(x)=152284393)
+8400 val loss 5.7760
+8400 val perplexity 322.4603
+8400 train 5.819298 (lr=2.3469e-05) (hash(x)=152284393)
+8500 val loss 5.7384
+8500 val perplexity 310.5595
+8500 train 5.599001 (lr=2.1697e-05) (hash(x)=147044705)
+8500 val loss 5.7701
+8500 val perplexity 320.5808
+8500 train 5.625921 (lr=2.3143e-05) (hash(x)=147044705)
+8600 val loss 5.7350
+8600 val perplexity 309.5278
+8600 train 5.594783 (lr=2.1391e-05) (hash(x)=147762608)
+8600 val loss 5.7662
+8600 val perplexity 319.3280
+8600 train 5.629746 (lr=2.2817e-05) (hash(x)=147762608)
+8700 val loss 5.7243
+8700 val perplexity 306.2231
+8700 train 5.737029 (lr=2.1085e-05) (hash(x)=141994567)
+8700 val loss 5.7582
+8700 val perplexity 316.7644
+8700 train 5.775930 (lr=2.2490e-05) (hash(x)=141994567)
+8800 val loss 5.7175
+8800 val perplexity 304.1554
+8800 train 5.835638 (lr=2.0778e-05) (hash(x)=152552896)
+8800 val loss 5.7513
+8800 val perplexity 314.6034
+8800 train 5.866969 (lr=2.2163e-05) (hash(x)=152552896)
+8900 val loss 5.7123
+8900 val perplexity 302.5586
+8900 train 5.556566 (lr=2.0472e-05) (hash(x)=139714454)
+8900 val loss 5.7443
+8900 val perplexity 312.4159
+8900 train 5.590520 (lr=2.1837e-05) (hash(x)=139714454)
+9000 val loss 5.7048
+9000 val perplexity 300.2996
+9000 train 5.737811 (lr=2.0165e-05) (hash(x)=152274419)
+9000 val loss 5.7367
+9000 val perplexity 310.0361
+9000 train 5.766550 (lr=2.1510e-05) (hash(x)=152274419)
+9100 val loss 5.7037
+9100 val perplexity 299.9624
+9100 train 5.813401 (lr=1.9859e-05) (hash(x)=160536885)
+9100 val loss 5.7347
+9100 val perplexity 309.4271
+9100 train 5.826126 (lr=2.1183e-05) (hash(x)=160536885)
+9200 val loss 5.6971
+9200 val perplexity 298.0017
+9200 train 5.508581 (lr=1.9553e-05) (hash(x)=138386899)
+9200 val loss 5.7295
+9200 val perplexity 307.8032
+9200 train 5.536484 (lr=2.0857e-05) (hash(x)=138386899)
+9300 val loss 5.6941
+9300 val perplexity 297.0971
+9300 train 5.746347 (lr=1.9248e-05) (hash(x)=149957457)
+9300 val loss 5.7248
+9300 val perplexity 306.3625
+9300 train 5.772715 (lr=2.0531e-05) (hash(x)=149957457)
+9400 val loss 5.6848
+9400 val perplexity 294.3666
+9400 train 5.670947 (lr=1.8942e-05) (hash(x)=161671548)
+9400 val loss 5.7138
+9400 val perplexity 303.0352
+9400 train 5.702243 (lr=2.0205e-05) (hash(x)=161671548)
+9500 val loss 5.6870
+9500 val perplexity 295.0154
+9500 train 5.817723 (lr=1.8638e-05) (hash(x)=171857400)
+9500 val loss 5.7160
+9500 val perplexity 303.6863
+9500 train 5.834106 (lr=1.9880e-05) (hash(x)=171857400)
+9600 val loss 5.6747
+9600 val perplexity 291.4071
+9600 train 5.456934 (lr=1.8334e-05) (hash(x)=135378471)
+9600 val loss 5.7046
+9600 val perplexity 300.2393
+9600 train 5.474633 (lr=1.9556e-05) (hash(x)=135378471)
+9700 val loss 5.6693
+9700 val perplexity 289.8326
+9700 train 5.607586 (lr=1.8031e-05) (hash(x)=149514591)
+9700 val loss 5.7020
+9700 val perplexity 299.4644
+9700 train 5.642845 (lr=1.9233e-05) (hash(x)=149514591)
+9800 val loss 5.6663
+9800 val perplexity 288.9694
+9800 train 5.640245 (lr=1.7728e-05) (hash(x)=152679072)
+9800 val loss 5.6972
+9800 val perplexity 298.0387
+9800 train 5.668010 (lr=1.8910e-05) (hash(x)=152679072)
+9900 val loss 5.6595
+9900 val perplexity 287.0002
+9900 train 5.624960 (lr=1.7427e-05) (hash(x)=150415193)
+9900 val loss 5.6929
+9900 val perplexity 296.7478
+9900 train 5.654785 (lr=1.8589e-05) (hash(x)=150415193)
+10000 val loss 5.6569
+10000 val perplexity 286.2522
+10000 val loss 5.6856
+10000 val perplexity 294.5803
+10000 train 5.507225 (lr=1.7126e-05) (hash(x)=146910087)
+10000 train 5.539617 (lr=1.8268e-05) (hash(x)=146910087)
+10100 val loss 5.6824
+10100 val perplexity 293.6487
+10100 val loss 5.6513
+10100 val perplexity 284.6697
+10100 train 5.601520 (lr=1.7949e-05) (hash(x)=152091050)
+10100 train 5.568396 (lr=1.6827e-05) (hash(x)=152091050)
+10200 val loss 5.6810
+10200 val perplexity 293.2429
+10200 val loss 5.6508
+10200 val perplexity 284.5271
+10200 train 5.529841 (lr=1.7631e-05) (hash(x)=143101306)
+10200 train 5.501368 (lr=1.6529e-05) (hash(x)=143101306)
+10300 val loss 5.6724
+10300 val perplexity 290.7270
+10300 train 5.686642 (lr=1.7315e-05) (hash(x)=150803985)
+10300 val loss 5.6468
+10300 val perplexity 283.3883
+10300 train 5.661108 (lr=1.6233e-05) (hash(x)=150803985)
+10400 val loss 5.6745
+10400 val perplexity 291.3510
+10400 train 5.633561 (lr=1.7000e-05) (hash(x)=148671575)
+10400 val loss 5.6431
+10400 val perplexity 282.3257
+10400 train 5.613005 (lr=1.5938e-05) (hash(x)=148671575)
+10500 val loss 5.6646
+10500 val perplexity 288.4691
+10500 train 5.477703 (lr=1.6687e-05) (hash(x)=146312794)
+10500 val loss 5.6361
+10500 val perplexity 280.3604
+10500 train 5.438739 (lr=1.5644e-05) (hash(x)=146312794)
+10600 val loss 5.6643
+10600 val perplexity 288.3757
+10600 train 5.490323 (lr=1.6376e-05) (hash(x)=137878985)
+10600 val loss 5.6384
+10600 val perplexity 281.0232
+10600 train 5.461352 (lr=1.5352e-05) (hash(x)=137878985)
+10700 val loss 5.6600
+10700 val perplexity 287.1475
+10700 train 5.472318 (lr=1.6066e-05) (hash(x)=148897486)
+10700 val loss 5.6302
+10700 val perplexity 278.7291
+10700 train 5.449248 (lr=1.5062e-05) (hash(x)=148897486)
+10800 val loss 5.6559
+10800 val perplexity 285.9793
+10800 train 5.805680 (lr=1.5758e-05) (hash(x)=147219580)
+10800 val loss 5.6267
+10800 val perplexity 277.7428
+10800 train 5.778777 (lr=1.4773e-05) (hash(x)=147219580)
+10900 val loss 5.6570
+10900 val perplexity 286.2779
+10900 train 5.681130 (lr=1.5453e-05) (hash(x)=152048929)
+10900 val loss 5.6266
+10900 val perplexity 277.7201
+10900 train 5.650803 (lr=1.4487e-05) (hash(x)=152048929)
+11000 val loss 5.6419
+11000 val perplexity 282.0003
+11000 train 5.529566 (lr=1.5149e-05) (hash(x)=146417066)
+11000 val loss 5.6135
+11000 val perplexity 274.1082
+11000 train 5.504382 (lr=1.4203e-05) (hash(x)=146417066)
+11100 val loss 5.6401
+11100 val perplexity 281.4987
+11100 train 5.493770 (lr=1.4848e-05) (hash(x)=148784436)
+11100 val loss 5.6129
+11100 val perplexity 273.9335
+11100 train 5.457813 (lr=1.3920e-05) (hash(x)=148784436)
+11200 val loss 5.6382
+11200 val perplexity 280.9495
+11200 train 5.628141 (lr=1.4550e-05) (hash(x)=144721870)
+11200 val loss 5.6098
+11200 val perplexity 273.0796
+11200 train 5.594026 (lr=1.3640e-05) (hash(x)=144721870)
+11300 val loss 5.6353
+11300 val perplexity 280.1469
+11300 train 5.545784 (lr=1.4253e-05) (hash(x)=149442981)
+11300 val loss 5.6067
+11300 val perplexity 272.2455
+11300 train 5.517210 (lr=1.3362e-05) (hash(x)=149442981)
+11400 val loss 5.6329
+11400 val perplexity 279.4653
+11400 train 5.514412 (lr=1.3960e-05) (hash(x)=154481228)
+11400 val loss 5.6055
+11400 val perplexity 271.9057
+11400 train 5.480088 (lr=1.3087e-05) (hash(x)=154481228)
+11500 val loss 5.6285
+11500 val perplexity 278.2540
+11500 train 5.319666 (lr=1.3668e-05) (hash(x)=115164581)
+11500 val loss 5.6020
+11500 val perplexity 270.9755
+11500 train 5.307278 (lr=1.2814e-05) (hash(x)=115164581)
+11600 val loss 5.6203
+11600 val perplexity 275.9729
+11600 train 5.839368 (lr=1.3380e-05) (hash(x)=155048082)
+11600 val loss 5.5953
+11600 val perplexity 269.1717
+11600 train 5.827364 (lr=1.2544e-05) (hash(x)=155048082)
+11700 val loss 5.6196
+11700 val perplexity 275.7854
+11700 train 5.684441 (lr=1.3095e-05) (hash(x)=152123770)
+11700 val loss 5.5941
+11700 val perplexity 268.8488
+11700 train 5.663855 (lr=1.2276e-05) (hash(x)=152123770)
+11800 val loss 5.6168
+11800 val perplexity 275.0091
+11800 train 5.545686 (lr=1.2812e-05) (hash(x)=152705857)
+11800 val loss 5.5908
+11800 val perplexity 267.9501
+11800 train 5.517049 (lr=1.2011e-05) (hash(x)=152705857)
+11900 val loss 5.6153
+11900 val perplexity 274.5948
+11900 train 5.349757 (lr=1.2532e-05) (hash(x)=137392362)
+11900 val loss 5.5877
+11900 val perplexity 267.1139
+11900 train 5.319537 (lr=1.1749e-05) (hash(x)=137392362)
+12000 val loss 5.6149
+12000 val perplexity 274.4831
+12000 train 5.422106 (lr=1.2256e-05) (hash(x)=143492562)
+12000 val loss 5.5900
+12000 val perplexity 267.7459
+12000 train 5.393394 (lr=1.1490e-05) (hash(x)=143492562)
+12100 val loss 5.6086
+12100 val perplexity 272.7590
+12100 train 5.589555 (lr=1.1983e-05) (hash(x)=141825431)
+12100 val loss 5.5840
+12100 val perplexity 266.1208
+12100 train 5.570032 (lr=1.1234e-05) (hash(x)=141825431)
+12200 val loss 5.6035
+12200 val perplexity 271.3845
+12200 train 5.514888 (lr=1.1713e-05) (hash(x)=141774370)
+12200 val loss 5.5780
+12200 val perplexity 264.5513
+12200 train 5.495852 (lr=1.0981e-05) (hash(x)=141774370)
+12300 val loss 5.5996
+12300 val perplexity 270.3284
+12300 train 5.632575 (lr=1.1446e-05) (hash(x)=152995173)
+12300 val loss 5.5736
+12300 val perplexity 263.3873
+12300 train 5.604387 (lr=1.0731e-05) (hash(x)=152995173)
+12400 val loss 5.6003
+12400 val perplexity 270.5031
+12400 train 5.425294 (lr=1.1183e-05) (hash(x)=144703471)
+12400 val loss 5.5743
+12400 val perplexity 263.5755
+12400 train 5.402729 (lr=1.0484e-05) (hash(x)=144703471)
+12500 val loss 5.5969
+12500 val perplexity 269.5911
+12500 train 5.497826 (lr=1.0924e-05) (hash(x)=144866179)
+12500 val loss 5.5720
+12500 val perplexity 262.9684
+12500 train 5.477533 (lr=1.0241e-05) (hash(x)=144866179)
+12600 val loss 5.5966
+12600 val perplexity 269.4970
+12600 train 5.607377 (lr=1.0668e-05) (hash(x)=157019663)
+12600 val loss 5.5686
+12600 val perplexity 262.0779
+12600 train 5.579659 (lr=1.0001e-05) (hash(x)=157019663)
+12700 val loss 5.5925
+12700 val perplexity 268.3951
+12700 train 5.546249 (lr=1.0416e-05) (hash(x)=152872020)
+12700 val loss 5.5681
+12700 val perplexity 261.9382
+12700 train 5.541395 (lr=9.7649e-06) (hash(x)=152872020)
+12800 val loss 5.5872
+12800 val perplexity 266.9830
+12800 train 5.617934 (lr=1.0168e-05) (hash(x)=158975636)
+12800 val loss 5.5627
+12800 val perplexity 260.5228
+12800 train 5.591032 (lr=9.5322e-06) (hash(x)=158975636)
+12900 val loss 5.5833
+12900 val perplexity 265.9386
+12900 train 5.631866 (lr=9.9233e-06) (hash(x)=143495701)
+12900 val loss 5.5572
+12900 val perplexity 259.1091
+12900 train 5.616541 (lr=9.3031e-06) (hash(x)=143495701)
+13000 val loss 5.5830
+13000 val perplexity 265.8771
+13000 train 5.510010 (lr=9.6829e-06) (hash(x)=151940005)
+13000 val loss 5.5570
+13000 val perplexity 259.0425
+13000 train 5.482976 (lr=9.0777e-06) (hash(x)=151940005)
+13100 val loss 5.5801
+13100 val perplexity 265.1006
+13100 train 5.866326 (lr=9.4466e-06) (hash(x)=155920713)
+13100 val loss 5.5544
+13100 val perplexity 258.3700
+13100 train 5.851393 (lr=8.8562e-06) (hash(x)=155920713)
+13200 val loss 5.5774
+13200 val perplexity 264.3724
+13200 train 5.770224 (lr=9.2144e-06) (hash(x)=152145878)
+13200 val loss 5.5530
+13200 val perplexity 258.0057
+13200 train 5.751560 (lr=8.6385e-06) (hash(x)=152145878)
+13300 val loss 5.5752
+13300 val perplexity 263.8147
+13300 train 5.402647 (lr=8.9865e-06) (hash(x)=134141709)
+13300 val loss 5.5511
+13300 val perplexity 257.5152
+13300 train 5.376328 (lr=8.4248e-06) (hash(x)=134141709)
+13400 val loss 5.5769
+13400 val perplexity 264.2384
+13400 train 5.506146 (lr=8.7628e-06) (hash(x)=146096299)
+13400 val loss 5.5543
+13400 val perplexity 258.3470
+13400 train 5.476796 (lr=8.2151e-06) (hash(x)=146096299)
+13500 val loss 5.5751
+13500 val perplexity 263.7630
+13500 train 5.330158 (lr=8.5435e-06) (hash(x)=144011431)
+13500 val loss 5.5502
+13500 val perplexity 257.2935
+13500 train 5.308886 (lr=8.0095e-06) (hash(x)=144011431)
+13600 val loss 5.5676
+13600 val perplexity 261.8164
+13600 train 5.802654 (lr=8.3286e-06) (hash(x)=163950932)
+13600 val loss 5.5438
+13600 val perplexity 255.6573
+13600 train 5.783780 (lr=7.8081e-06) (hash(x)=163950932)
+13700 val loss 5.5651
+13700 val perplexity 261.1624
+13700 train 5.712312 (lr=8.1183e-06) (hash(x)=156652406)
+13700 val loss 5.5410
+13700 val perplexity 254.9451
+13700 train 5.686758 (lr=7.6109e-06) (hash(x)=156652406)
+13800 val loss 5.5605
+13800 val perplexity 259.9570
+13800 train 5.476258 (lr=7.9125e-06) (hash(x)=145753166)
+13800 val loss 5.5367
+13800 val perplexity 253.8321
+13800 train 5.461190 (lr=7.4180e-06) (hash(x)=145753166)
+13900 val loss 5.5595
+13900 val perplexity 259.6913
+13900 train 5.424074 (lr=7.7114e-06) (hash(x)=147065551)
+13900 val loss 5.5352
+13900 val perplexity 253.4479
+13900 train 5.404427 (lr=7.2294e-06) (hash(x)=147065551)
+14000 val loss 5.5569
+14000 val perplexity 259.0065
+14000 train 6.028049 (lr=7.5149e-06) (hash(x)=179077542)
+14000 val loss 5.5337
+14000 val perplexity 253.0728
+14000 train 6.001717 (lr=7.0452e-06) (hash(x)=179077542)
+14100 val loss 5.5554
+14100 val perplexity 258.6318
+14100 train 5.623073 (lr=7.3233e-06) (hash(x)=171962802)
+14100 val loss 5.5331
+14100 val perplexity 252.9184
+14100 train 5.588490 (lr=6.8656e-06) (hash(x)=171962802)
+14200 val loss 5.5564
+14200 val perplexity 258.8788
+14200 train 5.642525 (lr=7.1365e-06) (hash(x)=153299913)
+14200 val loss 5.5331
+14200 val perplexity 252.9215
+14200 train 5.619051 (lr=6.6904e-06) (hash(x)=153299913)
+14300 val loss 5.5553
+14300 val perplexity 258.6111
+14300 train 5.481285 (lr=6.9546e-06) (hash(x)=154750565)
+14300 val loss 5.5332
+14300 val perplexity 252.9503
+14300 train 5.454321 (lr=6.5199e-06) (hash(x)=154750565)
+14400 val loss 5.5537
+14400 val perplexity 258.1823
+14400 train 5.830606 (lr=6.7776e-06) (hash(x)=154536694)
+14400 val loss 5.5308
+14400 val perplexity 252.3419
+14400 train 5.808904 (lr=6.3540e-06) (hash(x)=154536694)
+14500 val loss 5.5497
+14500 val perplexity 257.1502
+14500 train 5.613526 (lr=6.6057e-06) (hash(x)=151236383)
+14500 val loss 5.5264
+14500 val perplexity 251.2405
+14500 train 5.590186 (lr=6.1928e-06) (hash(x)=151236383)
+14600 val loss 5.5451
+14600 val perplexity 255.9854
+14600 train 5.570751 (lr=6.4389e-06) (hash(x)=152584272)
+14600 val loss 5.5217
+14600 val perplexity 250.0630
+14600 train 5.553623 (lr=6.0364e-06) (hash(x)=152584272)
+14700 val loss 5.5435
+14700 val perplexity 255.5691
+14700 train 5.501938 (lr=6.2771e-06) (hash(x)=175301826)
+14700 val loss 5.5211
+14700 val perplexity 249.9111
+14700 train 5.493054 (lr=5.8848e-06) (hash(x)=175301826)
+14800 val loss 5.5438
+14800 val perplexity 255.6586
+14800 train 5.460385 (lr=6.1206e-06) (hash(x)=148729408)
+14800 val loss 5.5212
+14800 val perplexity 249.9372
+14800 train 5.445827 (lr=5.7381e-06) (hash(x)=148729408)
+14900 val loss 5.5426
+14900 val perplexity 255.3284
+14900 train 5.746192 (lr=5.9693e-06) (hash(x)=150978283)
+14900 val loss 5.5200
+14900 val perplexity 249.6369
+14900 train 5.723087 (lr=5.5962e-06) (hash(x)=150978283)
+15000 val loss 5.5426
+15000 val perplexity 255.3446
+15000 train 5.486250 (lr=5.8233e-06) (hash(x)=153621389)
+15000 val loss 5.5219
+15000 val perplexity 250.1049
+15000 train 5.470625 (lr=5.4593e-06) (hash(x)=153621389)
+15100 val loss 5.5417
+15100 val perplexity 255.1073
+15100 train 5.458971 (lr=5.6826e-06) (hash(x)=150653793)
+15100 val loss 5.5205
+15100 val perplexity 249.7702
+15100 train 5.437407 (lr=5.3275e-06) (hash(x)=150653793)
+15200 val loss 5.5392
+15200 val perplexity 254.4813
+15200 train 5.726518 (lr=5.5473e-06) (hash(x)=140802934)
+15200 val loss 5.5179
+15200 val perplexity 249.0995
+15200 train 5.714012 (lr=5.2006e-06) (hash(x)=140802934)
+15300 val loss 5.5347
+15300 val perplexity 253.3340
+15300 train 5.516016 (lr=5.4175e-06) (hash(x)=152007556)
+15300 val loss 5.5123
+15300 val perplexity 247.7215
+15300 train 5.528864 (lr=5.0789e-06) (hash(x)=152007556)
+15400 val loss 5.5336
+15400 val perplexity 253.0472
+15400 train 5.484604 (lr=5.2931e-06) (hash(x)=149057446)
+15400 val loss 5.5114
+15400 val perplexity 247.4870
+15400 train 5.471828 (lr=4.9622e-06) (hash(x)=149057446)
+15500 val loss 5.5319
+15500 val perplexity 252.6117
+15500 train 5.448268 (lr=5.1742e-06) (hash(x)=147521774)
+15500 val loss 5.5098
+15500 val perplexity 247.1077
+15500 train 5.432381 (lr=4.8508e-06) (hash(x)=147521774)
+15600 val loss 5.5318
+15600 val perplexity 252.5913
+15600 train 5.508049 (lr=5.0608e-06) (hash(x)=148944405)
+15600 val loss 5.5085
+15600 val perplexity 246.7899
+15600 train 5.490562 (lr=4.7445e-06) (hash(x)=148944405)
+15700 val loss 5.5308
+15700 val perplexity 252.3476
+15700 train 5.481038 (lr=4.9531e-06) (hash(x)=148472532)
+15700 val loss 5.5086
+15700 val perplexity 246.8106
+15700 train 5.453821 (lr=4.6435e-06) (hash(x)=148472532)
+15800 val loss 5.5300
+15800 val perplexity 252.1432
+15800 train 5.296927 (lr=4.8509e-06) (hash(x)=149048780)
+15800 val loss 5.5086
+15800 val perplexity 246.8076
+15800 train 5.272242 (lr=4.5478e-06) (hash(x)=149048780)
+15900 val loss 5.5318
+15900 val perplexity 252.5956
+15900 train 5.496135 (lr=4.7545e-06) (hash(x)=150175778)
+15900 val loss 5.5102
+15900 val perplexity 247.2009
+15900 train 5.477364 (lr=4.4573e-06) (hash(x)=150175778)
+16000 val loss 5.5324
+16000 val perplexity 252.7528
+16000 train 5.306297 (lr=4.6637e-06) (hash(x)=145943831)
+16000 val loss 5.5102
+16000 val perplexity 247.2036
+16000 train 5.279723 (lr=4.3722e-06) (hash(x)=145943831)
+16100 val loss 5.5258
+16100 val perplexity 251.0751
+16100 train 5.708304 (lr=4.5786e-06) (hash(x)=134314842)
+16100 val loss 5.5040
+16100 val perplexity 245.6816
+16100 train 5.692306 (lr=4.2924e-06) (hash(x)=134314842)
+16200 val loss 5.5230
+16200 val perplexity 250.3804
+16200 train 5.557369 (lr=4.4993e-06) (hash(x)=153755135)
+16200 val loss 5.5016
+16200 val perplexity 245.0840
+16200 train 5.526541 (lr=4.2180e-06) (hash(x)=153755135)
+16300 val loss 5.5230
+16300 val perplexity 250.3745
+16300 train 5.654873 (lr=4.4257e-06) (hash(x)=146916346)
+16300 val loss 5.5019
+16300 val perplexity 245.1468
+16300 train 5.636758 (lr=4.1491e-06) (hash(x)=146916346)
+16400 val loss 5.5236
+16400 val perplexity 250.5255
+16400 train 5.434968 (lr=4.3579e-06) (hash(x)=155023643)
+16400 val loss 5.5035
+16400 val perplexity 245.5604
+16400 train 5.417802 (lr=4.0856e-06) (hash(x)=155023643)
+16500 val loss 5.5213
+16500 val perplexity 249.9600
+16500 train 5.613109 (lr=4.2960e-06) (hash(x)=148910191)
+16500 val loss 5.5012
+16500 val perplexity 244.9902
+16500 train 5.604181 (lr=4.0275e-06) (hash(x)=148910191)
+16600 val loss 5.5208
+16600 val perplexity 249.8469
+16600 train 5.450554 (lr=4.2399e-06) (hash(x)=150107746)
+16600 val loss 5.5008
+16600 val perplexity 244.8920
+16600 train 5.428391 (lr=3.9749e-06) (hash(x)=150107746)
+16700 val loss 5.5178
+16700 val perplexity 249.0921
+16700 train 5.644423 (lr=4.1896e-06) (hash(x)=159943976)
+16700 val loss 5.4975
+16700 val perplexity 244.0741
+16700 train 5.637831 (lr=3.9278e-06) (hash(x)=159943976)
+16800 val loss 5.5155
+16800 val perplexity 248.5153
+16800 train 5.501436 (lr=4.1452e-06) (hash(x)=160255226)
+16800 val loss 5.4954
+16800 val perplexity 243.5715
+16800 train 5.489238 (lr=3.8862e-06) (hash(x)=160255226)
+16900 val loss 5.5147
+16900 val perplexity 248.3036
+16900 train 5.508539 (lr=4.1067e-06) (hash(x)=157121826)
+16900 val loss 5.4945
+16900 val perplexity 243.3470
+16900 train 5.483582 (lr=3.8501e-06) (hash(x)=157121826)
+17000 val loss 5.5137
+17000 val perplexity 248.0701
+17000 train 5.639942 (lr=4.0741e-06) (hash(x)=159298029)
+17000 val loss 5.4932
+17000 val perplexity 243.0335
+17000 train 5.625374 (lr=3.8195e-06) (hash(x)=159298029)
+17100 val loss 5.5142
+17100 val perplexity 248.1973
+17100 train 5.479031 (lr=4.0475e-06) (hash(x)=149355903)
+17100 val loss 5.4936
+17100 val perplexity 243.1309
+17100 train 5.459307 (lr=3.7945e-06) (hash(x)=149355903)
+17200 val loss 5.5136
+17200 val perplexity 248.0547
+17200 train 5.324676 (lr=4.0267e-06) (hash(x)=144467909)
+17200 val loss 5.4942
+17200 val perplexity 243.2700
+17200 train 5.309157 (lr=3.7750e-06) (hash(x)=144467909)
+17300 val loss 5.5118
+17300 val perplexity 247.5893
+17300 train 5.678904 (lr=4.0119e-06) (hash(x)=157150350)
+17300 val loss 5.4913
+17300 val perplexity 242.5806
+17300 train 5.669414 (lr=3.7611e-06) (hash(x)=157150350)
+17400 val loss 5.5102
+17400 val perplexity 247.2052
+17400 train 5.547256 (lr=4.0030e-06) (hash(x)=154825212)
+17400 val loss 5.4896
+17400 val perplexity 242.1596
+17400 train 5.547105 (lr=3.7528e-06) (hash(x)=154825212)
+17499 val loss 5.5081
+17499 val perplexity 246.6936
+17499 val loss 5.4884
+17499 val perplexity 241.8788

attention_kindselective_n_heads4_seed1341/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dce8e25bfdc37d23fa7b903810e5a54ee63c135f64ec57d3a077ced4103acdcd
+oid sha256:570880b33301af8f4c646004f7ff6e8647c49c2ab85328d0b4b473002a544e3b
 size 92843394

attention_kindselective_n_heads4_seed1341/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c56419707c1fb4e769e2eba281bb02522eb64cd6672b7ac86009a4da1e54cc69
+oid sha256:a67d72ac0752c52af487c1bfa2cdedb30dc47cde5668d29bcd157543c08c58bc
 size 179406214