Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1341", "n_embd": 256}

attention_kindselective_n_heads4_seed1341/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.3107
 0 val perplexity 81693.5234
-0 train 11.308768 (lr=1.5000e-07) (hash(x)=145079536)
-100 val loss 9.9830
-100 val perplexity 21656.2090
-100 train 9.924451 (lr=1.5150e-05) (hash(x)=154745873)
-200 val loss 9.2262
-200 val perplexity 10159.3574
-200 train 9.254306 (lr=3.0000e-05) (hash(x)=155800595)
-300 val loss 8.0887
-300 val perplexity 3257.6125
-300 train 8.057108 (lr=2.9998e-05) (hash(x)=148595389)
-400 val loss 7.7043
-400 val perplexity 2217.7786
-400 train 7.619694 (lr=2.9991e-05) (hash(x)=145606733)
-500 val loss 7.5578
-500 val perplexity 1915.6061
-500 train 7.391018 (lr=2.9980e-05) (hash(x)=138221231)
-600 val loss 7.4932
-600 val perplexity 1795.7026
-600 train 7.504253 (lr=2.9964e-05) (hash(x)=150367139)
-700 val loss 7.4564
-700 val perplexity 1730.8922
-700 train 7.611702 (lr=2.9944e-05) (hash(x)=155579314)
-800 val loss 7.4182
-800 val perplexity 1665.9652
-800 train 7.493585 (lr=2.9920e-05) (hash(x)=155747374)
-900 val loss 7.3806
-900 val perplexity 1604.6064
-900 train 7.583446 (lr=2.9891e-05) (hash(x)=159334575)
-1000 val loss 7.3464
-1000 val perplexity 1550.5729
-1000 train 7.226415 (lr=2.9858e-05) (hash(x)=140604760)
-1100 val loss 7.3188
-1100 val perplexity 1508.4453
-1100 train 7.163220 (lr=2.9820e-05) (hash(x)=146208052)
-1200 val loss 7.2921
-1200 val perplexity 1468.6635
-1200 train 7.210934 (lr=2.9778e-05) (hash(x)=148404734)
-1300 val loss 7.2604
-1300 val perplexity 1422.8925
-1300 train 7.194634 (lr=2.9732e-05) (hash(x)=155681970)
-1400 val loss 7.2315
-1400 val perplexity 1382.2393
-1400 train 7.185406 (lr=2.9681e-05) (hash(x)=148115934)
-1500 val loss 7.1955
-1500 val perplexity 1333.4130
-1500 train 7.345188 (lr=2.9626e-05) (hash(x)=157074034)
-1600 val loss 7.1643
-1600 val perplexity 1292.4220
-1600 train 6.927887 (lr=2.9566e-05) (hash(x)=137464699)
-1700 val loss 7.1353
-1700 val perplexity 1255.4856
-1700 train 7.315463 (lr=2.9502e-05) (hash(x)=166955614)
-1800 val loss 7.1089
-1800 val perplexity 1222.8240
-1800 train 6.936053 (lr=2.9434e-05) (hash(x)=143886042)
-1900 val loss 7.0820
-1900 val perplexity 1190.2955
-1900 train 7.707833 (lr=2.9362e-05) (hash(x)=193610391)
-2000 val loss 7.0571
-2000 val perplexity 1161.1222
-2000 train 7.232715 (lr=2.9285e-05) (hash(x)=163799796)
-2100 val loss 7.0157
-2100 val perplexity 1113.9966
-2100 train 6.991386 (lr=2.9204e-05) (hash(x)=154107339)
-2200 val loss 6.9856
-2200 val perplexity 1080.9934
-2200 train 6.980697 (lr=2.9119e-05) (hash(x)=144592844)
-2300 val loss 6.9576
-2300 val perplexity 1051.1511
-2300 train 7.036345 (lr=2.9030e-05) (hash(x)=154535861)
-2400 val loss 6.9332
-2400 val perplexity 1025.8033
-2400 train 7.687595 (lr=2.8937e-05) (hash(x)=204706354)
-2500 val loss 6.9026
-2500 val perplexity 994.8754
-2500 train 6.700395 (lr=2.8840e-05) (hash(x)=143783202)
-2600 val loss 6.8699
-2600 val perplexity 962.8300
-2600 train 6.765809 (lr=2.8738e-05) (hash(x)=150862210)
-2700 val loss 6.8375
-2700 val perplexity 932.1981
-2700 train 6.894982 (lr=2.8633e-05) (hash(x)=150753426)
-2800 val loss 6.8110
-2800 val perplexity 907.7455
-2800 train 6.813390 (lr=2.8523e-05) (hash(x)=147766811)
-2900 val loss 6.7701
-2900 val perplexity 871.4382
-2900 train 6.708093 (lr=2.8410e-05) (hash(x)=151177814)
-3000 val loss 6.7345
-3000 val perplexity 840.8976
-3000 train 6.587459 (lr=2.8292e-05) (hash(x)=135925327)
-3100 val loss 6.7059
-3100 val perplexity 817.2025
-3100 train 6.357706 (lr=2.8171e-05) (hash(x)=124842353)
-3200 val loss 6.6830
-3200 val perplexity 798.7388
-3200 train 6.789571 (lr=2.8046e-05) (hash(x)=160440642)
-3300 val loss 6.6456
-3300 val perplexity 769.3922
-3300 train 6.545945 (lr=2.7917e-05) (hash(x)=143017131)
-3400 val loss 6.6180
-3400 val perplexity 748.4457
-3400 train 6.906633 (lr=2.7784e-05) (hash(x)=148143474)
-3500 val loss 6.6012
-3500 val perplexity 735.9523
-3500 train 6.420564 (lr=2.7648e-05) (hash(x)=148732639)
-3600 val loss 6.5736
-3600 val perplexity 715.9206
-3600 train 6.569450 (lr=2.7508e-05) (hash(x)=146333803)
-3700 val loss 6.5550
-3700 val perplexity 702.7313
-3700 train 6.353135 (lr=2.7364e-05) (hash(x)=143611445)
-3800 val loss 6.5419
-3800 val perplexity 693.6258
-3800 train 6.329458 (lr=2.7216e-05) (hash(x)=141272115)
-3900 val loss 6.5180
-3900 val perplexity 677.2001
-3900 train 6.447794 (lr=2.7066e-05) (hash(x)=149161848)
-4000 val loss 6.4837
-4000 val perplexity 654.3748
-4000 train 6.561291 (lr=2.6911e-05) (hash(x)=152036955)
-4100 val loss 6.4627
-4100 val perplexity 640.7753
-4100 train 6.516588 (lr=2.6754e-05) (hash(x)=153260968)
-4200 val loss 6.4425
-4200 val perplexity 627.9695
-4200 train 6.316812 (lr=2.6592e-05) (hash(x)=147629478)
-4300 val loss 6.4259
-4300 val perplexity 617.6288
-4300 train 6.424664 (lr=2.6428e-05) (hash(x)=161701080)
-4400 val loss 6.4103
-4400 val perplexity 608.0597
-4400 train 6.460846 (lr=2.6260e-05) (hash(x)=155290524)
-4500 val loss 6.4012
-4500 val perplexity 602.5555
-4500 train 6.182449 (lr=2.6089e-05) (hash(x)=144726593)
-4600 val loss 6.3826
-4600 val perplexity 591.4775
-4600 train 6.321618 (lr=2.5915e-05) (hash(x)=149517202)
-4700 val loss 6.3610
-4700 val perplexity 578.8070
-4700 train 6.363839 (lr=2.5738e-05) (hash(x)=139403008)
-4800 val loss 6.3395
-4800 val perplexity 566.5151
-4800 train 6.422396 (lr=2.5558e-05) (hash(x)=153387238)
-4900 val loss 6.3396
-4900 val perplexity 566.5673
-4900 train 6.089847 (lr=2.5374e-05) (hash(x)=139881569)
-5000 val loss 6.3176
-5000 val perplexity 554.2572
-5000 train 6.305876 (lr=2.5188e-05) (hash(x)=152602580)
-5100 val loss 6.3054
-5100 val perplexity 547.5158
-5100 train 6.356030 (lr=2.4999e-05) (hash(x)=150598460)
-5200 val loss 6.2945
-5200 val perplexity 541.5656
-5200 train 6.180711 (lr=2.4807e-05) (hash(x)=147660496)
-5300 val loss 6.2825
-5300 val perplexity 535.1163
-5300 train 6.018123 (lr=2.4613e-05) (hash(x)=147693848)
-5400 val loss 6.2812
-5400 val perplexity 534.4260
-5400 train 6.027461 (lr=2.4415e-05) (hash(x)=136048145)
-5500 val loss 6.2582
-5500 val perplexity 522.2893
-5500 train 6.254835 (lr=2.4215e-05) (hash(x)=140313123)
-5600 val loss 6.2472
-5600 val perplexity 516.5477
-5600 train 6.269135 (lr=2.4013e-05) (hash(x)=156997993)
-5700 val loss 6.2285
-5700 val perplexity 506.9784
-5700 train 6.237016 (lr=2.3808e-05) (hash(x)=143513880)
-5800 val loss 6.2168
-5800 val perplexity 501.1166
-5800 train 6.343710 (lr=2.3601e-05) (hash(x)=153736628)
-5900 val loss 6.2070
-5900 val perplexity 496.1949
-5900 train 6.360643 (lr=2.3391e-05) (hash(x)=166385417)
-6000 val loss 6.1948
-6000 val perplexity 490.2095
-6000 train 6.430104 (lr=2.3179e-05) (hash(x)=153216517)
-6100 val loss 6.1878
-6100 val perplexity 486.7515
-6100 train 6.067350 (lr=2.2965e-05) (hash(x)=145387545)
-6200 val loss 6.1868
-6200 val perplexity 486.2852
-6200 train 6.303199 (lr=2.2749e-05) (hash(x)=156306460)
-6300 val loss 6.1642
-6300 val perplexity 475.4185
-6300 train 6.087062 (lr=2.2530e-05) (hash(x)=148610447)
-6400 val loss 6.1500
-6400 val perplexity 468.7014
-6400 train 6.373208 (lr=2.2310e-05) (hash(x)=158343535)
-6500 val loss 6.1425
-6500 val perplexity 465.1938
-6500 train 6.307102 (lr=2.2088e-05) (hash(x)=157742557)
-6600 val loss 6.1336
-6600 val perplexity 461.1154
-6600 train 6.029664 (lr=2.1864e-05) (hash(x)=150457308)
-6700 val loss 6.1283
-6700 val perplexity 458.6432
-6700 train 5.993783 (lr=2.1638e-05) (hash(x)=142426205)
-6800 val loss 6.1207
-6800 val perplexity 455.1693
-6800 train 5.913719 (lr=2.1410e-05) (hash(x)=151522525)
-6900 val loss 6.1069
-6900 val perplexity 448.9227
-6900 train 6.053385 (lr=2.1181e-05) (hash(x)=152029630)
-7000 val loss 6.0944
-7000 val perplexity 443.3823
-7000 train 5.944612 (lr=2.0951e-05) (hash(x)=142978954)
-7100 val loss 6.0889
-7100 val perplexity 440.9485
-7100 train 5.960811 (lr=2.0718e-05) (hash(x)=148064359)
-7200 val loss 6.0795
-7200 val perplexity 436.8317
-7200 train 5.983203 (lr=2.0485e-05) (hash(x)=155771845)
-7300 val loss 6.0744
-7300 val perplexity 434.5692
-7300 train 5.878072 (lr=2.0250e-05) (hash(x)=141639844)
-7400 val loss 6.0725
-7400 val perplexity 433.7572
-7400 train 6.120244 (lr=2.0014e-05) (hash(x)=153466021)
-7500 val loss 6.0606
-7500 val perplexity 428.6192
-7500 train 6.057049 (lr=1.9777e-05) (hash(x)=148843587)
-7600 val loss 6.0469
-7600 val perplexity 422.8022
-7600 train 6.166252 (lr=1.9538e-05) (hash(x)=152053933)
-7700 val loss 6.0409
-7700 val perplexity 420.2540
-7700 train 5.897380 (lr=1.9299e-05) (hash(x)=141564003)
-7800 val loss 6.0309
-7800 val perplexity 416.0959
-7800 train 5.900531 (lr=1.9059e-05) (hash(x)=148617843)
-7900 val loss 6.0284
-7900 val perplexity 415.0530
-7900 train 5.994560 (lr=1.8817e-05) (hash(x)=146417668)
-8000 val loss 6.0244
-8000 val perplexity 413.3800
-8000 train 5.822050 (lr=1.8576e-05) (hash(x)=150165488)
-8100 val loss 6.0155
-8100 val perplexity 409.7361
-8100 train 6.002255 (lr=1.8333e-05) (hash(x)=144818855)
-8200 val loss 6.0087
-8200 val perplexity 406.9713
-8200 train 6.095023 (lr=1.8090e-05) (hash(x)=143074954)
-8300 val loss 5.9975
-8300 val perplexity 402.4019
-8300 train 5.877024 (lr=1.7846e-05) (hash(x)=145172494)
-8400 val loss 5.9932
-8400 val perplexity 400.7092
-8400 train 6.021608 (lr=1.7602e-05) (hash(x)=152284393)
-8500 val loss 5.9849
-8500 val perplexity 397.3773
-8500 train 5.836091 (lr=1.7357e-05) (hash(x)=147044705)
-8600 val loss 5.9824
-8600 val perplexity 396.3830
-8600 train 5.866980 (lr=1.7113e-05) (hash(x)=147762608)
-8700 val loss 5.9753
-8700 val perplexity 393.5761
-8700 train 5.968902 (lr=1.6868e-05) (hash(x)=141994567)
-8800 val loss 5.9670
-8800 val perplexity 390.3279
-8800 train 6.055788 (lr=1.6623e-05) (hash(x)=152552896)
-8900 val loss 5.9606
-8900 val perplexity 387.8401
-8900 train 5.794694 (lr=1.6377e-05) (hash(x)=139714454)
-9000 val loss 5.9507
-9000 val perplexity 384.0324
-9000 train 5.985847 (lr=1.6132e-05) (hash(x)=152274419)
-9100 val loss 5.9520
-9100 val perplexity 384.5033
-9100 train 6.051697 (lr=1.5887e-05) (hash(x)=160536885)
-9200 val loss 5.9469
-9200 val perplexity 382.5706
-9200 train 5.750583 (lr=1.5643e-05) (hash(x)=138386899)
-9300 val loss 5.9390
-9300 val perplexity 379.5516
-9300 train 5.991132 (lr=1.5398e-05) (hash(x)=149957457)
-9400 val loss 5.9296
-9400 val perplexity 375.9923
-9400 train 5.968359 (lr=1.5154e-05) (hash(x)=161671548)
-9500 val loss 5.9283
-9500 val perplexity 375.4995
-9500 train 6.068425 (lr=1.4910e-05) (hash(x)=171857400)
-9600 val loss 5.9188
-9600 val perplexity 371.9698
-9600 train 5.667602 (lr=1.4667e-05) (hash(x)=135378471)
-9700 val loss 5.9142
-9700 val perplexity 370.2644
-9700 train 5.849272 (lr=1.4424e-05) (hash(x)=149514591)
-9800 val loss 5.9104
-9800 val perplexity 368.8677
-9800 train 5.887805 (lr=1.4183e-05) (hash(x)=152679072)
-9900 val loss 5.9027
-9900 val perplexity 366.0232
-9900 train 5.846486 (lr=1.3941e-05) (hash(x)=150415193)
-10000 val loss 5.8980
-10000 val perplexity 364.3136
-10000 train 5.735629 (lr=1.3701e-05) (hash(x)=146910087)
-10100 val loss 5.8937
-10100 val perplexity 362.7524
-10100 train 5.806296 (lr=1.3462e-05) (hash(x)=152091050)
-10200 val loss 5.8943
-10200 val perplexity 362.9474
-10200 train 5.742750 (lr=1.3223e-05) (hash(x)=143101306)
-10300 val loss 5.8847
-10300 val perplexity 359.4808
-10300 train 5.894190 (lr=1.2986e-05) (hash(x)=150803985)
-10400 val loss 5.8856
-10400 val perplexity 359.8033
-10400 train 5.868137 (lr=1.2750e-05) (hash(x)=148671575)
-10500 val loss 5.8745
-10500 val perplexity 355.8593
-10500 train 5.674866 (lr=1.2515e-05) (hash(x)=146312794)
-10600 val loss 5.8748
-10600 val perplexity 355.9517
-10600 train 5.699982 (lr=1.2282e-05) (hash(x)=137878985)
-10700 val loss 5.8703
-10700 val perplexity 354.3533
-10700 train 5.673525 (lr=1.2049e-05) (hash(x)=148897486)
-10800 val loss 5.8628
-10800 val perplexity 351.6937
-10800 train 5.969318 (lr=1.1819e-05) (hash(x)=147219580)
-10900 val loss 5.8686
-10900 val perplexity 353.7402
-10900 train 5.893168 (lr=1.1590e-05) (hash(x)=152048929)
-11000 val loss 5.8514
-11000 val perplexity 347.7291
-11000 train 5.751215 (lr=1.1362e-05) (hash(x)=146417066)
-11100 val loss 5.8479
-11100 val perplexity 346.4909
-11100 train 5.713552 (lr=1.1136e-05) (hash(x)=148784436)
-11200 val loss 5.8447
-11200 val perplexity 345.3947
-11200 train 5.803506 (lr=1.0912e-05) (hash(x)=144721870)
-11300 val loss 5.8406
-11300 val perplexity 343.9958
-11300 train 5.762826 (lr=1.0690e-05) (hash(x)=149442981)
-11400 val loss 5.8382
-11400 val perplexity 343.1647
-11400 train 5.735843 (lr=1.0470e-05) (hash(x)=154481228)
-11500 val loss 5.8336
-11500 val perplexity 341.5893
-11500 train 5.523773 (lr=1.0251e-05) (hash(x)=115164581)
-11600 val loss 5.8279
-11600 val perplexity 339.6416
-11600 train 6.049501 (lr=1.0035e-05) (hash(x)=155048082)
-11700 val loss 5.8249
-11700 val perplexity 338.6386
-11700 train 5.881338 (lr=9.8209e-06) (hash(x)=152123770)
-11800 val loss 5.8224
-11800 val perplexity 337.7885
-11800 train 5.759107 (lr=9.6090e-06) (hash(x)=152705857)
-11900 val loss 5.8188
-11900 val perplexity 336.5702
-11900 train 5.538026 (lr=9.3993e-06) (hash(x)=137392362)
-12000 val loss 5.8193
-12000 val perplexity 336.7239
-12000 train 5.628447 (lr=9.1920e-06) (hash(x)=143492562)
-12100 val loss 5.8126
-12100 val perplexity 334.4966
-12100 train 5.782037 (lr=8.9871e-06) (hash(x)=141825431)
-12200 val loss 5.8071
-12200 val perplexity 332.6457
-12200 train 5.707171 (lr=8.7846e-06) (hash(x)=141774370)
-12300 val loss 5.8049
-12300 val perplexity 331.9221
-12300 train 5.825204 (lr=8.5847e-06) (hash(x)=152995173)
-12400 val loss 5.8022
-12400 val perplexity 331.0165
-12400 train 5.623291 (lr=8.3875e-06) (hash(x)=144703471)
-12500 val loss 5.7999
-12500 val perplexity 330.2648
-12500 train 5.694048 (lr=8.1929e-06) (hash(x)=144866179)
-12600 val loss 5.7973
-12600 val perplexity 329.4210
-12600 train 5.801395 (lr=8.0010e-06) (hash(x)=157019663)
-12700 val loss 5.7957
-12700 val perplexity 328.8947
-12700 train 5.754167 (lr=7.8119e-06) (hash(x)=152872020)
-12800 val loss 5.7910
-12800 val perplexity 327.3514
-12800 train 5.827247 (lr=7.6257e-06) (hash(x)=158975636)
-12900 val loss 5.7854
-12900 val perplexity 325.5142
-12900 train 5.825149 (lr=7.4425e-06) (hash(x)=143495701)
-13000 val loss 5.7849
-13000 val perplexity 325.3500
-13000 train 5.703864 (lr=7.2622e-06) (hash(x)=151940005)
-13100 val loss 5.7801
-13100 val perplexity 323.7766
-13100 train 6.062990 (lr=7.0850e-06) (hash(x)=155920713)
-13200 val loss 5.7798
-13200 val perplexity 323.7076
-13200 train 5.957345 (lr=6.9108e-06) (hash(x)=152145878)
-13300 val loss 5.7770
-13300 val perplexity 322.7914
-13300 train 5.568100 (lr=6.7398e-06) (hash(x)=134141709)
-13400 val loss 5.7791
-13400 val perplexity 323.4539
-13400 train 5.697982 (lr=6.5721e-06) (hash(x)=146096299)
-13500 val loss 5.7754
-13500 val perplexity 322.2620
-13500 train 5.530023 (lr=6.4076e-06) (hash(x)=144011431)
-13600 val loss 5.7691
-13600 val perplexity 320.2527
-13600 train 6.011714 (lr=6.2465e-06) (hash(x)=163950932)
-13700 val loss 5.7677
-13700 val perplexity 319.8127
-13700 train 5.914011 (lr=6.0887e-06) (hash(x)=156652406)
-13800 val loss 5.7617
-13800 val perplexity 317.8942
-13800 train 5.671328 (lr=5.9344e-06) (hash(x)=145753166)
-13900 val loss 5.7614
-13900 val perplexity 317.7904
-13900 train 5.626029 (lr=5.7835e-06) (hash(x)=147065551)
-14000 val loss 5.7574
-14000 val perplexity 316.5275
-14000 train 6.217151 (lr=5.6362e-06) (hash(x)=179077542)
-14100 val loss 5.7576
-14100 val perplexity 316.5848
-14100 train 5.831569 (lr=5.4925e-06) (hash(x)=171962802)
-14200 val loss 5.7569
-14200 val perplexity 316.3727
-14200 train 5.824154 (lr=5.3524e-06) (hash(x)=153299913)
-14300 val loss 5.7570
-14300 val perplexity 316.4114
-14300 train 5.687628 (lr=5.2159e-06) (hash(x)=154750565)
-14400 val loss 5.7550
-14400 val perplexity 315.7674
-14400 train 6.040416 (lr=5.0832e-06) (hash(x)=154536694)
-14500 val loss 5.7488
-14500 val perplexity 313.8270
-14500 train 5.798073 (lr=4.9543e-06) (hash(x)=151236383)
-14600 val loss 5.7461
-14600 val perplexity 312.9566
-14600 train 5.754922 (lr=4.8291e-06) (hash(x)=152584272)
-14700 val loss 5.7443
-14700 val perplexity 312.4029
-14700 train 5.761162 (lr=4.7079e-06) (hash(x)=175301826)
-14800 val loss 5.7434
-14800 val perplexity 312.1136
-14800 train 5.661438 (lr=4.5905e-06) (hash(x)=148729408)
-14900 val loss 5.7431
-14900 val perplexity 312.0179
-14900 train 5.918625 (lr=4.4770e-06) (hash(x)=150978283)
-15000 val loss 5.7427
-15000 val perplexity 311.9147
-15000 train 5.683867 (lr=4.3675e-06) (hash(x)=153621389)
-15100 val loss 5.7424
-15100 val perplexity 311.8216
-15100 train 5.667899 (lr=4.2620e-06) (hash(x)=150653793)
-15200 val loss 5.7405
-15200 val perplexity 311.2079
-15200 train 5.915138 (lr=4.1605e-06) (hash(x)=140802934)
-15300 val loss 5.7337
-15300 val perplexity 309.1233
-15300 train 5.712039 (lr=4.0631e-06) (hash(x)=152007556)
-15400 val loss 5.7335
-15400 val perplexity 309.0579
-15400 train 5.686319 (lr=3.9698e-06) (hash(x)=149057446)
-15500 val loss 5.7319
-15500 val perplexity 308.5640
-15500 train 5.655775 (lr=3.8806e-06) (hash(x)=147521774)
-15600 val loss 5.7299
-15600 val perplexity 307.9506
-15600 train 5.693157 (lr=3.7956e-06) (hash(x)=148944405)
-15700 val loss 5.7300
-15700 val perplexity 307.9562
-15700 train 5.683930 (lr=3.7148e-06) (hash(x)=148472532)
-15800 val loss 5.7297
-15800 val perplexity 307.8897
-15800 train 5.504209 (lr=3.6382e-06) (hash(x)=149048780)
-15900 val loss 5.7301
-15900 val perplexity 308.0150
-15900 train 5.690699 (lr=3.5658e-06) (hash(x)=150175778)
-16000 val loss 5.7306
-16000 val perplexity 308.1541
-16000 train 5.519766 (lr=3.4977e-06) (hash(x)=145943831)
-16100 val loss 5.7238
-16100 val perplexity 306.0658
-16100 train 5.874838 (lr=3.4339e-06) (hash(x)=134314842)
-16200 val loss 5.7219
-16200 val perplexity 305.4991
-16200 train 5.747156 (lr=3.3744e-06) (hash(x)=153755135)
-16300 val loss 5.7224
-16300 val perplexity 305.6330
-16300 train 5.851801 (lr=3.3193e-06) (hash(x)=146916346)
-16400 val loss 5.7225
-16400 val perplexity 305.6758
-16400 train 5.643525 (lr=3.2684e-06) (hash(x)=155023643)
-16500 val loss 5.7201
-16500 val perplexity 304.9290
-16500 train 5.794312 (lr=3.2220e-06) (hash(x)=148910191)
-16600 val loss 5.7190
-16600 val perplexity 304.6099
-16600 train 5.634387 (lr=3.1799e-06) (hash(x)=150107746)
-16700 val loss 5.7152
-16700 val perplexity 303.4561
-16700 train 5.852514 (lr=3.1422e-06) (hash(x)=159943976)
-16800 val loss 5.7141
-16800 val perplexity 303.1232
-16800 train 5.711351 (lr=3.1089e-06) (hash(x)=160255226)
-16900 val loss 5.7129
-16900 val perplexity 302.7485
-16900 train 5.699970 (lr=3.0801e-06) (hash(x)=157121826)
-17000 val loss 5.7126
-17000 val perplexity 302.6575
-17000 train 5.846539 (lr=3.0556e-06) (hash(x)=159298029)
-17100 val loss 5.7119
-17100 val perplexity 302.4565
-17100 train 5.660579 (lr=3.0356e-06) (hash(x)=149355903)
-17200 val loss 5.7121
-17200 val perplexity 302.5085
-17200 train 5.529926 (lr=3.0200e-06) (hash(x)=144467909)
-17300 val loss 5.7097
-17300 val perplexity 301.7834
-17300 train 5.881879 (lr=3.0089e-06) (hash(x)=157150350)
-17400 val loss 5.7080
-17400 val perplexity 301.2650
-17400 train 5.758003 (lr=3.0022e-06) (hash(x)=154825212)
-17499 val loss 5.7072
-17499 val perplexity 301.0173
+0 train 11.308768 (lr=1.3750e-07) (hash(x)=145079536)
+100 val loss 9.9993
+100 val perplexity 22010.1719
+100 train 9.939669 (lr=1.3888e-05) (hash(x)=154745873)
+200 val loss 9.1864
+200 val perplexity 9763.2383
+200 train 9.216236 (lr=2.7500e-05) (hash(x)=155800595)
+300 val loss 8.0336
+300 val perplexity 3082.9048
+300 train 8.000475 (lr=2.7498e-05) (hash(x)=148595389)
+400 val loss 7.6898
+400 val perplexity 2185.9587
+400 train 7.606212 (lr=2.7492e-05) (hash(x)=145606733)
+500 val loss 7.5730
+500 val perplexity 1944.9144
+500 train 7.413569 (lr=2.7482e-05) (hash(x)=138221231)
+600 val loss 7.5082
+600 val perplexity 1822.9774
+600 train 7.518361 (lr=2.7467e-05) (hash(x)=150367139)
+700 val loss 7.4621
+700 val perplexity 1740.8705
+700 train 7.613705 (lr=2.7449e-05) (hash(x)=155579314)
+800 val loss 7.4197
+800 val perplexity 1668.4584
+800 train 7.498192 (lr=2.7427e-05) (hash(x)=155747374)
+900 val loss 7.3766
+900 val perplexity 1598.2067
+900 train 7.581522 (lr=2.7400e-05) (hash(x)=159334575)
+1000 val loss 7.3366
+1000 val perplexity 1535.4816
+1000 train 7.210344 (lr=2.7370e-05) (hash(x)=140604760)
+1100 val loss 7.3040
+1100 val perplexity 1486.2832
+1100 train 7.140453 (lr=2.7335e-05) (hash(x)=146208052)
+1200 val loss 7.2680
+1200 val perplexity 1433.7283
+1200 train 7.167972 (lr=2.7297e-05) (hash(x)=148404734)
+1300 val loss 7.2364
+1300 val perplexity 1389.1176
+1300 train 7.170488 (lr=2.7254e-05) (hash(x)=155681970)
+1400 val loss 7.2065
+1400 val perplexity 1348.1545
+1400 train 7.154507 (lr=2.7207e-05) (hash(x)=148115934)
+1500 val loss 7.1673
+1500 val perplexity 1296.3444
+1500 train 7.319304 (lr=2.7157e-05) (hash(x)=157074034)
+1600 val loss 7.1201
+1600 val perplexity 1236.6083
+1600 train 6.882675 (lr=2.7102e-05) (hash(x)=137464699)
+1700 val loss 7.0777
+1700 val perplexity 1185.2509
+1700 train 7.243387 (lr=2.7044e-05) (hash(x)=166955614)
+1800 val loss 7.0408
+1800 val perplexity 1142.3480
+1800 train 6.867510 (lr=2.6981e-05) (hash(x)=143886042)
+1900 val loss 7.0111
+1900 val perplexity 1108.8199
+1900 train 7.622176 (lr=2.6915e-05) (hash(x)=193610391)
+2000 val loss 6.9824
+2000 val perplexity 1077.5485
+2000 train 7.160104 (lr=2.6845e-05) (hash(x)=163799796)
+2100 val loss 6.9365
+2100 val perplexity 1029.1138
+2100 train 6.915351 (lr=2.6771e-05) (hash(x)=154107339)
+2200 val loss 6.9076
+2200 val perplexity 999.8671
+2200 train 6.908937 (lr=2.6693e-05) (hash(x)=144592844)
+2300 val loss 6.8779
+2300 val perplexity 970.5944
+2300 train 6.958189 (lr=2.6611e-05) (hash(x)=154535861)
+2400 val loss 6.8486
+2400 val perplexity 942.5140
+2400 train 7.593205 (lr=2.6525e-05) (hash(x)=204706354)
+2500 val loss 6.8168
+2500 val perplexity 913.0473
+2500 train 6.615187 (lr=2.6436e-05) (hash(x)=143783202)
+2600 val loss 6.7786
+2600 val perplexity 878.8146
+2600 train 6.676283 (lr=2.6343e-05) (hash(x)=150862210)
+2700 val loss 6.7375
+2700 val perplexity 843.4412
+2700 train 6.790683 (lr=2.6246e-05) (hash(x)=150753426)
+2800 val loss 6.7151
+2800 val perplexity 824.7335
+2800 train 6.720888 (lr=2.6146e-05) (hash(x)=147766811)
+2900 val loss 6.6649
+2900 val perplexity 784.4006
+2900 train 6.601873 (lr=2.6042e-05) (hash(x)=151177814)
+3000 val loss 6.6305
+3000 val perplexity 757.8395
+3000 train 6.486729 (lr=2.5934e-05) (hash(x)=135925327)
+3100 val loss 6.6070
+3100 val perplexity 740.2267
+3100 train 6.250475 (lr=2.5823e-05) (hash(x)=124842353)
+3200 val loss 6.5818
+3200 val perplexity 721.8633
+3200 train 6.691461 (lr=2.5709e-05) (hash(x)=160440642)
+3300 val loss 6.5474
+3300 val perplexity 697.4006
+3300 train 6.458886 (lr=2.5590e-05) (hash(x)=143017131)
+3400 val loss 6.5191
+3400 val perplexity 677.9949
+3400 train 6.827440 (lr=2.5469e-05) (hash(x)=148143474)
+3500 val loss 6.4999
+3500 val perplexity 665.0741
+3500 train 6.319547 (lr=2.5344e-05) (hash(x)=148732639)
+3600 val loss 6.4765
+3600 val perplexity 649.7057
+3600 train 6.477383 (lr=2.5215e-05) (hash(x)=146333803)
+3700 val loss 6.4615
+3700 val perplexity 640.0327
+3700 train 6.259657 (lr=2.5083e-05) (hash(x)=143611445)
+3800 val loss 6.4465
+3800 val perplexity 630.4874
+3800 train 6.230040 (lr=2.4948e-05) (hash(x)=141272115)
+3900 val loss 6.4254
+3900 val perplexity 617.3164
+3900 train 6.358279 (lr=2.4810e-05) (hash(x)=149161848)
+4000 val loss 6.3938
+4000 val perplexity 598.1159
+4000 train 6.475472 (lr=2.4669e-05) (hash(x)=152036955)
+4100 val loss 6.3766
+4100 val perplexity 587.9465
+4100 train 6.431635 (lr=2.4524e-05) (hash(x)=153260968)
+4200 val loss 6.3591
+4200 val perplexity 577.7065
+4200 train 6.244727 (lr=2.4376e-05) (hash(x)=147629478)
+4300 val loss 6.3432
+4300 val perplexity 568.6074
+4300 train 6.326876 (lr=2.4226e-05) (hash(x)=161701080)
+4400 val loss 6.3301
+4400 val perplexity 561.1898
+4400 train 6.374499 (lr=2.4072e-05) (hash(x)=155290524)
+4500 val loss 6.3202
+4500 val perplexity 555.6777
+4500 train 6.102327 (lr=2.3915e-05) (hash(x)=144726593)
+4600 val loss 6.3096
+4600 val perplexity 549.8514
+4600 train 6.254610 (lr=2.3755e-05) (hash(x)=149517202)
+4700 val loss 6.2907
+4700 val perplexity 539.5439
+4700 train 6.293215 (lr=2.3593e-05) (hash(x)=139403008)
+4800 val loss 6.2714
+4800 val perplexity 529.2137
+4800 train 6.360953 (lr=2.3428e-05) (hash(x)=153387238)
+4900 val loss 6.2676
+4900 val perplexity 527.2242
+4900 train 6.016084 (lr=2.3260e-05) (hash(x)=139881569)
+5000 val loss 6.2394
+5000 val perplexity 512.5274
+5000 train 6.236556 (lr=2.3089e-05) (hash(x)=152602580)
+5100 val loss 6.2321
+5100 val perplexity 508.8243
+5100 train 6.281746 (lr=2.2916e-05) (hash(x)=150598460)
+5200 val loss 6.2226
+5200 val perplexity 503.9968
+5200 train 6.106723 (lr=2.2740e-05) (hash(x)=147660496)
+5300 val loss 6.2120
+5300 val perplexity 498.7130
+5300 train 5.953794 (lr=2.2561e-05) (hash(x)=147693848)
+5400 val loss 6.2089
+5400 val perplexity 497.1427
+5400 train 5.962179 (lr=2.2381e-05) (hash(x)=136048145)
+5500 val loss 6.1849
+5500 val perplexity 485.3567
+5500 train 6.196786 (lr=2.2197e-05) (hash(x)=140313123)
+5600 val loss 6.1751
+5600 val perplexity 480.6118
+5600 train 6.196830 (lr=2.2012e-05) (hash(x)=156997993)
+5700 val loss 6.1596
+5700 val perplexity 473.2623
+5700 train 6.171885 (lr=2.1824e-05) (hash(x)=143513880)
+5800 val loss 6.1521
+5800 val perplexity 469.7099
+5800 train 6.271136 (lr=2.1634e-05) (hash(x)=153736628)
+5900 val loss 6.1446
+5900 val perplexity 466.1858
+5900 train 6.286342 (lr=2.1442e-05) (hash(x)=166385417)
+6000 val loss 6.1329
+6000 val perplexity 460.7750
+6000 train 6.367750 (lr=2.1248e-05) (hash(x)=153216517)
+6100 val loss 6.1302
+6100 val perplexity 459.5361
+6100 train 6.014255 (lr=2.1051e-05) (hash(x)=145387545)
+6200 val loss 6.1215
+6200 val perplexity 455.5325
+6200 train 6.245870 (lr=2.0853e-05) (hash(x)=156306460)
+6300 val loss 6.1079
+6300 val perplexity 449.4084
+6300 train 6.042010 (lr=2.0653e-05) (hash(x)=148610447)
+6400 val loss 6.0925
+6400 val perplexity 442.5284
+6400 train 6.320738 (lr=2.0451e-05) (hash(x)=158343535)
+6500 val loss 6.0817
+6500 val perplexity 437.7669
+6500 train 6.248390 (lr=2.0247e-05) (hash(x)=157742557)
+6600 val loss 6.0811
+6600 val perplexity 437.5079
+6600 train 5.983360 (lr=2.0042e-05) (hash(x)=150457308)
+6700 val loss 6.0731
+6700 val perplexity 434.0280
+6700 train 5.936596 (lr=1.9835e-05) (hash(x)=142426205)
+6800 val loss 6.0693
+6800 val perplexity 432.3716
+6800 train 5.861384 (lr=1.9626e-05) (hash(x)=151522525)
+6900 val loss 6.0550
+6900 val perplexity 426.2327
+6900 train 6.003469 (lr=1.9416e-05) (hash(x)=152029630)
+7000 val loss 6.0406
+7000 val perplexity 420.1532
+7000 train 5.885346 (lr=1.9205e-05) (hash(x)=142978954)
+7100 val loss 6.0321
+7100 val perplexity 416.6035
+7100 train 5.914243 (lr=1.8992e-05) (hash(x)=148064359)
+7200 val loss 6.0269
+7200 val perplexity 414.4176
+7200 train 5.927231 (lr=1.8778e-05) (hash(x)=155771845)
+7300 val loss 6.0232
+7300 val perplexity 412.8852
+7300 train 5.828673 (lr=1.8562e-05) (hash(x)=141639844)
+7400 val loss 6.0229
+7400 val perplexity 412.7574
+7400 train 6.078304 (lr=1.8346e-05) (hash(x)=153466021)
+7500 val loss 6.0097
+7500 val perplexity 407.3585
+7500 train 6.001205 (lr=1.8128e-05) (hash(x)=148843587)
+7600 val loss 5.9989
+7600 val perplexity 402.9885
+7600 train 6.119777 (lr=1.7910e-05) (hash(x)=152053933)
+7700 val loss 5.9926
+7700 val perplexity 400.4386
+7700 train 5.853600 (lr=1.7691e-05) (hash(x)=141564003)
+7800 val loss 5.9839
+7800 val perplexity 396.9968
+7800 train 5.855370 (lr=1.7470e-05) (hash(x)=148617843)
+7900 val loss 5.9810
+7900 val perplexity 395.8490
+7900 train 5.949779 (lr=1.7249e-05) (hash(x)=146417668)
+8000 val loss 5.9751
+8000 val perplexity 393.5120
+8000 train 5.772148 (lr=1.7028e-05) (hash(x)=150165488)
+8100 val loss 5.9658
+8100 val perplexity 389.8839
+8100 train 5.963167 (lr=1.6805e-05) (hash(x)=144818855)
+8200 val loss 5.9597
+8200 val perplexity 387.4996
+8200 train 6.048904 (lr=1.6582e-05) (hash(x)=143074954)
+8300 val loss 5.9525
+8300 val perplexity 384.7203
+8300 train 5.833498 (lr=1.6359e-05) (hash(x)=145172494)
+8400 val loss 5.9454
+8400 val perplexity 381.9796
+8400 train 5.982568 (lr=1.6135e-05) (hash(x)=152284393)
+8500 val loss 5.9396
+8500 val perplexity 379.7978
+8500 train 5.799961 (lr=1.5911e-05) (hash(x)=147044705)
+8600 val loss 5.9406
+8600 val perplexity 380.1599
+8600 train 5.820881 (lr=1.5687e-05) (hash(x)=147762608)
+8700 val loss 5.9296
+8700 val perplexity 375.9991
+8700 train 5.935435 (lr=1.5462e-05) (hash(x)=141994567)
+8800 val loss 5.9242
+8800 val perplexity 373.9842
+8800 train 6.012501 (lr=1.5237e-05) (hash(x)=152552896)
+8900 val loss 5.9175
+8900 val perplexity 371.4999
+8900 train 5.755026 (lr=1.5013e-05) (hash(x)=139714454)
+9000 val loss 5.9095
+9000 val perplexity 368.5270
+9000 train 5.943279 (lr=1.4788e-05) (hash(x)=152274419)
+9100 val loss 5.9073
+9100 val perplexity 367.7229
+9100 train 5.999683 (lr=1.4563e-05) (hash(x)=160536885)
+9200 val loss 5.9038
+9200 val perplexity 366.4267
+9200 train 5.710702 (lr=1.4339e-05) (hash(x)=138386899)
+9300 val loss 5.8969
+9300 val perplexity 363.8964
+9300 train 5.944336 (lr=1.4115e-05) (hash(x)=149957457)
+9400 val loss 5.8869
+9400 val perplexity 360.2753
+9400 train 5.909703 (lr=1.3891e-05) (hash(x)=161671548)
+9500 val loss 5.8875
+9500 val perplexity 360.5039
+9500 train 6.022851 (lr=1.3668e-05) (hash(x)=171857400)
+9600 val loss 5.8794
+9600 val perplexity 357.5925
+9600 train 5.638875 (lr=1.3445e-05) (hash(x)=135378471)
+9700 val loss 5.8734
+9700 val perplexity 355.4576
+9700 train 5.805750 (lr=1.3222e-05) (hash(x)=149514591)
+9800 val loss 5.8676
+9800 val perplexity 353.3880
+9800 train 5.845808 (lr=1.3001e-05) (hash(x)=152679072)
+9900 val loss 5.8620
+9900 val perplexity 351.4184
+9900 train 5.809219 (lr=1.2780e-05) (hash(x)=150415193)
+10000 val loss 5.8582
+10000 val perplexity 350.1083
+10000 train 5.701046 (lr=1.2559e-05) (hash(x)=146910087)
+10100 val loss 5.8528
+10100 val perplexity 348.1940
+10100 train 5.768382 (lr=1.2340e-05) (hash(x)=152091050)
+10200 val loss 5.8537
+10200 val perplexity 348.5076
+10200 train 5.697745 (lr=1.2122e-05) (hash(x)=143101306)
+10300 val loss 5.8457
+10300 val perplexity 345.7297
+10300 train 5.859143 (lr=1.1904e-05) (hash(x)=150803985)
+10400 val loss 5.8446
+10400 val perplexity 345.3572
+10400 train 5.820405 (lr=1.1688e-05) (hash(x)=148671575)
+10500 val loss 5.8366
+10500 val perplexity 342.6211
+10500 train 5.637980 (lr=1.1472e-05) (hash(x)=146312794)
+10600 val loss 5.8366
+10600 val perplexity 342.6121
+10600 train 5.661836 (lr=1.1258e-05) (hash(x)=137878985)
+10700 val loss 5.8311
+10700 val perplexity 340.7318
+10700 train 5.640622 (lr=1.1045e-05) (hash(x)=148897486)
+10800 val loss 5.8273
+10800 val perplexity 339.4301
+10800 train 5.929305 (lr=1.0834e-05) (hash(x)=147219580)
+10900 val loss 5.8305
+10900 val perplexity 340.5177
+10900 train 5.855745 (lr=1.0624e-05) (hash(x)=152048929)
+11000 val loss 5.8134
+11000 val perplexity 334.7551
+11000 train 5.701807 (lr=1.0415e-05) (hash(x)=146417066)
+11100 val loss 5.8115
+11100 val perplexity 334.1171
+11100 train 5.671261 (lr=1.0208e-05) (hash(x)=148784436)
+11200 val loss 5.8088
+11200 val perplexity 333.2312
+11200 train 5.778788 (lr=1.0003e-05) (hash(x)=144721870)
+11300 val loss 5.8068
+11300 val perplexity 332.5641
+11300 train 5.729158 (lr=9.7991e-06) (hash(x)=149442981)
+11400 val loss 5.8033
+11400 val perplexity 331.3796
+11400 train 5.692762 (lr=9.5972e-06) (hash(x)=154481228)
+11500 val loss 5.7970
+11500 val perplexity 329.2994
+11500 train 5.479829 (lr=9.3970e-06) (hash(x)=115164581)
+11600 val loss 5.7922
+11600 val perplexity 327.7303
+11600 train 6.014553 (lr=9.1988e-06) (hash(x)=155048082)
+11700 val loss 5.7885
+11700 val perplexity 326.5278
+11700 train 5.846733 (lr=9.0025e-06) (hash(x)=152123770)
+11800 val loss 5.7866
+11800 val perplexity 325.8922
+11800 train 5.729887 (lr=8.8082e-06) (hash(x)=152705857)
+11900 val loss 5.7835
+11900 val perplexity 324.8806
+11900 train 5.503841 (lr=8.6160e-06) (hash(x)=137392362)
+12000 val loss 5.7836
+12000 val perplexity 324.9294
+12000 train 5.593571 (lr=8.4260e-06) (hash(x)=143492562)
+12100 val loss 5.7785
+12100 val perplexity 323.2764
+12100 train 5.755323 (lr=8.2381e-06) (hash(x)=141825431)
+12200 val loss 5.7731
+12200 val perplexity 321.5305
+12200 train 5.674329 (lr=8.0526e-06) (hash(x)=141774370)
+12300 val loss 5.7699
+12300 val perplexity 320.4902
+12300 train 5.789728 (lr=7.8694e-06) (hash(x)=152995173)
+12400 val loss 5.7699
+12400 val perplexity 320.5191
+12400 train 5.592089 (lr=7.6885e-06) (hash(x)=144703471)
+12500 val loss 5.7671
+12500 val perplexity 319.6013
+12500 train 5.667903 (lr=7.5101e-06) (hash(x)=144866179)
+12600 val loss 5.7646
+12600 val perplexity 318.8022
+12600 train 5.770620 (lr=7.3342e-06) (hash(x)=157019663)
+12700 val loss 5.7630
+12700 val perplexity 318.2904
+12700 train 5.726803 (lr=7.1609e-06) (hash(x)=152872020)
+12800 val loss 5.7580
+12800 val perplexity 316.7254
+12800 train 5.795256 (lr=6.9903e-06) (hash(x)=158975636)
+12900 val loss 5.7524
+12900 val perplexity 314.9327
+12900 train 5.794280 (lr=6.8223e-06) (hash(x)=143495701)
+13000 val loss 5.7524
+13000 val perplexity 314.9420
+13000 train 5.674680 (lr=6.6570e-06) (hash(x)=151940005)
+13100 val loss 5.7502
+13100 val perplexity 314.2474
+13100 train 6.036286 (lr=6.4945e-06) (hash(x)=155920713)
+13200 val loss 5.7482
+13200 val perplexity 313.6359
+13200 train 5.937761 (lr=6.3349e-06) (hash(x)=152145878)
+13300 val loss 5.7452
+13300 val perplexity 312.6820
+13300 train 5.543325 (lr=6.1782e-06) (hash(x)=134141709)
+13400 val loss 5.7481
+13400 val perplexity 313.5864
+13400 train 5.669959 (lr=6.0244e-06) (hash(x)=146096299)
+13500 val loss 5.7450
+13500 val perplexity 312.6364
+13500 train 5.501187 (lr=5.8737e-06) (hash(x)=144011431)
+13600 val loss 5.7382
+13600 val perplexity 310.4945
+13600 train 5.982531 (lr=5.7259e-06) (hash(x)=163950932)
+13700 val loss 5.7341
+13700 val perplexity 309.2354
+13700 train 5.881189 (lr=5.5813e-06) (hash(x)=156652406)
+13800 val loss 5.7304
+13800 val perplexity 308.0932
+13800 train 5.646419 (lr=5.4398e-06) (hash(x)=145753166)
+13900 val loss 5.7296
+13900 val perplexity 307.8456
+13900 train 5.600583 (lr=5.3016e-06) (hash(x)=147065551)
+14000 val loss 5.7269
+14000 val perplexity 307.0269
+14000 train 6.192235 (lr=5.1665e-06) (hash(x)=179077542)
+14100 val loss 5.7259
+14100 val perplexity 306.7160
+14100 train 5.803774 (lr=5.0348e-06) (hash(x)=171962802)
+14200 val loss 5.7267
+14200 val perplexity 306.9601
+14200 train 5.801540 (lr=4.9063e-06) (hash(x)=153299913)
+14300 val loss 5.7263
+14300 val perplexity 306.8460
+14300 train 5.652976 (lr=4.7813e-06) (hash(x)=154750565)
+14400 val loss 5.7242
+14400 val perplexity 306.1801
+14400 train 6.001330 (lr=4.6596e-06) (hash(x)=154536694)
+14500 val loss 5.7192
+14500 val perplexity 304.6466
+14500 train 5.773301 (lr=4.5414e-06) (hash(x)=151236383)
+14600 val loss 5.7168
+14600 val perplexity 303.9378
+14600 train 5.725638 (lr=4.4267e-06) (hash(x)=152584272)
+14700 val loss 5.7140
+14700 val perplexity 303.0956
+14700 train 5.713636 (lr=4.3155e-06) (hash(x)=175301826)
+14800 val loss 5.7137
+14800 val perplexity 302.9854
+14800 train 5.628159 (lr=4.2079e-06) (hash(x)=148729408)
+14900 val loss 5.7130
+14900 val perplexity 302.7845
+14900 train 5.891885 (lr=4.1039e-06) (hash(x)=150978283)
+15000 val loss 5.7144
+15000 val perplexity 303.2153
+15000 train 5.661406 (lr=4.0035e-06) (hash(x)=153621389)
+15100 val loss 5.7129
+15100 val perplexity 302.7448
+15100 train 5.637873 (lr=3.9068e-06) (hash(x)=150653793)
+15200 val loss 5.7099
+15200 val perplexity 301.8424
+15200 train 5.894018 (lr=3.8138e-06) (hash(x)=140802934)
+15300 val loss 5.7049
+15300 val perplexity 300.3208
+15300 train 5.675510 (lr=3.7245e-06) (hash(x)=152007556)
+15400 val loss 5.7041
+15400 val perplexity 300.0896
+15400 train 5.662685 (lr=3.6390e-06) (hash(x)=149057446)
+15500 val loss 5.7029
+15500 val perplexity 299.7485
+15500 train 5.633412 (lr=3.5572e-06) (hash(x)=147521774)
+15600 val loss 5.7019
+15600 val perplexity 299.4391
+15600 train 5.676160 (lr=3.4793e-06) (hash(x)=148944405)
+15700 val loss 5.7012
+15700 val perplexity 299.2253
+15700 train 5.650658 (lr=3.4052e-06) (hash(x)=148472532)
+15800 val loss 5.7003
+15800 val perplexity 298.9456
+15800 train 5.472708 (lr=3.3350e-06) (hash(x)=149048780)
+15900 val loss 5.7025
+15900 val perplexity 299.6013
+15900 train 5.669323 (lr=3.2687e-06) (hash(x)=150175778)
+16000 val loss 5.7021
+16000 val perplexity 299.4836
+16000 train 5.492558 (lr=3.2063e-06) (hash(x)=145943831)
+16100 val loss 5.6955
+16100 val perplexity 297.5219
+16100 train 5.860049 (lr=3.1478e-06) (hash(x)=134314842)
+16200 val loss 5.6942
+16200 val perplexity 297.1338
+16200 train 5.722423 (lr=3.0932e-06) (hash(x)=153755135)
+16300 val loss 5.6933
+16300 val perplexity 296.8820
+16300 train 5.815263 (lr=3.0427e-06) (hash(x)=146916346)
+16400 val loss 5.6944
+16400 val perplexity 297.2124
+16400 train 5.611971 (lr=2.9961e-06) (hash(x)=155023643)
+16500 val loss 5.6925
+16500 val perplexity 296.6199
+16500 train 5.773914 (lr=2.9535e-06) (hash(x)=148910191)
+16600 val loss 5.6919
+16600 val perplexity 296.4496
+16600 train 5.614566 (lr=2.9149e-06) (hash(x)=150107746)
+16700 val loss 5.6883
+16700 val perplexity 295.3911
+16700 train 5.811312 (lr=2.8804e-06) (hash(x)=159943976)
+16800 val loss 5.6868
+16800 val perplexity 294.9484
+16800 train 5.690961 (lr=2.8498e-06) (hash(x)=160255226)
+16900 val loss 5.6852
+16900 val perplexity 294.4747
+16900 train 5.677330 (lr=2.8234e-06) (hash(x)=157121826)
+17000 val loss 5.6848
+17000 val perplexity 294.3450
+17000 train 5.815355 (lr=2.8010e-06) (hash(x)=159298029)
+17100 val loss 5.6850
+17100 val perplexity 294.4066
+17100 train 5.642745 (lr=2.7826e-06) (hash(x)=149355903)
+17200 val loss 5.6852
+17200 val perplexity 294.4778
+17200 train 5.502587 (lr=2.7684e-06) (hash(x)=144467909)
+17300 val loss 5.6827
+17300 val perplexity 293.7423
+17300 train 5.851804 (lr=2.7582e-06) (hash(x)=157150350)
+17400 val loss 5.6817
+17400 val perplexity 293.4457
+17400 train 5.723725 (lr=2.7520e-06) (hash(x)=154825212)
+17499 val loss 5.6803
+17499 val perplexity 293.0353

attention_kindselective_n_heads4_seed1341/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d0a2f2b27fd5eb105bf18c7fb787a33f930c1969ed34839a6a4e3d80c229c920
+oid sha256:1c7cd902b46cbc421e2c48ee77f0e0aa4b5befdcf357d1af96e2d5e9ccd48c45
 size 92843394

attention_kindselective_n_heads4_seed1341/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a3c839abef22afa1e6ca2833950a7cb0f4fa92ec28ef715594bf1dda52b98965
+oid sha256:4a0cb68dbad563d91d661387bf39ace20ba219489270a1c9461a784234115623
 size 179406214