Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1342/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1342", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1342", "n_embd": 256}

attention_kindselective_n_heads4_seed1342/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2907
 0 val perplexity 80077.4453
-0 train 11.286213 (lr=1.5000e-07) (hash(x)=150332693)
-100 val loss 9.9351
-100 val perplexity 20642.3594
-100 train 9.958242 (lr=1.5150e-05) (hash(x)=148948662)
-200 val loss 9.0369
-200 val perplexity 8407.2920
-200 train 8.958506 (lr=3.0000e-05) (hash(x)=149277319)
-300 val loss 7.9533
-300 val perplexity 2844.9207
-300 train 7.853283 (lr=2.9998e-05) (hash(x)=146429920)
-400 val loss 7.6313
-400 val perplexity 2061.7974
-400 train 7.745924 (lr=2.9991e-05) (hash(x)=155934075)
-500 val loss 7.5210
-500 val perplexity 1846.3237
-500 train 7.530861 (lr=2.9980e-05) (hash(x)=157510366)
-600 val loss 7.4547
-600 val perplexity 1727.9935
-600 train 7.567689 (lr=2.9964e-05) (hash(x)=165402628)
-700 val loss 7.3864
-700 val perplexity 1613.9584
-700 train 7.445437 (lr=2.9944e-05) (hash(x)=166427433)
-800 val loss 7.3378
-800 val perplexity 1537.3270
-800 train 7.138241 (lr=2.9920e-05) (hash(x)=139168939)
-900 val loss 7.2903
-900 val perplexity 1466.0508
-900 train 7.199311 (lr=2.9891e-05) (hash(x)=144698667)
-1000 val loss 7.2443
-1000 val perplexity 1400.1667
-1000 train 7.230426 (lr=2.9858e-05) (hash(x)=147134338)
-1100 val loss 7.2013
-1100 val perplexity 1341.2264
-1100 train 7.165863 (lr=2.9820e-05) (hash(x)=147172640)
-1200 val loss 7.1529
-1200 val perplexity 1277.8662
-1200 train 7.153451 (lr=2.9778e-05) (hash(x)=159086459)
-1300 val loss 7.1123
-1300 val perplexity 1226.9387
-1300 train 6.877395 (lr=2.9732e-05) (hash(x)=141578940)
-1400 val loss 7.0736
-1400 val perplexity 1180.4430
-1400 train 7.141044 (lr=2.9681e-05) (hash(x)=152399099)
-1500 val loss 7.0265
-1500 val perplexity 1126.0300
-1500 train 7.104778 (lr=2.9626e-05) (hash(x)=158569712)
-1600 val loss 6.9794
-1600 val perplexity 1074.2599
-1600 train 7.075292 (lr=2.9566e-05) (hash(x)=156548843)
-1700 val loss 6.9464
-1700 val perplexity 1039.4301
-1700 train 6.948898 (lr=2.9502e-05) (hash(x)=152608173)
-1800 val loss 6.9093
-1800 val perplexity 1001.5162
-1800 train 6.986165 (lr=2.9434e-05) (hash(x)=157889911)
-1900 val loss 6.8804
-1900 val perplexity 973.0444
-1900 train 6.833936 (lr=2.9362e-05) (hash(x)=140839139)
-2000 val loss 6.8449
-2000 val perplexity 939.1225
-2000 train 6.733738 (lr=2.9285e-05) (hash(x)=142943707)
-2100 val loss 6.8209
-2100 val perplexity 916.8447
-2100 train 6.793978 (lr=2.9204e-05) (hash(x)=151025344)
-2200 val loss 6.7804
-2200 val perplexity 880.4022
-2200 train 6.775433 (lr=2.9119e-05) (hash(x)=148938689)
-2300 val loss 6.7393
-2300 val perplexity 844.9785
-2300 train 6.695773 (lr=2.9030e-05) (hash(x)=150321226)
-2400 val loss 6.6971
-2400 val perplexity 810.0812
-2400 train 6.919641 (lr=2.8937e-05) (hash(x)=161962903)
-2500 val loss 6.6544
-2500 val perplexity 776.1724
-2500 train 6.579124 (lr=2.8840e-05) (hash(x)=153085222)
-2600 val loss 6.6196
-2600 val perplexity 749.6511
-2600 train 6.666867 (lr=2.8738e-05) (hash(x)=145979235)
-2700 val loss 6.5862
-2700 val perplexity 725.0187
-2700 train 6.645664 (lr=2.8633e-05) (hash(x)=149091945)
-2800 val loss 6.5506
-2800 val perplexity 699.6385
-2800 train 6.620130 (lr=2.8523e-05) (hash(x)=159157507)
-2900 val loss 6.5289
-2900 val perplexity 684.6458
-2900 train 6.537559 (lr=2.8410e-05) (hash(x)=155377499)
-3000 val loss 6.4989
-3000 val perplexity 664.4088
-3000 train 6.464838 (lr=2.8292e-05) (hash(x)=145943130)
-3100 val loss 6.4650
-3100 val perplexity 642.2914
-3100 train 6.520201 (lr=2.8171e-05) (hash(x)=153264973)
-3200 val loss 6.4402
-3200 val perplexity 626.5162
-3200 train 6.270744 (lr=2.8046e-05) (hash(x)=143249173)
-3300 val loss 6.4214
-3300 val perplexity 614.8851
-3300 train 6.430489 (lr=2.7917e-05) (hash(x)=155558767)
-3400 val loss 6.3902
-3400 val perplexity 596.0014
-3400 train 6.332700 (lr=2.7784e-05) (hash(x)=144137265)
-3500 val loss 6.3664
-3500 val perplexity 581.9711
-3500 train 6.553246 (lr=2.7648e-05) (hash(x)=154379201)
-3600 val loss 6.3509
-3600 val perplexity 572.9807
-3600 train 6.470469 (lr=2.7508e-05) (hash(x)=158728918)
-3700 val loss 6.3331
-3700 val perplexity 562.8910
-3700 train 6.070055 (lr=2.7364e-05) (hash(x)=139902844)
-3800 val loss 6.3185
-3800 val perplexity 554.7255
-3800 train 6.389813 (lr=2.7216e-05) (hash(x)=160738928)
-3900 val loss 6.2904
-3900 val perplexity 539.3649
-3900 train 6.331081 (lr=2.7066e-05) (hash(x)=156122233)
-4000 val loss 6.2721
-4000 val perplexity 529.5984
-4000 train 6.095728 (lr=2.6911e-05) (hash(x)=136691174)
-4100 val loss 6.2625
-4100 val perplexity 524.5030
-4100 train 6.228635 (lr=2.6754e-05) (hash(x)=150135523)
-4200 val loss 6.2536
-4200 val perplexity 519.8756
-4200 train 6.243220 (lr=2.6592e-05) (hash(x)=154053307)
-4300 val loss 6.2403
-4300 val perplexity 513.0081
-4300 train 6.015927 (lr=2.6428e-05) (hash(x)=136076439)
-4400 val loss 6.2177
-4400 val perplexity 501.5292
-4400 train 6.107508 (lr=2.6260e-05) (hash(x)=147368042)
-4500 val loss 6.2021
-4500 val perplexity 493.7754
-4500 train 6.297653 (lr=2.6089e-05) (hash(x)=163726147)
-4600 val loss 6.1854
-4600 val perplexity 485.5912
-4600 train 6.203592 (lr=2.5915e-05) (hash(x)=151996388)
-4700 val loss 6.1712
-4700 val perplexity 478.7798
-4700 train 6.213857 (lr=2.5738e-05) (hash(x)=148952763)
-4800 val loss 6.1600
-4800 val perplexity 473.4117
-4800 train 6.077888 (lr=2.5558e-05) (hash(x)=139747769)
-4900 val loss 6.1494
-4900 val perplexity 468.4167
-4900 train 6.010595 (lr=2.5374e-05) (hash(x)=145011383)
-5000 val loss 6.1393
-5000 val perplexity 463.7233
-5000 train 6.162326 (lr=2.5188e-05) (hash(x)=153449428)
-5100 val loss 6.1294
-5100 val perplexity 459.1557
-5100 train 6.172811 (lr=2.4999e-05) (hash(x)=161397672)
-5200 val loss 6.1093
-5200 val perplexity 450.0402
-5200 train 6.097622 (lr=2.4807e-05) (hash(x)=164813156)
-5300 val loss 6.0982
-5300 val perplexity 445.0724
-5300 train 6.152255 (lr=2.4613e-05) (hash(x)=146155759)
-5400 val loss 6.0879
-5400 val perplexity 440.5091
-5400 train 6.156091 (lr=2.4415e-05) (hash(x)=152337652)
-5500 val loss 6.0800
-5500 val perplexity 437.0487
-5500 train 5.936502 (lr=2.4215e-05) (hash(x)=148346631)
-5600 val loss 6.0698
-5600 val perplexity 432.5997
-5600 train 5.978254 (lr=2.4013e-05) (hash(x)=146652463)
-5700 val loss 6.0636
-5700 val perplexity 429.9003
-5700 train 6.000385 (lr=2.3808e-05) (hash(x)=147553632)
-5800 val loss 6.0496
-5800 val perplexity 423.9548
-5800 train 6.145557 (lr=2.3601e-05) (hash(x)=153512314)
-5900 val loss 6.0372
-5900 val perplexity 418.7395
-5900 train 6.004522 (lr=2.3391e-05) (hash(x)=137351707)
-6000 val loss 6.0272
-6000 val perplexity 414.5595
-6000 train 6.050917 (lr=2.3179e-05) (hash(x)=143375055)
-6100 val loss 6.0245
-6100 val perplexity 413.4346
-6100 train 6.047802 (lr=2.2965e-05) (hash(x)=144781857)
-6200 val loss 6.0119
-6200 val perplexity 408.2610
-6200 train 6.255000 (lr=2.2749e-05) (hash(x)=167556812)
-6300 val loss 6.0016
-6300 val perplexity 404.0685
-6300 train 6.156122 (lr=2.2530e-05) (hash(x)=162290597)
-6400 val loss 5.9908
-6400 val perplexity 399.7199
-6400 train 5.886322 (lr=2.2310e-05) (hash(x)=144836361)
-6500 val loss 5.9783
-6500 val perplexity 394.7825
-6500 train 5.851570 (lr=2.2088e-05) (hash(x)=150949131)
-6600 val loss 5.9938
-6600 val perplexity 400.9352
-6600 train 6.061947 (lr=2.1864e-05) (hash(x)=151386312)
-6700 val loss 5.9631
-6700 val perplexity 388.8249
-6700 train 6.041219 (lr=2.1638e-05) (hash(x)=159840718)
-6800 val loss 5.9529
-6800 val perplexity 384.8733
-6800 train 6.046275 (lr=2.1410e-05) (hash(x)=158843242)
-6900 val loss 5.9471
-6900 val perplexity 382.6416
-6900 train 6.188788 (lr=2.1181e-05) (hash(x)=160544793)
-7000 val loss 5.9385
-7000 val perplexity 379.3763
-7000 train 5.780041 (lr=2.0951e-05) (hash(x)=139031423)
-7100 val loss 5.9357
-7100 val perplexity 378.2969
-7100 train 5.874920 (lr=2.0718e-05) (hash(x)=148444150)
-7200 val loss 5.9244
-7200 val perplexity 374.0450
-7200 train 6.257408 (lr=2.0485e-05) (hash(x)=162348298)
-7300 val loss 5.9135
-7300 val perplexity 370.0092
-7300 train 5.969735 (lr=2.0250e-05) (hash(x)=160070640)
-7400 val loss 5.9077
-7400 val perplexity 367.8553
-7400 train 5.828381 (lr=2.0014e-05) (hash(x)=142907242)
-7500 val loss 5.9027
-7500 val perplexity 366.0135
-7500 train 5.909079 (lr=1.9777e-05) (hash(x)=155567735)
-7600 val loss 5.8932
-7600 val perplexity 362.5581
-7600 train 5.749373 (lr=1.9538e-05) (hash(x)=149026190)
-7700 val loss 5.8864
-7700 val perplexity 360.1118
-7700 train 6.437926 (lr=1.9299e-05) (hash(x)=172973386)
-7800 val loss 5.8815
-7800 val perplexity 358.3377
-7800 train 5.852612 (lr=1.9059e-05) (hash(x)=148166384)
-7900 val loss 5.8721
-7900 val perplexity 354.9937
-7900 train 6.311577 (lr=1.8817e-05) (hash(x)=174580801)
-8000 val loss 5.8638
-8000 val perplexity 352.0487
-8000 train 5.881599 (lr=1.8576e-05) (hash(x)=153576391)
-8100 val loss 5.8601
-8100 val perplexity 350.7613
-8100 train 5.759035 (lr=1.8333e-05) (hash(x)=143483378)
-8200 val loss 5.8529
-8200 val perplexity 348.2314
-8200 train 6.302994 (lr=1.8090e-05) (hash(x)=125210829)
-8300 val loss 5.8485
-8300 val perplexity 346.7147
-8300 train 5.778973 (lr=1.7846e-05) (hash(x)=152432601)
-8400 val loss 5.8504
-8400 val perplexity 347.3867
-8400 train 5.727871 (lr=1.7602e-05) (hash(x)=148759990)
-8500 val loss 5.8366
-8500 val perplexity 342.6263
-8500 train 5.820582 (lr=1.7357e-05) (hash(x)=156974533)
-8600 val loss 5.8261
-8600 val perplexity 339.0219
-8600 train 5.939592 (lr=1.7113e-05) (hash(x)=155441439)
-8700 val loss 5.8206
-8700 val perplexity 337.1772
-8700 train 5.801907 (lr=1.6868e-05) (hash(x)=149001504)
-8800 val loss 5.8160
-8800 val perplexity 335.6134
-8800 train 5.685404 (lr=1.6623e-05) (hash(x)=144721722)
-8900 val loss 5.8102
-8900 val perplexity 333.6932
-8900 train 5.717410 (lr=1.6377e-05) (hash(x)=151789511)
-9000 val loss 5.8033
-9000 val perplexity 331.3896
-9000 train 5.732379 (lr=1.6132e-05) (hash(x)=151307797)
-9100 val loss 5.7979
-9100 val perplexity 329.5970
-9100 train 5.928390 (lr=1.5887e-05) (hash(x)=155534554)
-9200 val loss 5.7888
-9200 val perplexity 326.6216
-9200 train 5.677572 (lr=1.5643e-05) (hash(x)=138467356)
-9300 val loss 5.7872
-9300 val perplexity 326.0889
-9300 train 5.718623 (lr=1.5398e-05) (hash(x)=148622510)
-9400 val loss 5.7821
-9400 val perplexity 324.4314
-9400 train 5.915169 (lr=1.5154e-05) (hash(x)=154343672)
-9500 val loss 5.7790
-9500 val perplexity 323.4408
-9500 train 5.763975 (lr=1.4910e-05) (hash(x)=156831628)
-9600 val loss 5.7742
-9600 val perplexity 321.8810
-9600 train 5.550517 (lr=1.4667e-05) (hash(x)=142190812)
-9700 val loss 5.7644
-9700 val perplexity 318.7634
-9700 train 5.670130 (lr=1.4424e-05) (hash(x)=146487376)
-9800 val loss 5.7629
-9800 val perplexity 318.2787
-9800 train 5.842342 (lr=1.4183e-05) (hash(x)=164178945)
-9900 val loss 5.7580
-9900 val perplexity 316.7204
-9900 train 5.622567 (lr=1.3941e-05) (hash(x)=142224918)
-10000 val loss 5.7564
-10000 val perplexity 316.2114
-10000 train 5.682513 (lr=1.3701e-05) (hash(x)=148509740)
-10100 val loss 5.7488
-10100 val perplexity 313.8015
-10100 train 5.813766 (lr=1.3462e-05) (hash(x)=153536520)
-10200 val loss 5.7386
-10200 val perplexity 310.6290
-10200 train 5.667438 (lr=1.3223e-05) (hash(x)=153740168)
-10300 val loss 5.7613
-10300 val perplexity 317.7690
-10300 train 5.717767 (lr=1.2986e-05) (hash(x)=160410260)
-10400 val loss 5.7330
-10400 val perplexity 308.9095
-10400 train 5.894064 (lr=1.2750e-05) (hash(x)=136788294)
-10500 val loss 5.7284
-10500 val perplexity 307.4727
-10500 train 5.576824 (lr=1.2515e-05) (hash(x)=153988760)
-10600 val loss 5.7258
-10600 val perplexity 306.6924
-10600 train 5.606709 (lr=1.2282e-05) (hash(x)=141896522)
-10700 val loss 5.7242
-10700 val perplexity 306.1778
-10700 train 6.063038 (lr=1.2049e-05) (hash(x)=147057702)
-10800 val loss 5.7162
-10800 val perplexity 303.7487
-10800 train 5.782267 (lr=1.1819e-05) (hash(x)=157542287)
-10900 val loss 5.7106
-10900 val perplexity 302.0526
-10900 train 5.783550 (lr=1.1590e-05) (hash(x)=153860040)
-11000 val loss 5.7061
-11000 val perplexity 300.7032
-11000 train 5.563945 (lr=1.1362e-05) (hash(x)=142932555)
-11100 val loss 5.7038
-11100 val perplexity 299.9987
-11100 train 5.739359 (lr=1.1136e-05) (hash(x)=146260760)
-11200 val loss 5.7056
-11200 val perplexity 300.5415
-11200 train 5.801177 (lr=1.0912e-05) (hash(x)=148970533)
-11300 val loss 5.6988
-11300 val perplexity 298.5046
-11300 train 5.713600 (lr=1.0690e-05) (hash(x)=149421645)
-11400 val loss 5.6921
-11400 val perplexity 296.5246
-11400 train 5.633717 (lr=1.0470e-05) (hash(x)=158728429)
-11500 val loss 5.6890
-11500 val perplexity 295.6100
-11500 train 5.784037 (lr=1.0251e-05) (hash(x)=139524335)
-11600 val loss 5.6881
-11600 val perplexity 295.3450
-11600 train 5.721812 (lr=1.0035e-05) (hash(x)=152760174)
-11700 val loss 5.6831
-11700 val perplexity 293.8480
-11700 train 5.586452 (lr=9.8209e-06) (hash(x)=140740682)
-11800 val loss 5.6822
-11800 val perplexity 293.5990
-11800 train 5.538492 (lr=9.6090e-06) (hash(x)=153665407)
-11900 val loss 5.6813
-11900 val perplexity 293.3422
-11900 train 5.654160 (lr=9.3993e-06) (hash(x)=163068424)
-12000 val loss 5.6737
-12000 val perplexity 291.1071
-12000 train 5.718170 (lr=9.1920e-06) (hash(x)=145166541)
-12100 val loss 5.6692
-12100 val perplexity 289.8016
-12100 train 5.626221 (lr=8.9871e-06) (hash(x)=143228042)
-12200 val loss 5.6683
-12200 val perplexity 289.5381
-12200 train 5.776905 (lr=8.7846e-06) (hash(x)=159222680)
-12300 val loss 5.6676
-12300 val perplexity 289.3356
-12300 train 5.611221 (lr=8.5847e-06) (hash(x)=145196157)
-12400 val loss 5.6638
-12400 val perplexity 288.2523
-12400 train 5.575829 (lr=8.3875e-06) (hash(x)=143033742)
-12500 val loss 5.6615
-12500 val perplexity 287.5922
-12500 train 5.694445 (lr=8.1929e-06) (hash(x)=151133346)
-12600 val loss 5.6591
-12600 val perplexity 286.8763
-12600 train 5.681115 (lr=8.0010e-06) (hash(x)=153607184)
-12700 val loss 5.6548
-12700 val perplexity 285.6623
-12700 train 5.569495 (lr=7.8119e-06) (hash(x)=136931911)
-12800 val loss 5.6528
-12800 val perplexity 285.0946
-12800 train 5.761200 (lr=7.6257e-06) (hash(x)=155105402)
-12900 val loss 5.6501
-12900 val perplexity 284.3116
-12900 train 5.479981 (lr=7.4425e-06) (hash(x)=144141636)
-13000 val loss 5.6512
-13000 val perplexity 284.6234
-13000 train 5.568130 (lr=7.2622e-06) (hash(x)=139380218)
-13100 val loss 5.6434
-13100 val perplexity 282.4235
-13100 train 5.737788 (lr=7.0850e-06) (hash(x)=157695174)
-13200 val loss 5.6419
-13200 val perplexity 281.9885
-13200 train 5.578032 (lr=6.9108e-06) (hash(x)=143697560)
-13300 val loss 5.6394
-13300 val perplexity 281.2981
-13300 train 5.478682 (lr=6.7398e-06) (hash(x)=156314952)
-13400 val loss 5.6395
-13400 val perplexity 281.3124
-13400 train 5.608853 (lr=6.5721e-06) (hash(x)=136720279)
-13500 val loss 5.6387
-13500 val perplexity 281.0854
-13500 train 5.592543 (lr=6.4076e-06) (hash(x)=142579957)
-13600 val loss 5.6389
-13600 val perplexity 281.1602
-13600 train 5.352937 (lr=6.2465e-06) (hash(x)=140719440)
-13700 val loss 5.6381
-13700 val perplexity 280.9380
-13700 train 5.736770 (lr=6.0887e-06) (hash(x)=164396003)
-13800 val loss 5.6320
-13800 val perplexity 279.2188
-13800 train 5.693521 (lr=5.9344e-06) (hash(x)=152712862)
-13900 val loss 5.6282
-13900 val perplexity 278.1716
-13900 train 5.652749 (lr=5.7835e-06) (hash(x)=160985139)
-14000 val loss 5.6259
-14000 val perplexity 277.5321
-14000 train 5.626481 (lr=5.6362e-06) (hash(x)=147429379)
-14100 val loss 5.6248
-14100 val perplexity 277.2046
-14100 train 5.689951 (lr=5.4925e-06) (hash(x)=131830470)
-14200 val loss 5.6258
-14200 val perplexity 277.4954
-14200 train 5.367467 (lr=5.3524e-06) (hash(x)=146383028)
-14300 val loss 5.6231
-14300 val perplexity 276.7373
-14300 train 5.671228 (lr=5.2159e-06) (hash(x)=157158747)
-14400 val loss 5.6251
-14400 val perplexity 277.3032
-14400 train 5.476326 (lr=5.0832e-06) (hash(x)=143911520)
-14500 val loss 5.6246
-14500 val perplexity 277.1737
-14500 train 5.399273 (lr=4.9543e-06) (hash(x)=159555347)
-14600 val loss 5.6189
-14600 val perplexity 275.5745
-14600 train 5.619324 (lr=4.8291e-06) (hash(x)=152957062)
-14700 val loss 5.6143
-14700 val perplexity 274.3109
-14700 train 5.683186 (lr=4.7079e-06) (hash(x)=156001594)
-14800 val loss 5.6143
-14800 val perplexity 274.3225
-14800 train 5.715827 (lr=4.5905e-06) (hash(x)=145885458)
-14900 val loss 5.6122
-14900 val perplexity 273.7392
-14900 train 5.676468 (lr=4.4770e-06) (hash(x)=144544291)
-15000 val loss 5.6122
-15000 val perplexity 273.7438
-15000 train 5.622100 (lr=4.3675e-06) (hash(x)=154606709)
-15100 val loss 5.6118
-15100 val perplexity 273.6255
-15100 train 5.653947 (lr=4.2620e-06) (hash(x)=156005324)
-15200 val loss 5.6125
-15200 val perplexity 273.8391
-15200 train 5.712396 (lr=4.1605e-06) (hash(x)=158847373)
-15300 val loss 5.6134
-15300 val perplexity 274.0727
-15300 train 5.397266 (lr=4.0631e-06) (hash(x)=159609594)
-15400 val loss 5.6084
-15400 val perplexity 272.7195
-15400 train 5.706025 (lr=3.9698e-06) (hash(x)=152428492)
-15500 val loss 5.6049
-15500 val perplexity 271.7659
-15500 train 5.666723 (lr=3.8806e-06) (hash(x)=162251192)
-15600 val loss 5.6044
-15600 val perplexity 271.6171
-15600 train 5.850659 (lr=3.7956e-06) (hash(x)=155072002)
-15700 val loss 5.6027
-15700 val perplexity 271.1621
-15700 train 5.619005 (lr=3.7148e-06) (hash(x)=151203094)
-15800 val loss 5.6023
-15800 val perplexity 271.0609
-15800 train 5.521658 (lr=3.6382e-06) (hash(x)=146125932)
-15900 val loss 5.6018
-15900 val perplexity 270.9266
-15900 train 5.488575 (lr=3.5658e-06) (hash(x)=148060195)
-16000 val loss 5.6027
-16000 val perplexity 271.1473
-16000 train 5.299024 (lr=3.4977e-06) (hash(x)=137406416)
-16100 val loss 5.6029
-16100 val perplexity 271.2044
-16100 train 5.664294 (lr=3.4339e-06) (hash(x)=160149755)
-16200 val loss 5.5976
-16200 val perplexity 269.7787
-16200 train 5.769825 (lr=3.3744e-06) (hash(x)=151371894)
-16300 val loss 5.5959
-16300 val perplexity 269.3203
-16300 train 5.638423 (lr=3.3193e-06) (hash(x)=151810252)
-16400 val loss 5.5946
-16400 val perplexity 268.9703
-16400 train 5.578471 (lr=3.2684e-06) (hash(x)=153088303)
-16500 val loss 5.5948
-16500 val perplexity 269.0238
-16500 train 5.474696 (lr=3.2220e-06) (hash(x)=151706615)
-16600 val loss 5.5926
-16600 val perplexity 268.4265
-16600 train 5.483826 (lr=3.1799e-06) (hash(x)=142110626)
-16700 val loss 5.5937
-16700 val perplexity 268.7342
-16700 train 5.486968 (lr=3.1422e-06) (hash(x)=153222439)
-16800 val loss 5.5921
-16800 val perplexity 268.2885
-16800 train 5.519623 (lr=3.1089e-06) (hash(x)=154964902)
-16900 val loss 5.5899
-16900 val perplexity 267.6995
-16900 train 5.766605 (lr=3.0801e-06) (hash(x)=158893204)
-17000 val loss 5.5877
-17000 val perplexity 267.1279
-17000 train 5.519545 (lr=3.0556e-06) (hash(x)=148929955)
-17100 val loss 5.5881
-17100 val perplexity 267.2181
-17100 train 5.599936 (lr=3.0356e-06) (hash(x)=147818677)
-17200 val loss 5.5868
-17200 val perplexity 266.8931
-17200 train 5.519412 (lr=3.0200e-06) (hash(x)=147119446)
-17300 val loss 5.5870
-17300 val perplexity 266.9319
-17300 train 5.630085 (lr=3.0089e-06) (hash(x)=150494142)
-17400 val loss 5.5871
-17400 val perplexity 266.9568
-17400 train 5.614751 (lr=3.0022e-06) (hash(x)=155918058)
-17499 val loss 5.5846
-17499 val perplexity 266.2846
+0 train 11.286213 (lr=1.3750e-07) (hash(x)=150332693)
+100 val loss 9.9514
+100 val perplexity 20981.1719
+100 train 9.972456 (lr=1.3888e-05) (hash(x)=148948662)
+200 val loss 9.0084
+200 val perplexity 8171.4985
+200 train 8.930535 (lr=2.7500e-05) (hash(x)=149277319)
+300 val loss 7.9062
+300 val perplexity 2714.1672
+300 train 7.803435 (lr=2.7498e-05) (hash(x)=146429920)
+400 val loss 7.5790
+400 val perplexity 1956.7255
+400 train 7.696357 (lr=2.7492e-05) (hash(x)=155934075)
+500 val loss 7.4743
+500 val perplexity 1762.2222
+500 train 7.482019 (lr=2.7482e-05) (hash(x)=157510366)
+600 val loss 7.4041
+600 val perplexity 1642.7472
+600 train 7.518789 (lr=2.7467e-05) (hash(x)=165402628)
+700 val loss 7.3399
+700 val perplexity 1540.5265
+700 train 7.399743 (lr=2.7449e-05) (hash(x)=166427433)
+800 val loss 7.2851
+800 val perplexity 1458.4285
+800 train 7.083396 (lr=2.7427e-05) (hash(x)=139168939)
+900 val loss 7.2414
+900 val perplexity 1396.0408
+900 train 7.144016 (lr=2.7400e-05) (hash(x)=144698667)
+1000 val loss 7.1911
+1000 val perplexity 1327.6097
+1000 train 7.171625 (lr=2.7370e-05) (hash(x)=147134338)
+1100 val loss 7.1443
+1100 val perplexity 1266.8485
+1100 train 7.110559 (lr=2.7335e-05) (hash(x)=147172640)
+1200 val loss 7.1005
+1200 val perplexity 1212.5867
+1200 train 7.100369 (lr=2.7297e-05) (hash(x)=159086459)
+1300 val loss 7.0613
+1300 val perplexity 1165.9291
+1300 train 6.819506 (lr=2.7254e-05) (hash(x)=141578940)
+1400 val loss 7.0270
+1400 val perplexity 1126.5918
+1400 train 7.090215 (lr=2.7207e-05) (hash(x)=152399099)
+1500 val loss 6.9761
+1500 val perplexity 1070.7262
+1500 train 7.055522 (lr=2.7157e-05) (hash(x)=158569712)
+1600 val loss 6.9304
+1600 val perplexity 1022.9405
+1600 train 7.023970 (lr=2.7102e-05) (hash(x)=156548843)
+1700 val loss 6.8955
+1700 val perplexity 987.8098
+1700 train 6.901347 (lr=2.7044e-05) (hash(x)=152608173)
+1800 val loss 6.8543
+1800 val perplexity 947.9863
+1800 train 6.928975 (lr=2.6981e-05) (hash(x)=157889911)
+1900 val loss 6.8293
+1900 val perplexity 924.5248
+1900 train 6.787612 (lr=2.6915e-05) (hash(x)=140839139)
+2000 val loss 6.7897
+2000 val perplexity 888.6237
+2000 train 6.679601 (lr=2.6845e-05) (hash(x)=142943707)
+2100 val loss 6.7547
+2100 val perplexity 858.0652
+2100 train 6.727079 (lr=2.6771e-05) (hash(x)=151025344)
+2200 val loss 6.7160
+2200 val perplexity 825.4786
+2200 train 6.709481 (lr=2.6693e-05) (hash(x)=148938689)
+2300 val loss 6.6813
+2300 val perplexity 797.3430
+2300 train 6.643854 (lr=2.6611e-05) (hash(x)=150321226)
+2400 val loss 6.6369
+2400 val perplexity 762.7330
+2400 train 6.865366 (lr=2.6525e-05) (hash(x)=161962903)
+2500 val loss 6.6012
+2500 val perplexity 735.9520
+2500 train 6.529111 (lr=2.6436e-05) (hash(x)=153085222)
+2600 val loss 6.5678
+2600 val perplexity 711.7835
+2600 train 6.628248 (lr=2.6343e-05) (hash(x)=145979235)
+2700 val loss 6.5422
+2700 val perplexity 693.8163
+2700 train 6.600425 (lr=2.6246e-05) (hash(x)=149091945)
+2800 val loss 6.5175
+2800 val perplexity 676.8688
+2800 train 6.596137 (lr=2.6146e-05) (hash(x)=159157507)
+2900 val loss 6.5001
+2900 val perplexity 665.1902
+2900 train 6.509790 (lr=2.6042e-05) (hash(x)=155377499)
+3000 val loss 6.4763
+3000 val perplexity 649.5642
+3000 train 6.439157 (lr=2.5934e-05) (hash(x)=145943130)
+3100 val loss 6.4548
+3100 val perplexity 635.7664
+3100 train 6.511821 (lr=2.5823e-05) (hash(x)=153264973)
+3200 val loss 6.4241
+3200 val perplexity 616.5283
+3200 train 6.256510 (lr=2.5709e-05) (hash(x)=143249173)
+3300 val loss 6.4090
+3300 val perplexity 607.2606
+3300 train 6.416599 (lr=2.5590e-05) (hash(x)=155558767)
+3400 val loss 6.3850
+3400 val perplexity 592.9050
+3400 train 6.332426 (lr=2.5469e-05) (hash(x)=144137265)
+3500 val loss 6.3681
+3500 val perplexity 582.9590
+3500 train 6.559334 (lr=2.5344e-05) (hash(x)=154379201)
+3600 val loss 6.3557
+3600 val perplexity 575.7748
+3600 train 6.476018 (lr=2.5215e-05) (hash(x)=158728918)
+3700 val loss 6.3384
+3700 val perplexity 565.8771
+3700 train 6.080819 (lr=2.5083e-05) (hash(x)=139902844)
+3800 val loss 6.3307
+3800 val perplexity 561.5490
+3800 train 6.400692 (lr=2.4948e-05) (hash(x)=160738928)
+3900 val loss 6.3023
+3900 val perplexity 545.8097
+3900 train 6.346352 (lr=2.4810e-05) (hash(x)=156122233)
+4000 val loss 6.2826
+4000 val perplexity 535.1808
+4000 train 6.109324 (lr=2.4669e-05) (hash(x)=136691174)
+4100 val loss 6.2762
+4100 val perplexity 531.7452
+4100 train 6.242570 (lr=2.4524e-05) (hash(x)=150135523)
+4200 val loss 6.2686
+4200 val perplexity 527.7131
+4200 train 6.256784 (lr=2.4376e-05) (hash(x)=154053307)
+4300 val loss 6.2503
+4300 val perplexity 518.1600
+4300 train 6.028021 (lr=2.4226e-05) (hash(x)=136076439)
+4400 val loss 6.2345
+4400 val perplexity 510.0574
+4400 train 6.124469 (lr=2.4072e-05) (hash(x)=147368042)
+4500 val loss 6.2168
+4500 val perplexity 501.1109
+4500 train 6.318371 (lr=2.3915e-05) (hash(x)=163726147)
+4600 val loss 6.2070
+4600 val perplexity 496.2216
+4600 train 6.221811 (lr=2.3755e-05) (hash(x)=151996388)
+4700 val loss 6.1921
+4700 val perplexity 488.8610
+4700 train 6.235697 (lr=2.3593e-05) (hash(x)=148952763)
+4800 val loss 6.1823
+4800 val perplexity 484.1152
+4800 train 6.098751 (lr=2.3428e-05) (hash(x)=139747769)
+4900 val loss 6.1781
+4900 val perplexity 482.0607
+4900 train 6.045214 (lr=2.3260e-05) (hash(x)=145011383)
+5000 val loss 6.1633
+5000 val perplexity 474.9830
+5000 train 6.185411 (lr=2.3089e-05) (hash(x)=153449428)
+5100 val loss 6.1545
+5100 val perplexity 470.8437
+5100 train 6.195387 (lr=2.2916e-05) (hash(x)=161397672)
+5200 val loss 6.1344
+5200 val perplexity 461.4544
+5200 train 6.126793 (lr=2.2740e-05) (hash(x)=164813156)
+5300 val loss 6.1258
+5300 val perplexity 457.5305
+5300 train 6.177435 (lr=2.2561e-05) (hash(x)=146155759)
+5400 val loss 6.1213
+5400 val perplexity 455.4430
+5400 train 6.183184 (lr=2.2381e-05) (hash(x)=152337652)
+5500 val loss 6.1119
+5500 val perplexity 451.1878
+5500 train 5.973808 (lr=2.2197e-05) (hash(x)=148346631)
+5600 val loss 6.1063
+5600 val perplexity 448.6686
+5600 train 6.012882 (lr=2.2012e-05) (hash(x)=146652463)
+5700 val loss 6.1002
+5700 val perplexity 445.9608
+5700 train 6.034775 (lr=2.1824e-05) (hash(x)=147553632)
+5800 val loss 6.0846
+5800 val perplexity 439.0377
+5800 train 6.185885 (lr=2.1634e-05) (hash(x)=153512314)
+5900 val loss 6.0728
+5900 val perplexity 433.8759
+5900 train 6.044499 (lr=2.1442e-05) (hash(x)=137351707)
+6000 val loss 6.0726
+6000 val perplexity 433.7897
+6000 train 6.099359 (lr=2.1248e-05) (hash(x)=143375055)
+6100 val loss 6.0634
+6100 val perplexity 429.8267
+6100 train 6.073904 (lr=2.1051e-05) (hash(x)=144781857)
+6200 val loss 6.0519
+6200 val perplexity 424.9072
+6200 train 6.308413 (lr=2.0853e-05) (hash(x)=167556812)
+6300 val loss 6.0446
+6300 val perplexity 421.8202
+6300 train 6.198883 (lr=2.0653e-05) (hash(x)=162290597)
+6400 val loss 6.0359
+6400 val perplexity 418.1750
+6400 train 5.934332 (lr=2.0451e-05) (hash(x)=144836361)
+6500 val loss 6.0226
+6500 val perplexity 412.6547
+6500 train 5.907002 (lr=2.0247e-05) (hash(x)=150949131)
+6600 val loss 6.0350
+6600 val perplexity 417.7888
+6600 train 6.098232 (lr=2.0042e-05) (hash(x)=151386312)
+6700 val loss 6.0056
+6700 val perplexity 405.7001
+6700 train 6.086429 (lr=1.9835e-05) (hash(x)=159840718)
+6800 val loss 5.9969
+6800 val perplexity 402.1692
+6800 train 6.088245 (lr=1.9626e-05) (hash(x)=158843242)
+6900 val loss 5.9943
+6900 val perplexity 401.1250
+6900 train 6.230082 (lr=1.9416e-05) (hash(x)=160544793)
+7000 val loss 5.9862
+7000 val perplexity 397.9082
+7000 train 5.832255 (lr=1.9205e-05) (hash(x)=139031423)
+7100 val loss 5.9803
+7100 val perplexity 395.5639
+7100 train 5.924754 (lr=1.8992e-05) (hash(x)=148444150)
+7200 val loss 5.9745
+7200 val perplexity 393.2809
+7200 train 6.303175 (lr=1.8778e-05) (hash(x)=162348298)
+7300 val loss 5.9657
+7300 val perplexity 389.8229
+7300 train 6.031035 (lr=1.8562e-05) (hash(x)=160070640)
+7400 val loss 5.9595
+7400 val perplexity 387.4172
+7400 train 5.878331 (lr=1.8346e-05) (hash(x)=142907242)
+7500 val loss 5.9573
+7500 val perplexity 386.5767
+7500 train 5.959059 (lr=1.8128e-05) (hash(x)=155567735)
+7600 val loss 5.9498
+7600 val perplexity 383.6834
+7600 train 5.799048 (lr=1.7910e-05) (hash(x)=149026190)
+7700 val loss 5.9417
+7700 val perplexity 380.5897
+7700 train 6.468088 (lr=1.7691e-05) (hash(x)=172973386)
+7800 val loss 5.9373
+7800 val perplexity 378.9236
+7800 train 5.907558 (lr=1.7470e-05) (hash(x)=148166384)
+7900 val loss 5.9337
+7900 val perplexity 377.5314
+7900 train 6.369796 (lr=1.7249e-05) (hash(x)=174580801)
+8000 val loss 5.9233
+8000 val perplexity 373.6542
+8000 train 5.941860 (lr=1.7028e-05) (hash(x)=153576391)
+8100 val loss 5.9166
+8100 val perplexity 371.1408
+8100 train 5.815281 (lr=1.6805e-05) (hash(x)=143483378)
+8200 val loss 5.9127
+8200 val perplexity 369.7064
+8200 train 6.327120 (lr=1.6582e-05) (hash(x)=125210829)
+8300 val loss 5.9088
+8300 val perplexity 368.2632
+8300 train 5.830599 (lr=1.6359e-05) (hash(x)=152432601)
+8400 val loss 5.9119
+8400 val perplexity 369.3940
+8400 train 5.790779 (lr=1.6135e-05) (hash(x)=148759990)
+8500 val loss 5.8946
+8500 val perplexity 363.0729
+8500 train 5.893033 (lr=1.5911e-05) (hash(x)=156974533)
+8600 val loss 5.8877
+8600 val perplexity 360.5785
+8600 train 5.997952 (lr=1.5687e-05) (hash(x)=155441439)
+8700 val loss 5.8845
+8700 val perplexity 359.4311
+8700 train 5.866372 (lr=1.5462e-05) (hash(x)=149001504)
+8800 val loss 5.8765
+8800 val perplexity 356.5466
+8800 train 5.753472 (lr=1.5237e-05) (hash(x)=144721722)
+8900 val loss 5.8741
+8900 val perplexity 355.7068
+8900 train 5.780489 (lr=1.5013e-05) (hash(x)=151789511)
+9000 val loss 5.8679
+9000 val perplexity 353.4932
+9000 train 5.797716 (lr=1.4788e-05) (hash(x)=151307797)
+9100 val loss 5.8624
+9100 val perplexity 351.5597
+9100 train 5.990904 (lr=1.4563e-05) (hash(x)=155534554)
+9200 val loss 5.8534
+9200 val perplexity 348.4138
+9200 train 5.737578 (lr=1.4339e-05) (hash(x)=138467356)
+9300 val loss 5.8519
+9300 val perplexity 347.8787
+9300 train 5.780663 (lr=1.4115e-05) (hash(x)=148622510)
+9400 val loss 5.8451
+9400 val perplexity 345.5435
+9400 train 5.971584 (lr=1.3891e-05) (hash(x)=154343672)
+9500 val loss 5.8430
+9500 val perplexity 344.8152
+9500 train 5.831597 (lr=1.3668e-05) (hash(x)=156831628)
+9600 val loss 5.8395
+9600 val perplexity 343.6098
+9600 train 5.615783 (lr=1.3445e-05) (hash(x)=142190812)
+9700 val loss 5.8304
+9700 val perplexity 340.5052
+9700 train 5.740179 (lr=1.3222e-05) (hash(x)=146487376)
+9800 val loss 5.8253
+9800 val perplexity 338.7470
+9800 train 5.915164 (lr=1.3001e-05) (hash(x)=164178945)
+9900 val loss 5.8230
+9900 val perplexity 337.9937
+9900 train 5.688615 (lr=1.2780e-05) (hash(x)=142224918)
+10000 val loss 5.8229
+10000 val perplexity 337.9587
+10000 train 5.753917 (lr=1.2559e-05) (hash(x)=148509740)
+10100 val loss 5.8134
+10100 val perplexity 334.7476
+10100 train 5.879743 (lr=1.2340e-05) (hash(x)=153536520)
+10200 val loss 5.8054
+10200 val perplexity 332.1046
+10200 train 5.738307 (lr=1.2122e-05) (hash(x)=153740168)
+10300 val loss 5.8274
+10300 val perplexity 339.4831
+10300 train 5.786556 (lr=1.1904e-05) (hash(x)=160410260)
+10400 val loss 5.7976
+10400 val perplexity 329.5164
+10400 train 5.951182 (lr=1.1688e-05) (hash(x)=136788294)
+10500 val loss 5.7929
+10500 val perplexity 327.9620
+10500 train 5.652078 (lr=1.1472e-05) (hash(x)=153988760)
+10600 val loss 5.7913
+10600 val perplexity 327.4321
+10600 train 5.665859 (lr=1.1258e-05) (hash(x)=141896522)
+10700 val loss 5.7906
+10700 val perplexity 327.2158
+10700 train 6.129151 (lr=1.1045e-05) (hash(x)=147057702)
+10800 val loss 5.7819
+10800 val perplexity 324.3701
+10800 train 5.848338 (lr=1.0834e-05) (hash(x)=157542287)
+10900 val loss 5.7773
+10900 val perplexity 322.8757
+10900 train 5.841738 (lr=1.0624e-05) (hash(x)=153860040)
+11000 val loss 5.7728
+11000 val perplexity 321.4455
+11000 train 5.637451 (lr=1.0415e-05) (hash(x)=142932555)
+11100 val loss 5.7708
+11100 val perplexity 320.7993
+11100 train 5.806400 (lr=1.0208e-05) (hash(x)=146260760)
+11200 val loss 5.7700
+11200 val perplexity 320.5267
+11200 train 5.858836 (lr=1.0003e-05) (hash(x)=148970533)
+11300 val loss 5.7650
+11300 val perplexity 318.9247
+11300 train 5.781595 (lr=9.7991e-06) (hash(x)=149421645)
+11400 val loss 5.7578
+11400 val perplexity 316.6360
+11400 train 5.698096 (lr=9.5972e-06) (hash(x)=158728429)
+11500 val loss 5.7544
+11500 val perplexity 315.5844
+11500 train 5.843369 (lr=9.3970e-06) (hash(x)=139524335)
+11600 val loss 5.7539
+11600 val perplexity 315.4190
+11600 train 5.796008 (lr=9.1988e-06) (hash(x)=152760174)
+11700 val loss 5.7492
+11700 val perplexity 313.9311
+11700 train 5.653507 (lr=9.0025e-06) (hash(x)=140740682)
+11800 val loss 5.7480
+11800 val perplexity 313.5658
+11800 train 5.604234 (lr=8.8082e-06) (hash(x)=153665407)
+11900 val loss 5.7477
+11900 val perplexity 313.4580
+11900 train 5.717497 (lr=8.6160e-06) (hash(x)=163068424)
+12000 val loss 5.7406
+12000 val perplexity 311.2459
+12000 train 5.797648 (lr=8.4260e-06) (hash(x)=145166541)
+12100 val loss 5.7374
+12100 val perplexity 310.2444
+12100 train 5.691184 (lr=8.2381e-06) (hash(x)=143228042)
+12200 val loss 5.7353
+12200 val perplexity 309.5932
+12200 train 5.842823 (lr=8.0526e-06) (hash(x)=159222680)
+12300 val loss 5.7341
+12300 val perplexity 309.2215
+12300 train 5.674417 (lr=7.8694e-06) (hash(x)=145196157)
+12400 val loss 5.7328
+12400 val perplexity 308.8237
+12400 train 5.636087 (lr=7.6885e-06) (hash(x)=143033742)
+12500 val loss 5.7302
+12500 val perplexity 308.0446
+12500 train 5.762257 (lr=7.5101e-06) (hash(x)=151133346)
+12600 val loss 5.7278
+12600 val perplexity 307.2995
+12600 train 5.752061 (lr=7.3342e-06) (hash(x)=153607184)
+12700 val loss 5.7241
+12700 val perplexity 306.1727
+12700 train 5.640927 (lr=7.1609e-06) (hash(x)=136931911)
+12800 val loss 5.7206
+12800 val perplexity 305.0846
+12800 train 5.832143 (lr=6.9903e-06) (hash(x)=155105402)
+12900 val loss 5.7191
+12900 val perplexity 304.6164
+12900 train 5.547976 (lr=6.8223e-06) (hash(x)=144141636)
+13000 val loss 5.7202
+13000 val perplexity 304.9768
+13000 train 5.634468 (lr=6.6570e-06) (hash(x)=139380218)
+13100 val loss 5.7123
+13100 val perplexity 302.5593
+13100 train 5.798109 (lr=6.4945e-06) (hash(x)=157695174)
+13200 val loss 5.7105
+13200 val perplexity 302.0205
+13200 train 5.648192 (lr=6.3349e-06) (hash(x)=143697560)
+13300 val loss 5.7079
+13300 val perplexity 301.2352
+13300 train 5.550069 (lr=6.1782e-06) (hash(x)=156314952)
+13400 val loss 5.7082
+13400 val perplexity 301.3254
+13400 train 5.694297 (lr=6.0244e-06) (hash(x)=136720279)
+13500 val loss 5.7072
+13500 val perplexity 301.0287
+13500 train 5.658339 (lr=5.8737e-06) (hash(x)=142579957)
+13600 val loss 5.7082
+13600 val perplexity 301.3250
+13600 train 5.420486 (lr=5.7259e-06) (hash(x)=140719440)
+13700 val loss 5.7064
+13700 val perplexity 300.7962
+13700 train 5.813454 (lr=5.5813e-06) (hash(x)=164396003)
+13800 val loss 5.7019
+13800 val perplexity 299.4438
+13800 train 5.761009 (lr=5.4398e-06) (hash(x)=152712862)
+13900 val loss 5.6972
+13900 val perplexity 298.0294
+13900 train 5.723367 (lr=5.3016e-06) (hash(x)=160985139)
+14000 val loss 5.6961
+14000 val perplexity 297.7088
+14000 train 5.682368 (lr=5.1665e-06) (hash(x)=147429379)
+14100 val loss 5.6939
+14100 val perplexity 297.0557
+14100 train 5.748112 (lr=5.0348e-06) (hash(x)=131830470)
+14200 val loss 5.6948
+14200 val perplexity 297.3153
+14200 train 5.442815 (lr=4.9063e-06) (hash(x)=146383028)
+14300 val loss 5.6928
+14300 val perplexity 296.7108
+14300 train 5.738539 (lr=4.7813e-06) (hash(x)=157158747)
+14400 val loss 5.6954
+14400 val perplexity 297.4931
+14400 train 5.549876 (lr=4.6596e-06) (hash(x)=143911520)
+14500 val loss 5.6943
+14500 val perplexity 297.1754
+14500 train 5.474485 (lr=4.5414e-06) (hash(x)=159555347)
+14600 val loss 5.6888
+14600 val perplexity 295.5297
+14600 train 5.689527 (lr=4.4267e-06) (hash(x)=152957062)
+14700 val loss 5.6849
+14700 val perplexity 294.3972
+14700 train 5.756591 (lr=4.3155e-06) (hash(x)=156001594)
+14800 val loss 5.6850
+14800 val perplexity 294.4080
+14800 train 5.783759 (lr=4.2079e-06) (hash(x)=145885458)
+14900 val loss 5.6814
+14900 val perplexity 293.3562
+14900 train 5.734191 (lr=4.1039e-06) (hash(x)=144544291)
+15000 val loss 5.6825
+15000 val perplexity 293.6803
+15000 train 5.689342 (lr=4.0035e-06) (hash(x)=154606709)
+15100 val loss 5.6817
+15100 val perplexity 293.4573
+15100 train 5.734373 (lr=3.9068e-06) (hash(x)=156005324)
+15200 val loss 5.6824
+15200 val perplexity 293.6678
+15200 train 5.786275 (lr=3.8138e-06) (hash(x)=158847373)
+15300 val loss 5.6821
+15300 val perplexity 293.5554
+15300 train 5.474887 (lr=3.7245e-06) (hash(x)=159609594)
+15400 val loss 5.6780
+15400 val perplexity 292.3782
+15400 train 5.767415 (lr=3.6390e-06) (hash(x)=152428492)
+15500 val loss 5.6752
+15500 val perplexity 291.5412
+15500 train 5.741548 (lr=3.5572e-06) (hash(x)=162251192)
+15600 val loss 5.6734
+15600 val perplexity 291.0144
+15600 train 5.920865 (lr=3.4793e-06) (hash(x)=155072002)
+15700 val loss 5.6726
+15700 val perplexity 290.8026
+15700 train 5.688998 (lr=3.4052e-06) (hash(x)=151203094)
+15800 val loss 5.6722
+15800 val perplexity 290.6788
+15800 train 5.592441 (lr=3.3350e-06) (hash(x)=146125932)
+15900 val loss 5.6719
+15900 val perplexity 290.5904
+15900 train 5.568439 (lr=3.2687e-06) (hash(x)=148060195)
+16000 val loss 5.6720
+16000 val perplexity 290.6114
+16000 train 5.368674 (lr=3.2063e-06) (hash(x)=137406416)
+16100 val loss 5.6725
+16100 val perplexity 290.7514
+16100 train 5.737967 (lr=3.1478e-06) (hash(x)=160149755)
+16200 val loss 5.6678
+16200 val perplexity 289.3838
+16200 train 5.834690 (lr=3.0932e-06) (hash(x)=151371894)
+16300 val loss 5.6665
+16300 val perplexity 289.0163
+16300 train 5.719723 (lr=3.0427e-06) (hash(x)=151810252)
+16400 val loss 5.6650
+16400 val perplexity 288.5883
+16400 train 5.651065 (lr=2.9961e-06) (hash(x)=153088303)
+16500 val loss 5.6643
+16500 val perplexity 288.3983
+16500 train 5.553295 (lr=2.9535e-06) (hash(x)=151706615)
+16600 val loss 5.6628
+16600 val perplexity 287.9627
+16600 train 5.548329 (lr=2.9149e-06) (hash(x)=142110626)
+16700 val loss 5.6636
+16700 val perplexity 288.1895
+16700 train 5.557631 (lr=2.8804e-06) (hash(x)=153222439)
+16800 val loss 5.6631
+16800 val perplexity 288.0464
+16800 train 5.597987 (lr=2.8498e-06) (hash(x)=154964902)
+16900 val loss 5.6597
+16900 val perplexity 287.0678
+16900 train 5.819545 (lr=2.8234e-06) (hash(x)=158893204)
+17000 val loss 5.6579
+17000 val perplexity 286.5460
+17000 train 5.582509 (lr=2.8010e-06) (hash(x)=148929955)
+17100 val loss 5.6586
+17100 val perplexity 286.7525
+17100 train 5.656537 (lr=2.7826e-06) (hash(x)=147818677)
+17200 val loss 5.6572
+17200 val perplexity 286.3454
+17200 train 5.592340 (lr=2.7684e-06) (hash(x)=147119446)
+17300 val loss 5.6573
+17300 val perplexity 286.3883
+17300 train 5.699736 (lr=2.7582e-06) (hash(x)=150494142)
+17400 val loss 5.6580
+17400 val perplexity 286.5887
+17400 train 5.689650 (lr=2.7520e-06) (hash(x)=155918058)
+17499 val loss 5.6557
+17499 val perplexity 285.9162

attention_kindselective_n_heads4_seed1342/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48e71907b9a9c5cb915194076023f3234729f187f59bea2b256a18008ea6605b
+oid sha256:f9def388577ca6684b4d2318222575b30520f226958f22a1a955916b8dc34087
 size 92843394

attention_kindselective_n_heads4_seed1342/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:751318665afe834c16da4ecbe1c1b7971a7477cea8c663e1ec88b6fc29605938
+oid sha256:3631c59d915628c94b2e90c73fd41a51ac337ccb57c78c9ceccbe5cf36299c38
 size 179406214