Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1339", "n_embd": 256}

attention_kindselective_n_heads4_seed1339/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2810
 0 val perplexity 79298.4922
-0 train 11.295413 (lr=1.5000e-07) (hash(x)=150724848)
-100 val loss 9.9471
-100 val perplexity 20892.3984
-100 train 9.995423 (lr=1.5150e-05) (hash(x)=149217061)
-200 val loss 8.9068
-200 val perplexity 7381.7563
-200 train 8.924874 (lr=3.0000e-05) (hash(x)=149910534)
-300 val loss 7.8970
-300 val perplexity 2689.2625
-300 train 7.817223 (lr=2.9998e-05) (hash(x)=142185643)
-400 val loss 7.6460
-400 val perplexity 2092.2935
-400 train 7.695558 (lr=2.9991e-05) (hash(x)=148123706)
-500 val loss 7.5708
-500 val perplexity 1940.7567
-500 train 7.445272 (lr=2.9980e-05) (hash(x)=149952383)
-600 val loss 7.5152
-600 val perplexity 1835.7777
-600 train 7.335600 (lr=2.9964e-05) (hash(x)=146678221)
-700 val loss 7.4498
-700 val perplexity 1719.4503
-700 train 7.483349 (lr=2.9944e-05) (hash(x)=156180736)
-800 val loss 7.4000
-800 val perplexity 1635.9183
-800 train 7.391922 (lr=2.9920e-05) (hash(x)=151700982)
-900 val loss 7.3614
-900 val perplexity 1574.0306
-900 train 7.154299 (lr=2.9891e-05) (hash(x)=147288467)
-1000 val loss 7.3255
-1000 val perplexity 1518.5808
-1000 train 7.366494 (lr=2.9858e-05) (hash(x)=156182087)
-1100 val loss 7.2923
-1100 val perplexity 1468.8792
-1100 train 7.162993 (lr=2.9820e-05) (hash(x)=147861550)
-1200 val loss 7.2558
-1200 val perplexity 1416.3641
-1200 train 7.121205 (lr=2.9778e-05) (hash(x)=149318660)
-1300 val loss 7.2265
-1300 val perplexity 1375.4232
-1300 train 7.061676 (lr=2.9732e-05) (hash(x)=151631103)
-1400 val loss 7.1858
-1400 val perplexity 1320.5341
-1400 train 7.228111 (lr=2.9681e-05) (hash(x)=150482428)
-1500 val loss 7.1244
-1500 val perplexity 1241.9030
-1500 train 6.974311 (lr=2.9626e-05) (hash(x)=144458617)
-1600 val loss 7.0693
-1600 val perplexity 1175.3258
-1600 train 6.943380 (lr=2.9566e-05) (hash(x)=143268605)
-1700 val loss 7.0222
-1700 val perplexity 1121.2941
-1700 train 6.952559 (lr=2.9502e-05) (hash(x)=150817011)
-1800 val loss 6.9806
-1800 val perplexity 1075.6172
-1800 train 6.963822 (lr=2.9434e-05) (hash(x)=152322423)
-1900 val loss 6.9419
-1900 val perplexity 1034.7129
-1900 train 6.733173 (lr=2.9362e-05) (hash(x)=143574126)
-2000 val loss 6.9049
-2000 val perplexity 997.1170
-2000 train 6.658395 (lr=2.9285e-05) (hash(x)=147904298)
-2100 val loss 6.8658
-2100 val perplexity 958.9514
-2100 train 6.606602 (lr=2.9204e-05) (hash(x)=144719400)
-2200 val loss 6.8183
-2200 val perplexity 914.4473
-2200 train 7.060718 (lr=2.9119e-05) (hash(x)=154343147)
-2300 val loss 6.7630
-2300 val perplexity 865.2518
-2300 train 6.803497 (lr=2.9030e-05) (hash(x)=154372635)
-2400 val loss 6.7219
-2400 val perplexity 830.3688
-2400 train 6.548813 (lr=2.8937e-05) (hash(x)=141843115)
-2500 val loss 6.6719
-2500 val perplexity 789.8730
-2500 train 6.569248 (lr=2.8840e-05) (hash(x)=149417679)
-2600 val loss 6.6304
-2600 val perplexity 757.7546
-2600 train 6.501026 (lr=2.8738e-05) (hash(x)=145279030)
-2700 val loss 6.5949
-2700 val perplexity 731.3845
-2700 train 6.507457 (lr=2.8633e-05) (hash(x)=146363911)
-2800 val loss 6.5743
-2800 val perplexity 716.4705
-2800 train 6.523441 (lr=2.8523e-05) (hash(x)=152507639)
-2900 val loss 6.5356
-2900 val perplexity 689.2171
-2900 train 6.322543 (lr=2.8410e-05) (hash(x)=139296191)
-3000 val loss 6.5133
-3000 val perplexity 674.0377
-3000 train 6.520536 (lr=2.8292e-05) (hash(x)=148473774)
-3100 val loss 6.4626
-3100 val perplexity 640.7552
-3100 train 6.553386 (lr=2.8171e-05) (hash(x)=154130527)
-3200 val loss 6.4329
-3200 val perplexity 621.9801
-3200 train 6.385654 (lr=2.8046e-05) (hash(x)=151117002)
-3300 val loss 6.4235
-3300 val perplexity 616.1671
-3300 train 6.582705 (lr=2.7917e-05) (hash(x)=151035383)
-3400 val loss 6.3875
-3400 val perplexity 594.3979
-3400 train 6.521036 (lr=2.7784e-05) (hash(x)=150614749)
-3500 val loss 6.3695
-3500 val perplexity 583.7698
-3500 train 6.374157 (lr=2.7648e-05) (hash(x)=151431391)
-3600 val loss 6.3467
-3600 val perplexity 570.5809
-3600 train 6.277895 (lr=2.7508e-05) (hash(x)=151140797)
-3700 val loss 6.3221
-3700 val perplexity 556.7333
-3700 train 6.505704 (lr=2.7364e-05) (hash(x)=154608340)
-3800 val loss 6.3033
-3800 val perplexity 546.3527
-3800 train 6.334675 (lr=2.7216e-05) (hash(x)=169641309)
-3900 val loss 6.2860
-3900 val perplexity 537.0226
-3900 train 6.250165 (lr=2.7066e-05) (hash(x)=150592479)
-4000 val loss 6.2703
-4000 val perplexity 528.6576
-4000 train 6.258526 (lr=2.6911e-05) (hash(x)=155495317)
-4100 val loss 6.2547
-4100 val perplexity 520.4630
-4100 train 6.449267 (lr=2.6754e-05) (hash(x)=170323970)
-4200 val loss 6.2507
-4200 val perplexity 518.3842
-4200 train 6.087318 (lr=2.6592e-05) (hash(x)=140854360)
-4300 val loss 6.2253
-4300 val perplexity 505.3736
-4300 train 6.244441 (lr=2.6428e-05) (hash(x)=148926644)
-4400 val loss 6.2046
-4400 val perplexity 495.0300
-4400 train 6.239708 (lr=2.6260e-05) (hash(x)=149997871)
-4500 val loss 6.1887
-4500 val perplexity 487.2027
-4500 train 6.140278 (lr=2.6089e-05) (hash(x)=149219289)
-4600 val loss 6.1760
-4600 val perplexity 481.0439
-4600 train 5.976910 (lr=2.5915e-05) (hash(x)=139836197)
-4700 val loss 6.1692
-4700 val perplexity 477.7905
-4700 train 6.123997 (lr=2.5738e-05) (hash(x)=150722883)
-4800 val loss 6.1552
-4800 val perplexity 471.1728
-4800 train 6.370835 (lr=2.5558e-05) (hash(x)=166864648)
-4900 val loss 6.1406
-4900 val perplexity 464.3523
-4900 train 6.248538 (lr=2.5374e-05) (hash(x)=157746743)
-5000 val loss 6.1230
-5000 val perplexity 456.2238
-5000 train 6.070064 (lr=2.5188e-05) (hash(x)=153032257)
-5100 val loss 6.1162
-5100 val perplexity 453.1526
-5100 train 6.186953 (lr=2.4999e-05) (hash(x)=151862371)
-5200 val loss 6.1029
-5200 val perplexity 447.1548
-5200 train 6.414737 (lr=2.4807e-05) (hash(x)=168862995)
-5300 val loss 6.1030
-5300 val perplexity 447.1857
-5300 train 6.067278 (lr=2.4613e-05) (hash(x)=148243567)
-5400 val loss 6.0834
-5400 val perplexity 438.5384
-5400 train 6.013880 (lr=2.4415e-05) (hash(x)=145381742)
-5500 val loss 6.0749
-5500 val perplexity 434.8040
-5500 train 6.234234 (lr=2.4215e-05) (hash(x)=155602174)
-5600 val loss 6.0558
-5600 val perplexity 426.5808
-5600 train 6.131358 (lr=2.4013e-05) (hash(x)=160637672)
-5700 val loss 6.0488
-5700 val perplexity 423.5999
-5700 train 5.978982 (lr=2.3808e-05) (hash(x)=152101131)
-5800 val loss 6.0364
-5800 val perplexity 418.3911
-5800 train 6.002306 (lr=2.3601e-05) (hash(x)=152327089)
-5900 val loss 6.0371
-5900 val perplexity 418.6625
-5900 train 5.995678 (lr=2.3391e-05) (hash(x)=151094543)
-6000 val loss 6.0246
-6000 val perplexity 413.4569
-6000 train 6.119165 (lr=2.3179e-05) (hash(x)=160233315)
-6100 val loss 6.0102
-6100 val perplexity 407.5835
-6100 train 6.113656 (lr=2.2965e-05) (hash(x)=158295337)
-6200 val loss 5.9963
-6200 val perplexity 401.9257
-6200 train 5.958003 (lr=2.2749e-05) (hash(x)=151269817)
-6300 val loss 5.9892
-6300 val perplexity 399.1044
-6300 train 6.144239 (lr=2.2530e-05) (hash(x)=156912756)
-6400 val loss 5.9820
-6400 val perplexity 396.2269
-6400 train 5.911047 (lr=2.2310e-05) (hash(x)=125779637)
-6500 val loss 5.9787
-6500 val perplexity 394.9186
-6500 train 5.875282 (lr=2.2088e-05) (hash(x)=148157661)
-6600 val loss 5.9746
-6600 val perplexity 393.2974
-6600 train 5.987086 (lr=2.1864e-05) (hash(x)=152820049)
-6700 val loss 5.9633
-6700 val perplexity 388.8889
-6700 train 5.899116 (lr=2.1638e-05) (hash(x)=155818717)
-6800 val loss 5.9481
-6800 val perplexity 383.0392
-6800 train 5.957938 (lr=2.1410e-05) (hash(x)=158833881)
-6900 val loss 5.9493
-6900 val perplexity 383.4933
-6900 train 5.839056 (lr=2.1181e-05) (hash(x)=164984909)
-7000 val loss 5.9369
-7000 val perplexity 378.7484
-7000 train 5.838758 (lr=2.0951e-05) (hash(x)=151172493)
-7100 val loss 5.9317
-7100 val perplexity 376.7919
-7100 train 5.992643 (lr=2.0718e-05) (hash(x)=154023633)
-7200 val loss 5.9226
-7200 val perplexity 373.3749
-7200 train 5.773759 (lr=2.0485e-05) (hash(x)=139351250)
-7300 val loss 5.9178
-7300 val perplexity 371.5756
-7300 train 5.878859 (lr=2.0250e-05) (hash(x)=158719458)
-7400 val loss 5.9039
-7400 val perplexity 366.4463
-7400 train 5.980419 (lr=2.0014e-05) (hash(x)=162250810)
-7500 val loss 5.8974
-7500 val perplexity 364.0830
-7500 train 5.782121 (lr=1.9777e-05) (hash(x)=142827557)
-7600 val loss 5.8903
-7600 val perplexity 361.5138
-7600 train 5.795710 (lr=1.9538e-05) (hash(x)=147830801)
-7700 val loss 5.8812
-7700 val perplexity 358.2511
-7700 train 5.979882 (lr=1.9299e-05) (hash(x)=165233314)
-7800 val loss 5.8823
-7800 val perplexity 358.6237
-7800 train 5.873540 (lr=1.9059e-05) (hash(x)=160654717)
-7900 val loss 5.8770
-7900 val perplexity 356.7522
-7900 train 5.746340 (lr=1.8817e-05) (hash(x)=140712586)
-8000 val loss 5.8659
-8000 val perplexity 352.8039
-8000 train 6.036586 (lr=1.8576e-05) (hash(x)=150309061)
-8100 val loss 5.8587
-8100 val perplexity 350.2709
-8100 train 5.856085 (lr=1.8333e-05) (hash(x)=158805466)
-8200 val loss 5.8494
-8200 val perplexity 347.0211
-8200 train 5.742389 (lr=1.8090e-05) (hash(x)=142814078)
-8300 val loss 5.8445
-8300 val perplexity 345.3150
-8300 train 5.809830 (lr=1.7846e-05) (hash(x)=148340417)
-8400 val loss 5.8435
-8400 val perplexity 344.9737
-8400 train 5.779657 (lr=1.7602e-05) (hash(x)=157132639)
-8500 val loss 5.8382
-8500 val perplexity 343.1600
-8500 train 5.765084 (lr=1.7357e-05) (hash(x)=152994240)
-8600 val loss 5.8261
-8600 val perplexity 339.0495
-8600 train 5.788753 (lr=1.7113e-05) (hash(x)=152210154)
-8700 val loss 5.8198
-8700 val perplexity 336.9041
-8700 train 5.916071 (lr=1.6868e-05) (hash(x)=149631247)
-8800 val loss 5.8151
-8800 val perplexity 335.3272
-8800 train 5.684615 (lr=1.6623e-05) (hash(x)=145617784)
-8900 val loss 5.8101
-8900 val perplexity 333.6368
-8900 train 5.917306 (lr=1.6377e-05) (hash(x)=155984970)
-9000 val loss 5.8076
-9000 val perplexity 332.8319
-9000 train 5.541238 (lr=1.6132e-05) (hash(x)=143433013)
-9100 val loss 5.8032
-9100 val perplexity 331.3690
-9100 train 5.712577 (lr=1.5887e-05) (hash(x)=152507533)
-9200 val loss 5.7958
-9200 val perplexity 328.9049
-9200 train 5.927247 (lr=1.5643e-05) (hash(x)=162400650)
-9300 val loss 5.7907
-9300 val perplexity 327.2459
-9300 train 5.836190 (lr=1.5398e-05) (hash(x)=148419068)
-9400 val loss 5.7857
-9400 val perplexity 325.6125
-9400 train 5.751761 (lr=1.5154e-05) (hash(x)=145025169)
-9500 val loss 5.7782
-9500 val perplexity 323.1750
-9500 train 5.749483 (lr=1.4910e-05) (hash(x)=145104235)
-9600 val loss 5.7757
-9600 val perplexity 322.3782
-9600 train 5.712863 (lr=1.4667e-05) (hash(x)=150499797)
-9700 val loss 5.7716
-9700 val perplexity 321.0546
-9700 train 5.580113 (lr=1.4424e-05) (hash(x)=139690022)
-9800 val loss 5.7661
-9800 val perplexity 319.3048
-9800 train 5.768727 (lr=1.4183e-05) (hash(x)=145645279)
-9900 val loss 5.7569
-9900 val perplexity 316.3550
-9900 train 5.687554 (lr=1.3941e-05) (hash(x)=150103432)
-10000 val loss 5.7571
-10000 val perplexity 316.4407
-10000 train 5.912628 (lr=1.3701e-05) (hash(x)=162309473)
-10100 val loss 5.7508
-10100 val perplexity 314.4393
-10100 train 5.638128 (lr=1.3462e-05) (hash(x)=146586772)
-10200 val loss 5.7462
-10200 val perplexity 312.9856
-10200 train 5.645834 (lr=1.3223e-05) (hash(x)=149256313)
-10300 val loss 5.7423
-10300 val perplexity 311.7654
-10300 train 5.703022 (lr=1.2986e-05) (hash(x)=151455083)
-10400 val loss 5.7415
-10400 val perplexity 311.5248
-10400 train 5.517107 (lr=1.2750e-05) (hash(x)=150210444)
-10500 val loss 5.7448
-10500 val perplexity 312.5583
-10500 train 5.516900 (lr=1.2515e-05) (hash(x)=152144428)
-10600 val loss 5.7347
-10600 val perplexity 309.4212
-10600 train 5.781371 (lr=1.2282e-05) (hash(x)=168628935)
-10700 val loss 5.7286
-10700 val perplexity 307.5286
-10700 train 5.520483 (lr=1.2049e-05) (hash(x)=164667785)
-10800 val loss 5.7195
-10800 val perplexity 304.7384
-10800 train 5.695211 (lr=1.1819e-05) (hash(x)=143621491)
-10900 val loss 5.7186
-10900 val perplexity 304.4742
-10900 train 5.612442 (lr=1.1590e-05) (hash(x)=143914826)
-11000 val loss 5.7168
-11000 val perplexity 303.9456
-11000 train 5.496383 (lr=1.1362e-05) (hash(x)=137663435)
-11100 val loss 5.7156
-11100 val perplexity 303.5701
-11100 train 5.597968 (lr=1.1136e-05) (hash(x)=149054007)
-11200 val loss 5.7137
-11200 val perplexity 302.9761
-11200 train 5.722148 (lr=1.0912e-05) (hash(x)=153473924)
-11300 val loss 5.7115
-11300 val perplexity 302.3359
-11300 train 5.504845 (lr=1.0690e-05) (hash(x)=158563275)
-11400 val loss 5.7127
-11400 val perplexity 302.6973
-11400 train 5.851794 (lr=1.0470e-05) (hash(x)=161016667)
-11500 val loss 5.7064
-11500 val perplexity 300.7805
-11500 train 5.754418 (lr=1.0251e-05) (hash(x)=157344822)
-11600 val loss 5.6957
-11600 val perplexity 297.5784
-11600 train 5.589668 (lr=1.0035e-05) (hash(x)=139977713)
-11700 val loss 5.6905
-11700 val perplexity 296.0485
-11700 train 5.684609 (lr=9.8209e-06) (hash(x)=156511738)
-11800 val loss 5.6911
-11800 val perplexity 296.2219
-11800 train 6.142660 (lr=9.6090e-06) (hash(x)=185372558)
-11900 val loss 5.6868
-11900 val perplexity 294.9570
-11900 train 5.536606 (lr=9.3993e-06) (hash(x)=138085470)
-12000 val loss 5.6874
-12000 val perplexity 295.1362
-12000 train 5.535735 (lr=9.1920e-06) (hash(x)=142928235)
-12100 val loss 5.6871
-12100 val perplexity 295.0487
-12100 train 5.533151 (lr=8.9871e-06) (hash(x)=151960396)
-12200 val loss 5.6783
-12200 val perplexity 292.4572
-12200 train 5.732773 (lr=8.7846e-06) (hash(x)=156794523)
-12300 val loss 5.6751
-12300 val perplexity 291.5198
-12300 train 5.482319 (lr=8.5847e-06) (hash(x)=138082358)
-12400 val loss 5.6712
-12400 val perplexity 290.3687
-12400 train 5.574787 (lr=8.3875e-06) (hash(x)=155255899)
-12500 val loss 5.6670
-12500 val perplexity 289.1582
-12500 train 5.633625 (lr=8.1929e-06) (hash(x)=150490528)
-12600 val loss 5.6654
-12600 val perplexity 288.7173
-12600 train 5.813358 (lr=8.0010e-06) (hash(x)=154831027)
-12700 val loss 5.6657
-12700 val perplexity 288.7786
-12700 train 5.622353 (lr=7.8119e-06) (hash(x)=147341052)
-12800 val loss 5.6650
-12800 val perplexity 288.5848
-12800 train 5.699083 (lr=7.6257e-06) (hash(x)=159333795)
-12900 val loss 5.6663
-12900 val perplexity 288.9516
-12900 train 5.509811 (lr=7.4425e-06) (hash(x)=143896867)
-13000 val loss 5.6566
-13000 val perplexity 286.1717
-13000 train 5.669372 (lr=7.2622e-06) (hash(x)=147161537)
-13100 val loss 5.6542
-13100 val perplexity 285.4856
-13100 train 5.788258 (lr=7.0850e-06) (hash(x)=147081918)
-13200 val loss 5.6536
-13200 val perplexity 285.3229
-13200 train 5.424483 (lr=6.9108e-06) (hash(x)=145079287)
-13300 val loss 5.6523
-13300 val perplexity 284.9567
-13300 train 5.450130 (lr=6.7398e-06) (hash(x)=142515041)
-13400 val loss 5.6506
-13400 val perplexity 284.4539
-13400 train 5.442078 (lr=6.5721e-06) (hash(x)=144081985)
-13500 val loss 5.6479
-13500 val perplexity 283.7066
-13500 train 5.731144 (lr=6.4076e-06) (hash(x)=149957292)
-13600 val loss 5.6435
-13600 val perplexity 282.4502
-13600 train 5.672267 (lr=6.2465e-06) (hash(x)=157123033)
-13700 val loss 5.6416
-13700 val perplexity 281.9233
-13700 train 5.625963 (lr=6.0887e-06) (hash(x)=148277247)
-13800 val loss 5.6432
-13800 val perplexity 282.3522
-13800 train 5.776651 (lr=5.9344e-06) (hash(x)=142769907)
-13900 val loss 5.6362
-13900 val perplexity 280.4048
-13900 train 5.587573 (lr=5.7835e-06) (hash(x)=144932004)
-14000 val loss 5.6349
-14000 val perplexity 280.0245
-14000 train 5.657510 (lr=5.6362e-06) (hash(x)=149532265)
-14100 val loss 5.6347
-14100 val perplexity 279.9716
-14100 train 5.452249 (lr=5.4925e-06) (hash(x)=149301374)
-14200 val loss 5.6345
-14200 val perplexity 279.9138
-14200 train 5.374617 (lr=5.3524e-06) (hash(x)=142871075)
-14300 val loss 5.6303
-14300 val perplexity 278.7531
-14300 train 5.782886 (lr=5.2159e-06) (hash(x)=166744960)
-14400 val loss 5.6266
-14400 val perplexity 277.7065
-14400 train 5.592690 (lr=5.0832e-06) (hash(x)=145950920)
-14500 val loss 5.6249
-14500 val perplexity 277.2531
-14500 train 5.620202 (lr=4.9543e-06) (hash(x)=141089155)
-14600 val loss 5.6226
-14600 val perplexity 276.6090
-14600 train 5.778128 (lr=4.8291e-06) (hash(x)=174516275)
-14700 val loss 5.6212
-14700 val perplexity 276.2231
-14700 train 5.430392 (lr=4.7079e-06) (hash(x)=137896065)
-14800 val loss 5.6209
-14800 val perplexity 276.1483
-14800 train 5.662985 (lr=4.5905e-06) (hash(x)=154396556)
-14900 val loss 5.6214
-14900 val perplexity 276.2749
-14900 train 5.470448 (lr=4.4770e-06) (hash(x)=149322394)
-15000 val loss 5.6214
-15000 val perplexity 276.2655
-15000 train 5.537586 (lr=4.3675e-06) (hash(x)=147410513)
-15100 val loss 5.6189
-15100 val perplexity 275.5881
-15100 train 5.530842 (lr=4.2620e-06) (hash(x)=145245634)
-15200 val loss 5.6146
-15200 val perplexity 274.4064
-15200 train 5.703465 (lr=4.1605e-06) (hash(x)=148341108)
-15300 val loss 5.6130
-15300 val perplexity 273.9719
-15300 train 5.423482 (lr=4.0631e-06) (hash(x)=140162437)
-15400 val loss 5.6127
-15400 val perplexity 273.8959
-15400 train 5.822367 (lr=3.9698e-06) (hash(x)=142166081)
-15500 val loss 5.6111
-15500 val perplexity 273.4330
-15500 train 5.532878 (lr=3.8806e-06) (hash(x)=141042196)
-15600 val loss 5.6105
-15600 val perplexity 273.2874
-15600 train 5.737755 (lr=3.7956e-06) (hash(x)=148672701)
-15700 val loss 5.6109
-15700 val perplexity 273.3769
-15700 train 5.517663 (lr=3.7148e-06) (hash(x)=149672385)
-15800 val loss 5.6109
-15800 val perplexity 273.4001
-15800 train 5.636478 (lr=3.6382e-06) (hash(x)=152379562)
-15900 val loss 5.6063
-15900 val perplexity 272.1482
-15900 train 5.659053 (lr=3.5658e-06) (hash(x)=154674265)
-16000 val loss 5.6055
-16000 val perplexity 271.9062
-16000 train 5.651720 (lr=3.4977e-06) (hash(x)=150368078)
-16100 val loss 5.6030
-16100 val perplexity 271.2387
-16100 train 5.712838 (lr=3.4339e-06) (hash(x)=160390561)
-16200 val loss 5.6025
-16200 val perplexity 271.1014
-16200 train 5.973002 (lr=3.3744e-06) (hash(x)=155218378)
-16300 val loss 5.6012
-16300 val perplexity 270.7619
-16300 train 5.792291 (lr=3.3193e-06) (hash(x)=160891077)
-16400 val loss 5.6012
-16400 val perplexity 270.7604
-16400 train 5.501259 (lr=3.2684e-06) (hash(x)=142789253)
-16500 val loss 5.6012
-16500 val perplexity 270.7393
-16500 train 5.344845 (lr=3.2220e-06) (hash(x)=141497841)
-16600 val loss 5.6001
-16600 val perplexity 270.4425
-16600 train 5.567000 (lr=3.1799e-06) (hash(x)=157947018)
-16700 val loss 5.5973
-16700 val perplexity 269.7056
-16700 train 5.557473 (lr=3.1422e-06) (hash(x)=146039309)
-16800 val loss 5.5967
-16800 val perplexity 269.5226
-16800 train 5.564966 (lr=3.1089e-06) (hash(x)=156381445)
-16900 val loss 5.5951
-16900 val perplexity 269.1154
-16900 train 5.553369 (lr=3.0801e-06) (hash(x)=146537354)
-17000 val loss 5.5950
-17000 val perplexity 269.0740
-17000 train 5.522299 (lr=3.0556e-06) (hash(x)=135939444)
-17100 val loss 5.5942
-17100 val perplexity 268.8705
-17100 train 5.673860 (lr=3.0356e-06) (hash(x)=147819652)
-17200 val loss 5.5945
-17200 val perplexity 268.9415
-17200 train 5.685050 (lr=3.0200e-06) (hash(x)=155307909)
-17300 val loss 5.5914
-17300 val perplexity 268.1093
-17300 train 5.543001 (lr=3.0089e-06) (hash(x)=151879548)
-17400 val loss 5.5902
-17400 val perplexity 267.7778
-17400 train 5.508076 (lr=3.0022e-06) (hash(x)=143641792)
-17499 val loss 5.5892
-17499 val perplexity 267.5307
+0 train 11.295413 (lr=1.3750e-07) (hash(x)=150724848)
+100 val loss 9.9603
+100 val perplexity 21170.0430
+100 train 10.007094 (lr=1.3888e-05) (hash(x)=149217061)
+200 val loss 8.9723
+200 val perplexity 7882.0664
+200 train 8.990173 (lr=2.7500e-05) (hash(x)=149910534)
+300 val loss 7.9449
+300 val perplexity 2821.2769
+300 train 7.869526 (lr=2.7498e-05) (hash(x)=142185643)
+400 val loss 7.6747
+400 val perplexity 2153.2439
+400 train 7.723728 (lr=2.7492e-05) (hash(x)=148123706)
+500 val loss 7.5919
+500 val perplexity 1982.0444
+500 train 7.469253 (lr=2.7482e-05) (hash(x)=149952383)
+600 val loss 7.5425
+600 val perplexity 1886.4757
+600 train 7.369303 (lr=2.7467e-05) (hash(x)=146678221)
+700 val loss 7.4943
+700 val perplexity 1797.8137
+700 train 7.524321 (lr=2.7449e-05) (hash(x)=156180736)
+800 val loss 7.4513
+800 val perplexity 1722.1606
+800 train 7.442773 (lr=2.7427e-05) (hash(x)=151700982)
+900 val loss 7.4230
+900 val perplexity 1673.9784
+900 train 7.216110 (lr=2.7400e-05) (hash(x)=147288467)
+1000 val loss 7.3971
+1000 val perplexity 1631.3278
+1000 train 7.439397 (lr=2.7370e-05) (hash(x)=156182087)
+1100 val loss 7.3783
+1100 val perplexity 1600.9114
+1100 train 7.255154 (lr=2.7335e-05) (hash(x)=147861550)
+1200 val loss 7.3593
+1200 val perplexity 1570.6859
+1200 train 7.232437 (lr=2.7297e-05) (hash(x)=149318660)
+1300 val loss 7.3489
+1300 val perplexity 1554.4735
+1300 train 7.209687 (lr=2.7254e-05) (hash(x)=151631103)
+1400 val loss 7.3331
+1400 val perplexity 1530.1715
+1400 train 7.381249 (lr=2.7207e-05) (hash(x)=150482428)
+1500 val loss 7.3048
+1500 val perplexity 1487.3849
+1500 train 7.156554 (lr=2.7157e-05) (hash(x)=144458617)
+1600 val loss 7.2813
+1600 val perplexity 1452.9476
+1600 train 7.155491 (lr=2.7102e-05) (hash(x)=143268605)
+1700 val loss 7.2596
+1700 val perplexity 1421.6182
+1700 train 7.191982 (lr=2.7044e-05) (hash(x)=150817011)
+1800 val loss 7.2414
+1800 val perplexity 1396.0886
+1800 train 7.228529 (lr=2.6981e-05) (hash(x)=152322423)
+1900 val loss 7.2235
+1900 val perplexity 1371.2922
+1900 train 7.044053 (lr=2.6915e-05) (hash(x)=143574126)
+2000 val loss 7.2147
+2000 val perplexity 1359.2687
+2000 train 6.985788 (lr=2.6845e-05) (hash(x)=147904298)
+2100 val loss 7.1918
+2100 val perplexity 1328.4425
+2100 train 6.953863 (lr=2.6771e-05) (hash(x)=144719400)
+2200 val loss 7.1646
+2200 val perplexity 1292.8639
+2200 train 7.402001 (lr=2.6693e-05) (hash(x)=154343147)
+2300 val loss 7.1374
+2300 val perplexity 1258.2095
+2300 train 7.186423 (lr=2.6611e-05) (hash(x)=154372635)
+2400 val loss 7.1253
+2400 val perplexity 1242.9806
+2400 train 6.955888 (lr=2.6525e-05) (hash(x)=141843115)
+2500 val loss 7.0979
+2500 val perplexity 1209.3766
+2500 train 6.993516 (lr=2.6436e-05) (hash(x)=149417679)
+2600 val loss 7.0803
+2600 val perplexity 1188.2710
+2600 train 6.944737 (lr=2.6343e-05) (hash(x)=145279030)
+2700 val loss 7.0618
+2700 val perplexity 1166.5103
+2700 train 6.953462 (lr=2.6246e-05) (hash(x)=146363911)
+2800 val loss 7.0477
+2800 val perplexity 1150.1772
+2800 train 7.019940 (lr=2.6146e-05) (hash(x)=152507639)
+2900 val loss 7.0327
+2900 val perplexity 1133.1031
+2900 train 6.825722 (lr=2.6042e-05) (hash(x)=139296191)
+3000 val loss 7.0194
+3000 val perplexity 1118.1536
+3000 train 7.043332 (lr=2.5934e-05) (hash(x)=148473774)
+3100 val loss 6.9894
+3100 val perplexity 1085.0210
+3100 train 7.095091 (lr=2.5823e-05) (hash(x)=154130527)
+3200 val loss 6.9654
+3200 val perplexity 1059.3645
+3200 train 6.898325 (lr=2.5709e-05) (hash(x)=151117002)
+3300 val loss 6.9591
+3300 val perplexity 1052.6724
+3300 train 7.079136 (lr=2.5590e-05) (hash(x)=151035383)
+3400 val loss 6.9313
+3400 val perplexity 1023.8008
+3400 train 7.011525 (lr=2.5469e-05) (hash(x)=150614749)
+3500 val loss 6.9130
+3500 val perplexity 1005.3008
+3500 train 6.961452 (lr=2.5344e-05) (hash(x)=151431391)
+3600 val loss 6.8958
+3600 val perplexity 988.1005
+3600 train 6.819849 (lr=2.5215e-05) (hash(x)=151140797)
+3700 val loss 6.8788
+3700 val perplexity 971.4802
+3700 train 7.120714 (lr=2.5083e-05) (hash(x)=154608340)
+3800 val loss 6.8610
+3800 val perplexity 954.3665
+3800 train 6.883213 (lr=2.4948e-05) (hash(x)=169641309)
+3900 val loss 6.8405
+3900 val perplexity 934.9172
+3900 train 6.791928 (lr=2.4810e-05) (hash(x)=150592479)
+4000 val loss 6.8152
+4000 val perplexity 911.6139
+4000 train 6.790205 (lr=2.4669e-05) (hash(x)=155495317)
+4100 val loss 6.8076
+4100 val perplexity 904.6551
+4100 train 7.009809 (lr=2.4524e-05) (hash(x)=170323970)
+4200 val loss 6.7861
+4200 val perplexity 885.4940
+4200 train 6.587790 (lr=2.4376e-05) (hash(x)=140854360)
+4300 val loss 6.7558
+4300 val perplexity 859.0294
+4300 train 6.750789 (lr=2.4226e-05) (hash(x)=148926644)
+4400 val loss 6.7324
+4400 val perplexity 839.1576
+4400 train 6.774224 (lr=2.4072e-05) (hash(x)=149997871)
+4500 val loss 6.7066
+4500 val perplexity 817.7622
+4500 train 6.658325 (lr=2.3915e-05) (hash(x)=149219289)
+4600 val loss 6.6926
+4600 val perplexity 806.4287
+4600 train 6.489277 (lr=2.3755e-05) (hash(x)=139836197)
+4700 val loss 6.6692
+4700 val perplexity 787.7410
+4700 train 6.632764 (lr=2.3593e-05) (hash(x)=150722883)
+4800 val loss 6.6471
+4800 val perplexity 770.5707
+4800 train 6.862972 (lr=2.3428e-05) (hash(x)=166864648)
+4900 val loss 6.6300
+4900 val perplexity 757.4689
+4900 train 6.746885 (lr=2.3260e-05) (hash(x)=157746743)
+5000 val loss 6.6030
+5000 val perplexity 737.3123
+5000 train 6.564852 (lr=2.3089e-05) (hash(x)=153032257)
+5100 val loss 6.5809
+5100 val perplexity 721.2192
+5100 train 6.662528 (lr=2.2916e-05) (hash(x)=151862371)
+5200 val loss 6.5598
+5200 val perplexity 706.0972
+5200 train 6.881927 (lr=2.2740e-05) (hash(x)=168862995)
+5300 val loss 6.5497
+5300 val perplexity 699.0560
+5300 train 6.526187 (lr=2.2561e-05) (hash(x)=148243567)
+5400 val loss 6.5310
+5400 val perplexity 686.0756
+5400 train 6.454970 (lr=2.2381e-05) (hash(x)=145381742)
+5500 val loss 6.5144
+5500 val perplexity 674.7809
+5500 train 6.669298 (lr=2.2197e-05) (hash(x)=155602174)
+5600 val loss 6.4878
+5600 val perplexity 657.1042
+5600 train 6.578669 (lr=2.2012e-05) (hash(x)=160637672)
+5700 val loss 6.4728
+5700 val perplexity 647.2852
+5700 train 6.409344 (lr=2.1824e-05) (hash(x)=152101131)
+5800 val loss 6.4536
+5800 val perplexity 634.9938
+5800 train 6.406909 (lr=2.1634e-05) (hash(x)=152327089)
+5900 val loss 6.4456
+5900 val perplexity 629.9174
+5900 train 6.379840 (lr=2.1442e-05) (hash(x)=151094543)
+6000 val loss 6.4294
+6000 val perplexity 619.8093
+6000 train 6.527921 (lr=2.1248e-05) (hash(x)=160233315)
+6100 val loss 6.4108
+6100 val perplexity 608.3984
+6100 train 6.511137 (lr=2.1051e-05) (hash(x)=158295337)
+6200 val loss 6.3918
+6200 val perplexity 596.9457
+6200 train 6.349186 (lr=2.0853e-05) (hash(x)=151269817)
+6300 val loss 6.3760
+6300 val perplexity 587.5455
+6300 train 6.490969 (lr=2.0653e-05) (hash(x)=156912756)
+6400 val loss 6.3643
+6400 val perplexity 580.7366
+6400 train 6.304334 (lr=2.0451e-05) (hash(x)=125779637)
+6500 val loss 6.3546
+6500 val perplexity 575.1245
+6500 train 6.241210 (lr=2.0247e-05) (hash(x)=148157661)
+6600 val loss 6.3439
+6600 val perplexity 569.0137
+6600 train 6.392389 (lr=2.0042e-05) (hash(x)=152820049)
+6700 val loss 6.3313
+6700 val perplexity 561.9103
+6700 train 6.277595 (lr=1.9835e-05) (hash(x)=155818717)
+6800 val loss 6.3142
+6800 val perplexity 552.3473
+6800 train 6.357372 (lr=1.9626e-05) (hash(x)=158833881)
+6900 val loss 6.3115
+6900 val perplexity 550.8728
+6900 train 6.280612 (lr=1.9416e-05) (hash(x)=164984909)
+7000 val loss 6.2944
+7000 val perplexity 541.5046
+7000 train 6.183699 (lr=1.9205e-05) (hash(x)=151172493)
+7100 val loss 6.2913
+7100 val perplexity 539.8703
+7100 train 6.301157 (lr=1.8992e-05) (hash(x)=154023633)
+7200 val loss 6.2841
+7200 val perplexity 535.9922
+7200 train 6.126408 (lr=1.8778e-05) (hash(x)=139351250)
+7300 val loss 6.2701
+7300 val perplexity 528.5300
+7300 train 6.216894 (lr=1.8562e-05) (hash(x)=158719458)
+7400 val loss 6.2545
+7400 val perplexity 520.3541
+7400 train 6.342453 (lr=1.8346e-05) (hash(x)=162250810)
+7500 val loss 6.2451
+7500 val perplexity 515.4594
+7500 train 6.107119 (lr=1.8128e-05) (hash(x)=142827557)
+7600 val loss 6.2375
+7600 val perplexity 511.5827
+7600 train 6.134127 (lr=1.7910e-05) (hash(x)=147830801)
+7700 val loss 6.2256
+7700 val perplexity 505.5025
+7700 train 6.331251 (lr=1.7691e-05) (hash(x)=165233314)
+7800 val loss 6.2179
+7800 val perplexity 501.6711
+7800 train 6.216796 (lr=1.7470e-05) (hash(x)=160654717)
+7900 val loss 6.2136
+7900 val perplexity 499.4775
+7900 train 6.051061 (lr=1.7249e-05) (hash(x)=140712586)
+8000 val loss 6.1968
+8000 val perplexity 491.1817
+8000 train 6.355695 (lr=1.7028e-05) (hash(x)=150309061)
+8100 val loss 6.1933
+8100 val perplexity 489.4791
+8100 train 6.204559 (lr=1.6805e-05) (hash(x)=158805466)
+8200 val loss 6.1826
+8200 val perplexity 484.2360
+8200 train 6.047708 (lr=1.6582e-05) (hash(x)=142814078)
+8300 val loss 6.1745
+8300 val perplexity 480.3483
+8300 train 6.138761 (lr=1.6359e-05) (hash(x)=148340417)
+8400 val loss 6.1701
+8400 val perplexity 478.2222
+8400 train 6.109375 (lr=1.6135e-05) (hash(x)=157132639)
+8500 val loss 6.1692
+8500 val perplexity 477.8114
+8500 train 6.108513 (lr=1.5911e-05) (hash(x)=152994240)
+8600 val loss 6.1531
+8600 val perplexity 470.1633
+8600 train 6.107152 (lr=1.5687e-05) (hash(x)=152210154)
+8700 val loss 6.1455
+8700 val perplexity 466.6085
+8700 train 6.205480 (lr=1.5462e-05) (hash(x)=149631247)
+8800 val loss 6.1372
+8800 val perplexity 462.7359
+8800 train 5.981178 (lr=1.5237e-05) (hash(x)=145617784)
+8900 val loss 6.1299
+8900 val perplexity 459.3880
+8900 train 6.226513 (lr=1.5013e-05) (hash(x)=155984970)
+9000 val loss 6.1254
+9000 val perplexity 457.3501
+9000 train 5.854463 (lr=1.4788e-05) (hash(x)=143433013)
+9100 val loss 6.1202
+9100 val perplexity 454.9649
+9100 train 6.005787 (lr=1.4563e-05) (hash(x)=152507533)
+9200 val loss 6.1142
+9200 val perplexity 452.2314
+9200 train 6.255028 (lr=1.4339e-05) (hash(x)=162400650)
+9300 val loss 6.1044
+9300 val perplexity 447.8143
+9300 train 6.153483 (lr=1.4115e-05) (hash(x)=148419068)
+9400 val loss 6.0967
+9400 val perplexity 444.3860
+9400 train 6.057858 (lr=1.3891e-05) (hash(x)=145025169)
+9500 val loss 6.0875
+9500 val perplexity 440.3331
+9500 train 6.061573 (lr=1.3668e-05) (hash(x)=145104235)
+9600 val loss 6.0901
+9600 val perplexity 441.4541
+9600 train 6.020576 (lr=1.3445e-05) (hash(x)=150499797)
+9700 val loss 6.0780
+9700 val perplexity 436.1382
+9700 train 5.881082 (lr=1.3222e-05) (hash(x)=139690022)
+9800 val loss 6.0699
+9800 val perplexity 432.6385
+9800 train 6.058245 (lr=1.3001e-05) (hash(x)=145645279)
+9900 val loss 6.0611
+9900 val perplexity 428.8531
+9900 train 5.971967 (lr=1.2780e-05) (hash(x)=150103432)
+10000 val loss 6.0610
+10000 val perplexity 428.7884
+10000 train 6.213492 (lr=1.2559e-05) (hash(x)=162309473)
+10100 val loss 6.0507
+10100 val perplexity 424.4024
+10100 train 5.937528 (lr=1.2340e-05) (hash(x)=146586772)
+10200 val loss 6.0450
+10200 val perplexity 422.0151
+10200 train 5.940793 (lr=1.2122e-05) (hash(x)=149256313)
+10300 val loss 6.0396
+10300 val perplexity 419.7351
+10300 train 6.018402 (lr=1.1904e-05) (hash(x)=151455083)
+10400 val loss 6.0374
+10400 val perplexity 418.8122
+10400 train 5.842232 (lr=1.1688e-05) (hash(x)=150210444)
+10500 val loss 6.0397
+10500 val perplexity 419.7683
+10500 train 5.804628 (lr=1.1472e-05) (hash(x)=152144428)
+10600 val loss 6.0296
+10600 val perplexity 415.5560
+10600 train 6.099782 (lr=1.1258e-05) (hash(x)=168628935)
+10700 val loss 6.0206
+10700 val perplexity 411.8316
+10700 train 5.880192 (lr=1.1045e-05) (hash(x)=164667785)
+10800 val loss 6.0117
+10800 val perplexity 408.1864
+10800 train 5.955902 (lr=1.0834e-05) (hash(x)=143621491)
+10900 val loss 6.0106
+10900 val perplexity 407.7130
+10900 train 5.899141 (lr=1.0624e-05) (hash(x)=143914826)
+11000 val loss 6.0058
+11000 val perplexity 405.7754
+11000 train 5.781573 (lr=1.0415e-05) (hash(x)=137663435)
+11100 val loss 6.0036
+11100 val perplexity 404.8896
+11100 train 5.873570 (lr=1.0208e-05) (hash(x)=149054007)
+11200 val loss 6.0013
+11200 val perplexity 403.9420
+11200 train 6.005395 (lr=1.0003e-05) (hash(x)=153473924)
+11300 val loss 6.0030
+11300 val perplexity 404.6429
+11300 train 5.808824 (lr=9.7991e-06) (hash(x)=158563275)
+11400 val loss 6.0014
+11400 val perplexity 403.9809
+11400 train 6.135341 (lr=9.5972e-06) (hash(x)=161016667)
+11500 val loss 5.9971
+11500 val perplexity 402.2647
+11500 train 6.046785 (lr=9.3970e-06) (hash(x)=157344822)
+11600 val loss 5.9819
+11600 val perplexity 396.1853
+11600 train 5.841331 (lr=9.1988e-06) (hash(x)=139977713)
+11700 val loss 5.9752
+11700 val perplexity 393.5630
+11700 train 5.986742 (lr=9.0025e-06) (hash(x)=156511738)
+11800 val loss 5.9721
+11800 val perplexity 392.3307
+11800 train 6.460408 (lr=8.8082e-06) (hash(x)=185372558)
+11900 val loss 5.9709
+11900 val perplexity 391.8408
+11900 train 5.801854 (lr=8.6160e-06) (hash(x)=138085470)
+12000 val loss 5.9692
+12000 val perplexity 391.1958
+12000 train 5.824976 (lr=8.4260e-06) (hash(x)=142928235)
+12100 val loss 5.9703
+12100 val perplexity 391.6365
+12100 train 5.825260 (lr=8.2381e-06) (hash(x)=151960396)
+12200 val loss 5.9584
+12200 val perplexity 386.9728
+12200 train 6.033947 (lr=8.0526e-06) (hash(x)=156794523)
+12300 val loss 5.9553
+12300 val perplexity 385.8007
+12300 train 5.767831 (lr=7.8694e-06) (hash(x)=138082358)
+12400 val loss 5.9486
+12400 val perplexity 383.2321
+12400 train 5.852021 (lr=7.6885e-06) (hash(x)=155255899)
+12500 val loss 5.9469
+12500 val perplexity 382.5752
+12500 train 5.925366 (lr=7.5101e-06) (hash(x)=150490528)
+12600 val loss 5.9437
+12600 val perplexity 381.3263
+12600 train 6.066210 (lr=7.3342e-06) (hash(x)=154831027)
+12700 val loss 5.9425
+12700 val perplexity 380.8722
+12700 train 5.882764 (lr=7.1609e-06) (hash(x)=147341052)
+12800 val loss 5.9403
+12800 val perplexity 380.0535
+12800 train 5.977871 (lr=6.9903e-06) (hash(x)=159333795)
+12900 val loss 5.9416
+12900 val perplexity 380.5322
+12900 train 5.784962 (lr=6.8223e-06) (hash(x)=143896867)
+13000 val loss 5.9327
+13000 val perplexity 377.1534
+13000 train 5.921026 (lr=6.6570e-06) (hash(x)=147161537)
+13100 val loss 5.9279
+13100 val perplexity 375.3620
+13100 train 6.057076 (lr=6.4945e-06) (hash(x)=147081918)
+13200 val loss 5.9254
+13200 val perplexity 374.4168
+13200 train 5.689006 (lr=6.3349e-06) (hash(x)=145079287)
+13300 val loss 5.9268
+13300 val perplexity 374.9709
+13300 train 5.729674 (lr=6.1782e-06) (hash(x)=142515041)
+13400 val loss 5.9236
+13400 val perplexity 373.7578
+13400 train 5.724186 (lr=6.0244e-06) (hash(x)=144081985)
+13500 val loss 5.9207
+13500 val perplexity 372.6601
+13500 train 5.990724 (lr=5.8737e-06) (hash(x)=149957292)
+13600 val loss 5.9153
+13600 val perplexity 370.6725
+13600 train 5.946911 (lr=5.7259e-06) (hash(x)=157123033)
+13700 val loss 5.9115
+13700 val perplexity 369.2667
+13700 train 5.899968 (lr=5.5813e-06) (hash(x)=148277247)
+13800 val loss 5.9118
+13800 val perplexity 369.3729
+13800 train 6.018199 (lr=5.4398e-06) (hash(x)=142769907)
+13900 val loss 5.9075
+13900 val perplexity 367.7686
+13900 train 5.817693 (lr=5.3016e-06) (hash(x)=144932004)
+14000 val loss 5.9045
+14000 val perplexity 366.6660
+14000 train 5.907049 (lr=5.1665e-06) (hash(x)=149532265)
+14100 val loss 5.9036
+14100 val perplexity 366.3682
+14100 train 5.748715 (lr=5.0348e-06) (hash(x)=149301374)
+14200 val loss 5.9043
+14200 val perplexity 366.5968
+14200 train 5.656896 (lr=4.9063e-06) (hash(x)=142871075)
+14300 val loss 5.8980
+14300 val perplexity 364.3106
+14300 train 6.041042 (lr=4.7813e-06) (hash(x)=166744960)
+14400 val loss 5.8939
+14400 val perplexity 362.8291
+14400 train 5.875367 (lr=4.6596e-06) (hash(x)=145950920)
+14500 val loss 5.8929
+14500 val perplexity 362.4689
+14500 train 5.894318 (lr=4.5414e-06) (hash(x)=141089155)
+14600 val loss 5.8899
+14600 val perplexity 361.3864
+14600 train 6.058316 (lr=4.4267e-06) (hash(x)=174516275)
+14700 val loss 5.8880
+14700 val perplexity 360.7004
+14700 train 5.669492 (lr=4.3155e-06) (hash(x)=137896065)
+14800 val loss 5.8868
+14800 val perplexity 360.2480
+14800 train 5.918042 (lr=4.2079e-06) (hash(x)=154396556)
+14900 val loss 5.8871
+14900 val perplexity 360.3737
+14900 train 5.747381 (lr=4.1039e-06) (hash(x)=149322394)
+15000 val loss 5.8871
+15000 val perplexity 360.3573
+15000 train 5.802734 (lr=4.0035e-06) (hash(x)=147410513)
+15100 val loss 5.8844
+15100 val perplexity 359.3901
+15100 train 5.803135 (lr=3.9068e-06) (hash(x)=145245634)
+15200 val loss 5.8800
+15200 val perplexity 357.8166
+15200 train 5.965070 (lr=3.8138e-06) (hash(x)=148341108)
+15300 val loss 5.8774
+15300 val perplexity 356.8876
+15300 train 5.697327 (lr=3.7245e-06) (hash(x)=140162437)
+15400 val loss 5.8762
+15400 val perplexity 356.4378
+15400 train 6.069530 (lr=3.6390e-06) (hash(x)=142166081)
+15500 val loss 5.8757
+15500 val perplexity 356.2906
+15500 train 5.752399 (lr=3.5572e-06) (hash(x)=141042196)
+15600 val loss 5.8742
+15600 val perplexity 355.7265
+15600 train 5.975054 (lr=3.4793e-06) (hash(x)=148672701)
+15700 val loss 5.8742
+15700 val perplexity 355.7508
+15700 train 5.772561 (lr=3.4052e-06) (hash(x)=149672385)
+15800 val loss 5.8731
+15800 val perplexity 355.3330
+15800 train 5.898124 (lr=3.3350e-06) (hash(x)=152379562)
+15900 val loss 5.8693
+15900 val perplexity 354.0105
+15900 train 5.903119 (lr=3.2687e-06) (hash(x)=154674265)
+16000 val loss 5.8670
+16000 val perplexity 353.1752
+16000 train 5.897068 (lr=3.2063e-06) (hash(x)=150368078)
+16100 val loss 5.8651
+16100 val perplexity 352.5308
+16100 train 5.962111 (lr=3.1478e-06) (hash(x)=160390561)
+16200 val loss 5.8642
+16200 val perplexity 352.1844
+16200 train 6.208712 (lr=3.0932e-06) (hash(x)=155218378)
+16300 val loss 5.8629
+16300 val perplexity 351.7262
+16300 train 6.052020 (lr=3.0427e-06) (hash(x)=160891077)
+16400 val loss 5.8618
+16400 val perplexity 351.3526
+16400 train 5.771412 (lr=2.9961e-06) (hash(x)=142789253)
+16500 val loss 5.8614
+16500 val perplexity 351.1985
+16500 train 5.603426 (lr=2.9535e-06) (hash(x)=141497841)
+16600 val loss 5.8614
+16600 val perplexity 351.2000
+16600 train 5.843324 (lr=2.9149e-06) (hash(x)=157947018)
+16700 val loss 5.8569
+16700 val perplexity 349.6255
+16700 train 5.805718 (lr=2.8804e-06) (hash(x)=146039309)
+16800 val loss 5.8567
+16800 val perplexity 349.5811
+16800 train 5.820101 (lr=2.8498e-06) (hash(x)=156381445)
+16900 val loss 5.8545
+16900 val perplexity 348.8135
+16900 train 5.816824 (lr=2.8234e-06) (hash(x)=146537354)
+17000 val loss 5.8537
+17000 val perplexity 348.5178
+17000 train 5.735730 (lr=2.8010e-06) (hash(x)=135939444)
+17100 val loss 5.8534
+17100 val perplexity 348.4101
+17100 train 5.909929 (lr=2.7826e-06) (hash(x)=147819652)
+17200 val loss 5.8540
+17200 val perplexity 348.6358
+17200 train 5.964046 (lr=2.7684e-06) (hash(x)=155307909)
+17300 val loss 5.8497
+17300 val perplexity 347.1329
+17300 train 5.801245 (lr=2.7582e-06) (hash(x)=151879548)
+17400 val loss 5.8487
+17400 val perplexity 346.7682
+17400 train 5.754910 (lr=2.7520e-06) (hash(x)=143641792)
+17499 val loss 5.8466
+17499 val perplexity 346.0586

attention_kindselective_n_heads4_seed1339/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b44ef25da5cba733c02f2c04a51eaa2be2a2661a5325c75a45021d78f5a41de
+oid sha256:7a8f3c5900182fece7b2ad89221ee3a19873525cfb62a9be93a17deb36b902e1
 size 92843394

attention_kindselective_n_heads4_seed1339/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b97dd8e4d328d85b60f03892bc042e1207bccec1f9f8130b2224e78a1a36fa23
+oid sha256:3bea0b6ffe448da1642694f3629ed45aa464f2db4f9c4a8bd07ebe572c330412
 size 179406214