Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1343/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1343", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_30720_4_1343", "n_embd": 256}

attention_kindselective_n_heads4_seed1343/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.3001
 0 val perplexity 80829.3594
-0 train 11.304919 (lr=1.5000e-07) (hash(x)=153418542)
-100 val loss 9.8883
-100 val perplexity 19698.2617
-100 train 9.855974 (lr=1.5150e-05) (hash(x)=154406429)
-200 val loss 8.7906
-200 val perplexity 6572.4741
-200 train 8.700301 (lr=3.0000e-05) (hash(x)=143722001)
-300 val loss 7.9352
-300 val perplexity 2793.9514
-300 train 7.877781 (lr=2.9998e-05) (hash(x)=152421341)
-400 val loss 7.6556
-400 val perplexity 2112.4343
-400 train 7.560735 (lr=2.9991e-05) (hash(x)=152989689)
-500 val loss 7.5524
-500 val perplexity 1905.2402
-500 train 7.480812 (lr=2.9980e-05) (hash(x)=145113544)
-600 val loss 7.4805
-600 val perplexity 1773.1368
-600 train 7.474395 (lr=2.9964e-05) (hash(x)=150071018)
-700 val loss 7.4305
-700 val perplexity 1686.5872
-700 train 7.521656 (lr=2.9944e-05) (hash(x)=156594182)
-800 val loss 7.4027
-800 val perplexity 1640.3590
-800 train 7.404630 (lr=2.9920e-05) (hash(x)=153358238)
-900 val loss 7.3667
-900 val perplexity 1582.4790
-900 train 7.198230 (lr=2.9891e-05) (hash(x)=146617394)
-1000 val loss 7.3379
-1000 val perplexity 1537.5419
-1000 train 7.275929 (lr=2.9858e-05) (hash(x)=148983354)
-1100 val loss 7.3000
-1100 val perplexity 1480.3101
-1100 train 7.329852 (lr=2.9820e-05) (hash(x)=146313352)
-1200 val loss 7.2790
-1200 val perplexity 1449.5948
-1200 train 7.231915 (lr=2.9778e-05) (hash(x)=150770333)
-1300 val loss 7.2209
-1300 val perplexity 1367.7261
-1300 train 7.161463 (lr=2.9732e-05) (hash(x)=153237933)
-1400 val loss 7.1820
-1400 val perplexity 1315.4723
-1400 train 7.019403 (lr=2.9681e-05) (hash(x)=145246201)
-1500 val loss 7.1543
-1500 val perplexity 1279.5875
-1500 train 7.149806 (lr=2.9626e-05) (hash(x)=154002540)
-1600 val loss 7.1365
-1600 val perplexity 1256.9694
-1600 train 6.916010 (lr=2.9566e-05) (hash(x)=142635842)
-1700 val loss 7.1051
-1700 val perplexity 1218.1798
-1700 train 7.146691 (lr=2.9502e-05) (hash(x)=151450555)
-1800 val loss 7.0845
-1800 val perplexity 1193.2864
-1800 train 6.996157 (lr=2.9434e-05) (hash(x)=148299832)
-1900 val loss 7.0558
-1900 val perplexity 1159.5553
-1900 train 7.217354 (lr=2.9362e-05) (hash(x)=155487476)
-2000 val loss 7.0480
-2000 val perplexity 1150.5250
-2000 train 7.025408 (lr=2.9285e-05) (hash(x)=152887721)
-2100 val loss 7.0169
-2100 val perplexity 1115.2950
-2100 train 6.724870 (lr=2.9204e-05) (hash(x)=143430603)
-2200 val loss 7.0010
-2200 val perplexity 1097.7700
-2200 train 6.989249 (lr=2.9119e-05) (hash(x)=153721194)
-2300 val loss 6.9800
-2300 val perplexity 1074.9148
-2300 train 7.102264 (lr=2.9030e-05) (hash(x)=148702429)
-2400 val loss 6.9560
-2400 val perplexity 1049.4778
-2400 train 6.998848 (lr=2.8937e-05) (hash(x)=153047184)
-2500 val loss 6.9328
-2500 val perplexity 1025.3099
-2500 train 6.808068 (lr=2.8840e-05) (hash(x)=149720123)
-2600 val loss 6.9113
-2600 val perplexity 1003.5928
-2600 train 6.947131 (lr=2.8738e-05) (hash(x)=152466045)
-2700 val loss 6.9011
-2700 val perplexity 993.4158
-2700 train 6.847266 (lr=2.8633e-05) (hash(x)=155796305)
-2800 val loss 6.8791
-2800 val perplexity 971.7216
-2800 train 6.721686 (lr=2.8523e-05) (hash(x)=151991552)
-2900 val loss 6.8777
-2900 val perplexity 970.3585
-2900 train 7.544370 (lr=2.8410e-05) (hash(x)=166537989)
-3000 val loss 6.8465
-3000 val perplexity 940.5893
-3000 train 6.846784 (lr=2.8292e-05) (hash(x)=147892594)
-3100 val loss 6.8313
-3100 val perplexity 926.3897
-3100 train 6.789685 (lr=2.8171e-05) (hash(x)=149536927)
-3200 val loss 6.8165
-3200 val perplexity 912.7487
-3200 train 6.529910 (lr=2.8046e-05) (hash(x)=135678663)
-3300 val loss 6.8121
-3300 val perplexity 908.7906
-3300 train 6.854458 (lr=2.7917e-05) (hash(x)=148523646)
-3400 val loss 6.8010
-3400 val perplexity 898.7062
-3400 train 6.861735 (lr=2.7784e-05) (hash(x)=150281778)
-3500 val loss 6.7779
-3500 val perplexity 878.2265
-3500 train 6.702899 (lr=2.7648e-05) (hash(x)=151527953)
-3600 val loss 6.7678
-3600 val perplexity 869.4343
-3600 train 7.013708 (lr=2.7508e-05) (hash(x)=141303739)
-3700 val loss 6.7572
-3700 val perplexity 860.2509
-3700 train 6.770426 (lr=2.7364e-05) (hash(x)=146778597)
-3800 val loss 6.7347
-3800 val perplexity 841.0608
-3800 train 6.702837 (lr=2.7216e-05) (hash(x)=146902797)
-3900 val loss 6.7188
-3900 val perplexity 827.8188
-3900 train 6.640366 (lr=2.7066e-05) (hash(x)=142022555)
-4000 val loss 6.7084
-4000 val perplexity 819.2586
-4000 train 6.622983 (lr=2.6911e-05) (hash(x)=151590309)
-4100 val loss 6.6857
-4100 val perplexity 800.8436
-4100 train 6.525273 (lr=2.6754e-05) (hash(x)=144127100)
-4200 val loss 6.6768
-4200 val perplexity 793.8104
-4200 train 6.748284 (lr=2.6592e-05) (hash(x)=149526774)
-4300 val loss 6.6665
-4300 val perplexity 785.6467
-4300 train 6.647405 (lr=2.6428e-05) (hash(x)=150924362)
-4400 val loss 6.6485
-4400 val perplexity 771.5940
-4400 train 6.821003 (lr=2.6260e-05) (hash(x)=155547282)
-4500 val loss 6.6379
-4500 val perplexity 763.4731
-4500 train 6.611216 (lr=2.6089e-05) (hash(x)=152479203)
-4600 val loss 6.6250
-4600 val perplexity 753.6772
-4600 train 6.524678 (lr=2.5915e-05) (hash(x)=146739760)
-4700 val loss 6.6198
-4700 val perplexity 749.8013
-4700 train 6.845607 (lr=2.5738e-05) (hash(x)=159977388)
-4800 val loss 6.6073
-4800 val perplexity 740.5098
-4800 train 6.487453 (lr=2.5558e-05) (hash(x)=142959739)
-4900 val loss 6.5901
-4900 val perplexity 727.8875
-4900 train 6.565074 (lr=2.5374e-05) (hash(x)=152253034)
-5000 val loss 6.5784
-5000 val perplexity 719.4218
-5000 train 6.547985 (lr=2.5188e-05) (hash(x)=140379747)
-5100 val loss 6.5660
-5100 val perplexity 710.5010
-5100 train 6.563121 (lr=2.4999e-05) (hash(x)=151826392)
-5200 val loss 6.5526
-5200 val perplexity 701.0939
-5200 train 6.488415 (lr=2.4807e-05) (hash(x)=152869594)
-5300 val loss 6.5369
-5300 val perplexity 690.1281
-5300 train 6.329997 (lr=2.4613e-05) (hash(x)=139093709)
-5400 val loss 6.5262
-5400 val perplexity 682.7764
-5400 train 6.384964 (lr=2.4415e-05) (hash(x)=145908738)
-5500 val loss 6.5242
-5500 val perplexity 681.4649
-5500 train 6.371226 (lr=2.4215e-05) (hash(x)=137913426)
-5600 val loss 6.4916
-5600 val perplexity 659.5744
-5600 train 6.317540 (lr=2.4013e-05) (hash(x)=140770390)
-5700 val loss 6.4865
-5700 val perplexity 656.2468
-5700 train 6.395054 (lr=2.3808e-05) (hash(x)=143728578)
-5800 val loss 6.4640
-5800 val perplexity 641.5987
-5800 train 6.571401 (lr=2.3601e-05) (hash(x)=154469787)
-5900 val loss 6.4519
-5900 val perplexity 633.9158
-5900 train 6.362564 (lr=2.3391e-05) (hash(x)=139316559)
-6000 val loss 6.4401
-6000 val perplexity 626.4839
-6000 train 6.436338 (lr=2.3179e-05) (hash(x)=152058488)
-6100 val loss 6.4282
-6100 val perplexity 619.0591
-6100 train 6.419155 (lr=2.2965e-05) (hash(x)=152958689)
-6200 val loss 6.4113
-6200 val perplexity 608.6703
-6200 train 6.381656 (lr=2.2749e-05) (hash(x)=150887864)
-6300 val loss 6.4057
-6300 val perplexity 605.2612
-6300 train 6.343083 (lr=2.2530e-05) (hash(x)=156290469)
-6400 val loss 6.3950
-6400 val perplexity 598.8619
-6400 train 6.678469 (lr=2.2310e-05) (hash(x)=159160159)
-6500 val loss 6.3797
-6500 val perplexity 589.7661
-6500 train 6.292921 (lr=2.2088e-05) (hash(x)=151239121)
-6600 val loss 6.3699
-6600 val perplexity 583.9717
-6600 train 6.366690 (lr=2.1864e-05) (hash(x)=151964101)
-6700 val loss 6.3644
-6700 val perplexity 580.7982
-6700 train 6.257575 (lr=2.1638e-05) (hash(x)=141291229)
-6800 val loss 6.3523
-6800 val perplexity 573.7996
-6800 train 6.283431 (lr=2.1410e-05) (hash(x)=160416800)
-6900 val loss 6.3442
-6900 val perplexity 569.1738
-6900 train 6.224077 (lr=2.1181e-05) (hash(x)=143382691)
-7000 val loss 6.3253
-7000 val perplexity 558.5463
-7000 train 6.237834 (lr=2.0951e-05) (hash(x)=147606572)
-7100 val loss 6.3138
-7100 val perplexity 552.1153
-7100 train 6.133327 (lr=2.0718e-05) (hash(x)=145901538)
-7200 val loss 6.3107
-7200 val perplexity 550.4401
-7200 train 6.134284 (lr=2.0485e-05) (hash(x)=139099153)
-7300 val loss 6.3096
-7300 val perplexity 549.8050
-7300 train 6.254177 (lr=2.0250e-05) (hash(x)=140434004)
-7400 val loss 6.2928
-7400 val perplexity 540.6658
-7400 train 6.412626 (lr=2.0014e-05) (hash(x)=155496146)
-7500 val loss 6.2809
-7500 val perplexity 534.2749
-7500 train 6.195351 (lr=1.9777e-05) (hash(x)=146940054)
-7600 val loss 6.2699
-7600 val perplexity 528.4411
-7600 train 6.516941 (lr=1.9538e-05) (hash(x)=165785723)
-7700 val loss 6.2614
-7700 val perplexity 523.9423
-7700 train 6.410962 (lr=1.9299e-05) (hash(x)=156339008)
-7800 val loss 6.2609
-7800 val perplexity 523.6866
-7800 train 6.255965 (lr=1.9059e-05) (hash(x)=148593556)
-7900 val loss 6.2599
-7900 val perplexity 523.1839
-7900 train 6.208652 (lr=1.8817e-05) (hash(x)=148079742)
-8000 val loss 6.2368
-8000 val perplexity 511.2038
-8000 train 6.185139 (lr=1.8576e-05) (hash(x)=159706718)
-8100 val loss 6.2373
-8100 val perplexity 511.4888
-8100 train 6.245315 (lr=1.8333e-05) (hash(x)=160203225)
-8200 val loss 6.2167
-8200 val perplexity 501.0641
-8200 train 6.175346 (lr=1.8090e-05) (hash(x)=146176536)
-8300 val loss 6.2092
-8300 val perplexity 497.2911
-8300 train 6.272688 (lr=1.7846e-05) (hash(x)=158883824)
-8400 val loss 6.2010
-8400 val perplexity 493.2605
-8400 train 6.003109 (lr=1.7602e-05) (hash(x)=137102572)
-8500 val loss 6.1948
-8500 val perplexity 490.1775
-8500 train 6.208079 (lr=1.7357e-05) (hash(x)=152812797)
-8600 val loss 6.1912
-8600 val perplexity 488.4562
-8600 train 6.055223 (lr=1.7113e-05) (hash(x)=148854244)
-8700 val loss 6.1987
-8700 val perplexity 492.0999
-8700 train 6.009529 (lr=1.6868e-05) (hash(x)=135590023)
-8800 val loss 6.1780
-8800 val perplexity 482.0101
-8800 train 6.104549 (lr=1.6623e-05) (hash(x)=144913509)
-8900 val loss 6.1720
-8900 val perplexity 479.1523
-8900 train 6.282070 (lr=1.6377e-05) (hash(x)=148045669)
-9000 val loss 6.1602
-9000 val perplexity 473.5063
-9000 train 6.194612 (lr=1.6132e-05) (hash(x)=146661746)
-9100 val loss 6.1541
-9100 val perplexity 470.6329
-9100 train 6.077746 (lr=1.5887e-05) (hash(x)=145354881)
-9200 val loss 6.1512
-9200 val perplexity 469.2866
-9200 train 6.007998 (lr=1.5643e-05) (hash(x)=150939310)
-9300 val loss 6.1439
-9300 val perplexity 465.8655
-9300 train 6.017556 (lr=1.5398e-05) (hash(x)=162258309)
-9400 val loss 6.1358
-9400 val perplexity 462.1187
-9400 train 6.142342 (lr=1.5154e-05) (hash(x)=154701830)
-9500 val loss 6.1309
-9500 val perplexity 459.8270
-9500 train 6.003320 (lr=1.4910e-05) (hash(x)=142241404)
-9600 val loss 6.1231
-9600 val perplexity 456.2891
-9600 train 6.089352 (lr=1.4667e-05) (hash(x)=148439858)
-9700 val loss 6.1225
-9700 val perplexity 455.9869
-9700 train 6.105500 (lr=1.4424e-05) (hash(x)=152204410)
-9800 val loss 6.1121
-9800 val perplexity 451.3029
-9800 train 5.930009 (lr=1.4183e-05) (hash(x)=141294630)
-9900 val loss 6.1118
-9900 val perplexity 451.1717
-9900 train 6.094609 (lr=1.3941e-05) (hash(x)=147390033)
-10000 val loss 6.1036
-10000 val perplexity 447.4525
-10000 train 6.077098 (lr=1.3701e-05) (hash(x)=144131604)
-10100 val loss 6.0939
-10100 val perplexity 443.1394
-10100 train 6.301743 (lr=1.3462e-05) (hash(x)=158814818)
-10200 val loss 6.0879
-10200 val perplexity 440.5142
-10200 train 6.152081 (lr=1.3223e-05) (hash(x)=153491388)
-10300 val loss 6.0845
-10300 val perplexity 439.0029
-10300 train 6.039337 (lr=1.2986e-05) (hash(x)=149600707)
-10400 val loss 6.0796
-10400 val perplexity 436.8394
-10400 train 5.983914 (lr=1.2750e-05) (hash(x)=152277064)
-10500 val loss 6.0791
-10500 val perplexity 436.6190
-10500 train 5.996013 (lr=1.2515e-05) (hash(x)=150447091)
-10600 val loss 6.0679
-10600 val perplexity 431.7908
-10600 train 6.079579 (lr=1.2282e-05) (hash(x)=151606822)
-10700 val loss 6.0601
-10700 val perplexity 428.3991
-10700 train 6.063359 (lr=1.2049e-05) (hash(x)=158730240)
-10800 val loss 6.0590
-10800 val perplexity 427.9493
-10800 train 6.112919 (lr=1.1819e-05) (hash(x)=163095518)
-10900 val loss 6.0536
-10900 val perplexity 425.6318
-10900 train 5.809468 (lr=1.1590e-05) (hash(x)=145260395)
-11000 val loss 6.0468
-11000 val perplexity 422.7498
-11000 train 6.053051 (lr=1.1362e-05) (hash(x)=155970264)
-11100 val loss 6.0466
-11100 val perplexity 422.6930
-11100 train 6.130270 (lr=1.1136e-05) (hash(x)=159871097)
-11200 val loss 6.0358
-11200 val perplexity 418.1324
-11200 train 6.133372 (lr=1.0912e-05) (hash(x)=161135244)
-11300 val loss 6.0310
-11300 val perplexity 416.1445
-11300 train 6.038279 (lr=1.0690e-05) (hash(x)=150649336)
-11400 val loss 6.0320
-11400 val perplexity 416.5644
-11400 train 5.988773 (lr=1.0470e-05) (hash(x)=152230013)
-11500 val loss 6.0225
-11500 val perplexity 412.6224
-11500 train 6.093239 (lr=1.0251e-05) (hash(x)=163300613)
-11600 val loss 6.0230
-11600 val perplexity 412.8220
-11600 train 6.006931 (lr=1.0035e-05) (hash(x)=151203838)
-11700 val loss 6.0172
-11700 val perplexity 410.4170
-11700 train 6.031619 (lr=9.8209e-06) (hash(x)=155255111)
-11800 val loss 6.0196
-11800 val perplexity 411.4132
-11800 train 5.976803 (lr=9.6090e-06) (hash(x)=139380668)
-11900 val loss 6.0059
-11900 val perplexity 405.8001
-11900 train 5.918542 (lr=9.3993e-06) (hash(x)=144033405)
-12000 val loss 6.0045
-12000 val perplexity 405.2554
-12000 train 5.923520 (lr=9.1920e-06) (hash(x)=148689574)
-12100 val loss 6.0011
-12100 val perplexity 403.8767
-12100 train 5.959810 (lr=8.9871e-06) (hash(x)=143272239)
-12200 val loss 5.9979
-12200 val perplexity 402.5737
-12200 train 6.155020 (lr=8.7846e-06) (hash(x)=157671609)
-12300 val loss 5.9938
-12300 val perplexity 400.9520
-12300 train 5.999459 (lr=8.5847e-06) (hash(x)=150363247)
-12400 val loss 5.9882
-12400 val perplexity 398.6854
-12400 train 5.878437 (lr=8.3875e-06) (hash(x)=146857787)
-12500 val loss 5.9853
-12500 val perplexity 397.5401
-12500 train 5.934603 (lr=8.1929e-06) (hash(x)=153327320)
-12600 val loss 5.9825
-12600 val perplexity 396.4457
-12600 train 5.696634 (lr=8.0010e-06) (hash(x)=143612983)
-12700 val loss 5.9797
-12700 val perplexity 395.3220
-12700 train 5.917475 (lr=7.8119e-06) (hash(x)=150739132)
-12800 val loss 5.9765
-12800 val perplexity 394.0561
-12800 train 5.823044 (lr=7.6257e-06) (hash(x)=147986204)
-12900 val loss 5.9739
-12900 val perplexity 393.0287
-12900 train 6.124288 (lr=7.4425e-06) (hash(x)=155218833)
-13000 val loss 5.9681
-13000 val perplexity 390.7678
-13000 train 5.953165 (lr=7.2622e-06) (hash(x)=148867960)
-13100 val loss 5.9659
-13100 val perplexity 389.9185
-13100 train 5.990927 (lr=7.0850e-06) (hash(x)=160101478)
-13200 val loss 5.9653
-13200 val perplexity 389.6800
-13200 train 5.816887 (lr=6.9108e-06) (hash(x)=151080097)
-13300 val loss 5.9652
-13300 val perplexity 389.6155
-13300 train 5.955846 (lr=6.7398e-06) (hash(x)=150403039)
-13400 val loss 5.9606
-13400 val perplexity 387.8403
-13400 train 5.873844 (lr=6.5721e-06) (hash(x)=129728571)
-13500 val loss 5.9550
-13500 val perplexity 385.6749
-13500 train 5.867978 (lr=6.4076e-06) (hash(x)=148988625)
-13600 val loss 5.9525
-13600 val perplexity 384.7157
-13600 train 5.740644 (lr=6.2465e-06) (hash(x)=149790861)
-13700 val loss 5.9515
-13700 val perplexity 384.3409
-13700 train 5.894429 (lr=6.0887e-06) (hash(x)=153505062)
-13800 val loss 5.9558
-13800 val perplexity 385.9948
-13800 train 5.856753 (lr=5.9344e-06) (hash(x)=143845463)
-13900 val loss 5.9507
-13900 val perplexity 384.0300
-13900 train 5.741137 (lr=5.7835e-06) (hash(x)=143378076)
-14000 val loss 5.9437
-14000 val perplexity 381.3362
-14000 train 5.902184 (lr=5.6362e-06) (hash(x)=148681755)
-14100 val loss 5.9412
-14100 val perplexity 380.4063
-14100 train 5.825752 (lr=5.4925e-06) (hash(x)=145530465)
-14200 val loss 5.9403
-14200 val perplexity 380.0658
-14200 train 5.892087 (lr=5.3524e-06) (hash(x)=162139713)
-14300 val loss 5.9374
-14300 val perplexity 378.9445
-14300 train 5.937052 (lr=5.2159e-06) (hash(x)=141997596)
-14400 val loss 5.9368
-14400 val perplexity 378.7086
-14400 train 5.904932 (lr=5.0832e-06) (hash(x)=146156313)
-14500 val loss 5.9374
-14500 val perplexity 378.9321
-14500 train 5.706226 (lr=4.9543e-06) (hash(x)=145298178)
-14600 val loss 5.9318
-14600 val perplexity 376.8298
-14600 train 6.012121 (lr=4.8291e-06) (hash(x)=148232610)
-14700 val loss 5.9287
-14700 val perplexity 375.6682
-14700 train 5.810235 (lr=4.7079e-06) (hash(x)=151079115)
-14800 val loss 5.9270
-14800 val perplexity 375.0097
-14800 train 5.894715 (lr=4.5905e-06) (hash(x)=151340983)
-14900 val loss 5.9259
-14900 val perplexity 374.6101
-14900 train 6.131800 (lr=4.4770e-06) (hash(x)=150213803)
-15000 val loss 5.9256
-15000 val perplexity 374.5093
-15000 train 5.835275 (lr=4.3675e-06) (hash(x)=157652334)
-15100 val loss 5.9243
-15100 val perplexity 374.0276
-15100 train 5.657528 (lr=4.2620e-06) (hash(x)=145078577)
-15200 val loss 5.9208
-15200 val perplexity 372.7132
-15200 train 5.872759 (lr=4.1605e-06) (hash(x)=155898006)
-15300 val loss 5.9194
-15300 val perplexity 372.1980
-15300 train 5.947790 (lr=4.0631e-06) (hash(x)=149346671)
-15400 val loss 5.9169
-15400 val perplexity 371.2449
-15400 train 5.776122 (lr=3.9698e-06) (hash(x)=145708113)
-15500 val loss 5.9142
-15500 val perplexity 370.2740
-15500 train 6.114147 (lr=3.8806e-06) (hash(x)=165102970)
-15600 val loss 5.9156
-15600 val perplexity 370.7627
-15600 train 5.694969 (lr=3.7956e-06) (hash(x)=139106517)
-15700 val loss 5.9140
-15700 val perplexity 370.1714
-15700 train 5.733104 (lr=3.7148e-06) (hash(x)=141680838)
-15800 val loss 5.9110
-15800 val perplexity 369.0799
-15800 train 5.789922 (lr=3.6382e-06) (hash(x)=136152310)
-15900 val loss 5.9096
-15900 val perplexity 368.5567
-15900 train 5.971681 (lr=3.5658e-06) (hash(x)=151104640)
-16000 val loss 5.9071
-16000 val perplexity 367.6357
-16000 train 5.857261 (lr=3.4977e-06) (hash(x)=153333281)
-16100 val loss 5.9064
-16100 val perplexity 367.3925
-16100 train 5.863892 (lr=3.4339e-06) (hash(x)=145628059)
-16200 val loss 5.9072
-16200 val perplexity 367.6785
-16200 train 5.618507 (lr=3.3744e-06) (hash(x)=137220041)
-16300 val loss 5.9045
-16300 val perplexity 366.6861
-16300 train 6.059216 (lr=3.3193e-06) (hash(x)=161215791)
-16400 val loss 5.9041
-16400 val perplexity 366.5234
-16400 train 5.621179 (lr=3.2684e-06) (hash(x)=136194680)
-16500 val loss 5.9042
-16500 val perplexity 366.5812
-16500 train 5.709507 (lr=3.2220e-06) (hash(x)=152209568)
-16600 val loss 5.9005
-16600 val perplexity 365.2301
-16600 train 6.007233 (lr=3.1799e-06) (hash(x)=157345721)
-16700 val loss 5.8983
-16700 val perplexity 364.4319
-16700 train 5.996146 (lr=3.1422e-06) (hash(x)=146883642)
-16800 val loss 5.8986
-16800 val perplexity 364.5212
-16800 train 5.857209 (lr=3.1089e-06) (hash(x)=153037972)
-16900 val loss 5.8978
-16900 val perplexity 364.2220
-16900 train 5.685570 (lr=3.0801e-06) (hash(x)=150691367)
-17000 val loss 5.8971
-17000 val perplexity 363.9819
-17000 train 5.644271 (lr=3.0556e-06) (hash(x)=148975206)
-17100 val loss 5.8978
-17100 val perplexity 364.2212
-17100 train 5.614702 (lr=3.0356e-06) (hash(x)=142026221)
-17200 val loss 5.8946
-17200 val perplexity 363.0641
-17200 train 5.936298 (lr=3.0200e-06) (hash(x)=138211880)
-17300 val loss 5.8919
-17300 val perplexity 362.0752
-17300 train 5.743488 (lr=3.0089e-06) (hash(x)=143366666)
-17400 val loss 5.8922
-17400 val perplexity 362.2161
-17400 train 5.879615 (lr=3.0022e-06) (hash(x)=150791766)
-17499 val loss 5.8915
-17499 val perplexity 361.9645
+0 train 11.304919 (lr=1.7500e-07) (hash(x)=153418542)
+100 val loss 9.8579
+100 val perplexity 19109.1309
+100 train 9.824990 (lr=1.7675e-05) (hash(x)=154406429)
+200 val loss 8.8331
+200 val perplexity 6857.3115
+200 train 8.746761 (lr=3.5000e-05) (hash(x)=143722001)
+300 val loss 7.9027
+300 val perplexity 2704.6018
+300 train 7.843596 (lr=3.4997e-05) (hash(x)=152421341)
+400 val loss 7.5710
+400 val perplexity 1941.1111
+400 train 7.456883 (lr=3.4990e-05) (hash(x)=152989689)
+500 val loss 7.4553
+500 val perplexity 1729.0040
+500 train 7.379359 (lr=3.4977e-05) (hash(x)=145113544)
+600 val loss 7.3587
+600 val perplexity 1569.8698
+600 train 7.348912 (lr=3.4958e-05) (hash(x)=150071018)
+700 val loss 7.2927
+700 val perplexity 1469.5853
+700 train 7.403689 (lr=3.4935e-05) (hash(x)=156594182)
+800 val loss 7.2374
+800 val perplexity 1390.4237
+800 train 7.242920 (lr=3.4907e-05) (hash(x)=153358238)
+900 val loss 7.1837
+900 val perplexity 1317.7135
+900 train 7.002797 (lr=3.4873e-05) (hash(x)=146617394)
+1000 val loss 7.1280
+1000 val perplexity 1246.3452
+1000 train 7.063962 (lr=3.4834e-05) (hash(x)=148983354)
+1100 val loss 7.0634
+1100 val perplexity 1168.3807
+1100 train 7.084983 (lr=3.4790e-05) (hash(x)=146313352)
+1200 val loss 7.0047
+1200 val perplexity 1101.7477
+1200 train 6.932942 (lr=3.4741e-05) (hash(x)=150770333)
+1300 val loss 6.9237
+1300 val perplexity 1016.0912
+1300 train 6.875015 (lr=3.4687e-05) (hash(x)=153237933)
+1400 val loss 6.8673
+1400 val perplexity 960.3375
+1400 train 6.700178 (lr=3.4628e-05) (hash(x)=145246201)
+1500 val loss 6.8174
+1500 val perplexity 913.5900
+1500 train 6.794487 (lr=3.4563e-05) (hash(x)=154002540)
+1600 val loss 6.7607
+1600 val perplexity 863.2115
+1600 train 6.541095 (lr=3.4494e-05) (hash(x)=142635842)
+1700 val loss 6.7104
+1700 val perplexity 820.9233
+1700 train 6.758193 (lr=3.4419e-05) (hash(x)=151450555)
+1800 val loss 6.6554
+1800 val perplexity 776.9322
+1800 train 6.578032 (lr=3.4340e-05) (hash(x)=148299832)
+1900 val loss 6.6127
+1900 val perplexity 744.5060
+1900 train 6.748245 (lr=3.4255e-05) (hash(x)=155487476)
+2000 val loss 6.5803
+2000 val perplexity 720.7286
+2000 train 6.563023 (lr=3.4166e-05) (hash(x)=152887721)
+2100 val loss 6.5454
+2100 val perplexity 696.0361
+2100 train 6.236790 (lr=3.4072e-05) (hash(x)=143430603)
+2200 val loss 6.5212
+2200 val perplexity 679.4134
+2200 train 6.524914 (lr=3.3973e-05) (hash(x)=153721194)
+2300 val loss 6.4924
+2300 val perplexity 660.1232
+2300 train 6.651157 (lr=3.3869e-05) (hash(x)=148702429)
+2400 val loss 6.4638
+2400 val perplexity 641.5140
+2400 train 6.519583 (lr=3.3760e-05) (hash(x)=153047184)
+2500 val loss 6.4378
+2500 val perplexity 625.0129
+2500 train 6.324356 (lr=3.3646e-05) (hash(x)=149720123)
+2600 val loss 6.4174
+2600 val perplexity 612.4166
+2600 train 6.436531 (lr=3.3528e-05) (hash(x)=152466045)
+2700 val loss 6.4221
+2700 val perplexity 615.3074
+2700 train 6.333328 (lr=3.3405e-05) (hash(x)=155796305)
+2800 val loss 6.3789
+2800 val perplexity 589.3065
+2800 train 6.213628 (lr=3.3277e-05) (hash(x)=151991552)
+2900 val loss 6.3549
+2900 val perplexity 575.2861
+2900 train 6.934335 (lr=3.3144e-05) (hash(x)=166537989)
+3000 val loss 6.3344
+3000 val perplexity 563.6105
+3000 train 6.332823 (lr=3.3008e-05) (hash(x)=147892594)
+3100 val loss 6.3253
+3100 val perplexity 558.5463
+3100 train 6.284938 (lr=3.2866e-05) (hash(x)=149536927)
+3200 val loss 6.3026
+3200 val perplexity 545.9706
+3200 train 6.015903 (lr=3.2720e-05) (hash(x)=135678663)
+3300 val loss 6.2866
+3300 val perplexity 537.3096
+3300 train 6.366162 (lr=3.2570e-05) (hash(x)=148523646)
+3400 val loss 6.2729
+3400 val perplexity 530.0279
+3400 train 6.336071 (lr=3.2415e-05) (hash(x)=150281778)
+3500 val loss 6.2572
+3500 val perplexity 521.7344
+3500 train 6.212903 (lr=3.2256e-05) (hash(x)=151527953)
+3600 val loss 6.2510
+3600 val perplexity 518.5555
+3600 train 6.464981 (lr=3.2092e-05) (hash(x)=141303739)
+3700 val loss 6.2310
+3700 val perplexity 508.2712
+3700 train 6.281719 (lr=3.1924e-05) (hash(x)=146778597)
+3800 val loss 6.2172
+3800 val perplexity 501.2825
+3800 train 6.171426 (lr=3.1753e-05) (hash(x)=146902797)
+3900 val loss 6.2040
+3900 val perplexity 494.7110
+3900 train 6.130551 (lr=3.1577e-05) (hash(x)=142022555)
+4000 val loss 6.1886
+4000 val perplexity 487.1476
+4000 train 6.098490 (lr=3.1397e-05) (hash(x)=151590309)
+4100 val loss 6.1778
+4100 val perplexity 481.9368
+4100 train 6.023282 (lr=3.1212e-05) (hash(x)=144127100)
+4200 val loss 6.1688
+4200 val perplexity 477.6228
+4200 train 6.234536 (lr=3.1024e-05) (hash(x)=149526774)
+4300 val loss 6.1632
+4300 val perplexity 474.9631
+4300 train 6.102820 (lr=3.0833e-05) (hash(x)=150924362)
+4400 val loss 6.1428
+4400 val perplexity 465.3742
+4400 train 6.281572 (lr=3.0637e-05) (hash(x)=155547282)
+4500 val loss 6.1375
+4500 val perplexity 462.8767
+4500 train 6.125348 (lr=3.0437e-05) (hash(x)=152479203)
+4600 val loss 6.1183
+4600 val perplexity 454.0975
+4600 train 6.030381 (lr=3.0234e-05) (hash(x)=146739760)
+4700 val loss 6.1079
+4700 val perplexity 449.4118
+4700 train 6.328187 (lr=3.0027e-05) (hash(x)=159977388)
+4800 val loss 6.0983
+4800 val perplexity 445.0985
+4800 train 5.991615 (lr=2.9817e-05) (hash(x)=142959739)
+4900 val loss 6.0819
+4900 val perplexity 437.8497
+4900 train 6.032333 (lr=2.9603e-05) (hash(x)=152253034)
+5000 val loss 6.0800
+5000 val perplexity 437.0294
+5000 train 6.051497 (lr=2.9386e-05) (hash(x)=140379747)
+5100 val loss 6.0608
+5100 val perplexity 428.7304
+5100 train 6.091932 (lr=2.9166e-05) (hash(x)=151826392)
+5200 val loss 6.0533
+5200 val perplexity 425.5295
+5200 train 6.021602 (lr=2.8942e-05) (hash(x)=152869594)
+5300 val loss 6.0463
+5300 val perplexity 422.5276
+5300 train 5.831899 (lr=2.8715e-05) (hash(x)=139093709)
+5400 val loss 6.0359
+5400 val perplexity 418.1806
+5400 train 5.915012 (lr=2.8484e-05) (hash(x)=145908738)
+5500 val loss 6.0321
+5500 val perplexity 416.5761
+5500 train 5.895004 (lr=2.8251e-05) (hash(x)=137913426)
+5600 val loss 6.0108
+5600 val perplexity 407.8055
+5600 train 5.881226 (lr=2.8015e-05) (hash(x)=140770390)
+5700 val loss 6.0147
+5700 val perplexity 409.4116
+5700 train 5.930855 (lr=2.7776e-05) (hash(x)=143728578)
+5800 val loss 5.9952
+5800 val perplexity 401.5169
+5800 train 6.058795 (lr=2.7534e-05) (hash(x)=154469787)
+5900 val loss 5.9872
+5900 val perplexity 398.3034
+5900 train 5.950858 (lr=2.7290e-05) (hash(x)=139316559)
+6000 val loss 5.9789
+6000 val perplexity 395.0224
+6000 train 5.991045 (lr=2.7042e-05) (hash(x)=152058488)
+6100 val loss 5.9701
+6100 val perplexity 391.5618
+6100 train 5.985957 (lr=2.6792e-05) (hash(x)=152958689)
+6200 val loss 5.9554
+6200 val perplexity 385.8266
+6200 train 5.911655 (lr=2.6540e-05) (hash(x)=150887864)
+6300 val loss 5.9482
+6300 val perplexity 383.0494
+6300 train 5.901646 (lr=2.6285e-05) (hash(x)=156290469)
+6400 val loss 5.9369
+6400 val perplexity 378.7653
+6400 train 6.258915 (lr=2.6028e-05) (hash(x)=159160159)
+6500 val loss 5.9340
+6500 val perplexity 377.6659
+6500 train 5.852345 (lr=2.5769e-05) (hash(x)=151239121)
+6600 val loss 5.9260
+6600 val perplexity 374.6520
+6600 train 5.947871 (lr=2.5508e-05) (hash(x)=151964101)
+6700 val loss 5.9197
+6700 val perplexity 372.3041
+6700 train 5.845406 (lr=2.5244e-05) (hash(x)=141291229)
+6800 val loss 5.9132
+6800 val perplexity 369.8967
+6800 train 5.825259 (lr=2.4979e-05) (hash(x)=160416800)
+6900 val loss 5.9017
+6900 val perplexity 365.6624
+6900 train 5.810899 (lr=2.4711e-05) (hash(x)=143382691)
+7000 val loss 5.8909
+7000 val perplexity 361.7261
+7000 train 5.795605 (lr=2.4442e-05) (hash(x)=147606572)
+7100 val loss 5.8815
+7100 val perplexity 358.3473
+7100 train 5.707331 (lr=2.4171e-05) (hash(x)=145901538)
+7200 val loss 5.8815
+7200 val perplexity 358.3601
+7200 train 5.719882 (lr=2.3899e-05) (hash(x)=139099153)
+7300 val loss 5.8714
+7300 val perplexity 354.7415
+7300 train 5.836294 (lr=2.3625e-05) (hash(x)=140434004)
+7400 val loss 5.8686
+7400 val perplexity 353.7577
+7400 train 5.989874 (lr=2.3349e-05) (hash(x)=155496146)
+7500 val loss 5.8615
+7500 val perplexity 351.2433
+7500 train 5.792509 (lr=2.3073e-05) (hash(x)=146940054)
+7600 val loss 5.8472
+7600 val perplexity 346.2521
+7600 train 6.075488 (lr=2.2795e-05) (hash(x)=165785723)
+7700 val loss 5.8365
+7700 val perplexity 342.5613
+7700 train 5.990919 (lr=2.2515e-05) (hash(x)=156339008)
+7800 val loss 5.8346
+7800 val perplexity 341.9387
+7800 train 5.831989 (lr=2.2235e-05) (hash(x)=148593556)
+7900 val loss 5.8383
+7900 val perplexity 343.1989
+7900 train 5.761374 (lr=2.1954e-05) (hash(x)=148079742)
+8000 val loss 5.8212
+8000 val perplexity 337.3739
+8000 train 5.754209 (lr=2.1671e-05) (hash(x)=159706718)
+8100 val loss 5.8161
+8100 val perplexity 335.6689
+8100 train 5.803094 (lr=2.1388e-05) (hash(x)=160203225)
+8200 val loss 5.8042
+8200 val perplexity 331.6880
+8200 train 5.745097 (lr=2.1105e-05) (hash(x)=146176536)
+8300 val loss 5.7950
+8300 val perplexity 328.6607
+8300 train 5.881962 (lr=2.0820e-05) (hash(x)=158883824)
+8400 val loss 5.7912
+8400 val perplexity 327.4128
+8400 train 5.611819 (lr=2.0536e-05) (hash(x)=137102572)
+8500 val loss 5.7839
+8500 val perplexity 325.0120
+8500 train 5.818129 (lr=2.0250e-05) (hash(x)=152812797)
+8600 val loss 5.7820
+8600 val perplexity 324.4048
+8600 train 5.625070 (lr=1.9965e-05) (hash(x)=148854244)
+8700 val loss 5.7930
+8700 val perplexity 328.0061
+8700 train 5.611191 (lr=1.9679e-05) (hash(x)=135590023)
+8800 val loss 5.7679
+8800 val perplexity 319.8712
+8800 train 5.704375 (lr=1.9393e-05) (hash(x)=144913509)
+8900 val loss 5.7618
+8900 val perplexity 317.9144
+8900 train 5.900129 (lr=1.9107e-05) (hash(x)=148045669)
+9000 val loss 5.7572
+9000 val perplexity 316.4612
+9000 train 5.788160 (lr=1.8821e-05) (hash(x)=146661746)
+9100 val loss 5.7500
+9100 val perplexity 314.1996
+9100 train 5.697944 (lr=1.8535e-05) (hash(x)=145354881)
+9200 val loss 5.7463
+9200 val perplexity 313.0410
+9200 train 5.611643 (lr=1.8250e-05) (hash(x)=150939310)
+9300 val loss 5.7398
+9300 val perplexity 311.0170
+9300 train 5.541914 (lr=1.7964e-05) (hash(x)=162258309)
+9400 val loss 5.7344
+9400 val perplexity 309.3314
+9400 train 5.737438 (lr=1.7680e-05) (hash(x)=154701830)
+9500 val loss 5.7263
+9500 val perplexity 306.8179
+9500 train 5.623687 (lr=1.7395e-05) (hash(x)=142241404)
+9600 val loss 5.7201
+9600 val perplexity 304.9395
+9600 train 5.693145 (lr=1.7112e-05) (hash(x)=148439858)
+9700 val loss 5.7193
+9700 val perplexity 304.7034
+9700 train 5.717032 (lr=1.6829e-05) (hash(x)=152204410)
+9800 val loss 5.7127
+9800 val perplexity 302.6905
+9800 train 5.571279 (lr=1.6546e-05) (hash(x)=141294630)
+9900 val loss 5.7124
+9900 val perplexity 302.6070
+9900 train 5.687763 (lr=1.6265e-05) (hash(x)=147390033)
+10000 val loss 5.7070
+10000 val perplexity 300.9729
+10000 train 5.728780 (lr=1.5985e-05) (hash(x)=144131604)
+10100 val loss 5.6945
+10100 val perplexity 297.2338
+10100 train 5.905474 (lr=1.5705e-05) (hash(x)=158814818)
+10200 val loss 5.6932
+10200 val perplexity 296.8331
+10200 train 5.781689 (lr=1.5427e-05) (hash(x)=153491388)
+10300 val loss 5.6900
+10300 val perplexity 295.8885
+10300 train 5.668776 (lr=1.5151e-05) (hash(x)=149600707)
+10400 val loss 5.6850
+10400 val perplexity 294.4254
+10400 train 5.614613 (lr=1.4875e-05) (hash(x)=152277064)
+10500 val loss 5.6834
+10500 val perplexity 293.9567
+10500 train 5.584078 (lr=1.4601e-05) (hash(x)=150447091)
+10600 val loss 5.6768
+10600 val perplexity 292.0200
+10600 train 5.693406 (lr=1.4329e-05) (hash(x)=151606822)
+10700 val loss 5.6694
+10700 val perplexity 289.8598
+10700 train 5.642289 (lr=1.4058e-05) (hash(x)=158730240)
+10800 val loss 5.6663
+10800 val perplexity 288.9665
+10800 train 5.702177 (lr=1.3789e-05) (hash(x)=163095518)
+10900 val loss 5.6634
+10900 val perplexity 288.1227
+10900 train 5.426070 (lr=1.3521e-05) (hash(x)=145260395)
+11000 val loss 5.6586
+11000 val perplexity 286.7513
+11000 train 5.655396 (lr=1.3256e-05) (hash(x)=155970264)
+11100 val loss 5.6565
+11100 val perplexity 286.1584
+11100 train 5.748341 (lr=1.2992e-05) (hash(x)=159871097)
+11200 val loss 5.6514
+11200 val perplexity 284.6913
+11200 train 5.730143 (lr=1.2731e-05) (hash(x)=161135244)
+11300 val loss 5.6451
+11300 val perplexity 282.8916
+11300 train 5.687905 (lr=1.2472e-05) (hash(x)=150649336)
+11400 val loss 5.6426
+11400 val perplexity 282.1888
+11400 train 5.608679 (lr=1.2215e-05) (hash(x)=152230013)
+11500 val loss 5.6394
+11500 val perplexity 281.3017
+11500 train 5.663232 (lr=1.1960e-05) (hash(x)=163300613)
+11600 val loss 5.6377
+11600 val perplexity 280.8187
+11600 train 5.610705 (lr=1.1708e-05) (hash(x)=151203838)
+11700 val loss 5.6325
+11700 val perplexity 279.3708
+11700 train 5.657187 (lr=1.1458e-05) (hash(x)=155255111)
+11800 val loss 5.6326
+11800 val perplexity 279.3909
+11800 train 5.592705 (lr=1.1210e-05) (hash(x)=139380668)
+11900 val loss 5.6238
+11900 val perplexity 276.9400
+11900 train 5.564804 (lr=1.0966e-05) (hash(x)=144033405)
+12000 val loss 5.6209
+12000 val perplexity 276.1438
+12000 train 5.544782 (lr=1.0724e-05) (hash(x)=148689574)
+12100 val loss 5.6226
+12100 val perplexity 276.5996
+12100 train 5.551520 (lr=1.0485e-05) (hash(x)=143272239)
+12200 val loss 5.6176
+12200 val perplexity 275.2220
+12200 train 5.656269 (lr=1.0249e-05) (hash(x)=157671609)
+12300 val loss 5.6146
+12300 val perplexity 274.3976
+12300 train 5.639105 (lr=1.0016e-05) (hash(x)=150363247)
+12400 val loss 5.6096
+12400 val perplexity 273.0280
+12400 train 5.499437 (lr=9.7854e-06) (hash(x)=146857787)
+12500 val loss 5.6083
+12500 val perplexity 272.6883
+12500 train 5.576447 (lr=9.5583e-06) (hash(x)=153327320)
+12600 val loss 5.6047
+12600 val perplexity 271.6973
+12600 train 5.337243 (lr=9.3345e-06) (hash(x)=143612983)
+12700 val loss 5.6017
+12700 val perplexity 270.8927
+12700 train 5.535681 (lr=9.1139e-06) (hash(x)=150739132)
+12800 val loss 5.6011
+12800 val perplexity 270.7128
+12800 train 5.455906 (lr=8.8967e-06) (hash(x)=147986204)
+12900 val loss 5.5987
+12900 val perplexity 270.0814
+12900 train 5.773186 (lr=8.6829e-06) (hash(x)=155218833)
+13000 val loss 5.5935
+13000 val perplexity 268.6810
+13000 train 5.588228 (lr=8.4726e-06) (hash(x)=148867960)
+13100 val loss 5.5914
+13100 val perplexity 268.1018
+13100 train 5.612458 (lr=8.2658e-06) (hash(x)=160101478)
+13200 val loss 5.5920
+13200 val perplexity 268.2831
+13200 train 5.452489 (lr=8.0626e-06) (hash(x)=151080097)
+13300 val loss 5.5899
+13300 val perplexity 267.7130
+13300 train 5.557879 (lr=7.8632e-06) (hash(x)=150403039)
+13400 val loss 5.5862
+13400 val perplexity 266.7191
+13400 train 5.541445 (lr=7.6674e-06) (hash(x)=129728571)
+13500 val loss 5.5814
+13500 val perplexity 265.4492
+13500 train 5.501922 (lr=7.4756e-06) (hash(x)=148988625)
+13600 val loss 5.5807
+13600 val perplexity 265.2461
+13600 train 5.370127 (lr=7.2875e-06) (hash(x)=149790861)
+13700 val loss 5.5796
+13700 val perplexity 264.9609
+13700 train 5.498874 (lr=7.1035e-06) (hash(x)=153505062)
+13800 val loss 5.5790
+13800 val perplexity 264.7946
+13800 train 5.486200 (lr=6.9234e-06) (hash(x)=143845463)
+13900 val loss 5.5776
+13900 val perplexity 264.4264
+13900 train 5.383967 (lr=6.7474e-06) (hash(x)=143378076)
+14000 val loss 5.5723
+14000 val perplexity 263.0425
+14000 train 5.532637 (lr=6.5756e-06) (hash(x)=148681755)
+14100 val loss 5.5704
+14100 val perplexity 262.5340
+14100 train 5.456738 (lr=6.4079e-06) (hash(x)=145530465)
+14200 val loss 5.5713
+14200 val perplexity 262.7664
+14200 train 5.505947 (lr=6.2444e-06) (hash(x)=162139713)
+14300 val loss 5.5674
+14300 val perplexity 261.7432
+14300 train 5.574866 (lr=6.0852e-06) (hash(x)=141997596)
+14400 val loss 5.5670
+14400 val perplexity 261.6435
+14400 train 5.542020 (lr=5.9304e-06) (hash(x)=146156313)
+14500 val loss 5.5668
+14500 val perplexity 261.5967
+14500 train 5.344202 (lr=5.7800e-06) (hash(x)=145298178)
+14600 val loss 5.5618
+14600 val perplexity 260.2844
+14600 train 5.648545 (lr=5.6340e-06) (hash(x)=148232610)
+14700 val loss 5.5608
+14700 val perplexity 260.0356
+14700 train 5.436423 (lr=5.4925e-06) (hash(x)=151079115)
+14800 val loss 5.5602
+14800 val perplexity 259.8626
+14800 train 5.553704 (lr=5.3555e-06) (hash(x)=151340983)
+14900 val loss 5.5582
+14900 val perplexity 259.3505
+14900 train 5.774446 (lr=5.2231e-06) (hash(x)=150213803)
+15000 val loss 5.5574
+15000 val perplexity 259.1565
+15000 train 5.464648 (lr=5.0954e-06) (hash(x)=157652334)
+15100 val loss 5.5565
+15100 val perplexity 258.9096
+15100 train 5.274495 (lr=4.9723e-06) (hash(x)=145078577)
+15200 val loss 5.5530
+15200 val perplexity 258.0162
+15200 train 5.499384 (lr=4.8539e-06) (hash(x)=155898006)
+15300 val loss 5.5518
+15300 val perplexity 257.6959
+15300 train 5.588014 (lr=4.7403e-06) (hash(x)=149346671)
+15400 val loss 5.5496
+15400 val perplexity 257.1445
+15400 train 5.391021 (lr=4.6314e-06) (hash(x)=145708113)
+15500 val loss 5.5477
+15500 val perplexity 256.6455
+15500 train 5.739224 (lr=4.5274e-06) (hash(x)=165102970)
+15600 val loss 5.5488
+15600 val perplexity 256.9364
+15600 train 5.333233 (lr=4.4282e-06) (hash(x)=139106517)
+15700 val loss 5.5474
+15700 val perplexity 256.5752
+15700 train 5.435939 (lr=4.3339e-06) (hash(x)=141680838)
+15800 val loss 5.5448
+15800 val perplexity 255.9047
+15800 train 5.447368 (lr=4.2446e-06) (hash(x)=136152310)
+15900 val loss 5.5426
+15900 val perplexity 255.3536
+15900 train 5.602579 (lr=4.1601e-06) (hash(x)=151104640)
+16000 val loss 5.5418
+16000 val perplexity 255.1399
+16000 train 5.491660 (lr=4.0807e-06) (hash(x)=153333281)
+16100 val loss 5.5407
+16100 val perplexity 254.8457
+16100 train 5.496932 (lr=4.0063e-06) (hash(x)=145628059)
+16200 val loss 5.5414
+16200 val perplexity 255.0395
+16200 train 5.268761 (lr=3.9368e-06) (hash(x)=137220041)
+16300 val loss 5.5398
+16300 val perplexity 254.6356
+16300 train 5.710855 (lr=3.8725e-06) (hash(x)=161215791)
+16400 val loss 5.5392
+16400 val perplexity 254.4846
+16400 train 5.249424 (lr=3.8132e-06) (hash(x)=136194680)
+16500 val loss 5.5397
+16500 val perplexity 254.6127
+16500 train 5.334385 (lr=3.7590e-06) (hash(x)=152209568)
+16600 val loss 5.5366
+16600 val perplexity 253.8228
+16600 train 5.652360 (lr=3.7099e-06) (hash(x)=157345721)
+16700 val loss 5.5351
+16700 val perplexity 253.4332
+16700 train 5.666698 (lr=3.6659e-06) (hash(x)=146883642)
+16800 val loss 5.5349
+16800 val perplexity 253.3878
+16800 train 5.509064 (lr=3.6271e-06) (hash(x)=153037972)
+16900 val loss 5.5340
+16900 val perplexity 253.1516
+16900 train 5.297623 (lr=3.5934e-06) (hash(x)=150691367)
+17000 val loss 5.5341
+17000 val perplexity 253.1721
+17000 train 5.288759 (lr=3.5649e-06) (hash(x)=148975206)
+17100 val loss 5.5351
+17100 val perplexity 253.4207
+17100 train 5.278723 (lr=3.5415e-06) (hash(x)=142026221)
+17200 val loss 5.5315
+17200 val perplexity 252.5322
+17200 train 5.597274 (lr=3.5234e-06) (hash(x)=138211880)
+17300 val loss 5.5293
+17300 val perplexity 251.9741
+17300 train 5.385768 (lr=3.5104e-06) (hash(x)=143366666)
+17400 val loss 5.5290
+17400 val perplexity 251.8806
+17400 train 5.530194 (lr=3.5026e-06) (hash(x)=150791766)
+17499 val loss 5.5286
+17499 val perplexity 251.7872

attention_kindselective_n_heads4_seed1343/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cae2222b2a438aae1ef4db8da06094fd8f0026a9823a8246b18a21cb587edf18
+oid sha256:27081619b463bbaad238902544341ba0a158e6c08674f088292609d0df8be2db
 size 92843394

attention_kindselective_n_heads4_seed1343/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9055b35115695594360e1b670807584e8d24d02e2da189450f14cae00c740d99
+oid sha256:73b852ab286f52955da2ba1e0c51cb905db4b5700fae874d7533bc7e4896228d
 size 179406214