Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1343/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1343", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1343", "n_embd": 256}

attention_kindselective_n_heads4_seed1343/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.3001
-0 val perplexity 80829.3594
-0 train 11.304919 (lr=1.3750e-07) (hash(x)=153418542)
-100 val loss 9.9044
-100 val perplexity 20017.5859
-100 train 9.872492 (lr=1.3888e-05) (hash(x)=154406429)
-200 val loss 9.0083
-200 val perplexity 8170.6182
-200 train 8.930150 (lr=2.7500e-05) (hash(x)=143722001)
-300 val loss 8.0266
-300 val perplexity 3061.2393
-300 train 7.972954 (lr=2.7498e-05) (hash(x)=152421341)
-400 val loss 7.6457
-400 val perplexity 2091.6472
-400 train 7.543222 (lr=2.7492e-05) (hash(x)=152989689)
-500 val loss 7.5404
-500 val perplexity 1882.6754
-500 train 7.470467 (lr=2.7482e-05) (hash(x)=145113544)
-600 val loss 7.4701
-600 val perplexity 1754.6958
-600 train 7.464890 (lr=2.7467e-05) (hash(x)=150071018)
-700 val loss 7.4072
-700 val perplexity 1647.8019
-700 train 7.498733 (lr=2.7449e-05) (hash(x)=156594182)
-800 val loss 7.3685
-800 val perplexity 1585.1940
-800 train 7.370912 (lr=2.7427e-05) (hash(x)=153358238)
-900 val loss 7.3397
-900 val perplexity 1540.2643
-900 train 7.168626 (lr=2.7400e-05) (hash(x)=146617394)
-1000 val loss 7.3128
-1000 val perplexity 1499.4252
-1000 train 7.256218 (lr=2.7370e-05) (hash(x)=148983354)
-1100 val loss 7.2795
-1100 val perplexity 1450.2510
-1100 train 7.309369 (lr=2.7335e-05) (hash(x)=146313352)
-1200 val loss 7.2520
-1200 val perplexity 1410.9708
-1200 train 7.212644 (lr=2.7297e-05) (hash(x)=150770333)
-1300 val loss 7.2062
-1300 val perplexity 1347.7965
-1300 train 7.149687 (lr=2.7254e-05) (hash(x)=153237933)
-1400 val loss 7.1656
-1400 val perplexity 1294.1826
-1400 train 7.001445 (lr=2.7207e-05) (hash(x)=145246201)
-1500 val loss 7.1301
-1500 val perplexity 1249.0450
-1500 train 7.126824 (lr=2.7157e-05) (hash(x)=154002540)
-1600 val loss 7.0984
-1600 val perplexity 1210.0066
-1600 train 6.874021 (lr=2.7102e-05) (hash(x)=142635842)
-1700 val loss 7.0601
-1700 val perplexity 1164.6145
-1700 train 7.108422 (lr=2.7044e-05) (hash(x)=151450555)
-1800 val loss 7.0224
-1800 val perplexity 1121.4412
-1800 train 6.941932 (lr=2.6981e-05) (hash(x)=148299832)
-1900 val loss 6.9836
-1900 val perplexity 1078.8276
-1900 train 7.138231 (lr=2.6915e-05) (hash(x)=155487476)
-2000 val loss 6.9545
-2000 val perplexity 1047.8662
-2000 train 6.934770 (lr=2.6845e-05) (hash(x)=152887721)
-2100 val loss 6.9112
-2100 val perplexity 1003.4163
-2100 train 6.617202 (lr=2.6771e-05) (hash(x)=143430603)
-2200 val loss 6.8834
-2200 val perplexity 975.9449
-2200 train 6.881304 (lr=2.6693e-05) (hash(x)=153721194)
-2300 val loss 6.8433
-2300 val perplexity 937.5591
-2300 train 6.975938 (lr=2.6611e-05) (hash(x)=148702429)
-2400 val loss 6.8007
-2400 val perplexity 898.5173
-2400 train 6.848701 (lr=2.6525e-05) (hash(x)=153047184)
-2500 val loss 6.7660
-2500 val perplexity 867.8541
-2500 train 6.643397 (lr=2.6436e-05) (hash(x)=149720123)
-2600 val loss 6.7336
-2600 val perplexity 840.1345
-2600 train 6.765120 (lr=2.6343e-05) (hash(x)=152466045)
-2700 val loss 6.7126
-2700 val perplexity 822.6757
-2700 train 6.643747 (lr=2.6246e-05) (hash(x)=155796305)
-2800 val loss 6.6753
-2800 val perplexity 792.6136
-2800 train 6.515200 (lr=2.6146e-05) (hash(x)=151991552)
-2900 val loss 6.6396
-2900 val perplexity 764.7943
-2900 train 7.280913 (lr=2.6042e-05) (hash(x)=166537989)
-3000 val loss 6.6073
-3000 val perplexity 740.4809
-3000 train 6.607131 (lr=2.5934e-05) (hash(x)=147892594)
-3100 val loss 6.5759
-3100 val perplexity 717.6121
-3100 train 6.537065 (lr=2.5823e-05) (hash(x)=149536927)
-3200 val loss 6.5500
-3200 val perplexity 699.2293
-3200 train 6.262515 (lr=2.5709e-05) (hash(x)=135678663)
-3300 val loss 6.5233
-3300 val perplexity 680.8391
-3300 train 6.577153 (lr=2.5590e-05) (hash(x)=148523646)
-3400 val loss 6.5011
-3400 val perplexity 665.8766
-3400 train 6.573230 (lr=2.5469e-05) (hash(x)=150281778)
-3500 val loss 6.4749
-3500 val perplexity 648.6446
-3500 train 6.416248 (lr=2.5344e-05) (hash(x)=151527953)
-3600 val loss 6.4579
-3600 val perplexity 637.7144
-3600 train 6.692443 (lr=2.5215e-05) (hash(x)=141303739)
-3700 val loss 6.4379
-3700 val perplexity 625.0728
-3700 train 6.471801 (lr=2.5083e-05) (hash(x)=146778597)
-3800 val loss 6.4130
-3800 val perplexity 609.7248
-3800 train 6.366365 (lr=2.4948e-05) (hash(x)=146902797)
-3900 val loss 6.3946
-3900 val perplexity 598.6218
-3900 train 6.325142 (lr=2.4810e-05) (hash(x)=142022555)
-4000 val loss 6.3738
-4000 val perplexity 586.2914
-4000 train 6.296466 (lr=2.4669e-05) (hash(x)=151590309)
-4100 val loss 6.3588
-4100 val perplexity 577.5575
-4100 train 6.199865 (lr=2.4524e-05) (hash(x)=144127100)
-4200 val loss 6.3470
-4200 val perplexity 570.7855
-4200 train 6.413688 (lr=2.4376e-05) (hash(x)=149526774)
-4300 val loss 6.3291
-4300 val perplexity 560.6296
-4300 train 6.277426 (lr=2.4226e-05) (hash(x)=150924362)
-4400 val loss 6.3049
-4400 val perplexity 547.2457
-4400 train 6.464039 (lr=2.4072e-05) (hash(x)=155547282)
-4500 val loss 6.2960
-4500 val perplexity 542.4135
-4500 train 6.273605 (lr=2.3915e-05) (hash(x)=152479203)
-4600 val loss 6.2818
-4600 val perplexity 534.7637
-4600 train 6.199425 (lr=2.3755e-05) (hash(x)=146739760)
-4700 val loss 6.2702
-4700 val perplexity 528.5671
-4700 train 6.492648 (lr=2.3593e-05) (hash(x)=159977388)
-4800 val loss 6.2597
-4800 val perplexity 523.0594
-4800 train 6.143291 (lr=2.3428e-05) (hash(x)=142959739)
-4900 val loss 6.2422
-4900 val perplexity 513.9914
-4900 train 6.195841 (lr=2.3260e-05) (hash(x)=152253034)
-5000 val loss 6.2314
-5000 val perplexity 508.4516
-5000 train 6.202939 (lr=2.3089e-05) (hash(x)=140379747)
-5100 val loss 6.2158
-5100 val perplexity 500.5816
-5100 train 6.235268 (lr=2.2916e-05) (hash(x)=151826392)
-5200 val loss 6.2073
-5200 val perplexity 496.3683
-5200 train 6.160838 (lr=2.2740e-05) (hash(x)=152869594)
-5300 val loss 6.1968
-5300 val perplexity 491.1580
-5300 train 5.984524 (lr=2.2561e-05) (hash(x)=139093709)
-5400 val loss 6.1878
-5400 val perplexity 486.7791
-5400 train 6.060760 (lr=2.2381e-05) (hash(x)=145908738)
-5500 val loss 6.1784
-5500 val perplexity 482.2269
-5500 train 6.033603 (lr=2.2197e-05) (hash(x)=137913426)
-5600 val loss 6.1617
-5600 val perplexity 474.2398
-5600 train 6.011183 (lr=2.2012e-05) (hash(x)=140770390)
-5700 val loss 6.1576
-5700 val perplexity 472.2713
-5700 train 6.076570 (lr=2.1824e-05) (hash(x)=143728578)
-5800 val loss 6.1446
-5800 val perplexity 466.1837
-5800 train 6.218975 (lr=2.1634e-05) (hash(x)=154469787)
-5900 val loss 6.1377
-5900 val perplexity 462.9820
-5900 train 6.090513 (lr=2.1442e-05) (hash(x)=139316559)
-6000 val loss 6.1299
-6000 val perplexity 459.3950
-6000 train 6.138595 (lr=2.1248e-05) (hash(x)=152058488)
-6100 val loss 6.1255
-6100 val perplexity 457.3819
-6100 train 6.128792 (lr=2.1051e-05) (hash(x)=152958689)
-6200 val loss 6.1035
-6200 val perplexity 447.4265
-6200 train 6.071043 (lr=2.0853e-05) (hash(x)=150887864)
-6300 val loss 6.0974
-6300 val perplexity 444.6889
-6300 train 6.045360 (lr=2.0653e-05) (hash(x)=156290469)
-6400 val loss 6.0854
-6400 val perplexity 439.3943
-6400 train 6.386467 (lr=2.0451e-05) (hash(x)=159160159)
-6500 val loss 6.0784
-6500 val perplexity 436.3256
-6500 train 5.996751 (lr=2.0247e-05) (hash(x)=151239121)
-6600 val loss 6.0738
-6600 val perplexity 434.3249
-6600 train 6.088350 (lr=2.0042e-05) (hash(x)=151964101)
-6700 val loss 6.0651
-6700 val perplexity 430.5475
-6700 train 5.970629 (lr=1.9835e-05) (hash(x)=141291229)
-6800 val loss 6.0589
-6800 val perplexity 427.8987
-6800 train 5.985006 (lr=1.9626e-05) (hash(x)=160416800)
-6900 val loss 6.0504
-6900 val perplexity 424.2713
-6900 train 5.953206 (lr=1.9416e-05) (hash(x)=143382691)
-7000 val loss 6.0369
-7000 val perplexity 418.5984
-7000 train 5.939549 (lr=1.9205e-05) (hash(x)=147606572)
-7100 val loss 6.0279
-7100 val perplexity 414.8367
-7100 train 5.859169 (lr=1.8992e-05) (hash(x)=145901538)
-7200 val loss 6.0257
-7200 val perplexity 413.9467
-7200 train 5.854834 (lr=1.8778e-05) (hash(x)=139099153)
-7300 val loss 6.0190
-7300 val perplexity 411.1549
-7300 train 5.979025 (lr=1.8562e-05) (hash(x)=140434004)
-7400 val loss 6.0171
-7400 val perplexity 410.3832
-7400 train 6.131842 (lr=1.8346e-05) (hash(x)=155496146)
-7500 val loss 6.0081
-7500 val perplexity 406.6947
-7500 train 5.932770 (lr=1.8128e-05) (hash(x)=146940054)
-7600 val loss 5.9919
-7600 val perplexity 400.1565
-7600 train 6.220881 (lr=1.7910e-05) (hash(x)=165785723)
-7700 val loss 5.9845
-7700 val perplexity 397.2233
-7700 train 6.136736 (lr=1.7691e-05) (hash(x)=156339008)
-7800 val loss 5.9830
-7800 val perplexity 396.6223
-7800 train 5.985073 (lr=1.7470e-05) (hash(x)=148593556)
-7900 val loss 5.9865
-7900 val perplexity 398.0033
-7900 train 5.919378 (lr=1.7249e-05) (hash(x)=148079742)
-8000 val loss 5.9645
-8000 val perplexity 389.3672
-8000 train 5.894415 (lr=1.7028e-05) (hash(x)=159706718)
-8100 val loss 5.9625
-8100 val perplexity 388.5824
-8100 train 5.958558 (lr=1.6805e-05) (hash(x)=160203225)
-8200 val loss 5.9512
-8200 val perplexity 384.2123
-8200 train 5.893342 (lr=1.6582e-05) (hash(x)=146176536)
-8300 val loss 5.9434
-8300 val perplexity 381.2125
-8300 train 6.014031 (lr=1.6359e-05) (hash(x)=158883824)
-8400 val loss 5.9337
-8400 val perplexity 377.5597
-8400 train 5.745647 (lr=1.6135e-05) (hash(x)=137102572)
-8500 val loss 5.9282
-8500 val perplexity 375.4724
-8500 train 5.958540 (lr=1.5911e-05) (hash(x)=152812797)
-8600 val loss 5.9255
-8600 val perplexity 374.4656
-8600 train 5.783076 (lr=1.5687e-05) (hash(x)=148854244)
-8700 val loss 5.9364
-8700 val perplexity 378.5663
-8700 train 5.752574 (lr=1.5462e-05) (hash(x)=135590023)
-8800 val loss 5.9116
-8800 val perplexity 369.2888
-8800 train 5.844384 (lr=1.5237e-05) (hash(x)=144913509)
-8900 val loss 5.9079
-8900 val perplexity 367.9172
-8900 train 6.030931 (lr=1.5013e-05) (hash(x)=148045669)
-9000 val loss 5.9000
-9000 val perplexity 365.0257
-9000 train 5.921853 (lr=1.4788e-05) (hash(x)=146661746)
-9100 val loss 5.8940
-9100 val perplexity 362.8599
-9100 train 5.834559 (lr=1.4563e-05) (hash(x)=145354881)
-9200 val loss 5.8897
-9200 val perplexity 361.2867
-9200 train 5.755416 (lr=1.4339e-05) (hash(x)=150939310)
-9300 val loss 5.8864
-9300 val perplexity 360.0898
-9300 train 5.723167 (lr=1.4115e-05) (hash(x)=162258309)
-9400 val loss 5.8789
-9400 val perplexity 357.4210
-9400 train 5.885062 (lr=1.3891e-05) (hash(x)=154701830)
-9500 val loss 5.8721
-9500 val perplexity 355.0111
-9500 train 5.761203 (lr=1.3668e-05) (hash(x)=142241404)
-9600 val loss 5.8658
-9600 val perplexity 352.7511
-9600 train 5.832513 (lr=1.3445e-05) (hash(x)=148439858)
-9700 val loss 5.8651
-9700 val perplexity 352.5332
-9700 train 5.865323 (lr=1.3222e-05) (hash(x)=152204410)
-9800 val loss 5.8561
-9800 val perplexity 349.3512
-9800 train 5.703402 (lr=1.3001e-05) (hash(x)=141294630)
-9900 val loss 5.8539
-9900 val perplexity 348.6057
-9900 train 5.826562 (lr=1.2780e-05) (hash(x)=147390033)
-10000 val loss 5.8480
-10000 val perplexity 346.5345
-10000 train 5.846817 (lr=1.2559e-05) (hash(x)=144131604)
-10100 val loss 5.8387
-10100 val perplexity 343.3346
-10100 train 6.040196 (lr=1.2340e-05) (hash(x)=158814818)
-10200 val loss 5.8349
-10200 val perplexity 342.0135
-10200 train 5.914101 (lr=1.2122e-05) (hash(x)=153491388)
-10300 val loss 5.8303
-10300 val perplexity 340.4523
-10300 train 5.807222 (lr=1.1904e-05) (hash(x)=149600707)
-10400 val loss 5.8256
-10400 val perplexity 338.8573
-10400 train 5.740230 (lr=1.1688e-05) (hash(x)=152277064)
-10500 val loss 5.8238
-10500 val perplexity 338.2697
-10500 train 5.728241 (lr=1.1472e-05) (hash(x)=150447091)
-10600 val loss 5.8134
-10600 val perplexity 334.7462
-10600 train 5.837738 (lr=1.1258e-05) (hash(x)=151606822)
-10700 val loss 5.8070
-10700 val perplexity 332.6146
-10700 train 5.796923 (lr=1.1045e-05) (hash(x)=158730240)
-10800 val loss 5.8067
-10800 val perplexity 332.5079
-10800 train 5.856279 (lr=1.0834e-05) (hash(x)=163095518)
-10900 val loss 5.8021
-10900 val perplexity 330.9915
-10900 train 5.560940 (lr=1.0624e-05) (hash(x)=145260395)
-11000 val loss 5.7968
-11000 val perplexity 329.2379
-11000 train 5.802653 (lr=1.0415e-05) (hash(x)=155970264)
-11100 val loss 5.7948
-11100 val perplexity 328.5754
-11100 train 5.889194 (lr=1.0208e-05) (hash(x)=159871097)
-11200 val loss 5.7875
-11200 val perplexity 326.1909
-11200 train 5.876340 (lr=1.0003e-05) (hash(x)=161135244)
-11300 val loss 5.7826
-11300 val perplexity 324.6163
-11300 train 5.817194 (lr=9.7991e-06) (hash(x)=150649336)
-11400 val loss 5.7799
-11400 val perplexity 323.7270
-11400 train 5.749750 (lr=9.5972e-06) (hash(x)=152230013)
-11500 val loss 5.7759
-11500 val perplexity 322.4249
-11500 train 5.821681 (lr=9.3970e-06) (hash(x)=163300613)
-11600 val loss 5.7760
-11600 val perplexity 322.4666
-11600 train 5.755337 (lr=9.1988e-06) (hash(x)=151203838)
-11700 val loss 5.7722
-11700 val perplexity 321.2419
-11700 train 5.793112 (lr=9.0025e-06) (hash(x)=155255111)
-11800 val loss 5.7741
-11800 val perplexity 321.8658
-11800 train 5.731073 (lr=8.8082e-06) (hash(x)=139380668)
-11900 val loss 5.7614
-11900 val perplexity 317.8024
-11900 train 5.690895 (lr=8.6160e-06) (hash(x)=144033405)
-12000 val loss 5.7601
-12000 val perplexity 317.3673
-12000 train 5.683661 (lr=8.4260e-06) (hash(x)=148689574)
-12100 val loss 5.7576
-12100 val perplexity 316.5753
-12100 train 5.693658 (lr=8.2381e-06) (hash(x)=143272239)
-12200 val loss 5.7536
-12200 val perplexity 315.3213
-12200 train 5.826474 (lr=8.0526e-06) (hash(x)=157671609)
-12300 val loss 5.7511
-12300 val perplexity 314.5443
-12300 train 5.774444 (lr=7.8694e-06) (hash(x)=150363247)
-12400 val loss 5.7458
-12400 val perplexity 312.8600
-12400 train 5.633980 (lr=7.6885e-06) (hash(x)=146857787)
-12500 val loss 5.7446
-12500 val perplexity 312.5057
-12500 train 5.712029 (lr=7.5101e-06) (hash(x)=153327320)
-12600 val loss 5.7406
-12600 val perplexity 311.2592
-12600 train 5.462293 (lr=7.3342e-06) (hash(x)=143612983)
-12700 val loss 5.7379
-12700 val perplexity 310.4186
-12700 train 5.676553 (lr=7.1609e-06) (hash(x)=150739132)
-12800 val loss 5.7364
-12800 val perplexity 309.9375
-12800 train 5.584580 (lr=6.9903e-06) (hash(x)=147986204)
-12900 val loss 5.7343
-12900 val perplexity 309.3034
-12900 train 5.899726 (lr=6.8223e-06) (hash(x)=155218833)
-13000 val loss 5.7284
-13000 val perplexity 307.4839
-13000 train 5.723590 (lr=6.6570e-06) (hash(x)=148867960)
-13100 val loss 5.7259
-13100 val perplexity 306.7019
-13100 train 5.747935 (lr=6.4945e-06) (hash(x)=160101478)
-13200 val loss 5.7260
-13200 val perplexity 306.7493
-13200 train 5.585583 (lr=6.3349e-06) (hash(x)=151080097)
-13300 val loss 5.7252
-13300 val perplexity 306.4901
-13300 train 5.698304 (lr=6.1782e-06) (hash(x)=150403039)
-13400 val loss 5.7210
-13400 val perplexity 305.2061
-13400 train 5.663198 (lr=6.0244e-06) (hash(x)=129728571)
-13500 val loss 5.7166
-13500 val perplexity 303.8716
-13500 train 5.632593 (lr=5.8737e-06) (hash(x)=148988625)
-13600 val loss 5.7159
-13600 val perplexity 303.6613
-13600 train 5.503659 (lr=5.7259e-06) (hash(x)=149790861)
-13700 val loss 5.7148
-13700 val perplexity 303.3160
-13700 train 5.644720 (lr=5.5813e-06) (hash(x)=153505062)
-13800 val loss 5.7140
-13800 val perplexity 303.0839
-13800 train 5.621727 (lr=5.4398e-06) (hash(x)=143845463)
-13900 val loss 5.7126
-13900 val perplexity 302.6628
-13900 train 5.516610 (lr=5.3016e-06) (hash(x)=143378076)
-14000 val loss 5.7074
-14000 val perplexity 301.0799
-14000 train 5.666134 (lr=5.1665e-06) (hash(x)=148681755)
-14100 val loss 5.7058
-14100 val perplexity 300.6119
-14100 train 5.593862 (lr=5.0348e-06) (hash(x)=145530465)
-14200 val loss 5.7041
-14200 val perplexity 300.1066
-14200 train 5.649960 (lr=4.9063e-06) (hash(x)=162139713)
-14300 val loss 5.7019
-14300 val perplexity 299.4321
-14300 train 5.708417 (lr=4.7813e-06) (hash(x)=141997596)
-14400 val loss 5.7009
-14400 val perplexity 299.1285
-14400 train 5.672005 (lr=4.6596e-06) (hash(x)=146156313)
-14500 val loss 5.7000
-14500 val perplexity 298.8767
-14500 train 5.478105 (lr=4.5414e-06) (hash(x)=145298178)
-14600 val loss 5.6955
-14600 val perplexity 297.5303
-14600 train 5.779585 (lr=4.4267e-06) (hash(x)=148232610)
-14700 val loss 5.6937
-14700 val perplexity 296.9908
-14700 train 5.574780 (lr=4.3155e-06) (hash(x)=151079115)
-14800 val loss 5.6924
-14800 val perplexity 296.6122
-14800 train 5.674882 (lr=4.2079e-06) (hash(x)=151340983)
-14900 val loss 5.6911
-14900 val perplexity 296.2318
-14900 train 5.897525 (lr=4.1039e-06) (hash(x)=150213803)
-15000 val loss 5.6910
-15000 val perplexity 296.1945
-15000 train 5.598524 (lr=4.0035e-06) (hash(x)=157652334)
-15100 val loss 5.6897
-15100 val perplexity 295.8021
-15100 train 5.417470 (lr=3.9068e-06) (hash(x)=145078577)
-15200 val loss 5.6870
-15200 val perplexity 295.0057
-15200 train 5.639482 (lr=3.8138e-06) (hash(x)=155898006)
-15300 val loss 5.6855
-15300 val perplexity 294.5604
-15300 train 5.726047 (lr=3.7245e-06) (hash(x)=149346671)
-15400 val loss 5.6834
-15400 val perplexity 293.9359
-15400 train 5.534364 (lr=3.6390e-06) (hash(x)=145708113)
-15500 val loss 5.6821
-15500 val perplexity 293.5533
-15500 train 5.875023 (lr=3.5572e-06) (hash(x)=165102970)
-15600 val loss 5.6821
-15600 val perplexity 293.5759
-15600 train 5.470008 (lr=3.4793e-06) (hash(x)=139106517)
-15700 val loss 5.6807
-15700 val perplexity 293.1580
-15700 train 5.543227 (lr=3.4052e-06) (hash(x)=141680838)
-15800 val loss 5.6785
-15800 val perplexity 292.5013
-15800 train 5.570748 (lr=3.3350e-06) (hash(x)=136152310)
-15900 val loss 5.6766
-15900 val perplexity 291.9579
-15900 train 5.745454 (lr=3.2687e-06) (hash(x)=151104640)
-16000 val loss 5.6760
-16000 val perplexity 291.7779
-16000 train 5.618743 (lr=3.2063e-06) (hash(x)=153333281)
-16100 val loss 5.6743
-16100 val perplexity 291.2915
-16100 train 5.631518 (lr=3.1478e-06) (hash(x)=145628059)
-16200 val loss 5.6748
-16200 val perplexity 291.4199
-16200 train 5.398105 (lr=3.0932e-06) (hash(x)=137220041)
-16300 val loss 5.6739
-16300 val perplexity 291.1725
-16300 train 5.843124 (lr=3.0427e-06) (hash(x)=161215791)
-16400 val loss 5.6729
-16400 val perplexity 290.8825
-16400 train 5.388996 (lr=2.9961e-06) (hash(x)=136194680)
-16500 val loss 5.6734
-16500 val perplexity 291.0279
-16500 train 5.465913 (lr=2.9535e-06) (hash(x)=152209568)
-16600 val loss 5.6698
-16600 val perplexity 289.9884
-16600 train 5.788033 (lr=2.9149e-06) (hash(x)=157345721)
-16700 val loss 5.6678
-16700 val perplexity 289.3843
-16700 train 5.794989 (lr=2.8804e-06) (hash(x)=146883642)
-16800 val loss 5.6680
-16800 val perplexity 289.4506
-16800 train 5.638534 (lr=2.8498e-06) (hash(x)=153037972)
-16900 val loss 5.6672
-16900 val perplexity 289.2180
-16900 train 5.444636 (lr=2.8234e-06) (hash(x)=150691367)
-17000 val loss 5.6672
-17000 val perplexity 289.2126
-17000 train 5.424174 (lr=2.8010e-06) (hash(x)=148975206)
-17100 val loss 5.6681
-17100 val perplexity 289.4754
-17100 train 5.402887 (lr=2.7826e-06) (hash(x)=142026221)
-17200 val loss 5.6642
-17200 val perplexity 288.3643
-17200 train 5.721988 (lr=2.7684e-06) (hash(x)=138211880)
-17300 val loss 5.6625
-17300 val perplexity 287.8575
-17300 train 5.518986 (lr=2.7582e-06) (hash(x)=143366666)
-17400 val loss 5.6624
-17400 val perplexity 287.8243
-17400 train 5.650568 (lr=2.7520e-06) (hash(x)=150791766)
-17499 val loss 5.6616
-17499 val perplexity 287.6108
+0 val perplexity 80830.1328
+0 train 11.304919 (lr=1.6250e-07) (hash(x)=153418542)
+100 val loss 9.8733
+100 val perplexity 19405.9473
+100 train 9.841567 (lr=1.6412e-05) (hash(x)=154406429)
+200 val loss 8.7130
+200 val perplexity 6081.5020
+200 train 8.618789 (lr=3.2500e-05) (hash(x)=143722001)
+300 val loss 7.7544
+300 val perplexity 2331.8059
+300 train 7.689346 (lr=3.2498e-05) (hash(x)=152421341)
+400 val loss 7.5469
+400 val perplexity 1894.8176
+400 train 7.435925 (lr=3.2490e-05) (hash(x)=152989689)
+500 val loss 7.4583
+500 val perplexity 1734.2018
+500 train 7.384192 (lr=3.2478e-05) (hash(x)=145113544)
+600 val loss 7.3776
+600 val perplexity 1599.7095
+600 train 7.368221 (lr=3.2461e-05) (hash(x)=150071018)
+700 val loss 7.3126
+700 val perplexity 1499.1150
+700 train 7.416527 (lr=3.2440e-05) (hash(x)=156594182)
+800 val loss 7.2558
+800 val perplexity 1416.3041
+800 train 7.261938 (lr=3.2413e-05) (hash(x)=153358238)
+900 val loss 7.1929
+900 val perplexity 1329.8951
+900 train 7.011195 (lr=3.2382e-05) (hash(x)=146617394)
+1000 val loss 7.1334
+1000 val perplexity 1253.1919
+1000 train 7.066751 (lr=3.2346e-05) (hash(x)=148983354)
+1100 val loss 7.0718
+1100 val perplexity 1178.2454
+1100 train 7.096290 (lr=3.2305e-05) (hash(x)=146313352)
+1200 val loss 7.0079
+1200 val perplexity 1105.3306
+1200 train 6.949758 (lr=3.2260e-05) (hash(x)=150770333)
+1300 val loss 6.9429
+1300 val perplexity 1035.7495
+1300 train 6.894914 (lr=3.2209e-05) (hash(x)=153237933)
+1400 val loss 6.8895
+1400 val perplexity 981.9400
+1400 train 6.719554 (lr=3.2154e-05) (hash(x)=145246201)
+1500 val loss 6.8387
+1500 val perplexity 933.2887
+1500 train 6.812776 (lr=3.2094e-05) (hash(x)=154002540)
+1600 val loss 6.7915
+1600 val perplexity 890.2463
+1600 train 6.566239 (lr=3.2030e-05) (hash(x)=142635842)
+1700 val loss 6.7368
+1700 val perplexity 842.8735
+1700 train 6.787238 (lr=3.1961e-05) (hash(x)=151450555)
+1800 val loss 6.6812
+1800 val perplexity 797.2631
+1800 train 6.601893 (lr=3.1887e-05) (hash(x)=148299832)
+1900 val loss 6.6343
+1900 val perplexity 760.7756
+1900 train 6.770863 (lr=3.1809e-05) (hash(x)=155487476)
+2000 val loss 6.5992
+2000 val perplexity 734.4827
+2000 train 6.578701 (lr=3.1726e-05) (hash(x)=152887721)
+2100 val loss 6.5663
+2100 val perplexity 710.7366
+2100 train 6.259625 (lr=3.1638e-05) (hash(x)=143430603)
+2200 val loss 6.5374
+2200 val perplexity 690.4644
+2200 train 6.537328 (lr=3.1546e-05) (hash(x)=153721194)
+2300 val loss 6.5074
+2300 val perplexity 670.0813
+2300 train 6.664390 (lr=3.1449e-05) (hash(x)=148702429)
+2400 val loss 6.4757
+2400 val perplexity 649.1904
+2400 train 6.536206 (lr=3.1348e-05) (hash(x)=153047184)
+2500 val loss 6.4434
+2500 val perplexity 628.5614
+2500 train 6.321926 (lr=3.1243e-05) (hash(x)=149720123)
+2600 val loss 6.4200
+2600 val perplexity 614.0228
+2600 train 6.436261 (lr=3.1133e-05) (hash(x)=152466045)
+2700 val loss 6.4165
+2700 val perplexity 611.8614
+2700 train 6.326521 (lr=3.1019e-05) (hash(x)=155796305)
+2800 val loss 6.3826
+2800 val perplexity 591.4885
+2800 train 6.222201 (lr=3.0900e-05) (hash(x)=151991552)
+2900 val loss 6.3610
+2900 val perplexity 578.8026
+2900 train 6.974368 (lr=3.0777e-05) (hash(x)=166537989)
+3000 val loss 6.3340
+3000 val perplexity 563.3945
+3000 train 6.334172 (lr=3.0650e-05) (hash(x)=147892594)
+3100 val loss 6.3147
+3100 val perplexity 552.6521
+3100 train 6.278732 (lr=3.0518e-05) (hash(x)=149536927)
+3200 val loss 6.3047
+3200 val perplexity 547.1489
+3200 train 6.019104 (lr=3.0383e-05) (hash(x)=135678663)
+3300 val loss 6.2794
+3300 val perplexity 533.4483
+3300 train 6.361428 (lr=3.0243e-05) (hash(x)=148523646)
+3400 val loss 6.2695
+3400 val perplexity 528.1934
+3400 train 6.349548 (lr=3.0099e-05) (hash(x)=150281778)
+3500 val loss 6.2489
+3500 val perplexity 517.4524
+3500 train 6.206536 (lr=2.9952e-05) (hash(x)=151527953)
+3600 val loss 6.2416
+3600 val perplexity 513.6918
+3600 train 6.464660 (lr=2.9800e-05) (hash(x)=141303739)
+3700 val loss 6.2224
+3700 val perplexity 503.8872
+3700 train 6.266064 (lr=2.9644e-05) (hash(x)=146778597)
+3800 val loss 6.2020
+3800 val perplexity 493.7431
+3800 train 6.149168 (lr=2.9485e-05) (hash(x)=146902797)
+3900 val loss 6.1910
+3900 val perplexity 488.3286
+3900 train 6.114597 (lr=2.9321e-05) (hash(x)=142022555)
+4000 val loss 6.1740
+4000 val perplexity 480.0992
+4000 train 6.090962 (lr=2.9154e-05) (hash(x)=151590309)
+4100 val loss 6.1610
+4100 val perplexity 473.9238
+4100 train 6.004887 (lr=2.8983e-05) (hash(x)=144127100)
+4200 val loss 6.1519
+4200 val perplexity 469.6165
+4200 train 6.226448 (lr=2.8808e-05) (hash(x)=149526774)
+4300 val loss 6.1462
+4300 val perplexity 466.9373
+4300 train 6.081618 (lr=2.8630e-05) (hash(x)=150924362)
+4400 val loss 6.1223
+4400 val perplexity 455.9291
+4400 train 6.270722 (lr=2.8448e-05) (hash(x)=155547282)
+4500 val loss 6.1144
+4500 val perplexity 452.3334
+4500 train 6.097814 (lr=2.8263e-05) (hash(x)=152479203)
+4600 val loss 6.0978
+4600 val perplexity 444.8783
+4600 train 6.011952 (lr=2.8075e-05) (hash(x)=146739760)
+4700 val loss 6.0940
+4700 val perplexity 443.1834
+4700 train 6.322186 (lr=2.7883e-05) (hash(x)=159977388)
+4800 val loss 6.0822
+4800 val perplexity 437.9828
+4800 train 5.978403 (lr=2.7687e-05) (hash(x)=142959739)
+4900 val loss 6.0686
+4900 val perplexity 432.0666
+4900 train 6.018283 (lr=2.7489e-05) (hash(x)=152253034)
+5000 val loss 6.0667
+5000 val perplexity 431.2595
+5000 train 6.036760 (lr=2.7287e-05) (hash(x)=140379747)
+5100 val loss 6.0461
+5100 val perplexity 422.4584
+5100 train 6.082677 (lr=2.7082e-05) (hash(x)=151826392)
+5200 val loss 6.0384
+5200 val perplexity 419.2076
+5200 train 6.012521 (lr=2.6874e-05) (hash(x)=152869594)
+5300 val loss 6.0266
+5300 val perplexity 414.3012
+5300 train 5.810596 (lr=2.6664e-05) (hash(x)=139093709)
+5400 val loss 6.0199
+5400 val perplexity 411.5291
+5400 train 5.901548 (lr=2.6450e-05) (hash(x)=145908738)
+5500 val loss 6.0176
+5500 val perplexity 410.5985
+5500 train 5.885014 (lr=2.6233e-05) (hash(x)=137913426)
+5600 val loss 5.9932
+5600 val perplexity 400.7062
+5600 train 5.863060 (lr=2.6014e-05) (hash(x)=140770390)
+5700 val loss 5.9972
+5700 val perplexity 402.3169
+5700 train 5.915018 (lr=2.5792e-05) (hash(x)=143728578)
+5800 val loss 5.9781
+5800 val perplexity 394.6902
+5800 train 6.046650 (lr=2.5567e-05) (hash(x)=154469787)
+5900 val loss 5.9709
+5900 val perplexity 391.8690
+5900 train 5.934379 (lr=2.5340e-05) (hash(x)=139316559)
+6000 val loss 5.9608
+6000 val perplexity 387.9297
+6000 train 5.969604 (lr=2.5111e-05) (hash(x)=152058488)
+6100 val loss 5.9539
+6100 val perplexity 385.2404
+6100 train 5.971652 (lr=2.4879e-05) (hash(x)=152958689)
+6200 val loss 5.9376
+6200 val perplexity 379.0083
+6200 train 5.906252 (lr=2.4644e-05) (hash(x)=150887864)
+6300 val loss 5.9309
+6300 val perplexity 376.4814
+6300 train 5.876676 (lr=2.4408e-05) (hash(x)=156290469)
+6400 val loss 5.9232
+6400 val perplexity 373.5906
+6400 train 6.241812 (lr=2.4169e-05) (hash(x)=159160159)
+6500 val loss 5.9175
+6500 val perplexity 371.4980
+6500 train 5.838713 (lr=2.3928e-05) (hash(x)=151239121)
+6600 val loss 5.9117
+6600 val perplexity 369.3306
+6600 train 5.939512 (lr=2.3686e-05) (hash(x)=151964101)
+6700 val loss 5.9059
+6700 val perplexity 367.1886
+6700 train 5.826228 (lr=2.3441e-05) (hash(x)=141291229)
+6800 val loss 5.8966
+6800 val perplexity 363.7803
+6800 train 5.808872 (lr=2.3195e-05) (hash(x)=160416800)
+6900 val loss 5.8907
+6900 val perplexity 361.6555
+6900 train 5.790020 (lr=2.2946e-05) (hash(x)=143382691)
+7000 val loss 5.8768
+7000 val perplexity 356.6765
+7000 train 5.781026 (lr=2.2696e-05) (hash(x)=147606572)
+7100 val loss 5.8661
+7100 val perplexity 352.8714
+7100 train 5.686009 (lr=2.2445e-05) (hash(x)=145901538)
+7200 val loss 5.8655
+7200 val perplexity 352.6436
+7200 train 5.694974 (lr=2.2192e-05) (hash(x)=139099153)
+7300 val loss 5.8601
+7300 val perplexity 350.7533
+7300 train 5.822430 (lr=2.1937e-05) (hash(x)=140434004)
+7400 val loss 5.8545
+7400 val perplexity 348.8091
+7400 train 5.978949 (lr=2.1682e-05) (hash(x)=155496146)
+7500 val loss 5.8469
+7500 val perplexity 346.1687
+7500 train 5.778997 (lr=2.1425e-05) (hash(x)=146940054)
+7600 val loss 5.8329
+7600 val perplexity 341.3336
+7600 train 6.061818 (lr=2.1166e-05) (hash(x)=165785723)
+7700 val loss 5.8242
+7700 val perplexity 338.3779
+7700 train 5.981597 (lr=2.0907e-05) (hash(x)=156339008)
+7800 val loss 5.8223
+7800 val perplexity 337.7568
+7800 train 5.822869 (lr=2.0647e-05) (hash(x)=148593556)
+7900 val loss 5.8218
+7900 val perplexity 337.5886
+7900 train 5.741560 (lr=2.0386e-05) (hash(x)=148079742)
+8000 val loss 5.8046
+8000 val perplexity 331.8263
+8000 train 5.730910 (lr=2.0123e-05) (hash(x)=159706718)
+8100 val loss 5.8052
+8100 val perplexity 332.0050
+8100 train 5.795191 (lr=1.9861e-05) (hash(x)=160203225)
+8200 val loss 5.7912
+8200 val perplexity 327.4006
+8200 train 5.732122 (lr=1.9597e-05) (hash(x)=146176536)
+8300 val loss 5.7817
+8300 val perplexity 324.3087
+8300 train 5.866216 (lr=1.9333e-05) (hash(x)=158883824)
+8400 val loss 5.7747
+8400 val perplexity 322.0548
+8400 train 5.596129 (lr=1.9069e-05) (hash(x)=137102572)
+8500 val loss 5.7715
+8500 val perplexity 321.0264
+8500 train 5.804103 (lr=1.8804e-05) (hash(x)=152812797)
+8600 val loss 5.7680
+8600 val perplexity 319.9102
+8600 train 5.620030 (lr=1.8539e-05) (hash(x)=148854244)
+8700 val loss 5.7815
+8700 val perplexity 324.2577
+8700 train 5.606668 (lr=1.8273e-05) (hash(x)=135590023)
+8800 val loss 5.7541
+8800 val perplexity 315.4715
+8800 train 5.694954 (lr=1.8008e-05) (hash(x)=144913509)
+8900 val loss 5.7496
+8900 val perplexity 314.0553
+8900 train 5.889195 (lr=1.7742e-05) (hash(x)=148045669)
+9000 val loss 5.7432
+9000 val perplexity 312.0630
+9000 train 5.769840 (lr=1.7477e-05) (hash(x)=146661746)
+9100 val loss 5.7376
+9100 val perplexity 310.3078
+9100 train 5.681479 (lr=1.7211e-05) (hash(x)=145354881)
+9200 val loss 5.7336
+9200 val perplexity 309.0737
+9200 train 5.599654 (lr=1.6946e-05) (hash(x)=150939310)
+9300 val loss 5.7302
+9300 val perplexity 308.0255
+9300 train 5.550713 (lr=1.6681e-05) (hash(x)=162258309)
+9400 val loss 5.7229
+9400 val perplexity 305.7889
+9400 train 5.730636 (lr=1.6417e-05) (hash(x)=154701830)
+9500 val loss 5.7150
+9500 val perplexity 303.3853
+9500 train 5.604954 (lr=1.6153e-05) (hash(x)=142241404)
+9600 val loss 5.7098
+9600 val perplexity 301.8016
+9600 train 5.674274 (lr=1.5889e-05) (hash(x)=148439858)
+9700 val loss 5.7103
+9700 val perplexity 301.9514
+9700 train 5.708298 (lr=1.5627e-05) (hash(x)=152204410)
+9800 val loss 5.7045
+9800 val perplexity 300.2228
+9800 train 5.564878 (lr=1.5364e-05) (hash(x)=141294630)
+9900 val loss 5.7013
+9900 val perplexity 299.2693
+9900 train 5.683287 (lr=1.5103e-05) (hash(x)=147390033)
+10000 val loss 5.6947
+10000 val perplexity 297.2863
+10000 train 5.717478 (lr=1.4843e-05) (hash(x)=144131604)
+10100 val loss 5.6850
+10100 val perplexity 294.4181
+10100 train 5.896329 (lr=1.4584e-05) (hash(x)=158814818)
+10200 val loss 5.6829
+10200 val perplexity 293.8097
+10200 train 5.765366 (lr=1.4325e-05) (hash(x)=153491388)
+10300 val loss 5.6788
+10300 val perplexity 292.6093
+10300 train 5.659386 (lr=1.4068e-05) (hash(x)=149600707)
+10400 val loss 5.6739
+10400 val perplexity 291.1534
+10400 train 5.596166 (lr=1.3813e-05) (hash(x)=152277064)
+10500 val loss 5.6734
+10500 val perplexity 291.0283
+10500 train 5.570152 (lr=1.3558e-05) (hash(x)=150447091)
+10600 val loss 5.6657
+10600 val perplexity 288.7822
+10600 train 5.689522 (lr=1.3305e-05) (hash(x)=151606822)
+10700 val loss 5.6573
+10700 val perplexity 286.3630
+10700 train 5.630720 (lr=1.3054e-05) (hash(x)=158730240)
+10800 val loss 5.6556
+10800 val perplexity 285.8786
+10800 train 5.691805 (lr=1.2804e-05) (hash(x)=163095518)
+10900 val loss 5.6542
+10900 val perplexity 285.4816
+10900 train 5.412553 (lr=1.2555e-05) (hash(x)=145260395)
+11000 val loss 5.6501
+11000 val perplexity 284.3220
+11000 train 5.639949 (lr=1.2309e-05) (hash(x)=155970264)
+11100 val loss 5.6468
+11100 val perplexity 283.3775
+11100 train 5.728524 (lr=1.2064e-05) (hash(x)=159871097)
+11200 val loss 5.6414
+11200 val perplexity 281.8456
+11200 train 5.719944 (lr=1.1822e-05) (hash(x)=161135244)
+11300 val loss 5.6364
+11300 val perplexity 280.4393
+11300 train 5.679232 (lr=1.1581e-05) (hash(x)=150649336)
+11400 val loss 5.6341
+11400 val perplexity 279.8134
+11400 train 5.601008 (lr=1.1342e-05) (hash(x)=152230013)
+11500 val loss 5.6322
+11500 val perplexity 279.2714
+11500 train 5.649804 (lr=1.1106e-05) (hash(x)=163300613)
+11600 val loss 5.6292
+11600 val perplexity 278.4306
+11600 train 5.601301 (lr=1.0871e-05) (hash(x)=151203838)
+11700 val loss 5.6256
+11700 val perplexity 277.4498
+11700 train 5.643506 (lr=1.0639e-05) (hash(x)=155255111)
+11800 val loss 5.6279
+11800 val perplexity 278.0896
+11800 train 5.592367 (lr=1.0410e-05) (hash(x)=139380668)
+11900 val loss 5.6163
+11900 val perplexity 274.8596
+11900 train 5.549178 (lr=1.0183e-05) (hash(x)=144033405)
+12000 val loss 5.6132
+12000 val perplexity 274.0146
+12000 train 5.540679 (lr=9.9580e-06) (hash(x)=148689574)
+12100 val loss 5.6138
+12100 val perplexity 274.1830
+12100 train 5.543606 (lr=9.7360e-06) (hash(x)=143272239)
+12200 val loss 5.6096
+12200 val perplexity 273.0461
+12200 train 5.664837 (lr=9.5167e-06) (hash(x)=157671609)
+12300 val loss 5.6067
+12300 val perplexity 272.2368
+12300 train 5.630793 (lr=9.3001e-06) (hash(x)=150363247)
+12400 val loss 5.6037
+12400 val perplexity 271.4229
+12400 train 5.490661 (lr=9.0864e-06) (hash(x)=146857787)
+12500 val loss 5.6006
+12500 val perplexity 270.5929
+12500 train 5.567979 (lr=8.8756e-06) (hash(x)=153327320)
+12600 val loss 5.5969
+12600 val perplexity 269.5787
+12600 train 5.323852 (lr=8.6677e-06) (hash(x)=143612983)
+12700 val loss 5.5961
+12700 val perplexity 269.3662
+12700 train 5.522414 (lr=8.4629e-06) (hash(x)=150739132)
+12800 val loss 5.5931
+12800 val perplexity 268.5558
+12800 train 5.449065 (lr=8.2612e-06) (hash(x)=147986204)
+12900 val loss 5.5924
+12900 val perplexity 268.3826
+12900 train 5.753481 (lr=8.0627e-06) (hash(x)=155218833)
+13000 val loss 5.5863
+13000 val perplexity 266.7436
+13000 train 5.575964 (lr=7.8674e-06) (hash(x)=148867960)
+13100 val loss 5.5839
+13100 val perplexity 266.1086
+13100 train 5.614095 (lr=7.6754e-06) (hash(x)=160101478)
+13200 val loss 5.5855
+13200 val perplexity 266.5360
+13200 train 5.439250 (lr=7.4867e-06) (hash(x)=151080097)
+13300 val loss 5.5827
+13300 val perplexity 265.8013
+13300 train 5.558959 (lr=7.3015e-06) (hash(x)=150403039)
+13400 val loss 5.5794
+13400 val perplexity 264.9090
+13400 train 5.523631 (lr=7.1198e-06) (hash(x)=129728571)
+13500 val loss 5.5762
+13500 val perplexity 264.0602
+13500 train 5.492384 (lr=6.9416e-06) (hash(x)=148988625)
+13600 val loss 5.5735
+13600 val perplexity 263.3569
+13600 train 5.356260 (lr=6.7670e-06) (hash(x)=149790861)
+13700 val loss 5.5729
+13700 val perplexity 263.2009
+13700 train 5.489488 (lr=6.5961e-06) (hash(x)=153505062)
+13800 val loss 5.5742
+13800 val perplexity 263.5341
+13800 train 5.476908 (lr=6.4289e-06) (hash(x)=143845463)
+13900 val loss 5.5713
+13900 val perplexity 262.7654
+13900 train 5.380024 (lr=6.2655e-06) (hash(x)=143378076)
+14000 val loss 5.5657
+14000 val perplexity 261.3139
+14000 train 5.531826 (lr=6.1059e-06) (hash(x)=148681755)
+14100 val loss 5.5644
+14100 val perplexity 260.9744
+14100 train 5.451090 (lr=5.9502e-06) (hash(x)=145530465)
+14200 val loss 5.5646
+14200 val perplexity 261.0121
+14200 train 5.493296 (lr=5.7984e-06) (hash(x)=162139713)
+14300 val loss 5.5621
+14300 val perplexity 260.3621
+14300 train 5.566136 (lr=5.6506e-06) (hash(x)=141997596)
+14400 val loss 5.5614
+14400 val perplexity 260.1755
+14400 train 5.537404 (lr=5.5068e-06) (hash(x)=146156313)
+14500 val loss 5.5598
+14500 val perplexity 259.7654
+14500 train 5.328027 (lr=5.3671e-06) (hash(x)=145298178)
+14600 val loss 5.5561
+14600 val perplexity 258.8107
+14600 train 5.642579 (lr=5.2316e-06) (hash(x)=148232610)
+14700 val loss 5.5549
+14700 val perplexity 258.4908
+14700 train 5.430353 (lr=5.1002e-06) (hash(x)=151079115)
+14800 val loss 5.5524
+14800 val perplexity 257.8659
+14800 train 5.545774 (lr=4.9730e-06) (hash(x)=151340983)
+14900 val loss 5.5513
+14900 val perplexity 257.5683
+14900 train 5.763885 (lr=4.8501e-06) (hash(x)=150213803)
+15000 val loss 5.5509
+15000 val perplexity 257.4775
+15000 train 5.467402 (lr=4.7314e-06) (hash(x)=157652334)
+15100 val loss 5.5498
+15100 val perplexity 257.1843
+15100 train 5.265310 (lr=4.6171e-06) (hash(x)=145078577)
+15200 val loss 5.5471
+15200 val perplexity 256.4839
+15200 train 5.499277 (lr=4.5072e-06) (hash(x)=155898006)
+15300 val loss 5.5461
+15300 val perplexity 256.2384
+15300 train 5.575391 (lr=4.4017e-06) (hash(x)=149346671)
+15400 val loss 5.5443
+15400 val perplexity 255.7670
+15400 train 5.384352 (lr=4.3006e-06) (hash(x)=145708113)
+15500 val loss 5.5425
+15500 val perplexity 255.3121
+15500 train 5.733283 (lr=4.2040e-06) (hash(x)=165102970)
+15600 val loss 5.5436
+15600 val perplexity 255.6082
+15600 train 5.327137 (lr=4.1119e-06) (hash(x)=139106517)
+15700 val loss 5.5419
+15700 val perplexity 255.1559
+15700 train 5.405306 (lr=4.0244e-06) (hash(x)=141680838)
+15800 val loss 5.5391
+15800 val perplexity 254.4541
+15800 train 5.448102 (lr=3.9414e-06) (hash(x)=136152310)
+15900 val loss 5.5371
+15900 val perplexity 253.9308
+15900 train 5.600040 (lr=3.8630e-06) (hash(x)=151104640)
+16000 val loss 5.5373
+16000 val perplexity 253.9962
+16000 train 5.485271 (lr=3.7892e-06) (hash(x)=153333281)
+16100 val loss 5.5352
+16100 val perplexity 253.4691
+16100 train 5.494695 (lr=3.7201e-06) (hash(x)=145628059)
+16200 val loss 5.5362
+16200 val perplexity 253.7112
+16200 train 5.259943 (lr=3.6556e-06) (hash(x)=137220041)
+16300 val loss 5.5350
+16300 val perplexity 253.4003
+16300 train 5.710714 (lr=3.5959e-06) (hash(x)=161215791)
+16400 val loss 5.5336
+16400 val perplexity 253.0495
+16400 train 5.240963 (lr=3.5408e-06) (hash(x)=136194680)
+16500 val loss 5.5343
+16500 val perplexity 253.2211
+16500 train 5.320570 (lr=3.4905e-06) (hash(x)=152209568)
+16600 val loss 5.5314
+16600 val perplexity 252.4937
+16600 train 5.647983 (lr=3.4449e-06) (hash(x)=157345721)
+16700 val loss 5.5294
+16700 val perplexity 251.9874
+16700 train 5.649169 (lr=3.4041e-06) (hash(x)=146883642)
+16800 val loss 5.5290
+16800 val perplexity 251.8968
+16800 train 5.496567 (lr=3.3680e-06) (hash(x)=153037972)
+16900 val loss 5.5281
+16900 val perplexity 251.6766
+16900 train 5.297307 (lr=3.3367e-06) (hash(x)=150691367)
+17000 val loss 5.5288
+17000 val perplexity 251.8328
+17000 train 5.280123 (lr=3.3102e-06) (hash(x)=148975206)
+17100 val loss 5.5289
+17100 val perplexity 251.8759
+17100 train 5.266790 (lr=3.2886e-06) (hash(x)=142026221)
+17200 val loss 5.5261
+17200 val perplexity 251.1566
+17200 train 5.593449 (lr=3.2717e-06) (hash(x)=138211880)
+17300 val loss 5.5241
+17300 val perplexity 250.6509
+17300 train 5.383330 (lr=3.2596e-06) (hash(x)=143366666)
+17400 val loss 5.5229
+17400 val perplexity 250.3676
+17400 train 5.523359 (lr=3.2524e-06) (hash(x)=150791766)
+17499 val loss 5.5233
+17499 val perplexity 250.4611

attention_kindselective_n_heads4_seed1343/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3803a8a02a4eef5e6266bf1ae9e90cba514f507edd246d6c5a4cfb561f938f2d
+oid sha256:5f2d81873e4940064147bfc8eab16fd75bb372ba77dafef56dc7143ac3593856
 size 92843394

attention_kindselective_n_heads4_seed1343/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1909b34facdb00da31a90a3685379e7ef706df56e3b6259af0d1102d544286dc
+oid sha256:0fdfbc009957bc5ddfdf9c620e1209c1ea61ac8db143f251f024ce1b657e4137
 size 179406214