Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1343/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1343", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1343", "n_embd": 256}

attention_kindselective_n_heads4_seed1343/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.3001
-0 val perplexity 80830.1328
-0 train 11.304919 (lr=1.6250e-07) (hash(x)=153418542)
-100 val loss 9.8733
-100 val perplexity 19405.9473
-100 train 9.841567 (lr=1.6412e-05) (hash(x)=154406429)
-200 val loss 8.7130
-200 val perplexity 6081.5020
-200 train 8.618789 (lr=3.2500e-05) (hash(x)=143722001)
-300 val loss 7.7544
-300 val perplexity 2331.8059
-300 train 7.689346 (lr=3.2498e-05) (hash(x)=152421341)
-400 val loss 7.5469
-400 val perplexity 1894.8176
-400 train 7.435925 (lr=3.2490e-05) (hash(x)=152989689)
-500 val loss 7.4583
-500 val perplexity 1734.2018
-500 train 7.384192 (lr=3.2478e-05) (hash(x)=145113544)
-600 val loss 7.3776
-600 val perplexity 1599.7095
-600 train 7.368221 (lr=3.2461e-05) (hash(x)=150071018)
-700 val loss 7.3126
-700 val perplexity 1499.1150
-700 train 7.416527 (lr=3.2440e-05) (hash(x)=156594182)
-800 val loss 7.2558
-800 val perplexity 1416.3041
-800 train 7.261938 (lr=3.2413e-05) (hash(x)=153358238)
-900 val loss 7.1929
-900 val perplexity 1329.8951
-900 train 7.011195 (lr=3.2382e-05) (hash(x)=146617394)
-1000 val loss 7.1334
-1000 val perplexity 1253.1919
-1000 train 7.066751 (lr=3.2346e-05) (hash(x)=148983354)
-1100 val loss 7.0718
-1100 val perplexity 1178.2454
-1100 train 7.096290 (lr=3.2305e-05) (hash(x)=146313352)
-1200 val loss 7.0079
-1200 val perplexity 1105.3306
-1200 train 6.949758 (lr=3.2260e-05) (hash(x)=150770333)
-1300 val loss 6.9429
-1300 val perplexity 1035.7495
-1300 train 6.894914 (lr=3.2209e-05) (hash(x)=153237933)
-1400 val loss 6.8895
-1400 val perplexity 981.9400
-1400 train 6.719554 (lr=3.2154e-05) (hash(x)=145246201)
-1500 val loss 6.8387
-1500 val perplexity 933.2887
-1500 train 6.812776 (lr=3.2094e-05) (hash(x)=154002540)
-1600 val loss 6.7915
-1600 val perplexity 890.2463
-1600 train 6.566239 (lr=3.2030e-05) (hash(x)=142635842)
-1700 val loss 6.7368
-1700 val perplexity 842.8735
-1700 train 6.787238 (lr=3.1961e-05) (hash(x)=151450555)
-1800 val loss 6.6812
-1800 val perplexity 797.2631
-1800 train 6.601893 (lr=3.1887e-05) (hash(x)=148299832)
-1900 val loss 6.6343
-1900 val perplexity 760.7756
-1900 train 6.770863 (lr=3.1809e-05) (hash(x)=155487476)
-2000 val loss 6.5992
-2000 val perplexity 734.4827
-2000 train 6.578701 (lr=3.1726e-05) (hash(x)=152887721)
-2100 val loss 6.5663
-2100 val perplexity 710.7366
-2100 train 6.259625 (lr=3.1638e-05) (hash(x)=143430603)
-2200 val loss 6.5374
-2200 val perplexity 690.4644
-2200 train 6.537328 (lr=3.1546e-05) (hash(x)=153721194)
-2300 val loss 6.5074
-2300 val perplexity 670.0813
-2300 train 6.664390 (lr=3.1449e-05) (hash(x)=148702429)
-2400 val loss 6.4757
-2400 val perplexity 649.1904
-2400 train 6.536206 (lr=3.1348e-05) (hash(x)=153047184)
-2500 val loss 6.4434
-2500 val perplexity 628.5614
-2500 train 6.321926 (lr=3.1243e-05) (hash(x)=149720123)
-2600 val loss 6.4200
-2600 val perplexity 614.0228
-2600 train 6.436261 (lr=3.1133e-05) (hash(x)=152466045)
-2700 val loss 6.4165
-2700 val perplexity 611.8614
-2700 train 6.326521 (lr=3.1019e-05) (hash(x)=155796305)
-2800 val loss 6.3826
-2800 val perplexity 591.4885
-2800 train 6.222201 (lr=3.0900e-05) (hash(x)=151991552)
-2900 val loss 6.3610
-2900 val perplexity 578.8026
-2900 train 6.974368 (lr=3.0777e-05) (hash(x)=166537989)
-3000 val loss 6.3340
-3000 val perplexity 563.3945
-3000 train 6.334172 (lr=3.0650e-05) (hash(x)=147892594)
-3100 val loss 6.3147
-3100 val perplexity 552.6521
-3100 train 6.278732 (lr=3.0518e-05) (hash(x)=149536927)
-3200 val loss 6.3047
-3200 val perplexity 547.1489
-3200 train 6.019104 (lr=3.0383e-05) (hash(x)=135678663)
-3300 val loss 6.2794
-3300 val perplexity 533.4483
-3300 train 6.361428 (lr=3.0243e-05) (hash(x)=148523646)
-3400 val loss 6.2695
-3400 val perplexity 528.1934
-3400 train 6.349548 (lr=3.0099e-05) (hash(x)=150281778)
-3500 val loss 6.2489
-3500 val perplexity 517.4524
-3500 train 6.206536 (lr=2.9952e-05) (hash(x)=151527953)
-3600 val loss 6.2416
-3600 val perplexity 513.6918
-3600 train 6.464660 (lr=2.9800e-05) (hash(x)=141303739)
-3700 val loss 6.2224
-3700 val perplexity 503.8872
-3700 train 6.266064 (lr=2.9644e-05) (hash(x)=146778597)
-3800 val loss 6.2020
-3800 val perplexity 493.7431
-3800 train 6.149168 (lr=2.9485e-05) (hash(x)=146902797)
-3900 val loss 6.1910
-3900 val perplexity 488.3286
-3900 train 6.114597 (lr=2.9321e-05) (hash(x)=142022555)
-4000 val loss 6.1740
-4000 val perplexity 480.0992
-4000 train 6.090962 (lr=2.9154e-05) (hash(x)=151590309)
-4100 val loss 6.1610
-4100 val perplexity 473.9238
-4100 train 6.004887 (lr=2.8983e-05) (hash(x)=144127100)
-4200 val loss 6.1519
-4200 val perplexity 469.6165
-4200 train 6.226448 (lr=2.8808e-05) (hash(x)=149526774)
-4300 val loss 6.1462
-4300 val perplexity 466.9373
-4300 train 6.081618 (lr=2.8630e-05) (hash(x)=150924362)
-4400 val loss 6.1223
-4400 val perplexity 455.9291
-4400 train 6.270722 (lr=2.8448e-05) (hash(x)=155547282)
-4500 val loss 6.1144
-4500 val perplexity 452.3334
-4500 train 6.097814 (lr=2.8263e-05) (hash(x)=152479203)
-4600 val loss 6.0978
-4600 val perplexity 444.8783
-4600 train 6.011952 (lr=2.8075e-05) (hash(x)=146739760)
-4700 val loss 6.0940
-4700 val perplexity 443.1834
-4700 train 6.322186 (lr=2.7883e-05) (hash(x)=159977388)
-4800 val loss 6.0822
-4800 val perplexity 437.9828
-4800 train 5.978403 (lr=2.7687e-05) (hash(x)=142959739)
-4900 val loss 6.0686
-4900 val perplexity 432.0666
-4900 train 6.018283 (lr=2.7489e-05) (hash(x)=152253034)
-5000 val loss 6.0667
-5000 val perplexity 431.2595
-5000 train 6.036760 (lr=2.7287e-05) (hash(x)=140379747)
-5100 val loss 6.0461
-5100 val perplexity 422.4584
-5100 train 6.082677 (lr=2.7082e-05) (hash(x)=151826392)
-5200 val loss 6.0384
-5200 val perplexity 419.2076
-5200 train 6.012521 (lr=2.6874e-05) (hash(x)=152869594)
-5300 val loss 6.0266
-5300 val perplexity 414.3012
-5300 train 5.810596 (lr=2.6664e-05) (hash(x)=139093709)
-5400 val loss 6.0199
-5400 val perplexity 411.5291
-5400 train 5.901548 (lr=2.6450e-05) (hash(x)=145908738)
-5500 val loss 6.0176
-5500 val perplexity 410.5985
-5500 train 5.885014 (lr=2.6233e-05) (hash(x)=137913426)
-5600 val loss 5.9932
-5600 val perplexity 400.7062
-5600 train 5.863060 (lr=2.6014e-05) (hash(x)=140770390)
-5700 val loss 5.9972
-5700 val perplexity 402.3169
-5700 train 5.915018 (lr=2.5792e-05) (hash(x)=143728578)
-5800 val loss 5.9781
-5800 val perplexity 394.6902
-5800 train 6.046650 (lr=2.5567e-05) (hash(x)=154469787)
-5900 val loss 5.9709
-5900 val perplexity 391.8690
-5900 train 5.934379 (lr=2.5340e-05) (hash(x)=139316559)
-6000 val loss 5.9608
-6000 val perplexity 387.9297
-6000 train 5.969604 (lr=2.5111e-05) (hash(x)=152058488)
-6100 val loss 5.9539
-6100 val perplexity 385.2404
-6100 train 5.971652 (lr=2.4879e-05) (hash(x)=152958689)
-6200 val loss 5.9376
-6200 val perplexity 379.0083
-6200 train 5.906252 (lr=2.4644e-05) (hash(x)=150887864)
-6300 val loss 5.9309
-6300 val perplexity 376.4814
-6300 train 5.876676 (lr=2.4408e-05) (hash(x)=156290469)
-6400 val loss 5.9232
-6400 val perplexity 373.5906
-6400 train 6.241812 (lr=2.4169e-05) (hash(x)=159160159)
-6500 val loss 5.9175
-6500 val perplexity 371.4980
-6500 train 5.838713 (lr=2.3928e-05) (hash(x)=151239121)
-6600 val loss 5.9117
-6600 val perplexity 369.3306
-6600 train 5.939512 (lr=2.3686e-05) (hash(x)=151964101)
-6700 val loss 5.9059
-6700 val perplexity 367.1886
-6700 train 5.826228 (lr=2.3441e-05) (hash(x)=141291229)
-6800 val loss 5.8966
-6800 val perplexity 363.7803
-6800 train 5.808872 (lr=2.3195e-05) (hash(x)=160416800)
-6900 val loss 5.8907
-6900 val perplexity 361.6555
-6900 train 5.790020 (lr=2.2946e-05) (hash(x)=143382691)
-7000 val loss 5.8768
-7000 val perplexity 356.6765
-7000 train 5.781026 (lr=2.2696e-05) (hash(x)=147606572)
-7100 val loss 5.8661
-7100 val perplexity 352.8714
-7100 train 5.686009 (lr=2.2445e-05) (hash(x)=145901538)
-7200 val loss 5.8655
-7200 val perplexity 352.6436
-7200 train 5.694974 (lr=2.2192e-05) (hash(x)=139099153)
-7300 val loss 5.8601
-7300 val perplexity 350.7533
-7300 train 5.822430 (lr=2.1937e-05) (hash(x)=140434004)
-7400 val loss 5.8545
-7400 val perplexity 348.8091
-7400 train 5.978949 (lr=2.1682e-05) (hash(x)=155496146)
-7500 val loss 5.8469
-7500 val perplexity 346.1687
-7500 train 5.778997 (lr=2.1425e-05) (hash(x)=146940054)
-7600 val loss 5.8329
-7600 val perplexity 341.3336
-7600 train 6.061818 (lr=2.1166e-05) (hash(x)=165785723)
-7700 val loss 5.8242
-7700 val perplexity 338.3779
-7700 train 5.981597 (lr=2.0907e-05) (hash(x)=156339008)
-7800 val loss 5.8223
-7800 val perplexity 337.7568
-7800 train 5.822869 (lr=2.0647e-05) (hash(x)=148593556)
-7900 val loss 5.8218
-7900 val perplexity 337.5886
-7900 train 5.741560 (lr=2.0386e-05) (hash(x)=148079742)
-8000 val loss 5.8046
-8000 val perplexity 331.8263
-8000 train 5.730910 (lr=2.0123e-05) (hash(x)=159706718)
-8100 val loss 5.8052
-8100 val perplexity 332.0050
-8100 train 5.795191 (lr=1.9861e-05) (hash(x)=160203225)
-8200 val loss 5.7912
-8200 val perplexity 327.4006
-8200 train 5.732122 (lr=1.9597e-05) (hash(x)=146176536)
-8300 val loss 5.7817
-8300 val perplexity 324.3087
-8300 train 5.866216 (lr=1.9333e-05) (hash(x)=158883824)
-8400 val loss 5.7747
-8400 val perplexity 322.0548
-8400 train 5.596129 (lr=1.9069e-05) (hash(x)=137102572)
-8500 val loss 5.7715
-8500 val perplexity 321.0264
-8500 train 5.804103 (lr=1.8804e-05) (hash(x)=152812797)
-8600 val loss 5.7680
-8600 val perplexity 319.9102
-8600 train 5.620030 (lr=1.8539e-05) (hash(x)=148854244)
-8700 val loss 5.7815
-8700 val perplexity 324.2577
-8700 train 5.606668 (lr=1.8273e-05) (hash(x)=135590023)
-8800 val loss 5.7541
-8800 val perplexity 315.4715
-8800 train 5.694954 (lr=1.8008e-05) (hash(x)=144913509)
-8900 val loss 5.7496
-8900 val perplexity 314.0553
-8900 train 5.889195 (lr=1.7742e-05) (hash(x)=148045669)
-9000 val loss 5.7432
-9000 val perplexity 312.0630
-9000 train 5.769840 (lr=1.7477e-05) (hash(x)=146661746)
-9100 val loss 5.7376
-9100 val perplexity 310.3078
-9100 train 5.681479 (lr=1.7211e-05) (hash(x)=145354881)
-9200 val loss 5.7336
-9200 val perplexity 309.0737
-9200 train 5.599654 (lr=1.6946e-05) (hash(x)=150939310)
-9300 val loss 5.7302
-9300 val perplexity 308.0255
-9300 train 5.550713 (lr=1.6681e-05) (hash(x)=162258309)
-9400 val loss 5.7229
-9400 val perplexity 305.7889
-9400 train 5.730636 (lr=1.6417e-05) (hash(x)=154701830)
-9500 val loss 5.7150
-9500 val perplexity 303.3853
-9500 train 5.604954 (lr=1.6153e-05) (hash(x)=142241404)
-9600 val loss 5.7098
-9600 val perplexity 301.8016
-9600 train 5.674274 (lr=1.5889e-05) (hash(x)=148439858)
-9700 val loss 5.7103
-9700 val perplexity 301.9514
-9700 train 5.708298 (lr=1.5627e-05) (hash(x)=152204410)
-9800 val loss 5.7045
-9800 val perplexity 300.2228
-9800 train 5.564878 (lr=1.5364e-05) (hash(x)=141294630)
-9900 val loss 5.7013
-9900 val perplexity 299.2693
-9900 train 5.683287 (lr=1.5103e-05) (hash(x)=147390033)
-10000 val loss 5.6947
-10000 val perplexity 297.2863
-10000 train 5.717478 (lr=1.4843e-05) (hash(x)=144131604)
-10100 val loss 5.6850
-10100 val perplexity 294.4181
-10100 train 5.896329 (lr=1.4584e-05) (hash(x)=158814818)
-10200 val loss 5.6829
-10200 val perplexity 293.8097
-10200 train 5.765366 (lr=1.4325e-05) (hash(x)=153491388)
-10300 val loss 5.6788
-10300 val perplexity 292.6093
-10300 train 5.659386 (lr=1.4068e-05) (hash(x)=149600707)
-10400 val loss 5.6739
-10400 val perplexity 291.1534
-10400 train 5.596166 (lr=1.3813e-05) (hash(x)=152277064)
-10500 val loss 5.6734
-10500 val perplexity 291.0283
-10500 train 5.570152 (lr=1.3558e-05) (hash(x)=150447091)
-10600 val loss 5.6657
-10600 val perplexity 288.7822
-10600 train 5.689522 (lr=1.3305e-05) (hash(x)=151606822)
-10700 val loss 5.6573
-10700 val perplexity 286.3630
-10700 train 5.630720 (lr=1.3054e-05) (hash(x)=158730240)
-10800 val loss 5.6556
-10800 val perplexity 285.8786
-10800 train 5.691805 (lr=1.2804e-05) (hash(x)=163095518)
-10900 val loss 5.6542
-10900 val perplexity 285.4816
-10900 train 5.412553 (lr=1.2555e-05) (hash(x)=145260395)
-11000 val loss 5.6501
-11000 val perplexity 284.3220
-11000 train 5.639949 (lr=1.2309e-05) (hash(x)=155970264)
-11100 val loss 5.6468
-11100 val perplexity 283.3775
-11100 train 5.728524 (lr=1.2064e-05) (hash(x)=159871097)
-11200 val loss 5.6414
-11200 val perplexity 281.8456
-11200 train 5.719944 (lr=1.1822e-05) (hash(x)=161135244)
-11300 val loss 5.6364
-11300 val perplexity 280.4393
-11300 train 5.679232 (lr=1.1581e-05) (hash(x)=150649336)
-11400 val loss 5.6341
-11400 val perplexity 279.8134
-11400 train 5.601008 (lr=1.1342e-05) (hash(x)=152230013)
-11500 val loss 5.6322
-11500 val perplexity 279.2714
-11500 train 5.649804 (lr=1.1106e-05) (hash(x)=163300613)
-11600 val loss 5.6292
-11600 val perplexity 278.4306
-11600 train 5.601301 (lr=1.0871e-05) (hash(x)=151203838)
-11700 val loss 5.6256
-11700 val perplexity 277.4498
-11700 train 5.643506 (lr=1.0639e-05) (hash(x)=155255111)
-11800 val loss 5.6279
-11800 val perplexity 278.0896
-11800 train 5.592367 (lr=1.0410e-05) (hash(x)=139380668)
-11900 val loss 5.6163
-11900 val perplexity 274.8596
-11900 train 5.549178 (lr=1.0183e-05) (hash(x)=144033405)
-12000 val loss 5.6132
-12000 val perplexity 274.0146
-12000 train 5.540679 (lr=9.9580e-06) (hash(x)=148689574)
-12100 val loss 5.6138
-12100 val perplexity 274.1830
-12100 train 5.543606 (lr=9.7360e-06) (hash(x)=143272239)
-12200 val loss 5.6096
-12200 val perplexity 273.0461
-12200 train 5.664837 (lr=9.5167e-06) (hash(x)=157671609)
-12300 val loss 5.6067
-12300 val perplexity 272.2368
-12300 train 5.630793 (lr=9.3001e-06) (hash(x)=150363247)
-12400 val loss 5.6037
-12400 val perplexity 271.4229
-12400 train 5.490661 (lr=9.0864e-06) (hash(x)=146857787)
-12500 val loss 5.6006
-12500 val perplexity 270.5929
-12500 train 5.567979 (lr=8.8756e-06) (hash(x)=153327320)
-12600 val loss 5.5969
-12600 val perplexity 269.5787
-12600 train 5.323852 (lr=8.6677e-06) (hash(x)=143612983)
-12700 val loss 5.5961
-12700 val perplexity 269.3662
-12700 train 5.522414 (lr=8.4629e-06) (hash(x)=150739132)
-12800 val loss 5.5931
-12800 val perplexity 268.5558
-12800 train 5.449065 (lr=8.2612e-06) (hash(x)=147986204)
-12900 val loss 5.5924
-12900 val perplexity 268.3826
-12900 train 5.753481 (lr=8.0627e-06) (hash(x)=155218833)
-13000 val loss 5.5863
-13000 val perplexity 266.7436
-13000 train 5.575964 (lr=7.8674e-06) (hash(x)=148867960)
-13100 val loss 5.5839
-13100 val perplexity 266.1086
-13100 train 5.614095 (lr=7.6754e-06) (hash(x)=160101478)
-13200 val loss 5.5855
-13200 val perplexity 266.5360
-13200 train 5.439250 (lr=7.4867e-06) (hash(x)=151080097)
-13300 val loss 5.5827
-13300 val perplexity 265.8013
-13300 train 5.558959 (lr=7.3015e-06) (hash(x)=150403039)
-13400 val loss 5.5794
-13400 val perplexity 264.9090
-13400 train 5.523631 (lr=7.1198e-06) (hash(x)=129728571)
-13500 val loss 5.5762
-13500 val perplexity 264.0602
-13500 train 5.492384 (lr=6.9416e-06) (hash(x)=148988625)
-13600 val loss 5.5735
-13600 val perplexity 263.3569
-13600 train 5.356260 (lr=6.7670e-06) (hash(x)=149790861)
-13700 val loss 5.5729
-13700 val perplexity 263.2009
-13700 train 5.489488 (lr=6.5961e-06) (hash(x)=153505062)
-13800 val loss 5.5742
-13800 val perplexity 263.5341
-13800 train 5.476908 (lr=6.4289e-06) (hash(x)=143845463)
-13900 val loss 5.5713
-13900 val perplexity 262.7654
-13900 train 5.380024 (lr=6.2655e-06) (hash(x)=143378076)
-14000 val loss 5.5657
-14000 val perplexity 261.3139
-14000 train 5.531826 (lr=6.1059e-06) (hash(x)=148681755)
-14100 val loss 5.5644
-14100 val perplexity 260.9744
-14100 train 5.451090 (lr=5.9502e-06) (hash(x)=145530465)
-14200 val loss 5.5646
-14200 val perplexity 261.0121
-14200 train 5.493296 (lr=5.7984e-06) (hash(x)=162139713)
-14300 val loss 5.5621
-14300 val perplexity 260.3621
-14300 train 5.566136 (lr=5.6506e-06) (hash(x)=141997596)
-14400 val loss 5.5614
-14400 val perplexity 260.1755
-14400 train 5.537404 (lr=5.5068e-06) (hash(x)=146156313)
-14500 val loss 5.5598
-14500 val perplexity 259.7654
-14500 train 5.328027 (lr=5.3671e-06) (hash(x)=145298178)
-14600 val loss 5.5561
-14600 val perplexity 258.8107
-14600 train 5.642579 (lr=5.2316e-06) (hash(x)=148232610)
-14700 val loss 5.5549
-14700 val perplexity 258.4908
-14700 train 5.430353 (lr=5.1002e-06) (hash(x)=151079115)
-14800 val loss 5.5524
-14800 val perplexity 257.8659
-14800 train 5.545774 (lr=4.9730e-06) (hash(x)=151340983)
-14900 val loss 5.5513
-14900 val perplexity 257.5683
-14900 train 5.763885 (lr=4.8501e-06) (hash(x)=150213803)
-15000 val loss 5.5509
-15000 val perplexity 257.4775
-15000 train 5.467402 (lr=4.7314e-06) (hash(x)=157652334)
-15100 val loss 5.5498
-15100 val perplexity 257.1843
-15100 train 5.265310 (lr=4.6171e-06) (hash(x)=145078577)
-15200 val loss 5.5471
-15200 val perplexity 256.4839
-15200 train 5.499277 (lr=4.5072e-06) (hash(x)=155898006)
-15300 val loss 5.5461
-15300 val perplexity 256.2384
-15300 train 5.575391 (lr=4.4017e-06) (hash(x)=149346671)
-15400 val loss 5.5443
-15400 val perplexity 255.7670
-15400 train 5.384352 (lr=4.3006e-06) (hash(x)=145708113)
-15500 val loss 5.5425
-15500 val perplexity 255.3121
-15500 train 5.733283 (lr=4.2040e-06) (hash(x)=165102970)
-15600 val loss 5.5436
-15600 val perplexity 255.6082
-15600 train 5.327137 (lr=4.1119e-06) (hash(x)=139106517)
-15700 val loss 5.5419
-15700 val perplexity 255.1559
-15700 train 5.405306 (lr=4.0244e-06) (hash(x)=141680838)
-15800 val loss 5.5391
-15800 val perplexity 254.4541
-15800 train 5.448102 (lr=3.9414e-06) (hash(x)=136152310)
-15900 val loss 5.5371
-15900 val perplexity 253.9308
-15900 train 5.600040 (lr=3.8630e-06) (hash(x)=151104640)
-16000 val loss 5.5373
-16000 val perplexity 253.9962
-16000 train 5.485271 (lr=3.7892e-06) (hash(x)=153333281)
-16100 val loss 5.5352
-16100 val perplexity 253.4691
-16100 train 5.494695 (lr=3.7201e-06) (hash(x)=145628059)
-16200 val loss 5.5362
-16200 val perplexity 253.7112
-16200 train 5.259943 (lr=3.6556e-06) (hash(x)=137220041)
-16300 val loss 5.5350
-16300 val perplexity 253.4003
-16300 train 5.710714 (lr=3.5959e-06) (hash(x)=161215791)
-16400 val loss 5.5336
-16400 val perplexity 253.0495
-16400 train 5.240963 (lr=3.5408e-06) (hash(x)=136194680)
-16500 val loss 5.5343
-16500 val perplexity 253.2211
-16500 train 5.320570 (lr=3.4905e-06) (hash(x)=152209568)
-16600 val loss 5.5314
-16600 val perplexity 252.4937
-16600 train 5.647983 (lr=3.4449e-06) (hash(x)=157345721)
-16700 val loss 5.5294
-16700 val perplexity 251.9874
-16700 train 5.649169 (lr=3.4041e-06) (hash(x)=146883642)
-16800 val loss 5.5290
-16800 val perplexity 251.8968
-16800 train 5.496567 (lr=3.3680e-06) (hash(x)=153037972)
-16900 val loss 5.5281
-16900 val perplexity 251.6766
-16900 train 5.297307 (lr=3.3367e-06) (hash(x)=150691367)
-17000 val loss 5.5288
-17000 val perplexity 251.8328
-17000 train 5.280123 (lr=3.3102e-06) (hash(x)=148975206)
-17100 val loss 5.5289
-17100 val perplexity 251.8759
-17100 train 5.266790 (lr=3.2886e-06) (hash(x)=142026221)
-17200 val loss 5.5261
-17200 val perplexity 251.1566
-17200 train 5.593449 (lr=3.2717e-06) (hash(x)=138211880)
-17300 val loss 5.5241
-17300 val perplexity 250.6509
-17300 train 5.383330 (lr=3.2596e-06) (hash(x)=143366666)
-17400 val loss 5.5229
-17400 val perplexity 250.3676
-17400 train 5.523359 (lr=3.2524e-06) (hash(x)=150791766)
-17499 val loss 5.5233
-17499 val perplexity 250.4611
+0 val perplexity 80829.3594
+0 train 11.304919 (lr=1.5000e-07) (hash(x)=153418542)
+100 val loss 9.8883
+100 val perplexity 19698.2617
+100 train 9.855974 (lr=1.5150e-05) (hash(x)=154406429)
+200 val loss 8.7906
+200 val perplexity 6572.4741
+200 train 8.700301 (lr=3.0000e-05) (hash(x)=143722001)
+300 val loss 7.9352
+300 val perplexity 2793.9514
+300 train 7.877781 (lr=2.9998e-05) (hash(x)=152421341)
+400 val loss 7.6556
+400 val perplexity 2112.4343
+400 train 7.560735 (lr=2.9991e-05) (hash(x)=152989689)
+500 val loss 7.5524
+500 val perplexity 1905.2402
+500 train 7.480812 (lr=2.9980e-05) (hash(x)=145113544)
+600 val loss 7.4805
+600 val perplexity 1773.1368
+600 train 7.474395 (lr=2.9964e-05) (hash(x)=150071018)
+700 val loss 7.4305
+700 val perplexity 1686.5872
+700 train 7.521656 (lr=2.9944e-05) (hash(x)=156594182)
+800 val loss 7.4027
+800 val perplexity 1640.3590
+800 train 7.404630 (lr=2.9920e-05) (hash(x)=153358238)
+900 val loss 7.3667
+900 val perplexity 1582.4790
+900 train 7.198230 (lr=2.9891e-05) (hash(x)=146617394)
+1000 val loss 7.3379
+1000 val perplexity 1537.5419
+1000 train 7.275929 (lr=2.9858e-05) (hash(x)=148983354)
+1100 val loss 7.3000
+1100 val perplexity 1480.3101
+1100 train 7.329852 (lr=2.9820e-05) (hash(x)=146313352)
+1200 val loss 7.2790
+1200 val perplexity 1449.5948
+1200 train 7.231915 (lr=2.9778e-05) (hash(x)=150770333)
+1300 val loss 7.2209
+1300 val perplexity 1367.7261
+1300 train 7.161463 (lr=2.9732e-05) (hash(x)=153237933)
+1400 val loss 7.1820
+1400 val perplexity 1315.4723
+1400 train 7.019403 (lr=2.9681e-05) (hash(x)=145246201)
+1500 val loss 7.1543
+1500 val perplexity 1279.5875
+1500 train 7.149806 (lr=2.9626e-05) (hash(x)=154002540)
+1600 val loss 7.1365
+1600 val perplexity 1256.9694
+1600 train 6.916010 (lr=2.9566e-05) (hash(x)=142635842)
+1700 val loss 7.1051
+1700 val perplexity 1218.1798
+1700 train 7.146691 (lr=2.9502e-05) (hash(x)=151450555)
+1800 val loss 7.0845
+1800 val perplexity 1193.2864
+1800 train 6.996157 (lr=2.9434e-05) (hash(x)=148299832)
+1900 val loss 7.0558
+1900 val perplexity 1159.5553
+1900 train 7.217354 (lr=2.9362e-05) (hash(x)=155487476)
+2000 val loss 7.0480
+2000 val perplexity 1150.5250
+2000 train 7.025408 (lr=2.9285e-05) (hash(x)=152887721)
+2100 val loss 7.0169
+2100 val perplexity 1115.2950
+2100 train 6.724870 (lr=2.9204e-05) (hash(x)=143430603)
+2200 val loss 7.0010
+2200 val perplexity 1097.7700
+2200 train 6.989249 (lr=2.9119e-05) (hash(x)=153721194)
+2300 val loss 6.9800
+2300 val perplexity 1074.9148
+2300 train 7.102264 (lr=2.9030e-05) (hash(x)=148702429)
+2400 val loss 6.9560
+2400 val perplexity 1049.4778
+2400 train 6.998848 (lr=2.8937e-05) (hash(x)=153047184)
+2500 val loss 6.9328
+2500 val perplexity 1025.3099
+2500 train 6.808068 (lr=2.8840e-05) (hash(x)=149720123)
+2600 val loss 6.9113
+2600 val perplexity 1003.5928
+2600 train 6.947131 (lr=2.8738e-05) (hash(x)=152466045)
+2700 val loss 6.9011
+2700 val perplexity 993.4158
+2700 train 6.847266 (lr=2.8633e-05) (hash(x)=155796305)
+2800 val loss 6.8791
+2800 val perplexity 971.7216
+2800 train 6.721686 (lr=2.8523e-05) (hash(x)=151991552)
+2900 val loss 6.8777
+2900 val perplexity 970.3585
+2900 train 7.544370 (lr=2.8410e-05) (hash(x)=166537989)
+3000 val loss 6.8465
+3000 val perplexity 940.5893
+3000 train 6.846784 (lr=2.8292e-05) (hash(x)=147892594)
+3100 val loss 6.8313
+3100 val perplexity 926.3897
+3100 train 6.789685 (lr=2.8171e-05) (hash(x)=149536927)
+3200 val loss 6.8165
+3200 val perplexity 912.7487
+3200 train 6.529910 (lr=2.8046e-05) (hash(x)=135678663)
+3300 val loss 6.8121
+3300 val perplexity 908.7906
+3300 train 6.854458 (lr=2.7917e-05) (hash(x)=148523646)
+3400 val loss 6.8010
+3400 val perplexity 898.7062
+3400 train 6.861735 (lr=2.7784e-05) (hash(x)=150281778)
+3500 val loss 6.7779
+3500 val perplexity 878.2265
+3500 train 6.702899 (lr=2.7648e-05) (hash(x)=151527953)
+3600 val loss 6.7678
+3600 val perplexity 869.4343
+3600 train 7.013708 (lr=2.7508e-05) (hash(x)=141303739)
+3700 val loss 6.7572
+3700 val perplexity 860.2509
+3700 train 6.770426 (lr=2.7364e-05) (hash(x)=146778597)
+3800 val loss 6.7347
+3800 val perplexity 841.0608
+3800 train 6.702837 (lr=2.7216e-05) (hash(x)=146902797)
+3900 val loss 6.7188
+3900 val perplexity 827.8188
+3900 train 6.640366 (lr=2.7066e-05) (hash(x)=142022555)
+4000 val loss 6.7084
+4000 val perplexity 819.2586
+4000 train 6.622983 (lr=2.6911e-05) (hash(x)=151590309)
+4100 val loss 6.6857
+4100 val perplexity 800.8436
+4100 train 6.525273 (lr=2.6754e-05) (hash(x)=144127100)
+4200 val loss 6.6768
+4200 val perplexity 793.8104
+4200 train 6.748284 (lr=2.6592e-05) (hash(x)=149526774)
+4300 val loss 6.6665
+4300 val perplexity 785.6467
+4300 train 6.647405 (lr=2.6428e-05) (hash(x)=150924362)
+4400 val loss 6.6485
+4400 val perplexity 771.5940
+4400 train 6.821003 (lr=2.6260e-05) (hash(x)=155547282)
+4500 val loss 6.6379
+4500 val perplexity 763.4731
+4500 train 6.611216 (lr=2.6089e-05) (hash(x)=152479203)
+4600 val loss 6.6250
+4600 val perplexity 753.6772
+4600 train 6.524678 (lr=2.5915e-05) (hash(x)=146739760)
+4700 val loss 6.6198
+4700 val perplexity 749.8013
+4700 train 6.845607 (lr=2.5738e-05) (hash(x)=159977388)
+4800 val loss 6.6073
+4800 val perplexity 740.5098
+4800 train 6.487453 (lr=2.5558e-05) (hash(x)=142959739)
+4900 val loss 6.5901
+4900 val perplexity 727.8875
+4900 train 6.565074 (lr=2.5374e-05) (hash(x)=152253034)
+5000 val loss 6.5784
+5000 val perplexity 719.4218
+5000 train 6.547985 (lr=2.5188e-05) (hash(x)=140379747)
+5100 val loss 6.5660
+5100 val perplexity 710.5010
+5100 train 6.563121 (lr=2.4999e-05) (hash(x)=151826392)
+5200 val loss 6.5526
+5200 val perplexity 701.0939
+5200 train 6.488415 (lr=2.4807e-05) (hash(x)=152869594)
+5300 val loss 6.5369
+5300 val perplexity 690.1281
+5300 train 6.329997 (lr=2.4613e-05) (hash(x)=139093709)
+5400 val loss 6.5262
+5400 val perplexity 682.7764
+5400 train 6.384964 (lr=2.4415e-05) (hash(x)=145908738)
+5500 val loss 6.5242
+5500 val perplexity 681.4649
+5500 train 6.371226 (lr=2.4215e-05) (hash(x)=137913426)
+5600 val loss 6.4916
+5600 val perplexity 659.5744
+5600 train 6.317540 (lr=2.4013e-05) (hash(x)=140770390)
+5700 val loss 6.4865
+5700 val perplexity 656.2468
+5700 train 6.395054 (lr=2.3808e-05) (hash(x)=143728578)
+5800 val loss 6.4640
+5800 val perplexity 641.5987
+5800 train 6.571401 (lr=2.3601e-05) (hash(x)=154469787)
+5900 val loss 6.4519
+5900 val perplexity 633.9158
+5900 train 6.362564 (lr=2.3391e-05) (hash(x)=139316559)
+6000 val loss 6.4401
+6000 val perplexity 626.4839
+6000 train 6.436338 (lr=2.3179e-05) (hash(x)=152058488)
+6100 val loss 6.4282
+6100 val perplexity 619.0591
+6100 train 6.419155 (lr=2.2965e-05) (hash(x)=152958689)
+6200 val loss 6.4113
+6200 val perplexity 608.6703
+6200 train 6.381656 (lr=2.2749e-05) (hash(x)=150887864)
+6300 val loss 6.4057
+6300 val perplexity 605.2612
+6300 train 6.343083 (lr=2.2530e-05) (hash(x)=156290469)
+6400 val loss 6.3950
+6400 val perplexity 598.8619
+6400 train 6.678469 (lr=2.2310e-05) (hash(x)=159160159)
+6500 val loss 6.3797
+6500 val perplexity 589.7661
+6500 train 6.292921 (lr=2.2088e-05) (hash(x)=151239121)
+6600 val loss 6.3699
+6600 val perplexity 583.9717
+6600 train 6.366690 (lr=2.1864e-05) (hash(x)=151964101)
+6700 val loss 6.3644
+6700 val perplexity 580.7982
+6700 train 6.257575 (lr=2.1638e-05) (hash(x)=141291229)
+6800 val loss 6.3523
+6800 val perplexity 573.7996
+6800 train 6.283431 (lr=2.1410e-05) (hash(x)=160416800)
+6900 val loss 6.3442
+6900 val perplexity 569.1738
+6900 train 6.224077 (lr=2.1181e-05) (hash(x)=143382691)
+7000 val loss 6.3253
+7000 val perplexity 558.5463
+7000 train 6.237834 (lr=2.0951e-05) (hash(x)=147606572)
+7100 val loss 6.3138
+7100 val perplexity 552.1153
+7100 train 6.133327 (lr=2.0718e-05) (hash(x)=145901538)
+7200 val loss 6.3107
+7200 val perplexity 550.4401
+7200 train 6.134284 (lr=2.0485e-05) (hash(x)=139099153)
+7300 val loss 6.3096
+7300 val perplexity 549.8050
+7300 train 6.254177 (lr=2.0250e-05) (hash(x)=140434004)
+7400 val loss 6.2928
+7400 val perplexity 540.6658
+7400 train 6.412626 (lr=2.0014e-05) (hash(x)=155496146)
+7500 val loss 6.2809
+7500 val perplexity 534.2749
+7500 train 6.195351 (lr=1.9777e-05) (hash(x)=146940054)
+7600 val loss 6.2699
+7600 val perplexity 528.4411
+7600 train 6.516941 (lr=1.9538e-05) (hash(x)=165785723)
+7700 val loss 6.2614
+7700 val perplexity 523.9423
+7700 train 6.410962 (lr=1.9299e-05) (hash(x)=156339008)
+7800 val loss 6.2609
+7800 val perplexity 523.6866
+7800 train 6.255965 (lr=1.9059e-05) (hash(x)=148593556)
+7900 val loss 6.2599
+7900 val perplexity 523.1839
+7900 train 6.208652 (lr=1.8817e-05) (hash(x)=148079742)
+8000 val loss 6.2368
+8000 val perplexity 511.2038
+8000 train 6.185139 (lr=1.8576e-05) (hash(x)=159706718)
+8100 val loss 6.2373
+8100 val perplexity 511.4888
+8100 train 6.245315 (lr=1.8333e-05) (hash(x)=160203225)
+8200 val loss 6.2167
+8200 val perplexity 501.0641
+8200 train 6.175346 (lr=1.8090e-05) (hash(x)=146176536)
+8300 val loss 6.2092
+8300 val perplexity 497.2911
+8300 train 6.272688 (lr=1.7846e-05) (hash(x)=158883824)
+8400 val loss 6.2010
+8400 val perplexity 493.2605
+8400 train 6.003109 (lr=1.7602e-05) (hash(x)=137102572)
+8500 val loss 6.1948
+8500 val perplexity 490.1775
+8500 train 6.208079 (lr=1.7357e-05) (hash(x)=152812797)
+8600 val loss 6.1912
+8600 val perplexity 488.4562
+8600 train 6.055223 (lr=1.7113e-05) (hash(x)=148854244)
+8700 val loss 6.1987
+8700 val perplexity 492.0999
+8700 train 6.009529 (lr=1.6868e-05) (hash(x)=135590023)
+8800 val loss 6.1780
+8800 val perplexity 482.0101
+8800 train 6.104549 (lr=1.6623e-05) (hash(x)=144913509)
+8900 val loss 6.1720
+8900 val perplexity 479.1523
+8900 train 6.282070 (lr=1.6377e-05) (hash(x)=148045669)
+9000 val loss 6.1602
+9000 val perplexity 473.5063
+9000 train 6.194612 (lr=1.6132e-05) (hash(x)=146661746)
+9100 val loss 6.1541
+9100 val perplexity 470.6329
+9100 train 6.077746 (lr=1.5887e-05) (hash(x)=145354881)
+9200 val loss 6.1512
+9200 val perplexity 469.2866
+9200 train 6.007998 (lr=1.5643e-05) (hash(x)=150939310)
+9300 val loss 6.1439
+9300 val perplexity 465.8655
+9300 train 6.017556 (lr=1.5398e-05) (hash(x)=162258309)
+9400 val loss 6.1358
+9400 val perplexity 462.1187
+9400 train 6.142342 (lr=1.5154e-05) (hash(x)=154701830)
+9500 val loss 6.1309
+9500 val perplexity 459.8270
+9500 train 6.003320 (lr=1.4910e-05) (hash(x)=142241404)
+9600 val loss 6.1231
+9600 val perplexity 456.2891
+9600 train 6.089352 (lr=1.4667e-05) (hash(x)=148439858)
+9700 val loss 6.1225
+9700 val perplexity 455.9869
+9700 train 6.105500 (lr=1.4424e-05) (hash(x)=152204410)
+9800 val loss 6.1121
+9800 val perplexity 451.3029
+9800 train 5.930009 (lr=1.4183e-05) (hash(x)=141294630)
+9900 val loss 6.1118
+9900 val perplexity 451.1717
+9900 train 6.094609 (lr=1.3941e-05) (hash(x)=147390033)
+10000 val loss 6.1036
+10000 val perplexity 447.4525
+10000 train 6.077098 (lr=1.3701e-05) (hash(x)=144131604)
+10100 val loss 6.0939
+10100 val perplexity 443.1394
+10100 train 6.301743 (lr=1.3462e-05) (hash(x)=158814818)
+10200 val loss 6.0879
+10200 val perplexity 440.5142
+10200 train 6.152081 (lr=1.3223e-05) (hash(x)=153491388)
+10300 val loss 6.0845
+10300 val perplexity 439.0029
+10300 train 6.039337 (lr=1.2986e-05) (hash(x)=149600707)
+10400 val loss 6.0796
+10400 val perplexity 436.8394
+10400 train 5.983914 (lr=1.2750e-05) (hash(x)=152277064)
+10500 val loss 6.0791
+10500 val perplexity 436.6190
+10500 train 5.996013 (lr=1.2515e-05) (hash(x)=150447091)
+10600 val loss 6.0679
+10600 val perplexity 431.7908
+10600 train 6.079579 (lr=1.2282e-05) (hash(x)=151606822)
+10700 val loss 6.0601
+10700 val perplexity 428.3991
+10700 train 6.063359 (lr=1.2049e-05) (hash(x)=158730240)
+10800 val loss 6.0590
+10800 val perplexity 427.9493
+10800 train 6.112919 (lr=1.1819e-05) (hash(x)=163095518)
+10900 val loss 6.0536
+10900 val perplexity 425.6318
+10900 train 5.809468 (lr=1.1590e-05) (hash(x)=145260395)
+11000 val loss 6.0468
+11000 val perplexity 422.7498
+11000 train 6.053051 (lr=1.1362e-05) (hash(x)=155970264)
+11100 val loss 6.0466
+11100 val perplexity 422.6930
+11100 train 6.130270 (lr=1.1136e-05) (hash(x)=159871097)
+11200 val loss 6.0358
+11200 val perplexity 418.1324
+11200 train 6.133372 (lr=1.0912e-05) (hash(x)=161135244)
+11300 val loss 6.0310
+11300 val perplexity 416.1445
+11300 train 6.038279 (lr=1.0690e-05) (hash(x)=150649336)
+11400 val loss 6.0320
+11400 val perplexity 416.5644
+11400 train 5.988773 (lr=1.0470e-05) (hash(x)=152230013)
+11500 val loss 6.0225
+11500 val perplexity 412.6224
+11500 train 6.093239 (lr=1.0251e-05) (hash(x)=163300613)
+11600 val loss 6.0230
+11600 val perplexity 412.8220
+11600 train 6.006931 (lr=1.0035e-05) (hash(x)=151203838)
+11700 val loss 6.0172
+11700 val perplexity 410.4170
+11700 train 6.031619 (lr=9.8209e-06) (hash(x)=155255111)
+11800 val loss 6.0196
+11800 val perplexity 411.4132
+11800 train 5.976803 (lr=9.6090e-06) (hash(x)=139380668)
+11900 val loss 6.0059
+11900 val perplexity 405.8001
+11900 train 5.918542 (lr=9.3993e-06) (hash(x)=144033405)
+12000 val loss 6.0045
+12000 val perplexity 405.2554
+12000 train 5.923520 (lr=9.1920e-06) (hash(x)=148689574)
+12100 val loss 6.0011
+12100 val perplexity 403.8767
+12100 train 5.959810 (lr=8.9871e-06) (hash(x)=143272239)
+12200 val loss 5.9979
+12200 val perplexity 402.5737
+12200 train 6.155020 (lr=8.7846e-06) (hash(x)=157671609)
+12300 val loss 5.9938
+12300 val perplexity 400.9520
+12300 train 5.999459 (lr=8.5847e-06) (hash(x)=150363247)
+12400 val loss 5.9882
+12400 val perplexity 398.6854
+12400 train 5.878437 (lr=8.3875e-06) (hash(x)=146857787)
+12500 val loss 5.9853
+12500 val perplexity 397.5401
+12500 train 5.934603 (lr=8.1929e-06) (hash(x)=153327320)
+12600 val loss 5.9825
+12600 val perplexity 396.4457
+12600 train 5.696634 (lr=8.0010e-06) (hash(x)=143612983)
+12700 val loss 5.9797
+12700 val perplexity 395.3220
+12700 train 5.917475 (lr=7.8119e-06) (hash(x)=150739132)
+12800 val loss 5.9765
+12800 val perplexity 394.0561
+12800 train 5.823044 (lr=7.6257e-06) (hash(x)=147986204)
+12900 val loss 5.9739
+12900 val perplexity 393.0287
+12900 train 6.124288 (lr=7.4425e-06) (hash(x)=155218833)
+13000 val loss 5.9681
+13000 val perplexity 390.7678
+13000 train 5.953165 (lr=7.2622e-06) (hash(x)=148867960)
+13100 val loss 5.9659
+13100 val perplexity 389.9185
+13100 train 5.990927 (lr=7.0850e-06) (hash(x)=160101478)
+13200 val loss 5.9653
+13200 val perplexity 389.6800
+13200 train 5.816887 (lr=6.9108e-06) (hash(x)=151080097)
+13300 val loss 5.9652
+13300 val perplexity 389.6155
+13300 train 5.955846 (lr=6.7398e-06) (hash(x)=150403039)
+13400 val loss 5.9606
+13400 val perplexity 387.8403
+13400 train 5.873844 (lr=6.5721e-06) (hash(x)=129728571)
+13500 val loss 5.9550
+13500 val perplexity 385.6749
+13500 train 5.867978 (lr=6.4076e-06) (hash(x)=148988625)
+13600 val loss 5.9525
+13600 val perplexity 384.7157
+13600 train 5.740644 (lr=6.2465e-06) (hash(x)=149790861)
+13700 val loss 5.9515
+13700 val perplexity 384.3409
+13700 train 5.894429 (lr=6.0887e-06) (hash(x)=153505062)
+13800 val loss 5.9558
+13800 val perplexity 385.9948
+13800 train 5.856753 (lr=5.9344e-06) (hash(x)=143845463)
+13900 val loss 5.9507
+13900 val perplexity 384.0300
+13900 train 5.741137 (lr=5.7835e-06) (hash(x)=143378076)
+14000 val loss 5.9437
+14000 val perplexity 381.3362
+14000 train 5.902184 (lr=5.6362e-06) (hash(x)=148681755)
+14100 val loss 5.9412
+14100 val perplexity 380.4063
+14100 train 5.825752 (lr=5.4925e-06) (hash(x)=145530465)
+14200 val loss 5.9403
+14200 val perplexity 380.0658
+14200 train 5.892087 (lr=5.3524e-06) (hash(x)=162139713)
+14300 val loss 5.9374
+14300 val perplexity 378.9445
+14300 train 5.937052 (lr=5.2159e-06) (hash(x)=141997596)
+14400 val loss 5.9368
+14400 val perplexity 378.7086
+14400 train 5.904932 (lr=5.0832e-06) (hash(x)=146156313)
+14500 val loss 5.9374
+14500 val perplexity 378.9321
+14500 train 5.706226 (lr=4.9543e-06) (hash(x)=145298178)
+14600 val loss 5.9318
+14600 val perplexity 376.8298
+14600 train 6.012121 (lr=4.8291e-06) (hash(x)=148232610)
+14700 val loss 5.9287
+14700 val perplexity 375.6682
+14700 train 5.810235 (lr=4.7079e-06) (hash(x)=151079115)
+14800 val loss 5.9270
+14800 val perplexity 375.0097
+14800 train 5.894715 (lr=4.5905e-06) (hash(x)=151340983)
+14900 val loss 5.9259
+14900 val perplexity 374.6101
+14900 train 6.131800 (lr=4.4770e-06) (hash(x)=150213803)
+15000 val loss 5.9256
+15000 val perplexity 374.5093
+15000 train 5.835275 (lr=4.3675e-06) (hash(x)=157652334)
+15100 val loss 5.9243
+15100 val perplexity 374.0276
+15100 train 5.657528 (lr=4.2620e-06) (hash(x)=145078577)
+15200 val loss 5.9208
+15200 val perplexity 372.7132
+15200 train 5.872759 (lr=4.1605e-06) (hash(x)=155898006)
+15300 val loss 5.9194
+15300 val perplexity 372.1980
+15300 train 5.947790 (lr=4.0631e-06) (hash(x)=149346671)
+15400 val loss 5.9169
+15400 val perplexity 371.2449
+15400 train 5.776122 (lr=3.9698e-06) (hash(x)=145708113)
+15500 val loss 5.9142
+15500 val perplexity 370.2740
+15500 train 6.114147 (lr=3.8806e-06) (hash(x)=165102970)
+15600 val loss 5.9156
+15600 val perplexity 370.7627
+15600 train 5.694969 (lr=3.7956e-06) (hash(x)=139106517)
+15700 val loss 5.9140
+15700 val perplexity 370.1714
+15700 train 5.733104 (lr=3.7148e-06) (hash(x)=141680838)
+15800 val loss 5.9110
+15800 val perplexity 369.0799
+15800 train 5.789922 (lr=3.6382e-06) (hash(x)=136152310)
+15900 val loss 5.9096
+15900 val perplexity 368.5567
+15900 train 5.971681 (lr=3.5658e-06) (hash(x)=151104640)
+16000 val loss 5.9071
+16000 val perplexity 367.6357
+16000 train 5.857261 (lr=3.4977e-06) (hash(x)=153333281)
+16100 val loss 5.9064
+16100 val perplexity 367.3925
+16100 train 5.863892 (lr=3.4339e-06) (hash(x)=145628059)
+16200 val loss 5.9072
+16200 val perplexity 367.6785
+16200 train 5.618507 (lr=3.3744e-06) (hash(x)=137220041)
+16300 val loss 5.9045
+16300 val perplexity 366.6861
+16300 train 6.059216 (lr=3.3193e-06) (hash(x)=161215791)
+16400 val loss 5.9041
+16400 val perplexity 366.5234
+16400 train 5.621179 (lr=3.2684e-06) (hash(x)=136194680)
+16500 val loss 5.9042
+16500 val perplexity 366.5812
+16500 train 5.709507 (lr=3.2220e-06) (hash(x)=152209568)
+16600 val loss 5.9005
+16600 val perplexity 365.2301
+16600 train 6.007233 (lr=3.1799e-06) (hash(x)=157345721)
+16700 val loss 5.8983
+16700 val perplexity 364.4319
+16700 train 5.996146 (lr=3.1422e-06) (hash(x)=146883642)
+16800 val loss 5.8986
+16800 val perplexity 364.5212
+16800 train 5.857209 (lr=3.1089e-06) (hash(x)=153037972)
+16900 val loss 5.8978
+16900 val perplexity 364.2220
+16900 train 5.685570 (lr=3.0801e-06) (hash(x)=150691367)
+17000 val loss 5.8971
+17000 val perplexity 363.9819
+17000 train 5.644271 (lr=3.0556e-06) (hash(x)=148975206)
+17100 val loss 5.8978
+17100 val perplexity 364.2212
+17100 train 5.614702 (lr=3.0356e-06) (hash(x)=142026221)
+17200 val loss 5.8946
+17200 val perplexity 363.0641
+17200 train 5.936298 (lr=3.0200e-06) (hash(x)=138211880)
+17300 val loss 5.8919
+17300 val perplexity 362.0752
+17300 train 5.743488 (lr=3.0089e-06) (hash(x)=143366666)
+17400 val loss 5.8922
+17400 val perplexity 362.2161
+17400 train 5.879615 (lr=3.0022e-06) (hash(x)=150791766)
+17499 val loss 5.8915
+17499 val perplexity 361.9645

attention_kindselective_n_heads4_seed1343/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f2d81873e4940064147bfc8eab16fd75bb372ba77dafef56dc7143ac3593856
+oid sha256:cae2222b2a438aae1ef4db8da06094fd8f0026a9823a8246b18a21cb587edf18
 size 92843394

attention_kindselective_n_heads4_seed1343/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0fdfbc009957bc5ddfdf9c620e1209c1ea61ac8db143f251f024ce1b657e4137
+oid sha256:9055b35115695594360e1b670807584e8d24d02e2da189450f14cae00c740d99
 size 179406214