Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1339", "n_embd": 256}

attention_kindselective_n_heads4_seed1339/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2810
 0 val perplexity 79298.4922
-0 train 11.295413 (lr=1.6250e-07) (hash(x)=150724848)
-100 val loss 9.9363
-100 val perplexity 20667.9277
-100 train 9.984186 (lr=1.6412e-05) (hash(x)=149217061)
-200 val loss 8.9109
-200 val perplexity 7412.5137
-200 train 8.928561 (lr=3.2500e-05) (hash(x)=149910534)
-300 val loss 7.9375
-300 val perplexity 2800.2134
-300 train 7.859196 (lr=3.2498e-05) (hash(x)=142185643)
-400 val loss 7.6799
-400 val perplexity 2164.3118
-400 train 7.731187 (lr=3.2490e-05) (hash(x)=148123706)
-500 val loss 7.5834
-500 val perplexity 1965.3929
-500 train 7.456921 (lr=3.2478e-05) (hash(x)=149952383)
-600 val loss 7.5402
-600 val perplexity 1882.1863
-600 train 7.357423 (lr=3.2461e-05) (hash(x)=146678221)
-700 val loss 7.4786
-700 val perplexity 1769.6854
-700 train 7.511988 (lr=3.2440e-05) (hash(x)=156180736)
-800 val loss 7.4284
-800 val perplexity 1683.0771
-800 train 7.421630 (lr=3.2413e-05) (hash(x)=151700982)
-900 val loss 7.3999
-900 val perplexity 1635.8824
-900 train 7.194687 (lr=3.2382e-05) (hash(x)=147288467)
-1000 val loss 7.3629
-1000 val perplexity 1576.4763
-1000 train 7.404816 (lr=3.2346e-05) (hash(x)=156182087)
-1100 val loss 7.3335
-1100 val perplexity 1530.6554
-1100 train 7.207426 (lr=3.2305e-05) (hash(x)=147861550)
-1200 val loss 7.3059
-1200 val perplexity 1489.0640
-1200 train 7.170646 (lr=3.2260e-05) (hash(x)=149318660)
-1300 val loss 7.2851
-1300 val perplexity 1458.4640
-1300 train 7.124465 (lr=3.2209e-05) (hash(x)=151631103)
-1400 val loss 7.2523
-1400 val perplexity 1411.4155
-1400 train 7.294272 (lr=3.2154e-05) (hash(x)=150482428)
-1500 val loss 7.2043
-1500 val perplexity 1345.2277
-1500 train 7.051772 (lr=3.2094e-05) (hash(x)=144458617)
-1600 val loss 7.1622
-1600 val perplexity 1289.7428
-1600 train 7.036353 (lr=3.2030e-05) (hash(x)=143268605)
-1700 val loss 7.1200
-1700 val perplexity 1236.4585
-1700 train 7.047909 (lr=3.1961e-05) (hash(x)=150817011)
-1800 val loss 7.0871
-1800 val perplexity 1196.4166
-1800 train 7.066686 (lr=3.1887e-05) (hash(x)=152322423)
-1900 val loss 7.0412
-1900 val perplexity 1142.7028
-1900 train 6.837695 (lr=3.1809e-05) (hash(x)=143574126)
-2000 val loss 7.0072
-2000 val perplexity 1104.5946
-2000 train 6.765584 (lr=3.1726e-05) (hash(x)=147904298)
-2100 val loss 6.9793
-2100 val perplexity 1074.1523
-2100 train 6.724745 (lr=3.1638e-05) (hash(x)=144719400)
-2200 val loss 6.9346
-2200 val perplexity 1027.2444
-2200 train 7.184346 (lr=3.1546e-05) (hash(x)=154343147)
-2300 val loss 6.8918
-2300 val perplexity 984.1760
-2300 train 6.934226 (lr=3.1449e-05) (hash(x)=154372635)
-2400 val loss 6.8595
-2400 val perplexity 952.8591
-2400 train 6.681858 (lr=3.1348e-05) (hash(x)=141843115)
-2500 val loss 6.8220
-2500 val perplexity 917.7935
-2500 train 6.727233 (lr=3.1243e-05) (hash(x)=149417679)
-2600 val loss 6.7892
-2600 val perplexity 888.1670
-2600 train 6.649594 (lr=3.1133e-05) (hash(x)=145279030)
-2700 val loss 6.7601
-2700 val perplexity 862.7621
-2700 train 6.665741 (lr=3.1019e-05) (hash(x)=146363911)
-2800 val loss 6.7510
-2800 val perplexity 854.9482
-2800 train 6.706868 (lr=3.0900e-05) (hash(x)=152507639)
-2900 val loss 6.7043
-2900 val perplexity 815.9222
-2900 train 6.495276 (lr=3.0777e-05) (hash(x)=139296191)
-3000 val loss 6.6786
-3000 val perplexity 795.1898
-3000 train 6.690280 (lr=3.0650e-05) (hash(x)=148473774)
-3100 val loss 6.6301
-3100 val perplexity 757.5317
-3100 train 6.722582 (lr=3.0518e-05) (hash(x)=154130527)
-3200 val loss 6.6033
-3200 val perplexity 737.5317
-3200 train 6.548954 (lr=3.0383e-05) (hash(x)=151117002)
-3300 val loss 6.5959
-3300 val perplexity 732.1217
-3300 train 6.740818 (lr=3.0243e-05) (hash(x)=151035383)
-3400 val loss 6.5595
-3400 val perplexity 705.9531
-3400 train 6.679100 (lr=3.0099e-05) (hash(x)=150614749)
-3500 val loss 6.5290
-3500 val perplexity 684.6951
-3500 train 6.549811 (lr=2.9952e-05) (hash(x)=151431391)
-3600 val loss 6.5081
-3600 val perplexity 670.5336
-3600 train 6.444536 (lr=2.9800e-05) (hash(x)=151140797)
-3700 val loss 6.4810
-3700 val perplexity 652.6105
-3700 train 6.683108 (lr=2.9644e-05) (hash(x)=154608340)
-3800 val loss 6.4665
-3800 val perplexity 643.2121
-3800 train 6.493763 (lr=2.9485e-05) (hash(x)=169641309)
-3900 val loss 6.4394
-3900 val perplexity 626.0336
-3900 train 6.395096 (lr=2.9321e-05) (hash(x)=150592479)
-4000 val loss 6.4139
-4000 val perplexity 610.2711
-4000 train 6.398534 (lr=2.9154e-05) (hash(x)=155495317)
-4100 val loss 6.4077
-4100 val perplexity 606.4729
-4100 train 6.611394 (lr=2.8983e-05) (hash(x)=170323970)
-4200 val loss 6.4018
-4200 val perplexity 602.9016
-4200 train 6.223932 (lr=2.8808e-05) (hash(x)=140854360)
-4300 val loss 6.3646
-4300 val perplexity 580.9186
-4300 train 6.368628 (lr=2.8630e-05) (hash(x)=148926644)
-4400 val loss 6.3470
-4400 val perplexity 570.7972
-4400 train 6.378788 (lr=2.8448e-05) (hash(x)=149997871)
-4500 val loss 6.3283
-4500 val perplexity 560.2269
-4500 train 6.274667 (lr=2.8263e-05) (hash(x)=149219289)
-4600 val loss 6.3111
-4600 val perplexity 550.6545
-4600 train 6.106242 (lr=2.8075e-05) (hash(x)=139836197)
-4700 val loss 6.2991
-4700 val perplexity 544.1017
-4700 train 6.247629 (lr=2.7883e-05) (hash(x)=150722883)
-4800 val loss 6.2893
-4800 val perplexity 538.7761
-4800 train 6.492603 (lr=2.7687e-05) (hash(x)=166864648)
-4900 val loss 6.2657
-4900 val perplexity 526.1866
-4900 train 6.368227 (lr=2.7489e-05) (hash(x)=157746743)
-5000 val loss 6.2451
-5000 val perplexity 515.5007
-5000 train 6.195556 (lr=2.7287e-05) (hash(x)=153032257)
-5100 val loss 6.2342
-5100 val perplexity 509.8943
-5100 train 6.316526 (lr=2.7082e-05) (hash(x)=151862371)
-5200 val loss 6.2151
-5200 val perplexity 500.2562
-5200 train 6.555519 (lr=2.6874e-05) (hash(x)=168862995)
-5300 val loss 6.2200
-5300 val perplexity 502.7055
-5300 train 6.186482 (lr=2.6664e-05) (hash(x)=148243567)
-5400 val loss 6.1919
-5400 val perplexity 488.7558
-5400 train 6.124026 (lr=2.6450e-05) (hash(x)=145381742)
-5500 val loss 6.1883
-5500 val perplexity 487.0385
-5500 train 6.359857 (lr=2.6233e-05) (hash(x)=155602174)
-5600 val loss 6.1593
-5600 val perplexity 473.1143
-5600 train 6.246666 (lr=2.6014e-05) (hash(x)=160637672)
-5700 val loss 6.1523
-5700 val perplexity 469.7991
-5700 train 6.077770 (lr=2.5792e-05) (hash(x)=152101131)
-5800 val loss 6.1386
-5800 val perplexity 463.4019
-5800 train 6.099304 (lr=2.5567e-05) (hash(x)=152327089)
-5900 val loss 6.1331
-5900 val perplexity 460.8697
-5900 train 6.076286 (lr=2.5340e-05) (hash(x)=151094543)
-6000 val loss 6.1258
-6000 val perplexity 457.5008
-6000 train 6.232687 (lr=2.5111e-05) (hash(x)=160233315)
-6100 val loss 6.1109
-6100 val perplexity 450.7276
-6100 train 6.216568 (lr=2.4879e-05) (hash(x)=158295337)
-6200 val loss 6.0928
-6200 val perplexity 442.6566
-6200 train 6.044076 (lr=2.4644e-05) (hash(x)=151269817)
-6300 val loss 6.0799
-6300 val perplexity 436.9921
-6300 train 6.225775 (lr=2.4408e-05) (hash(x)=156912756)
-6400 val loss 6.0694
-6400 val perplexity 432.4186
-6400 train 5.995785 (lr=2.4169e-05) (hash(x)=125779637)
-6500 val loss 6.0646
-6500 val perplexity 430.3296
-6500 train 5.961404 (lr=2.3928e-05) (hash(x)=148157661)
-6600 val loss 6.0558
-6600 val perplexity 426.5678
-6600 train 6.084833 (lr=2.3686e-05) (hash(x)=152820049)
-6700 val loss 6.0447
-6700 val perplexity 421.8885
-6700 train 5.979776 (lr=2.3441e-05) (hash(x)=155818717)
-6800 val loss 6.0267
-6800 val perplexity 414.3600
-6800 train 6.053731 (lr=2.3195e-05) (hash(x)=158833881)
-6900 val loss 6.0245
-6900 val perplexity 413.4163
-6900 train 5.945818 (lr=2.2946e-05) (hash(x)=164984909)
-7000 val loss 6.0116
-7000 val perplexity 408.1465
-7000 train 5.903196 (lr=2.2696e-05) (hash(x)=151172493)
-7100 val loss 6.0139
-7100 val perplexity 409.0786
-7100 train 6.066981 (lr=2.2445e-05) (hash(x)=154023633)
-7200 val loss 5.9966
-7200 val perplexity 402.0624
-7200 train 5.848167 (lr=2.2192e-05) (hash(x)=139351250)
-7300 val loss 5.9881
-7300 val perplexity 398.6376
-7300 train 5.944324 (lr=2.1937e-05) (hash(x)=158719458)
-7400 val loss 5.9708
-7400 val perplexity 391.8035
-7400 train 6.046550 (lr=2.1682e-05) (hash(x)=162250810)
-7500 val loss 5.9644
-7500 val perplexity 389.3349
-7500 train 5.840899 (lr=2.1425e-05) (hash(x)=142827557)
-7600 val loss 5.9555
-7600 val perplexity 385.8814
-7600 train 5.863455 (lr=2.1166e-05) (hash(x)=147830801)
-7700 val loss 5.9490
-7700 val perplexity 383.3558
-7700 train 6.045110 (lr=2.0907e-05) (hash(x)=165233314)
-7800 val loss 5.9436
-7800 val perplexity 381.3161
-7800 train 5.926939 (lr=2.0647e-05) (hash(x)=160654717)
-7900 val loss 5.9398
-7900 val perplexity 379.8509
-7900 train 5.801051 (lr=2.0386e-05) (hash(x)=140712586)
-8000 val loss 5.9262
-8000 val perplexity 374.7310
-8000 train 6.088989 (lr=2.0123e-05) (hash(x)=150309061)
-8100 val loss 5.9236
-8100 val perplexity 373.7592
-8100 train 5.924323 (lr=1.9861e-05) (hash(x)=158805466)
-8200 val loss 5.9111
-8200 val perplexity 369.1290
-8200 train 5.796899 (lr=1.9597e-05) (hash(x)=142814078)
-8300 val loss 5.9062
-8300 val perplexity 367.3261
-8300 train 5.871168 (lr=1.9333e-05) (hash(x)=148340417)
-8400 val loss 5.9016
-8400 val perplexity 365.6312
-8400 train 5.841621 (lr=1.9069e-05) (hash(x)=157132639)
-8500 val loss 5.8938
-8500 val perplexity 362.7953
-8500 train 5.823905 (lr=1.8804e-05) (hash(x)=152994240)
-8600 val loss 5.8836
-8600 val perplexity 359.0916
-8600 train 5.847437 (lr=1.8539e-05) (hash(x)=152210154)
-8700 val loss 5.8735
-8700 val perplexity 355.4971
-8700 train 5.967282 (lr=1.8273e-05) (hash(x)=149631247)
-8800 val loss 5.8664
-8800 val perplexity 352.9633
-8800 train 5.732079 (lr=1.8008e-05) (hash(x)=145617784)
-8900 val loss 5.8604
-8900 val perplexity 350.8580
-8900 train 5.963461 (lr=1.7742e-05) (hash(x)=155984970)
-9000 val loss 5.8592
-9000 val perplexity 350.4601
-9000 train 5.591073 (lr=1.7477e-05) (hash(x)=143433013)
-9100 val loss 5.8540
-9100 val perplexity 348.6308
-9100 train 5.756921 (lr=1.7211e-05) (hash(x)=152507533)
-9200 val loss 5.8445
-9200 val perplexity 345.3170
-9200 train 5.973843 (lr=1.6946e-05) (hash(x)=162400650)
-9300 val loss 5.8391
-9300 val perplexity 343.4677
-9300 train 5.885237 (lr=1.6681e-05) (hash(x)=148419068)
-9400 val loss 5.8324
-9400 val perplexity 341.1773
-9400 train 5.801330 (lr=1.6417e-05) (hash(x)=145025169)
-9500 val loss 5.8231
-9500 val perplexity 338.0238
-9500 train 5.798188 (lr=1.6153e-05) (hash(x)=145104235)
-9600 val loss 5.8215
-9600 val perplexity 337.4738
-9600 train 5.755646 (lr=1.5889e-05) (hash(x)=150499797)
-9700 val loss 5.8166
-9700 val perplexity 335.8241
-9700 train 5.619888 (lr=1.5627e-05) (hash(x)=139690022)
-9800 val loss 5.8064
-9800 val perplexity 332.4281
-9800 train 5.808864 (lr=1.5364e-05) (hash(x)=145645279)
-9900 val loss 5.7980
-9900 val perplexity 329.6533
-9900 train 5.719332 (lr=1.5103e-05) (hash(x)=150103432)
-10000 val loss 5.7952
-10000 val perplexity 328.7140
-10000 train 5.949801 (lr=1.4843e-05) (hash(x)=162309473)
-10100 val loss 5.7908
-10100 val perplexity 327.2599
-10100 train 5.681758 (lr=1.4584e-05) (hash(x)=146586772)
-10200 val loss 5.7865
-10200 val perplexity 325.8835
-10200 train 5.690024 (lr=1.4325e-05) (hash(x)=149256313)
-10300 val loss 5.7816
-10300 val perplexity 324.2705
-10300 train 5.739097 (lr=1.4068e-05) (hash(x)=151455083)
-10400 val loss 5.7808
-10400 val perplexity 324.0317
-10400 train 5.552348 (lr=1.3813e-05) (hash(x)=150210444)
-10500 val loss 5.7806
-10500 val perplexity 323.9690
-10500 train 5.548230 (lr=1.3558e-05) (hash(x)=152144428)
-10600 val loss 5.7693
-10600 val perplexity 320.3099
-10600 train 5.816607 (lr=1.3305e-05) (hash(x)=168628935)
-10700 val loss 5.7633
-10700 val perplexity 318.3990
-10700 train 5.562486 (lr=1.3054e-05) (hash(x)=164667785)
-10800 val loss 5.7554
-10800 val perplexity 315.8957
-10800 train 5.717759 (lr=1.2804e-05) (hash(x)=143621491)
-10900 val loss 5.7534
-10900 val perplexity 315.2698
-10900 train 5.649728 (lr=1.2555e-05) (hash(x)=143914826)
-11000 val loss 5.7514
-11000 val perplexity 314.6223
-11000 train 5.527834 (lr=1.2309e-05) (hash(x)=137663435)
-11100 val loss 5.7517
-11100 val perplexity 314.7245
-11100 train 5.621320 (lr=1.2064e-05) (hash(x)=149054007)
-11200 val loss 5.7488
-11200 val perplexity 313.8074
-11200 train 5.756852 (lr=1.1822e-05) (hash(x)=153473924)
-11300 val loss 5.7443
-11300 val perplexity 312.4077
-11300 train 5.535771 (lr=1.1581e-05) (hash(x)=158563275)
-11400 val loss 5.7411
-11400 val perplexity 311.4039
-11400 train 5.877691 (lr=1.1342e-05) (hash(x)=161016667)
-11500 val loss 5.7400
-11500 val perplexity 311.0557
-11500 train 5.784426 (lr=1.1106e-05) (hash(x)=157344822)
-11600 val loss 5.7276
-11600 val perplexity 307.2221
-11600 train 5.616313 (lr=1.0871e-05) (hash(x)=139977713)
-11700 val loss 5.7197
-11700 val perplexity 304.8123
-11700 train 5.714229 (lr=1.0639e-05) (hash(x)=156511738)
-11800 val loss 5.7208
-11800 val perplexity 305.1424
-11800 train 6.174135 (lr=1.0410e-05) (hash(x)=185372558)
-11900 val loss 5.7162
-11900 val perplexity 303.7593
-11900 train 5.559208 (lr=1.0183e-05) (hash(x)=138085470)
-12000 val loss 5.7170
-12000 val perplexity 303.9993
-12000 train 5.562536 (lr=9.9580e-06) (hash(x)=142928235)
-12100 val loss 5.7150
-12100 val perplexity 303.3817
-12100 train 5.564174 (lr=9.7360e-06) (hash(x)=151960396)
-12200 val loss 5.7056
-12200 val perplexity 300.5520
-12200 train 5.767758 (lr=9.5167e-06) (hash(x)=156794523)
-12300 val loss 5.6999
-12300 val perplexity 298.8499
-12300 train 5.508867 (lr=9.3001e-06) (hash(x)=138082358)
-12400 val loss 5.6970
-12400 val perplexity 297.9801
-12400 train 5.598351 (lr=9.0864e-06) (hash(x)=155255899)
-12500 val loss 5.6931
-12500 val perplexity 296.8066
-12500 train 5.658473 (lr=8.8756e-06) (hash(x)=150490528)
-12600 val loss 5.6913
-12600 val perplexity 296.2926
-12600 train 5.849111 (lr=8.6677e-06) (hash(x)=154831027)
-12700 val loss 5.6911
-12700 val perplexity 296.2101
-12700 train 5.642239 (lr=8.4629e-06) (hash(x)=147341052)
-12800 val loss 5.6896
-12800 val perplexity 295.7809
-12800 train 5.728687 (lr=8.2612e-06) (hash(x)=159333795)
-12900 val loss 5.6913
-12900 val perplexity 296.2816
-12900 train 5.534995 (lr=8.0627e-06) (hash(x)=143896867)
-13000 val loss 5.6820
-13000 val perplexity 293.5359
-13000 train 5.691289 (lr=7.8674e-06) (hash(x)=147161537)
-13100 val loss 5.6788
-13100 val perplexity 292.6069
-13100 train 5.811933 (lr=7.6754e-06) (hash(x)=147081918)
-13200 val loss 5.6773
-13200 val perplexity 292.1565
-13200 train 5.445444 (lr=7.4867e-06) (hash(x)=145079287)
-13300 val loss 5.6772
-13300 val perplexity 292.1414
-13300 train 5.475905 (lr=7.3015e-06) (hash(x)=142515041)
-13400 val loss 5.6741
-13400 val perplexity 291.2227
-13400 train 5.459397 (lr=7.1198e-06) (hash(x)=144081985)
-13500 val loss 5.6716
-13500 val perplexity 290.4957
-13500 train 5.750075 (lr=6.9416e-06) (hash(x)=149957292)
-13600 val loss 5.6662
-13600 val perplexity 288.9391
-13600 train 5.702712 (lr=6.7670e-06) (hash(x)=157123033)
-13700 val loss 5.6631
-13700 val perplexity 288.0368
-13700 train 5.641377 (lr=6.5961e-06) (hash(x)=148277247)
-13800 val loss 5.6683
-13800 val perplexity 289.5473
-13800 train 5.796854 (lr=6.4289e-06) (hash(x)=142769907)
-13900 val loss 5.6588
-13900 val perplexity 286.8043
-13900 train 5.602085 (lr=6.2655e-06) (hash(x)=144932004)
-14000 val loss 5.6571
-14000 val perplexity 286.3113
-14000 train 5.670773 (lr=6.1059e-06) (hash(x)=149532265)
-14100 val loss 5.6563
-14100 val perplexity 286.0894
-14100 train 5.469543 (lr=5.9502e-06) (hash(x)=149301374)
-14200 val loss 5.6562
-14200 val perplexity 286.0475
-14200 train 5.395635 (lr=5.7984e-06) (hash(x)=142871075)
-14300 val loss 5.6512
-14300 val perplexity 284.6200
-14300 train 5.802889 (lr=5.6506e-06) (hash(x)=166744960)
-14400 val loss 5.6476
-14400 val perplexity 283.6026
-14400 train 5.615609 (lr=5.5068e-06) (hash(x)=145950920)
-14500 val loss 5.6455
-14500 val perplexity 283.0105
-14500 train 5.639256 (lr=5.3671e-06) (hash(x)=141089155)
-14600 val loss 5.6435
-14600 val perplexity 282.4373
-14600 train 5.801688 (lr=5.2316e-06) (hash(x)=174516275)
-14700 val loss 5.6425
-14700 val perplexity 282.1671
-14700 train 5.444544 (lr=5.1002e-06) (hash(x)=137896065)
-14800 val loss 5.6417
-14800 val perplexity 281.9483
-14800 train 5.677532 (lr=4.9730e-06) (hash(x)=154396556)
-14900 val loss 5.6416
-14900 val perplexity 281.9059
-14900 train 5.487867 (lr=4.8501e-06) (hash(x)=149322394)
-15000 val loss 5.6403
-15000 val perplexity 281.5419
-15000 train 5.556224 (lr=4.7314e-06) (hash(x)=147410513)
-15100 val loss 5.6387
-15100 val perplexity 281.0874
-15100 train 5.549696 (lr=4.6171e-06) (hash(x)=145245634)
-15200 val loss 5.6342
-15200 val perplexity 279.8425
-15200 train 5.724654 (lr=4.5072e-06) (hash(x)=148341108)
-15300 val loss 5.6318
-15300 val perplexity 279.1635
-15300 train 5.440813 (lr=4.4017e-06) (hash(x)=140162437)
-15400 val loss 5.6315
-15400 val perplexity 279.0840
-15400 train 5.841183 (lr=4.3006e-06) (hash(x)=142166081)
-15500 val loss 5.6299
-15500 val perplexity 278.6382
-15500 train 5.549793 (lr=4.2040e-06) (hash(x)=141042196)
-15600 val loss 5.6295
-15600 val perplexity 278.5304
-15600 train 5.751338 (lr=4.1119e-06) (hash(x)=148672701)
-15700 val loss 5.6293
-15700 val perplexity 278.4685
-15700 train 5.535214 (lr=4.0244e-06) (hash(x)=149672385)
-15800 val loss 5.6298
-15800 val perplexity 278.6071
-15800 train 5.662232 (lr=3.9414e-06) (hash(x)=152379562)
-15900 val loss 5.6254
-15900 val perplexity 277.3748
-15900 train 5.671198 (lr=3.8630e-06) (hash(x)=154674265)
-16000 val loss 5.6235
-16000 val perplexity 276.8669
-16000 train 5.658562 (lr=3.7892e-06) (hash(x)=150368078)
-16100 val loss 5.6212
-16100 val perplexity 276.2202
-16100 train 5.732701 (lr=3.7201e-06) (hash(x)=160390561)
-16200 val loss 5.6198
-16200 val perplexity 275.8415
-16200 train 5.990137 (lr=3.6556e-06) (hash(x)=155218378)
-16300 val loss 5.6190
-16300 val perplexity 275.6219
-16300 train 5.817885 (lr=3.5959e-06) (hash(x)=160891077)
-16400 val loss 5.6182
-16400 val perplexity 275.3945
-16400 train 5.519011 (lr=3.5408e-06) (hash(x)=142789253)
-16500 val loss 5.6190
-16500 val perplexity 275.6153
-16500 train 5.362583 (lr=3.4905e-06) (hash(x)=141497841)
-16600 val loss 5.6178
-16600 val perplexity 275.2920
-16600 train 5.576753 (lr=3.4449e-06) (hash(x)=157947018)
-16700 val loss 5.6149
-16700 val perplexity 274.4941
-16700 train 5.570585 (lr=3.4041e-06) (hash(x)=146039309)
-16800 val loss 5.6135
-16800 val perplexity 274.1042
-16800 train 5.582052 (lr=3.3680e-06) (hash(x)=156381445)
-16900 val loss 5.6119
-16900 val perplexity 273.6617
-16900 train 5.573091 (lr=3.3367e-06) (hash(x)=146537354)
-17000 val loss 5.6104
-17000 val perplexity 273.2666
-17000 train 5.533173 (lr=3.3102e-06) (hash(x)=135939444)
-17100 val loss 5.6107
-17100 val perplexity 273.3477
-17100 train 5.684841 (lr=3.2886e-06) (hash(x)=147819652)
-17200 val loss 5.6112
-17200 val perplexity 273.4625
-17200 train 5.705206 (lr=3.2717e-06) (hash(x)=155307909)
-17300 val loss 5.6073
-17300 val perplexity 272.4150
-17300 train 5.563191 (lr=3.2596e-06) (hash(x)=151879548)
-17400 val loss 5.6068
-17400 val perplexity 272.2817
-17400 train 5.529061 (lr=3.2524e-06) (hash(x)=143641792)
-17499 val loss 5.6043
-17499 val perplexity 271.5879
+0 train 11.295413 (lr=1.5000e-07) (hash(x)=150724848)
+100 val loss 9.9471
+100 val perplexity 20892.3984
+100 train 9.995423 (lr=1.5150e-05) (hash(x)=149217061)
+200 val loss 8.9068
+200 val perplexity 7381.7563
+200 train 8.924874 (lr=3.0000e-05) (hash(x)=149910534)
+300 val loss 7.8970
+300 val perplexity 2689.2625
+300 train 7.817223 (lr=2.9998e-05) (hash(x)=142185643)
+400 val loss 7.6460
+400 val perplexity 2092.2935
+400 train 7.695558 (lr=2.9991e-05) (hash(x)=148123706)
+500 val loss 7.5708
+500 val perplexity 1940.7567
+500 train 7.445272 (lr=2.9980e-05) (hash(x)=149952383)
+600 val loss 7.5152
+600 val perplexity 1835.7777
+600 train 7.335600 (lr=2.9964e-05) (hash(x)=146678221)
+700 val loss 7.4498
+700 val perplexity 1719.4503
+700 train 7.483349 (lr=2.9944e-05) (hash(x)=156180736)
+800 val loss 7.4000
+800 val perplexity 1635.9183
+800 train 7.391922 (lr=2.9920e-05) (hash(x)=151700982)
+900 val loss 7.3614
+900 val perplexity 1574.0306
+900 train 7.154299 (lr=2.9891e-05) (hash(x)=147288467)
+1000 val loss 7.3255
+1000 val perplexity 1518.5808
+1000 train 7.366494 (lr=2.9858e-05) (hash(x)=156182087)
+1100 val loss 7.2923
+1100 val perplexity 1468.8792
+1100 train 7.162993 (lr=2.9820e-05) (hash(x)=147861550)
+1200 val loss 7.2558
+1200 val perplexity 1416.3641
+1200 train 7.121205 (lr=2.9778e-05) (hash(x)=149318660)
+1300 val loss 7.2265
+1300 val perplexity 1375.4232
+1300 train 7.061676 (lr=2.9732e-05) (hash(x)=151631103)
+1400 val loss 7.1858
+1400 val perplexity 1320.5341
+1400 train 7.228111 (lr=2.9681e-05) (hash(x)=150482428)
+1500 val loss 7.1244
+1500 val perplexity 1241.9030
+1500 train 6.974311 (lr=2.9626e-05) (hash(x)=144458617)
+1600 val loss 7.0693
+1600 val perplexity 1175.3258
+1600 train 6.943380 (lr=2.9566e-05) (hash(x)=143268605)
+1700 val loss 7.0222
+1700 val perplexity 1121.2941
+1700 train 6.952559 (lr=2.9502e-05) (hash(x)=150817011)
+1800 val loss 6.9806
+1800 val perplexity 1075.6172
+1800 train 6.963822 (lr=2.9434e-05) (hash(x)=152322423)
+1900 val loss 6.9419
+1900 val perplexity 1034.7129
+1900 train 6.733173 (lr=2.9362e-05) (hash(x)=143574126)
+2000 val loss 6.9049
+2000 val perplexity 997.1170
+2000 train 6.658395 (lr=2.9285e-05) (hash(x)=147904298)
+2100 val loss 6.8658
+2100 val perplexity 958.9514
+2100 train 6.606602 (lr=2.9204e-05) (hash(x)=144719400)
+2200 val loss 6.8183
+2200 val perplexity 914.4473
+2200 train 7.060718 (lr=2.9119e-05) (hash(x)=154343147)
+2300 val loss 6.7630
+2300 val perplexity 865.2518
+2300 train 6.803497 (lr=2.9030e-05) (hash(x)=154372635)
+2400 val loss 6.7219
+2400 val perplexity 830.3688
+2400 train 6.548813 (lr=2.8937e-05) (hash(x)=141843115)
+2500 val loss 6.6719
+2500 val perplexity 789.8730
+2500 train 6.569248 (lr=2.8840e-05) (hash(x)=149417679)
+2600 val loss 6.6304
+2600 val perplexity 757.7546
+2600 train 6.501026 (lr=2.8738e-05) (hash(x)=145279030)
+2700 val loss 6.5949
+2700 val perplexity 731.3845
+2700 train 6.507457 (lr=2.8633e-05) (hash(x)=146363911)
+2800 val loss 6.5743
+2800 val perplexity 716.4705
+2800 train 6.523441 (lr=2.8523e-05) (hash(x)=152507639)
+2900 val loss 6.5356
+2900 val perplexity 689.2171
+2900 train 6.322543 (lr=2.8410e-05) (hash(x)=139296191)
+3000 val loss 6.5133
+3000 val perplexity 674.0377
+3000 train 6.520536 (lr=2.8292e-05) (hash(x)=148473774)
+3100 val loss 6.4626
+3100 val perplexity 640.7552
+3100 train 6.553386 (lr=2.8171e-05) (hash(x)=154130527)
+3200 val loss 6.4329
+3200 val perplexity 621.9801
+3200 train 6.385654 (lr=2.8046e-05) (hash(x)=151117002)
+3300 val loss 6.4235
+3300 val perplexity 616.1671
+3300 train 6.582705 (lr=2.7917e-05) (hash(x)=151035383)
+3400 val loss 6.3875
+3400 val perplexity 594.3979
+3400 train 6.521036 (lr=2.7784e-05) (hash(x)=150614749)
+3500 val loss 6.3695
+3500 val perplexity 583.7698
+3500 train 6.374157 (lr=2.7648e-05) (hash(x)=151431391)
+3600 val loss 6.3467
+3600 val perplexity 570.5809
+3600 train 6.277895 (lr=2.7508e-05) (hash(x)=151140797)
+3700 val loss 6.3221
+3700 val perplexity 556.7333
+3700 train 6.505704 (lr=2.7364e-05) (hash(x)=154608340)
+3800 val loss 6.3033
+3800 val perplexity 546.3527
+3800 train 6.334675 (lr=2.7216e-05) (hash(x)=169641309)
+3900 val loss 6.2860
+3900 val perplexity 537.0226
+3900 train 6.250165 (lr=2.7066e-05) (hash(x)=150592479)
+4000 val loss 6.2703
+4000 val perplexity 528.6576
+4000 train 6.258526 (lr=2.6911e-05) (hash(x)=155495317)
+4100 val loss 6.2547
+4100 val perplexity 520.4630
+4100 train 6.449267 (lr=2.6754e-05) (hash(x)=170323970)
+4200 val loss 6.2507
+4200 val perplexity 518.3842
+4200 train 6.087318 (lr=2.6592e-05) (hash(x)=140854360)
+4300 val loss 6.2253
+4300 val perplexity 505.3736
+4300 train 6.244441 (lr=2.6428e-05) (hash(x)=148926644)
+4400 val loss 6.2046
+4400 val perplexity 495.0300
+4400 train 6.239708 (lr=2.6260e-05) (hash(x)=149997871)
+4500 val loss 6.1887
+4500 val perplexity 487.2027
+4500 train 6.140278 (lr=2.6089e-05) (hash(x)=149219289)
+4600 val loss 6.1760
+4600 val perplexity 481.0439
+4600 train 5.976910 (lr=2.5915e-05) (hash(x)=139836197)
+4700 val loss 6.1692
+4700 val perplexity 477.7905
+4700 train 6.123997 (lr=2.5738e-05) (hash(x)=150722883)
+4800 val loss 6.1552
+4800 val perplexity 471.1728
+4800 train 6.370835 (lr=2.5558e-05) (hash(x)=166864648)
+4900 val loss 6.1406
+4900 val perplexity 464.3523
+4900 train 6.248538 (lr=2.5374e-05) (hash(x)=157746743)
+5000 val loss 6.1230
+5000 val perplexity 456.2238
+5000 train 6.070064 (lr=2.5188e-05) (hash(x)=153032257)
+5100 val loss 6.1162
+5100 val perplexity 453.1526
+5100 train 6.186953 (lr=2.4999e-05) (hash(x)=151862371)
+5200 val loss 6.1029
+5200 val perplexity 447.1548
+5200 train 6.414737 (lr=2.4807e-05) (hash(x)=168862995)
+5300 val loss 6.1030
+5300 val perplexity 447.1857
+5300 train 6.067278 (lr=2.4613e-05) (hash(x)=148243567)
+5400 val loss 6.0834
+5400 val perplexity 438.5384
+5400 train 6.013880 (lr=2.4415e-05) (hash(x)=145381742)
+5500 val loss 6.0749
+5500 val perplexity 434.8040
+5500 train 6.234234 (lr=2.4215e-05) (hash(x)=155602174)
+5600 val loss 6.0558
+5600 val perplexity 426.5808
+5600 train 6.131358 (lr=2.4013e-05) (hash(x)=160637672)
+5700 val loss 6.0488
+5700 val perplexity 423.5999
+5700 train 5.978982 (lr=2.3808e-05) (hash(x)=152101131)
+5800 val loss 6.0364
+5800 val perplexity 418.3911
+5800 train 6.002306 (lr=2.3601e-05) (hash(x)=152327089)
+5900 val loss 6.0371
+5900 val perplexity 418.6625
+5900 train 5.995678 (lr=2.3391e-05) (hash(x)=151094543)
+6000 val loss 6.0246
+6000 val perplexity 413.4569
+6000 train 6.119165 (lr=2.3179e-05) (hash(x)=160233315)
+6100 val loss 6.0102
+6100 val perplexity 407.5835
+6100 train 6.113656 (lr=2.2965e-05) (hash(x)=158295337)
+6200 val loss 5.9963
+6200 val perplexity 401.9257
+6200 train 5.958003 (lr=2.2749e-05) (hash(x)=151269817)
+6300 val loss 5.9892
+6300 val perplexity 399.1044
+6300 train 6.144239 (lr=2.2530e-05) (hash(x)=156912756)
+6400 val loss 5.9820
+6400 val perplexity 396.2269
+6400 train 5.911047 (lr=2.2310e-05) (hash(x)=125779637)
+6500 val loss 5.9787
+6500 val perplexity 394.9186
+6500 train 5.875282 (lr=2.2088e-05) (hash(x)=148157661)
+6600 val loss 5.9746
+6600 val perplexity 393.2974
+6600 train 5.987086 (lr=2.1864e-05) (hash(x)=152820049)
+6700 val loss 5.9633
+6700 val perplexity 388.8889
+6700 train 5.899116 (lr=2.1638e-05) (hash(x)=155818717)
+6800 val loss 5.9481
+6800 val perplexity 383.0392
+6800 train 5.957938 (lr=2.1410e-05) (hash(x)=158833881)
+6900 val loss 5.9493
+6900 val perplexity 383.4933
+6900 train 5.839056 (lr=2.1181e-05) (hash(x)=164984909)
+7000 val loss 5.9369
+7000 val perplexity 378.7484
+7000 train 5.838758 (lr=2.0951e-05) (hash(x)=151172493)
+7100 val loss 5.9317
+7100 val perplexity 376.7919
+7100 train 5.992643 (lr=2.0718e-05) (hash(x)=154023633)
+7200 val loss 5.9226
+7200 val perplexity 373.3749
+7200 train 5.773759 (lr=2.0485e-05) (hash(x)=139351250)
+7300 val loss 5.9178
+7300 val perplexity 371.5756
+7300 train 5.878859 (lr=2.0250e-05) (hash(x)=158719458)
+7400 val loss 5.9039
+7400 val perplexity 366.4463
+7400 train 5.980419 (lr=2.0014e-05) (hash(x)=162250810)
+7500 val loss 5.8974
+7500 val perplexity 364.0830
+7500 train 5.782121 (lr=1.9777e-05) (hash(x)=142827557)
+7600 val loss 5.8903
+7600 val perplexity 361.5138
+7600 train 5.795710 (lr=1.9538e-05) (hash(x)=147830801)
+7700 val loss 5.8812
+7700 val perplexity 358.2511
+7700 train 5.979882 (lr=1.9299e-05) (hash(x)=165233314)
+7800 val loss 5.8823
+7800 val perplexity 358.6237
+7800 train 5.873540 (lr=1.9059e-05) (hash(x)=160654717)
+7900 val loss 5.8770
+7900 val perplexity 356.7522
+7900 train 5.746340 (lr=1.8817e-05) (hash(x)=140712586)
+8000 val loss 5.8659
+8000 val perplexity 352.8039
+8000 train 6.036586 (lr=1.8576e-05) (hash(x)=150309061)
+8100 val loss 5.8587
+8100 val perplexity 350.2709
+8100 train 5.856085 (lr=1.8333e-05) (hash(x)=158805466)
+8200 val loss 5.8494
+8200 val perplexity 347.0211
+8200 train 5.742389 (lr=1.8090e-05) (hash(x)=142814078)
+8300 val loss 5.8445
+8300 val perplexity 345.3150
+8300 train 5.809830 (lr=1.7846e-05) (hash(x)=148340417)
+8400 val loss 5.8435
+8400 val perplexity 344.9737
+8400 train 5.779657 (lr=1.7602e-05) (hash(x)=157132639)
+8500 val loss 5.8382
+8500 val perplexity 343.1600
+8500 train 5.765084 (lr=1.7357e-05) (hash(x)=152994240)
+8600 val loss 5.8261
+8600 val perplexity 339.0495
+8600 train 5.788753 (lr=1.7113e-05) (hash(x)=152210154)
+8700 val loss 5.8198
+8700 val perplexity 336.9041
+8700 train 5.916071 (lr=1.6868e-05) (hash(x)=149631247)
+8800 val loss 5.8151
+8800 val perplexity 335.3272
+8800 train 5.684615 (lr=1.6623e-05) (hash(x)=145617784)
+8900 val loss 5.8101
+8900 val perplexity 333.6368
+8900 train 5.917306 (lr=1.6377e-05) (hash(x)=155984970)
+9000 val loss 5.8076
+9000 val perplexity 332.8319
+9000 train 5.541238 (lr=1.6132e-05) (hash(x)=143433013)
+9100 val loss 5.8032
+9100 val perplexity 331.3690
+9100 train 5.712577 (lr=1.5887e-05) (hash(x)=152507533)
+9200 val loss 5.7958
+9200 val perplexity 328.9049
+9200 train 5.927247 (lr=1.5643e-05) (hash(x)=162400650)
+9300 val loss 5.7907
+9300 val perplexity 327.2459
+9300 train 5.836190 (lr=1.5398e-05) (hash(x)=148419068)
+9400 val loss 5.7857
+9400 val perplexity 325.6125
+9400 train 5.751761 (lr=1.5154e-05) (hash(x)=145025169)
+9500 val loss 5.7782
+9500 val perplexity 323.1750
+9500 train 5.749483 (lr=1.4910e-05) (hash(x)=145104235)
+9600 val loss 5.7757
+9600 val perplexity 322.3782
+9600 train 5.712863 (lr=1.4667e-05) (hash(x)=150499797)
+9700 val loss 5.7716
+9700 val perplexity 321.0546
+9700 train 5.580113 (lr=1.4424e-05) (hash(x)=139690022)
+9800 val loss 5.7661
+9800 val perplexity 319.3048
+9800 train 5.768727 (lr=1.4183e-05) (hash(x)=145645279)
+9900 val loss 5.7569
+9900 val perplexity 316.3550
+9900 train 5.687554 (lr=1.3941e-05) (hash(x)=150103432)
+10000 val loss 5.7571
+10000 val perplexity 316.4407
+10000 train 5.912628 (lr=1.3701e-05) (hash(x)=162309473)
+10100 val loss 5.7508
+10100 val perplexity 314.4393
+10100 train 5.638128 (lr=1.3462e-05) (hash(x)=146586772)
+10200 val loss 5.7462
+10200 val perplexity 312.9856
+10200 train 5.645834 (lr=1.3223e-05) (hash(x)=149256313)
+10300 val loss 5.7423
+10300 val perplexity 311.7654
+10300 train 5.703022 (lr=1.2986e-05) (hash(x)=151455083)
+10400 val loss 5.7415
+10400 val perplexity 311.5248
+10400 train 5.517107 (lr=1.2750e-05) (hash(x)=150210444)
+10500 val loss 5.7448
+10500 val perplexity 312.5583
+10500 train 5.516900 (lr=1.2515e-05) (hash(x)=152144428)
+10600 val loss 5.7347
+10600 val perplexity 309.4212
+10600 train 5.781371 (lr=1.2282e-05) (hash(x)=168628935)
+10700 val loss 5.7286
+10700 val perplexity 307.5286
+10700 train 5.520483 (lr=1.2049e-05) (hash(x)=164667785)
+10800 val loss 5.7195
+10800 val perplexity 304.7384
+10800 train 5.695211 (lr=1.1819e-05) (hash(x)=143621491)
+10900 val loss 5.7186
+10900 val perplexity 304.4742
+10900 train 5.612442 (lr=1.1590e-05) (hash(x)=143914826)
+11000 val loss 5.7168
+11000 val perplexity 303.9456
+11000 train 5.496383 (lr=1.1362e-05) (hash(x)=137663435)
+11100 val loss 5.7156
+11100 val perplexity 303.5701
+11100 train 5.597968 (lr=1.1136e-05) (hash(x)=149054007)
+11200 val loss 5.7137
+11200 val perplexity 302.9761
+11200 train 5.722148 (lr=1.0912e-05) (hash(x)=153473924)
+11300 val loss 5.7115
+11300 val perplexity 302.3359
+11300 train 5.504845 (lr=1.0690e-05) (hash(x)=158563275)
+11400 val loss 5.7127
+11400 val perplexity 302.6973
+11400 train 5.851794 (lr=1.0470e-05) (hash(x)=161016667)
+11500 val loss 5.7064
+11500 val perplexity 300.7805
+11500 train 5.754418 (lr=1.0251e-05) (hash(x)=157344822)
+11600 val loss 5.6957
+11600 val perplexity 297.5784
+11600 train 5.589668 (lr=1.0035e-05) (hash(x)=139977713)
+11700 val loss 5.6905
+11700 val perplexity 296.0485
+11700 train 5.684609 (lr=9.8209e-06) (hash(x)=156511738)
+11800 val loss 5.6911
+11800 val perplexity 296.2219
+11800 train 6.142660 (lr=9.6090e-06) (hash(x)=185372558)
+11900 val loss 5.6868
+11900 val perplexity 294.9570
+11900 train 5.536606 (lr=9.3993e-06) (hash(x)=138085470)
+12000 val loss 5.6874
+12000 val perplexity 295.1362
+12000 train 5.535735 (lr=9.1920e-06) (hash(x)=142928235)
+12100 val loss 5.6871
+12100 val perplexity 295.0487
+12100 train 5.533151 (lr=8.9871e-06) (hash(x)=151960396)
+12200 val loss 5.6783
+12200 val perplexity 292.4572
+12200 train 5.732773 (lr=8.7846e-06) (hash(x)=156794523)
+12300 val loss 5.6751
+12300 val perplexity 291.5198
+12300 train 5.482319 (lr=8.5847e-06) (hash(x)=138082358)
+12400 val loss 5.6712
+12400 val perplexity 290.3687
+12400 train 5.574787 (lr=8.3875e-06) (hash(x)=155255899)
+12500 val loss 5.6670
+12500 val perplexity 289.1582
+12500 train 5.633625 (lr=8.1929e-06) (hash(x)=150490528)
+12600 val loss 5.6654
+12600 val perplexity 288.7173
+12600 train 5.813358 (lr=8.0010e-06) (hash(x)=154831027)
+12700 val loss 5.6657
+12700 val perplexity 288.7786
+12700 train 5.622353 (lr=7.8119e-06) (hash(x)=147341052)
+12800 val loss 5.6650
+12800 val perplexity 288.5848
+12800 train 5.699083 (lr=7.6257e-06) (hash(x)=159333795)
+12900 val loss 5.6663
+12900 val perplexity 288.9516
+12900 train 5.509811 (lr=7.4425e-06) (hash(x)=143896867)
+13000 val loss 5.6566
+13000 val perplexity 286.1717
+13000 train 5.669372 (lr=7.2622e-06) (hash(x)=147161537)
+13100 val loss 5.6542
+13100 val perplexity 285.4856
+13100 train 5.788258 (lr=7.0850e-06) (hash(x)=147081918)
+13200 val loss 5.6536
+13200 val perplexity 285.3229
+13200 train 5.424483 (lr=6.9108e-06) (hash(x)=145079287)
+13300 val loss 5.6523
+13300 val perplexity 284.9567
+13300 train 5.450130 (lr=6.7398e-06) (hash(x)=142515041)
+13400 val loss 5.6506
+13400 val perplexity 284.4539
+13400 train 5.442078 (lr=6.5721e-06) (hash(x)=144081985)
+13500 val loss 5.6479
+13500 val perplexity 283.7066
+13500 train 5.731144 (lr=6.4076e-06) (hash(x)=149957292)
+13600 val loss 5.6435
+13600 val perplexity 282.4502
+13600 train 5.672267 (lr=6.2465e-06) (hash(x)=157123033)
+13700 val loss 5.6416
+13700 val perplexity 281.9233
+13700 train 5.625963 (lr=6.0887e-06) (hash(x)=148277247)
+13800 val loss 5.6432
+13800 val perplexity 282.3522
+13800 train 5.776651 (lr=5.9344e-06) (hash(x)=142769907)
+13900 val loss 5.6362
+13900 val perplexity 280.4048
+13900 train 5.587573 (lr=5.7835e-06) (hash(x)=144932004)
+14000 val loss 5.6349
+14000 val perplexity 280.0245
+14000 train 5.657510 (lr=5.6362e-06) (hash(x)=149532265)
+14100 val loss 5.6347
+14100 val perplexity 279.9716
+14100 train 5.452249 (lr=5.4925e-06) (hash(x)=149301374)
+14200 val loss 5.6345
+14200 val perplexity 279.9138
+14200 train 5.374617 (lr=5.3524e-06) (hash(x)=142871075)
+14300 val loss 5.6303
+14300 val perplexity 278.7531
+14300 train 5.782886 (lr=5.2159e-06) (hash(x)=166744960)
+14400 val loss 5.6266
+14400 val perplexity 277.7065
+14400 train 5.592690 (lr=5.0832e-06) (hash(x)=145950920)
+14500 val loss 5.6249
+14500 val perplexity 277.2531
+14500 train 5.620202 (lr=4.9543e-06) (hash(x)=141089155)
+14600 val loss 5.6226
+14600 val perplexity 276.6090
+14600 train 5.778128 (lr=4.8291e-06) (hash(x)=174516275)
+14700 val loss 5.6212
+14700 val perplexity 276.2231
+14700 train 5.430392 (lr=4.7079e-06) (hash(x)=137896065)
+14800 val loss 5.6209
+14800 val perplexity 276.1483
+14800 train 5.662985 (lr=4.5905e-06) (hash(x)=154396556)
+14900 val loss 5.6214
+14900 val perplexity 276.2749
+14900 train 5.470448 (lr=4.4770e-06) (hash(x)=149322394)
+15000 val loss 5.6214
+15000 val perplexity 276.2655
+15000 train 5.537586 (lr=4.3675e-06) (hash(x)=147410513)
+15100 val loss 5.6189
+15100 val perplexity 275.5881
+15100 train 5.530842 (lr=4.2620e-06) (hash(x)=145245634)
+15200 val loss 5.6146
+15200 val perplexity 274.4064
+15200 train 5.703465 (lr=4.1605e-06) (hash(x)=148341108)
+15300 val loss 5.6130
+15300 val perplexity 273.9719
+15300 train 5.423482 (lr=4.0631e-06) (hash(x)=140162437)
+15400 val loss 5.6127
+15400 val perplexity 273.8959
+15400 train 5.822367 (lr=3.9698e-06) (hash(x)=142166081)
+15500 val loss 5.6111
+15500 val perplexity 273.4330
+15500 train 5.532878 (lr=3.8806e-06) (hash(x)=141042196)
+15600 val loss 5.6105
+15600 val perplexity 273.2874
+15600 train 5.737755 (lr=3.7956e-06) (hash(x)=148672701)
+15700 val loss 5.6109
+15700 val perplexity 273.3769
+15700 train 5.517663 (lr=3.7148e-06) (hash(x)=149672385)
+15800 val loss 5.6109
+15800 val perplexity 273.4001
+15800 train 5.636478 (lr=3.6382e-06) (hash(x)=152379562)
+15900 val loss 5.6063
+15900 val perplexity 272.1482
+15900 train 5.659053 (lr=3.5658e-06) (hash(x)=154674265)
+16000 val loss 5.6055
+16000 val perplexity 271.9062
+16000 train 5.651720 (lr=3.4977e-06) (hash(x)=150368078)
+16100 val loss 5.6030
+16100 val perplexity 271.2387
+16100 train 5.712838 (lr=3.4339e-06) (hash(x)=160390561)
+16200 val loss 5.6025
+16200 val perplexity 271.1014
+16200 train 5.973002 (lr=3.3744e-06) (hash(x)=155218378)
+16300 val loss 5.6012
+16300 val perplexity 270.7619
+16300 train 5.792291 (lr=3.3193e-06) (hash(x)=160891077)
+16400 val loss 5.6012
+16400 val perplexity 270.7604
+16400 train 5.501259 (lr=3.2684e-06) (hash(x)=142789253)
+16500 val loss 5.6012
+16500 val perplexity 270.7393
+16500 train 5.344845 (lr=3.2220e-06) (hash(x)=141497841)
+16600 val loss 5.6001
+16600 val perplexity 270.4425
+16600 train 5.567000 (lr=3.1799e-06) (hash(x)=157947018)
+16700 val loss 5.5973
+16700 val perplexity 269.7056
+16700 train 5.557473 (lr=3.1422e-06) (hash(x)=146039309)
+16800 val loss 5.5967
+16800 val perplexity 269.5226
+16800 train 5.564966 (lr=3.1089e-06) (hash(x)=156381445)
+16900 val loss 5.5951
+16900 val perplexity 269.1154
+16900 train 5.553369 (lr=3.0801e-06) (hash(x)=146537354)
+17000 val loss 5.5950
+17000 val perplexity 269.0740
+17000 train 5.522299 (lr=3.0556e-06) (hash(x)=135939444)
+17100 val loss 5.5942
+17100 val perplexity 268.8705
+17100 train 5.673860 (lr=3.0356e-06) (hash(x)=147819652)
+17200 val loss 5.5945
+17200 val perplexity 268.9415
+17200 train 5.685050 (lr=3.0200e-06) (hash(x)=155307909)
+17300 val loss 5.5914
+17300 val perplexity 268.1093
+17300 train 5.543001 (lr=3.0089e-06) (hash(x)=151879548)
+17400 val loss 5.5902
+17400 val perplexity 267.7778
+17400 train 5.508076 (lr=3.0022e-06) (hash(x)=143641792)
+17499 val loss 5.5892
+17499 val perplexity 267.5307

attention_kindselective_n_heads4_seed1339/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7af9fcbd1777940217e2ef7c944b0eda83764a7fd4dd1c87b97a28725052564
+oid sha256:8b44ef25da5cba733c02f2c04a51eaa2be2a2661a5325c75a45021d78f5a41de
 size 92843394

attention_kindselective_n_heads4_seed1339/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dfb3e48dec60661f0ba81a025ac7df2e450c39874152bc7fab11918abe8d4445
+oid sha256:b97dd8e4d328d85b60f03892bc042e1207bccec1f9f8130b2224e78a1a36fa23
 size 179406214