Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_8/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_8", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.05e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.05e-5_30720_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1339", "n_embd": 256}

attention_kindselective_n_heads4_seed1339/log2.txt

@@ -1,1055 +1,528 @@
 max_steps: 17500
 0 val loss 11.2810
 0 val perplexity 79298.4922
-0 val loss 11.2810
-0 val perplexity 79298.4922
-0 train 11.295413 (lr=1.5250e-07) (hash(x)=150724848)
-0 train 11.295413 (lr=1.4750e-07) (hash(x)=150724848)
-100 val loss 9.9403
-100 val perplexity 20750.7012
-100 train 9.987446 (lr=1.5403e-05) (hash(x)=149217061)
-100 val loss 9.9502
-100 val perplexity 20957.0137
-100 train 9.998073 (lr=1.4897e-05) (hash(x)=149217061)
-200 val loss 8.8749
-200 val perplexity 7150.1895
-200 train 8.892592 (lr=3.0500e-05) (hash(x)=149910534)
-200 val loss 9.0551
-200 val perplexity 8561.7783
-200 train 9.073477 (lr=2.9500e-05) (hash(x)=149910534)
-300 val loss 7.9272
-300 val perplexity 2771.6230
-300 train 7.850251 (lr=3.0498e-05) (hash(x)=142185643)
-300 val loss 7.9674
-300 val perplexity 2885.3694
-300 train 7.893691 (lr=2.9498e-05) (hash(x)=142185643)
-400 val loss 7.6951
-400 val perplexity 2197.5981
-400 train 7.746935 (lr=3.0491e-05) (hash(x)=148123706)
-400 val loss 7.6583
-400 val perplexity 2118.0957
-400 train 7.707627 (lr=2.9491e-05) (hash(x)=148123706)
-500 val loss 7.6221
-500 val perplexity 2042.7772
-500 train 7.506680 (lr=3.0480e-05) (hash(x)=149952383)
-500 val loss 7.5616
-500 val perplexity 1922.9458
-500 train 7.434763 (lr=2.9480e-05) (hash(x)=149952383)
-600 val loss 7.5631
-600 val perplexity 1925.8895
-600 train 7.386652 (lr=3.0464e-05) (hash(x)=146678221)
-600 val loss 7.4957
-600 val perplexity 1800.2955
-600 train 7.310193 (lr=2.9465e-05) (hash(x)=146678221)
-700 val loss 7.4984
-700 val perplexity 1805.0670
-700 train 7.530549 (lr=3.0443e-05) (hash(x)=156180736)
-700 val loss 7.4195
-700 val perplexity 1668.1918
-700 train 7.451057 (lr=2.9445e-05) (hash(x)=156180736)
-800 val loss 7.4494
-800 val perplexity 1718.8281
-800 train 7.440709 (lr=3.0419e-05) (hash(x)=151700982)
-800 val loss 7.3709
-800 val perplexity 1589.0681
-800 train 7.360362 (lr=2.9421e-05) (hash(x)=151700982)
-900 val loss 7.4097
-900 val perplexity 1651.8865
-900 train 7.204133 (lr=3.0389e-05) (hash(x)=147288467)
-900 val loss 7.3274
-900 val perplexity 1521.4858
-900 train 7.118642 (lr=2.9393e-05) (hash(x)=147288467)
-1000 val loss 7.3668
-1000 val perplexity 1582.6155
-1000 train 7.410699 (lr=3.0355e-05) (hash(x)=156182087)
-1000 val loss 7.2763
-1000 val perplexity 1445.6278
-1000 train 7.316601 (lr=2.9360e-05) (hash(x)=156182087)
-1100 val loss 7.3387
-1100 val perplexity 1538.7528
-1100 train 7.215382 (lr=3.0317e-05) (hash(x)=147861550)
-1100 val loss 7.2354
-1100 val perplexity 1387.6445
-1100 train 7.098856 (lr=2.9323e-05) (hash(x)=147861550)
-1200 val loss 7.3141
-1200 val perplexity 1501.2789
-1200 train 7.180576 (lr=3.0274e-05) (hash(x)=149318660)
-1200 val loss 7.1835
-1200 val perplexity 1317.5677
-1200 train 7.041782 (lr=2.9282e-05) (hash(x)=149318660)
-1300 val loss 7.2957
-1300 val perplexity 1473.8823
-1300 train 7.141894 (lr=3.0227e-05) (hash(x)=151631103)
-1300 val loss 7.1405
-1300 val perplexity 1262.1147
-1300 train 6.975276 (lr=2.9236e-05) (hash(x)=151631103)
-1400 val loss 7.2642
-1400 val perplexity 1428.1925
-1400 train 7.308764 (lr=3.0175e-05) (hash(x)=150482428)
-1400 val loss 7.0925
-1400 val perplexity 1202.9340
-1400 train 7.130383 (lr=2.9186e-05) (hash(x)=150482428)
-1500 val loss 7.2351
-1500 val perplexity 1387.3217
-1500 train 7.090758 (lr=3.0119e-05) (hash(x)=144458617)
-1500 val loss 7.0293
-1500 val perplexity 1129.2816
-1500 train 6.873682 (lr=2.9132e-05) (hash(x)=144458617)
-1600 val loss 7.2030
-1600 val perplexity 1343.5026
-1600 train 7.076006 (lr=3.0059e-05) (hash(x)=143268605)
-1600 val loss 6.9716
-1600 val perplexity 1065.9727
-1600 train 6.851519 (lr=2.9073e-05) (hash(x)=143268605)
-1700 val loss 7.1762
-1700 val perplexity 1307.9729
-1700 train 7.104409 (lr=2.9994e-05) (hash(x)=150817011)
-1700 val loss 6.9244
-1700 val perplexity 1016.7989
-1700 train 6.861414 (lr=2.9011e-05) (hash(x)=150817011)
-1800 val loss 7.1537
-1800 val perplexity 1278.8501
-1800 train 7.134459 (lr=2.9925e-05) (hash(x)=152322423)
-1800 val loss 6.8768
-1800 val perplexity 969.5588
-1800 train 6.861168 (lr=2.8944e-05) (hash(x)=152322423)
-1900 val loss 7.1213
-1900 val perplexity 1238.1111
-1900 train 6.933501 (lr=2.9851e-05) (hash(x)=143574126)
-1900 val loss 6.8384
-1900 val perplexity 932.9764
-1900 train 6.627684 (lr=2.8872e-05) (hash(x)=143574126)
-2000 val loss 7.1050
-2000 val perplexity 1218.1013
-2000 train 6.867764 (lr=2.9773e-05) (hash(x)=147904298)
-2000 val loss 6.7979
-2000 val perplexity 896.0020
-2000 train 6.545908 (lr=2.8797e-05) (hash(x)=147904298)
-2100 val loss 7.0871
-2100 val perplexity 1196.4924
-2100 train 6.842616 (lr=2.9691e-05) (hash(x)=144719400)
-2100 val loss 6.7609
-2100 val perplexity 863.3840
-2100 train 6.502030 (lr=2.8718e-05) (hash(x)=144719400)
-2200 val loss 7.0635
-2200 val perplexity 1168.5701
-2200 train 7.308959 (lr=2.9605e-05) (hash(x)=154343147)
-2200 val loss 6.7189
-2200 val perplexity 827.9282
-2200 train 6.959268 (lr=2.8634e-05) (hash(x)=154343147)
-2300 val loss 7.0304
-2300 val perplexity 1130.4938
-2300 train 7.076579 (lr=2.9514e-05) (hash(x)=154372635)
-2300 val loss 6.6580
-2300 val perplexity 778.9579
-2300 train 6.695727 (lr=2.8546e-05) (hash(x)=154372635)
-2400 val loss 7.0072
-2400 val perplexity 1104.5450
-2400 train 6.836390 (lr=2.9419e-05) (hash(x)=141843115)
-2400 val loss 6.6249
-2400 val perplexity 753.6154
-2400 train 6.451160 (lr=2.8455e-05) (hash(x)=141843115)
-2500 val loss 6.9812
-2500 val perplexity 1076.2590
-2500 train 6.881341 (lr=2.9320e-05) (hash(x)=149417679)
-2500 val loss 6.5807
-2500 val perplexity 721.0270
-2500 train 6.481295 (lr=2.8359e-05) (hash(x)=149417679)
-2600 val loss 6.9538
-2600 val perplexity 1047.1635
-2600 train 6.819181 (lr=2.9217e-05) (hash(x)=145279030)
-2600 val loss 6.5516
-2600 val perplexity 700.3401
-2600 train 6.423855 (lr=2.8259e-05) (hash(x)=145279030)
-2700 val loss 6.9304
-2700 val perplexity 1022.9283
-2700 train 6.835646 (lr=2.9110e-05) (hash(x)=146363911)
-2700 val loss 6.5250
-2700 val perplexity 681.9528
-2700 train 6.441555 (lr=2.8155e-05) (hash(x)=146363911)
-2800 val loss 6.9177
-2800 val perplexity 1010.0331
-2800 train 6.881118 (lr=2.8998e-05) (hash(x)=152507639)
-2800 val loss 6.5242
-2800 val perplexity 681.4077
-2800 train 6.475305 (lr=2.8048e-05) (hash(x)=152507639)
-2900 val loss 6.8858
-2900 val perplexity 978.2740
-2900 train 6.674061 (lr=2.8883e-05) (hash(x)=139296191)
-2900 val loss 6.4822
-2900 val perplexity 653.3870
-2900 train 6.268300 (lr=2.7936e-05) (hash(x)=139296191)
-3000 val loss 6.8598
-3000 val perplexity 953.1681
-3000 train 6.877350 (lr=2.8764e-05) (hash(x)=148473774)
-3000 val loss 6.4646
-3000 val perplexity 642.0195
-3000 train 6.477748 (lr=2.7821e-05) (hash(x)=148473774)
-3100 val loss 6.8173
-3100 val perplexity 913.4706
-3100 train 6.916183 (lr=2.8640e-05) (hash(x)=154130527)
-3100 val loss 6.4228
-3100 val perplexity 615.7438
-3100 train 6.511337 (lr=2.7701e-05) (hash(x)=154130527)
-3200 val loss 6.7796
-3200 val perplexity 879.7261
-3200 train 6.722135 (lr=2.8513e-05) (hash(x)=151117002)
-3200 val loss 6.3985
-3200 val perplexity 600.9621
-3200 train 6.346514 (lr=2.7578e-05) (hash(x)=151117002)
-3300 val loss 6.7729
-3300 val perplexity 873.8395
-3300 train 6.909571 (lr=2.8382e-05) (hash(x)=151035383)
-3300 val loss 6.3999
-3300 val perplexity 601.8144
-3300 train 6.554747 (lr=2.7452e-05) (hash(x)=151035383)
-3400 val loss 6.7306
-3400 val perplexity 837.6121
-3400 train 6.830935 (lr=2.8247e-05) (hash(x)=150614749)
-3400 val loss 6.3622
-3400 val perplexity 579.5051
-3400 train 6.500711 (lr=2.7321e-05) (hash(x)=150614749)
-3500 val loss 6.7059
-3500 val perplexity 817.2164
-3500 train 6.735514 (lr=2.8108e-05) (hash(x)=151431391)
-3500 val loss 6.3434
-3500 val perplexity 568.7427
-3500 train 6.345742 (lr=2.7187e-05) (hash(x)=151431391)
-3600 val loss 6.6839
-3600 val perplexity 799.4143
-3600 train 6.609454 (lr=2.7966e-05) (hash(x)=151140797)
-3600 val loss 6.3291
-3600 val perplexity 560.6556
-3600 train 6.262310 (lr=2.7049e-05) (hash(x)=151140797)
-3700 val loss 6.6507
-3700 val perplexity 773.3610
-3700 train 6.861124 (lr=2.7820e-05) (hash(x)=154608340)
-3700 val loss 6.3110
-3700 val perplexity 550.6154
-3700 train 6.497699 (lr=2.6908e-05) (hash(x)=154608340)
-3800 val loss 6.6230
-3800 val perplexity 752.2031
-3800 train 6.649333 (lr=2.7670e-05) (hash(x)=169641309)
-3800 val loss 6.2914
-3800 val perplexity 539.9333
-3800 train 6.326932 (lr=2.6763e-05) (hash(x)=169641309)
-3900 val loss 6.5989
-3900 val perplexity 734.2950
-3900 train 6.556187 (lr=2.7517e-05) (hash(x)=150592479)
-3900 val loss 6.2748
-3900 val perplexity 531.0274
-3900 train 6.237079 (lr=2.6615e-05) (hash(x)=150592479)
-4000 val loss 6.5761
-4000 val perplexity 717.7346
-4000 train 6.554400 (lr=2.7360e-05) (hash(x)=155495317)
-4000 val loss 6.2598
-4000 val perplexity 523.1260
-4000 train 6.247159 (lr=2.6463e-05) (hash(x)=155495317)
-4100 val loss 6.5603
-4100 val perplexity 706.4811
-4100 train 6.759033 (lr=2.7199e-05) (hash(x)=170323970)
-4200 val loss 6.5378
-4200 val perplexity 690.7957
-4100 val loss 6.2477
-4100 val perplexity 516.8098
-4200 train 6.356062 (lr=2.7036e-05) (hash(x)=140854360)
-4100 train 6.441728 (lr=2.6308e-05) (hash(x)=170323970)
-4300 val loss 6.5079
-4300 val perplexity 670.4118
-4300 train 6.510032 (lr=2.6868e-05) (hash(x)=148926644)
-4200 val loss 6.2416
-4200 val perplexity 513.6633
-4200 train 6.079424 (lr=2.6149e-05) (hash(x)=140854360)
-4400 val loss 6.4819
-4400 val perplexity 653.2350
-4400 train 6.516395 (lr=2.6698e-05) (hash(x)=149997871)
-4300 val loss 6.2183
-4300 val perplexity 501.8593
-4300 train 6.237810 (lr=2.5987e-05) (hash(x)=148926644)
-4500 val loss 6.4662
-4500 val perplexity 643.0361
-4500 train 6.419517 (lr=2.6524e-05) (hash(x)=149219289)
-4400 val loss 6.2000
-4400 val perplexity 492.7285
-4400 train 6.233898 (lr=2.5822e-05) (hash(x)=149997871)
-4600 val loss 6.4500
-4600 val perplexity 632.6786
-4600 train 6.247700 (lr=2.6347e-05) (hash(x)=139836197)
-4500 val loss 6.1882
-4500 val perplexity 486.9541
-4500 train 6.121775 (lr=2.5654e-05) (hash(x)=149219289)
-4700 val loss 6.4348
-4700 val perplexity 623.1877
-4700 train 6.393597 (lr=2.6167e-05) (hash(x)=150722883)
-4600 val loss 6.1793
-4600 val perplexity 482.6642
-4600 train 5.981081 (lr=2.5483e-05) (hash(x)=139836197)
-4800 val loss 6.4189
-4800 val perplexity 613.3000
-4800 train 6.624498 (lr=2.5983e-05) (hash(x)=166864648)
-4700 val loss 6.1676
-4700 val perplexity 477.0347
-4700 train 6.121356 (lr=2.5309e-05) (hash(x)=150722883)
-4900 val loss 6.3992
-4900 val perplexity 601.3419
-4900 train 6.507027 (lr=2.5797e-05) (hash(x)=157746743)
-4800 val loss 6.1581
-4800 val perplexity 472.5281
-4800 train 6.371707 (lr=2.5132e-05) (hash(x)=166864648)
-5000 val loss 6.3772
-5000 val perplexity 588.2783
-5000 train 6.331150 (lr=2.5608e-05) (hash(x)=153032257)
-4900 val loss 6.1408
-4900 val perplexity 464.4462
-4900 train 6.247256 (lr=2.4951e-05) (hash(x)=157746743)
-5100 val loss 6.3648
-5100 val perplexity 581.0560
-5100 train 6.449167 (lr=2.5416e-05) (hash(x)=151862371)
-5000 val loss 6.1246
-5000 val perplexity 456.9446
-5000 train 6.069416 (lr=2.4768e-05) (hash(x)=153032257)
-5200 val loss 6.3511
-5200 val perplexity 573.1337
-5200 train 6.675462 (lr=2.5221e-05) (hash(x)=168862995)
-5100 val loss 6.1151
-5100 val perplexity 452.6508
-5100 train 6.185919 (lr=2.4582e-05) (hash(x)=151862371)
-5300 val loss 6.3356
-5300 val perplexity 564.3328
-5300 train 6.305876 (lr=2.5023e-05) (hash(x)=148243567)
-5200 val loss 6.1025
-5200 val perplexity 446.9919
-5200 train 6.426899 (lr=2.4394e-05) (hash(x)=168862995)
-5400 val loss 6.3208
-5400 val perplexity 556.0337
-5400 train 6.246565 (lr=2.4822e-05) (hash(x)=145381742)
-5300 val loss 6.1000
-5300 val perplexity 445.8441
-5300 train 6.060615 (lr=2.4202e-05) (hash(x)=148243567)
-5500 val loss 6.3095
-5500 val perplexity 549.7858
-5500 train 6.458880 (lr=2.4619e-05) (hash(x)=155602174)
-5400 val loss 6.0834
-5400 val perplexity 438.5018
-5400 train 6.020972 (lr=2.4008e-05) (hash(x)=145381742)
-5600 val loss 6.2883
-5600 val perplexity 538.2455
-5600 train 6.374960 (lr=2.4413e-05) (hash(x)=160637672)
-5500 val loss 6.0733
-5500 val perplexity 434.1288
-5500 train 6.226335 (lr=2.3812e-05) (hash(x)=155602174)
-5700 val loss 6.2774
-5700 val perplexity 532.4090
-5700 train 6.205206 (lr=2.4205e-05) (hash(x)=152101131)
-5600 val loss 6.0568
-5600 val perplexity 427.0204
-5600 train 6.132084 (lr=2.3613e-05) (hash(x)=160637672)
-5800 val loss 6.2628
-5800 val perplexity 524.7109
-5800 train 6.217251 (lr=2.3994e-05) (hash(x)=152327089)
-5700 val loss 6.0494
-5700 val perplexity 423.8673
-5700 train 5.990386 (lr=2.3411e-05) (hash(x)=152101131)
-5900 val loss 6.2588
-5900 val perplexity 522.5915
-5900 train 6.198963 (lr=2.3781e-05) (hash(x)=151094543)
-5800 val loss 6.0375
-5800 val perplexity 418.8238
-5800 train 5.995212 (lr=2.3207e-05) (hash(x)=152327089)
-6000 val loss 6.2439
-6000 val perplexity 514.8661
-6000 train 6.343430 (lr=2.3565e-05) (hash(x)=160233315)
-5900 val loss 6.0362
-5900 val perplexity 418.3091
-5900 train 5.992724 (lr=2.3001e-05) (hash(x)=151094543)
-6100 val loss 6.2282
-6100 val perplexity 506.8423
-6100 train 6.328305 (lr=2.3348e-05) (hash(x)=158295337)
-6000 val loss 6.0256
-6000 val perplexity 413.8944
-6000 train 6.122548 (lr=2.2793e-05) (hash(x)=160233315)
-6200 val loss 6.2090
-6200 val perplexity 497.2017
-6200 train 6.173069 (lr=2.3128e-05) (hash(x)=151269817)
-6100 val loss 6.0158
-6100 val perplexity 409.8704
-6100 train 6.121073 (lr=2.2582e-05) (hash(x)=158295337)
-6300 val loss 6.1999
-6300 val perplexity 492.6811
-6300 train 6.335582 (lr=2.2906e-05) (hash(x)=156912756)
-6200 val loss 6.0026
-6200 val perplexity 404.4917
-6200 train 5.960115 (lr=2.2370e-05) (hash(x)=151269817)
-6400 val loss 6.1895
-6400 val perplexity 487.6138
-6400 train 6.121492 (lr=2.2682e-05) (hash(x)=125779637)
-6300 val loss 5.9935
-6300 val perplexity 400.8117
-6300 train 6.154233 (lr=2.2155e-05) (hash(x)=156912756)
-6500 val loss 6.1886
-6500 val perplexity 487.1752
-6500 train 6.085354 (lr=2.2456e-05) (hash(x)=148157661)
-6400 val loss 5.9882
-6400 val perplexity 398.6835
-6400 train 5.904276 (lr=2.1938e-05) (hash(x)=125779637)
-6600 val loss 6.1756
-6600 val perplexity 480.8834
-6600 train 6.213357 (lr=2.2228e-05) (hash(x)=152820049)
-6500 val loss 5.9839
-6500 val perplexity 396.9879
-6500 train 5.880041 (lr=2.1720e-05) (hash(x)=148157661)
-6700 val loss 6.1675
-6700 val perplexity 476.9749
-6700 train 6.110906 (lr=2.1999e-05) (hash(x)=155818717)
-6600 val loss 5.9731
-6600 val perplexity 392.7402
-6600 train 5.984869 (lr=2.1499e-05) (hash(x)=152820049)
-6800 val loss 6.1542
-6800 val perplexity 470.7045
-6800 train 6.195240 (lr=2.1767e-05) (hash(x)=158833881)
-6700 val loss 5.9640
-6700 val perplexity 389.1660
-6700 train 5.901100 (lr=2.1277e-05) (hash(x)=155818717)
-6900 val loss 6.1438
-6900 val perplexity 465.8134
-6900 train 6.073254 (lr=2.1534e-05) (hash(x)=164984909)
-6800 val loss 5.9525
-6800 val perplexity 384.7210
-6800 train 5.968745 (lr=2.1054e-05) (hash(x)=158833881)
-7000 val loss 6.1318
-7000 val perplexity 460.2508
-7000 train 6.027487 (lr=2.1300e-05) (hash(x)=151172493)
-6900 val loss 5.9492
-6900 val perplexity 383.4421
-6900 train 5.839891 (lr=2.0828e-05) (hash(x)=164984909)
-7100 val loss 6.1289
-7100 val perplexity 458.9186
-7100 train 6.169435 (lr=2.1064e-05) (hash(x)=154023633)
-7000 val loss 5.9397
-7000 val perplexity 379.8313
-7000 train 5.839072 (lr=2.0601e-05) (hash(x)=151172493)
-7200 val loss 6.1180
-7200 val perplexity 453.9522
-7200 train 5.968648 (lr=2.0826e-05) (hash(x)=139351250)
-7100 val loss 5.9385
-7100 val perplexity 379.3611
-7100 train 5.993623 (lr=2.0373e-05) (hash(x)=154023633)
-7300 val loss 6.1072
-7300 val perplexity 449.0635
-7300 train 6.059441 (lr=2.0587e-05) (hash(x)=158719458)
-7200 val loss 5.9277
-7200 val perplexity 375.3027
-7200 train 5.775423 (lr=2.0143e-05) (hash(x)=139351250)
-7400 val loss 6.0908
-7400 val perplexity 441.7955
-7400 train 6.171273 (lr=2.0347e-05) (hash(x)=162250810)
-7300 val loss 5.9222
-7300 val perplexity 373.2336
-7500 val loss 6.0842
-7500 val perplexity 438.8884
-7300 train 5.877505 (lr=1.9912e-05) (hash(x)=158719458)
-7500 train 5.957148 (lr=2.0106e-05) (hash(x)=142827557)
-7600 val loss 6.0757
-7600 val perplexity 435.1519
-7600 train 5.972519 (lr=1.9864e-05) (hash(x)=147830801)
-7400 val loss 5.9093
-7400 val perplexity 368.4441
-7400 train 5.981564 (lr=1.9680e-05) (hash(x)=162250810)
-7700 val loss 6.0687
-7700 val perplexity 432.1263
-7700 train 6.169074 (lr=1.9620e-05) (hash(x)=165233314)
-7500 val loss 5.9027
-7500 val perplexity 366.0380
-7500 train 5.781031 (lr=1.9447e-05) (hash(x)=142827557)
-7800 val loss 6.0619
-7800 val perplexity 429.1732
-7800 train 6.059940 (lr=1.9376e-05) (hash(x)=160654717)
-7600 val loss 5.8945
-7600 val perplexity 363.0433
-7600 train 5.797791 (lr=1.9213e-05) (hash(x)=147830801)
-7900 val loss 6.0558
-7900 val perplexity 426.5851
-7900 train 5.906787 (lr=1.9131e-05) (hash(x)=140712586)
-7700 val loss 5.8903
-7700 val perplexity 361.5111
-7700 train 5.995757 (lr=1.8977e-05) (hash(x)=165233314)
-8000 val loss 6.0414
-8000 val perplexity 420.5001
-8000 train 6.193912 (lr=1.8885e-05) (hash(x)=150309061)
-7800 val loss 5.8844
-7800 val perplexity 359.4033
-7800 train 5.873414 (lr=1.8741e-05) (hash(x)=160654717)
-8100 val loss 6.0381
-8100 val perplexity 419.0957
-8100 train 6.036276 (lr=1.8639e-05) (hash(x)=158805466)
-7900 val loss 5.8815
-7900 val perplexity 358.3615
-7900 train 5.745528 (lr=1.8504e-05) (hash(x)=140712586)
-8200 val loss 6.0279
-8200 val perplexity 414.8612
-8200 train 5.903849 (lr=1.8391e-05) (hash(x)=142814078)
-8000 val loss 5.8686
-8000 val perplexity 353.7522
-8000 train 6.030416 (lr=1.8266e-05) (hash(x)=150309061)
-8300 val loss 6.0211
-8300 val perplexity 412.0363
-8300 train 5.978704 (lr=1.8144e-05) (hash(x)=148340417)
-8100 val loss 5.8694
-8100 val perplexity 354.0227
-8100 train 5.860316 (lr=1.8027e-05) (hash(x)=158805466)
-8400 val loss 6.0145
-8400 val perplexity 409.3272
-8400 train 5.958896 (lr=1.7895e-05) (hash(x)=157132639)
-8200 val loss 5.8587
-8200 val perplexity 350.2765
-8200 train 5.747078 (lr=1.7788e-05) (hash(x)=142814078)
-8500 val loss 6.0134
-8500 val perplexity 408.8716
-8500 train 5.949043 (lr=1.7647e-05) (hash(x)=152994240)
-8300 val loss 5.8505
-8300 val perplexity 347.4053
-8300 train 5.815259 (lr=1.7549e-05) (hash(x)=148340417)
-8600 val loss 6.0015
-8600 val perplexity 404.0154
-8600 train 5.958991 (lr=1.7398e-05) (hash(x)=152210154)
-8400 val loss 5.8496
-8400 val perplexity 347.0857
-8400 train 5.783575 (lr=1.7309e-05) (hash(x)=157132639)
-8700 val loss 5.9926
-8700 val perplexity 400.4430
-8700 train 6.062593 (lr=1.7149e-05) (hash(x)=149631247)
-8500 val loss 5.8444
-8500 val perplexity 345.2999
-8500 train 5.779281 (lr=1.7068e-05) (hash(x)=152994240)
-8800 val loss 5.9882
-8800 val perplexity 398.7137
-8800 train 5.852299 (lr=1.6900e-05) (hash(x)=145617784)
-8600 val loss 5.8322
-8600 val perplexity 341.0950
-8600 train 5.794682 (lr=1.6827e-05) (hash(x)=152210154)
-8900 val loss 5.9829
-8900 val perplexity 396.6070
-8900 train 6.080505 (lr=1.6650e-05) (hash(x)=155984970)
-8700 val loss 5.8267
-8700 val perplexity 339.2507
-8700 train 5.919361 (lr=1.6587e-05) (hash(x)=149631247)
-9000 val loss 5.9800
-9000 val perplexity 395.4430
-9000 train 5.705452 (lr=1.6401e-05) (hash(x)=143433013)
-8800 val loss 5.8224
-8800 val perplexity 337.7725
-8800 train 5.695291 (lr=1.6346e-05) (hash(x)=145617784)
-9100 val loss 5.9749
-9100 val perplexity 393.4253
-9100 train 5.867756 (lr=1.6152e-05) (hash(x)=152507533)
-8900 val loss 5.8176
-8900 val perplexity 336.1762
-8900 train 5.918642 (lr=1.6104e-05) (hash(x)=155984970)
-9200 val loss 5.9658
-9200 val perplexity 389.8586
-9200 train 6.090382 (lr=1.5903e-05) (hash(x)=162400650)
-9000 val loss 5.8160
-9000 val perplexity 335.6179
-9000 train 5.550464 (lr=1.5863e-05) (hash(x)=143433013)
-9300 val loss 5.9627
-9300 val perplexity 388.6410
-9300 train 6.004210 (lr=1.5655e-05) (hash(x)=148419068)
-9100 val loss 5.8112
-9100 val perplexity 334.0226
-9100 train 5.715689 (lr=1.5623e-05) (hash(x)=152507533)
-9400 val loss 5.9547
-9400 val perplexity 385.5671
-9400 train 5.915373 (lr=1.5406e-05) (hash(x)=145025169)
-9200 val loss 5.8015
-9200 val perplexity 330.8046
-9200 train 5.924945 (lr=1.5382e-05) (hash(x)=162400650)
-9500 val loss 5.9468
-9500 val perplexity 382.5239
-9500 train 5.919880 (lr=1.5159e-05) (hash(x)=145104235)
-9300 val loss 5.7962
-9300 val perplexity 329.0339
-9300 train 5.840531 (lr=1.5141e-05) (hash(x)=148419068)
-9600 val loss 5.9431
-9600 val perplexity 381.1049
-9600 train 5.878403 (lr=1.4911e-05) (hash(x)=150499797)
-9400 val loss 5.7905
-9400 val perplexity 327.1928
-9400 train 5.752407 (lr=1.4901e-05) (hash(x)=145025169)
-9700 val loss 5.9373
-9700 val perplexity 378.9088
-9700 train 5.747884 (lr=1.4665e-05) (hash(x)=139690022)
-9500 val loss 5.7826
-9500 val perplexity 324.6118
-9500 train 5.753817 (lr=1.4662e-05) (hash(x)=145104235)
-9800 val loss 5.9307
-9800 val perplexity 376.4139
-9800 train 5.928053 (lr=1.4419e-05) (hash(x)=145645279)
-9600 val loss 5.7810
-9600 val perplexity 324.0819
-9600 train 5.721487 (lr=1.4423e-05) (hash(x)=150499797)
-9900 val loss 5.9231
-9900 val perplexity 373.5862
-9900 train 5.839291 (lr=1.4174e-05) (hash(x)=150103432)
-9700 val loss 5.7778
-9700 val perplexity 323.0622
-9700 train 5.585821 (lr=1.4184e-05) (hash(x)=139690022)
-10000 val loss 5.9204
-10000 val perplexity 372.5609
-10000 train 6.065903 (lr=1.3930e-05) (hash(x)=162309473)
-9800 val loss 5.7695
-9800 val perplexity 320.3742
-9800 train 5.767245 (lr=1.3946e-05) (hash(x)=145645279)
-10100 val loss 5.9139
-10100 val perplexity 370.1530
-10100 train 5.792250 (lr=1.3686e-05) (hash(x)=146586772)
-9900 val loss 5.7635
-9900 val perplexity 318.4589
-9900 train 5.692699 (lr=1.3709e-05) (hash(x)=150103432)
-10200 val loss 5.9105
-10200 val perplexity 368.9043
-10200 train 5.809811 (lr=1.3444e-05) (hash(x)=149256313)
-10000 val loss 5.7607
-10000 val perplexity 317.5830
-10000 train 5.915528 (lr=1.3473e-05) (hash(x)=162309473)
-10300 val loss 5.9035
-10300 val perplexity 366.3214
-10300 train 5.878336 (lr=1.3203e-05) (hash(x)=151455083)
-10100 val loss 5.7560
-10100 val perplexity 316.0764
-10100 train 5.648626 (lr=1.3237e-05) (hash(x)=146586772)
-10400 val loss 5.9043
-10400 val perplexity 366.6013
-10400 train 5.700070 (lr=1.2963e-05) (hash(x)=150210444)
-10200 val loss 5.7540
-10200 val perplexity 315.4593
-10200 train 5.656557 (lr=1.3003e-05) (hash(x)=149256313)
-10500 val loss 5.9076
-10500 val perplexity 367.8090
-10500 train 5.681701 (lr=1.2724e-05) (hash(x)=152144428)
-10300 val loss 5.7478
-10300 val perplexity 313.5120
-10300 train 5.710366 (lr=1.2770e-05) (hash(x)=151455083)
-10600 val loss 5.8944
-10600 val perplexity 362.9845
-10600 train 5.958439 (lr=1.2486e-05) (hash(x)=168628935)
-10400 val loss 5.7462
-10400 val perplexity 313.0086
-10400 train 5.515590 (lr=1.2538e-05) (hash(x)=150210444)
-10700 val loss 5.8878
-10700 val perplexity 360.6103
-10700 train 5.718093 (lr=1.2250e-05) (hash(x)=164667785)
-10500 val loss 5.7502
-10500 val perplexity 314.2516
-10500 train 5.522077 (lr=1.2307e-05) (hash(x)=152144428)
-10800 val loss 5.8794
-10800 val perplexity 357.5911
-10800 train 5.836078 (lr=1.2016e-05) (hash(x)=143621491)
-10600 val loss 5.7377
-10600 val perplexity 310.3452
-10600 train 5.779927 (lr=1.2077e-05) (hash(x)=168628935)
-10900 val loss 5.8775
-10900 val perplexity 356.9198
-10900 train 5.770650 (lr=1.1783e-05) (hash(x)=143914826)
-10700 val loss 5.7320
-10700 val perplexity 308.5768
-10700 train 5.524458 (lr=1.1849e-05) (hash(x)=164667785)
-11000 val loss 5.8748
-11000 val perplexity 355.9376
-11000 train 5.655579 (lr=1.1551e-05) (hash(x)=137663435)
-10800 val loss 5.7242
-10800 val perplexity 306.1740
-10800 train 5.697792 (lr=1.1622e-05) (hash(x)=143621491)
-11100 val loss 5.8723
-11100 val perplexity 355.0747
-11100 train 5.745441 (lr=1.1322e-05) (hash(x)=149054007)
-10900 val loss 5.7216
-10900 val perplexity 305.4018
-11200 val loss 5.8713
-11200 val perplexity 354.6989
-10900 train 5.616990 (lr=1.1396e-05) (hash(x)=143914826)
-11200 train 5.876592 (lr=1.1094e-05) (hash(x)=153473924)
-11300 val loss 5.8698
-11300 val perplexity 354.1694
-11300 train 5.669365 (lr=1.0868e-05) (hash(x)=158563275)
-11000 val loss 5.7216
-11000 val perplexity 305.3811
-11000 train 5.500835 (lr=1.1173e-05) (hash(x)=137663435)
-11400 val loss 5.8635
-11400 val perplexity 351.9668
-11400 train 6.001186 (lr=1.0644e-05) (hash(x)=161016667)
-11100 val loss 5.7197
-11100 val perplexity 304.8250
-11100 train 5.597692 (lr=1.0951e-05) (hash(x)=149054007)
-11500 val loss 5.8606
-11500 val perplexity 350.9504
-11500 train 5.906625 (lr=1.0422e-05) (hash(x)=157344822)
-11200 val loss 5.7178
-11200 val perplexity 304.2396
-11200 train 5.739293 (lr=1.0730e-05) (hash(x)=153473924)
-11600 val loss 5.8510
-11600 val perplexity 347.5726
-11600 train 5.727832 (lr=1.0202e-05) (hash(x)=139977713)
-11300 val loss 5.7177
-11300 val perplexity 304.2133
-11300 train 5.509121 (lr=1.0512e-05) (hash(x)=158563275)
-11700 val loss 5.8458
-11700 val perplexity 345.7836
-11700 train 5.846483 (lr=9.9846e-06) (hash(x)=156511738)
-11400 val loss 5.7134
-11400 val perplexity 302.8948
-11400 train 5.849710 (lr=1.0295e-05) (hash(x)=161016667)
-11800 val loss 5.8443
-11800 val perplexity 345.2447
-11800 train 6.304429 (lr=9.7691e-06) (hash(x)=185372558)
-11500 val loss 5.7119
-11500 val perplexity 302.4451
-11500 train 5.765417 (lr=1.0080e-05) (hash(x)=157344822)
-11900 val loss 5.8394
-11900 val perplexity 343.5626
-11900 train 5.682316 (lr=9.5560e-06) (hash(x)=138085470)
-11600 val loss 5.7009
-11600 val perplexity 299.1457
-11600 train 5.594313 (lr=9.8678e-06) (hash(x)=139977713)
-12000 val loss 5.8396
-12000 val perplexity 343.6371
-12000 train 5.694638 (lr=9.3452e-06) (hash(x)=142928235)
-11700 val loss 5.6955
-11700 val perplexity 297.5324
-11700 train 5.679156 (lr=9.6572e-06) (hash(x)=156511738)
-12100 val loss 5.8375
-12100 val perplexity 342.9313
-12100 train 5.686862 (lr=9.1369e-06) (hash(x)=151960396)
-11800 val loss 5.6963
-11800 val perplexity 297.7604
-11800 train 6.121418 (lr=9.4488e-06) (hash(x)=185372558)
-12200 val loss 5.8284
-12200 val perplexity 339.8134
-12200 train 5.894607 (lr=8.9310e-06) (hash(x)=156794523)
-11900 val loss 5.6916
-11900 val perplexity 296.3650
-11900 train 5.546271 (lr=9.2426e-06) (hash(x)=138085470)
-12300 val loss 5.8233
-12300 val perplexity 338.0966
-12300 train 5.643482 (lr=8.7278e-06) (hash(x)=138082358)
-12000 val loss 5.6926
-12000 val perplexity 296.6616
-12000 train 5.537786 (lr=9.0388e-06) (hash(x)=142928235)
-12400 val loss 5.8191
-12400 val perplexity 336.6854
-12400 train 5.724566 (lr=8.5273e-06) (hash(x)=155255899)
-12100 val loss 5.6904
-12100 val perplexity 296.0077
-12100 train 5.534543 (lr=8.8373e-06) (hash(x)=151960396)
-12500 val loss 5.8175
-12500 val perplexity 336.1197
-12500 train 5.788463 (lr=8.3294e-06) (hash(x)=150490528)
-12200 val loss 5.6833
-12200 val perplexity 293.9084
-12200 train 5.734580 (lr=8.6382e-06) (hash(x)=156794523)
-12600 val loss 5.8123
-12600 val perplexity 334.3896
-12600 train 5.961518 (lr=8.1343e-06) (hash(x)=154831027)
-12300 val loss 5.6791
-12300 val perplexity 292.6996
-12300 train 5.482205 (lr=8.4417e-06) (hash(x)=138082358)
-12700 val loss 5.8151
-12700 val perplexity 335.3237
-12700 train 5.761143 (lr=7.9421e-06) (hash(x)=147341052)
-12400 val loss 5.6767
-12400 val perplexity 291.9825
-12400 train 5.579998 (lr=8.2477e-06) (hash(x)=155255899)
-12800 val loss 5.8104
-12800 val perplexity 333.7692
-12800 train 5.853680 (lr=7.7528e-06) (hash(x)=159333795)
-12500 val loss 5.6720
-12500 val perplexity 290.6037
-12500 train 5.640311 (lr=8.0563e-06) (hash(x)=150490528)
-12900 val loss 5.8135
-12900 val perplexity 334.7743
-12900 train 5.666300 (lr=7.5665e-06) (hash(x)=143896867)
-12600 val loss 5.6706
-12600 val perplexity 290.1990
-12600 train 5.822594 (lr=7.8676e-06) (hash(x)=154831027)
-13000 val loss 5.8024
-13000 val perplexity 331.0936
-13000 train 5.798257 (lr=7.3832e-06) (hash(x)=147161537)
-12700 val loss 5.6706
-12700 val perplexity 290.2169
-12700 train 5.624063 (lr=7.6817e-06) (hash(x)=147341052)
-13100 val loss 5.7999
-13100 val perplexity 330.2825
-13100 train 5.934094 (lr=7.2030e-06) (hash(x)=147081918)
-12800 val loss 5.6685
-12800 val perplexity 289.5981
-12800 train 5.708333 (lr=7.4986e-06) (hash(x)=159333795)
-13200 val loss 5.7972
-13200 val perplexity 329.3883
-13200 train 5.561694 (lr=7.0260e-06) (hash(x)=145079287)
-12900 val loss 5.6704
-12900 val perplexity 290.1610
-12900 train 5.512172 (lr=7.3184e-06) (hash(x)=143896867)
-13300 val loss 5.7962
-13300 val perplexity 329.0331
-13300 train 5.605070 (lr=6.8522e-06) (hash(x)=142515041)
-13000 val loss 5.6615
-13000 val perplexity 287.5746
-13000 train 5.673651 (lr=7.1412e-06) (hash(x)=147161537)
-13400 val loss 5.7943
-13400 val perplexity 328.4106
-13400 train 5.588826 (lr=6.6816e-06) (hash(x)=144081985)
-13100 val loss 5.6591
-13100 val perplexity 286.8885
-13100 train 5.788761 (lr=6.9669e-06) (hash(x)=147081918)
-13500 val loss 5.7917
-13500 val perplexity 327.5791
-13500 train 5.864884 (lr=6.5144e-06) (hash(x)=149957292)
-13200 val loss 5.6571
-13200 val perplexity 286.3157
-13200 train 5.425736 (lr=6.7956e-06) (hash(x)=145079287)
-13600 val loss 5.7876
-13600 val perplexity 326.2422
-13600 train 5.820804 (lr=6.3506e-06) (hash(x)=157123033)
-13300 val loss 5.6566
-13300 val perplexity 286.1732
-13300 train 5.461178 (lr=6.6275e-06) (hash(x)=142515041)
-13700 val loss 5.7840
-13700 val perplexity 325.0585
-13700 train 5.768866 (lr=6.1902e-06) (hash(x)=148277247)
-13400 val loss 5.6556
-13400 val perplexity 285.8771
-13400 train 5.443344 (lr=6.4626e-06) (hash(x)=144081985)
-13800 val loss 5.7886
-13800 val perplexity 326.5713
-13800 train 5.907638 (lr=6.0333e-06) (hash(x)=142769907)
-13500 val loss 5.6538
-13500 val perplexity 285.3789
-13500 train 5.737462 (lr=6.3008e-06) (hash(x)=149957292)
-13900 val loss 5.7792
-13900 val perplexity 323.5025
-13900 train 5.708614 (lr=5.8799e-06) (hash(x)=144932004)
-13600 val loss 5.6490
-13600 val perplexity 284.0033
-13600 train 5.680844 (lr=6.1424e-06) (hash(x)=157123033)
-14000 val loss 5.7769
-14000 val perplexity 322.7484
-14000 train 5.786118 (lr=5.7301e-06) (hash(x)=149532265)
-13700 val loss 5.6453
-13700 val perplexity 282.9590
-13700 train 5.630872 (lr=5.9872e-06) (hash(x)=148277247)
-14100 val loss 5.7759
-14100 val perplexity 322.4438
-14100 train 5.607883 (lr=5.5840e-06) (hash(x)=149301374)
-13800 val loss 5.6474
-13800 val perplexity 283.5557
-13800 train 5.784145 (lr=5.8355e-06) (hash(x)=142769907)
-14200 val loss 5.7764
-14200 val perplexity 322.6044
-14200 train 5.530695 (lr=5.4416e-06) (hash(x)=142871075)
-13900 val loss 5.6412
-13900 val perplexity 281.7913
-13900 train 5.596690 (lr=5.6871e-06) (hash(x)=144932004)
-14300 val loss 5.7706
-14300 val perplexity 320.7310
-14300 train 5.922376 (lr=5.3029e-06) (hash(x)=166744960)
-14000 val loss 5.6400
-14000 val perplexity 281.4630
-14000 train 5.655642 (lr=5.5423e-06) (hash(x)=149532265)
-14400 val loss 5.7672
-14400 val perplexity 319.6311
-14400 train 5.744401 (lr=5.1679e-06) (hash(x)=145950920)
-14100 val loss 5.6402
-14100 val perplexity 281.5276
-14100 train 5.456847 (lr=5.4009e-06) (hash(x)=149301374)
-14500 val loss 5.7659
-14500 val perplexity 319.2237
-14500 train 5.762283 (lr=5.0368e-06) (hash(x)=141089155)
-14200 val loss 5.6397
-14200 val perplexity 281.3881
-14200 train 5.380307 (lr=5.2632e-06) (hash(x)=142871075)
-14600 val loss 5.7644
-14600 val perplexity 318.7477
-14600 train 5.922758 (lr=4.9096e-06) (hash(x)=174516275)
-14300 val loss 5.6348
-14300 val perplexity 279.9976
-14300 train 5.790681 (lr=5.1290e-06) (hash(x)=166744960)
-14700 val loss 5.7608
-14700 val perplexity 317.6062
-14700 train 5.558484 (lr=4.7863e-06) (hash(x)=137896065)
-14400 val loss 5.6315
-14400 val perplexity 279.0774
-14400 train 5.595186 (lr=4.9985e-06) (hash(x)=145950920)
-14800 val loss 5.7610
-14800 val perplexity 317.6509
-14800 train 5.790486 (lr=4.6670e-06) (hash(x)=154396556)
-14500 val loss 5.6296
-14500 val perplexity 278.5518
-14500 train 5.628878 (lr=4.8717e-06) (hash(x)=141089155)
-14900 val loss 5.7603
-14900 val perplexity 317.4471
-14900 train 5.616198 (lr=4.5516e-06) (hash(x)=149322394)
-15000 val loss 5.7602
-15000 val perplexity 317.3970
-14600 val loss 5.6288
-14600 val perplexity 278.3296
-15000 train 5.679364 (lr=4.4403e-06) (hash(x)=147410513)
-14600 train 5.799279 (lr=4.7487e-06) (hash(x)=174516275)
-15100 val loss 5.7566
-15100 val perplexity 316.2865
-15100 train 5.673219 (lr=4.3330e-06) (hash(x)=145245634)
-14700 val loss 5.6265
-14700 val perplexity 277.6753
-14700 train 5.430897 (lr=4.6294e-06) (hash(x)=137896065)
-15200 val loss 5.7537
-15200 val perplexity 315.3685
-15200 train 5.831272 (lr=4.2298e-06) (hash(x)=148341108)
-14800 val loss 5.6261
-14800 val perplexity 277.5859
-14800 train 5.665586 (lr=4.5139e-06) (hash(x)=154396556)
-15300 val loss 5.7518
-15300 val perplexity 314.7566
-15300 train 5.567430 (lr=4.1308e-06) (hash(x)=140162437)
-14900 val loss 5.6254
-14900 val perplexity 277.3864
-14900 train 5.466794 (lr=4.4024e-06) (hash(x)=149322394)
-15400 val loss 5.7498
-15400 val perplexity 314.1339
-15400 train 5.949061 (lr=4.0360e-06) (hash(x)=142166081)
-15000 val loss 5.6258
-15000 val perplexity 277.4995
-15000 train 5.546385 (lr=4.2947e-06) (hash(x)=147410513)
-15500 val loss 5.7493
-15500 val perplexity 313.9651
-15500 train 5.640825 (lr=3.9453e-06) (hash(x)=141042196)
-15100 val loss 5.6229
-15100 val perplexity 276.6991
-15100 train 5.538782 (lr=4.1909e-06) (hash(x)=145245634)
-15600 val loss 5.7487
-15600 val perplexity 313.7928
-15600 train 5.861061 (lr=3.8589e-06) (hash(x)=148672701)
-15200 val loss 5.6195
-15200 val perplexity 275.7560
-15200 train 5.712276 (lr=4.0912e-06) (hash(x)=148341108)
-15700 val loss 5.7477
-15700 val perplexity 313.4765
-15700 train 5.651585 (lr=3.7767e-06) (hash(x)=149672385)
-15300 val loss 5.6181
-15300 val perplexity 275.3652
-15300 train 5.435312 (lr=3.9954e-06) (hash(x)=140162437)
-15800 val loss 5.7477
-15800 val perplexity 313.4595
-15800 train 5.766627 (lr=3.6988e-06) (hash(x)=152379562)
-15400 val loss 5.6175
-15400 val perplexity 275.2077
-15400 train 5.828096 (lr=3.9036e-06) (hash(x)=142166081)
-15900 val loss 5.7442
-15900 val perplexity 312.3774
-15900 train 5.779916 (lr=3.6253e-06) (hash(x)=154674265)
-15500 val loss 5.6158
-15500 val perplexity 274.7468
-15500 train 5.539294 (lr=3.8160e-06) (hash(x)=141042196)
-16000 val loss 5.7422
-16000 val perplexity 311.7395
-16000 train 5.775348 (lr=3.5560e-06) (hash(x)=150368078)
-15600 val loss 5.6157
-15600 val perplexity 274.6968
-15600 train 5.745433 (lr=3.7324e-06) (hash(x)=148672701)
-16100 val loss 5.7397
-16100 val perplexity 310.9742
-16100 train 5.848284 (lr=3.4912e-06) (hash(x)=160390561)
-15700 val loss 5.6155
-15700 val perplexity 274.6542
-15700 train 5.520627 (lr=3.6529e-06) (hash(x)=149672385)
-16200 val loss 5.7392
-16200 val perplexity 310.8081
-16200 train 6.085945 (lr=3.4307e-06) (hash(x)=155218378)
-15800 val loss 5.6156
-15800 val perplexity 274.6676
-15800 train 5.647228 (lr=3.5776e-06) (hash(x)=152379562)
-16300 val loss 5.7374
-16300 val perplexity 310.2705
-16300 train 5.927204 (lr=3.3746e-06) (hash(x)=160891077)
-15900 val loss 5.6109
-15900 val perplexity 273.3817
-15900 train 5.652540 (lr=3.5064e-06) (hash(x)=154674265)
-16400 val loss 5.7363
-16400 val perplexity 309.9135
-16400 train 5.643398 (lr=3.3229e-06) (hash(x)=142789253)
-16000 val loss 5.6099
-16000 val perplexity 273.1302
-16000 train 5.655309 (lr=3.4394e-06) (hash(x)=150368078)
-16500 val loss 5.7368
-16500 val perplexity 310.0743
-16500 train 5.485014 (lr=3.2757e-06) (hash(x)=141497841)
-16100 val loss 5.6078
-16100 val perplexity 272.5380
-16100 train 5.721385 (lr=3.3767e-06) (hash(x)=160390561)
-16600 val loss 5.7353
-16600 val perplexity 309.6209
-16600 train 5.699547 (lr=3.2329e-06) (hash(x)=157947018)
-16200 val loss 5.6079
-16200 val perplexity 272.5835
-16200 train 5.967405 (lr=3.3182e-06) (hash(x)=155218378)
-16700 val loss 5.7326
-16700 val perplexity 308.7607
-16700 train 5.680963 (lr=3.1946e-06) (hash(x)=146039309)
-16300 val loss 5.6065
-16300 val perplexity 272.2005
-16300 train 5.800972 (lr=3.2639e-06) (hash(x)=160891077)
-16800 val loss 5.7315
-16800 val perplexity 308.4175
-16800 train 5.699938 (lr=3.1607e-06) (hash(x)=156381445)
-16400 val loss 5.6061
-16400 val perplexity 272.0685
-16400 train 5.499582 (lr=3.2140e-06) (hash(x)=142789253)
-16900 val loss 5.7299
-16900 val perplexity 307.9296
-16900 train 5.695052 (lr=3.1314e-06) (hash(x)=146537354)
-16500 val loss 5.6065
-16500 val perplexity 272.1959
-16500 train 5.352096 (lr=3.1683e-06) (hash(x)=141497841)
-17000 val loss 5.7289
-17000 val perplexity 307.6251
-17000 train 5.638675 (lr=3.1065e-06) (hash(x)=135939444)
-16600 val loss 5.6054
-16600 val perplexity 271.8837
-16600 train 5.569555 (lr=3.1269e-06) (hash(x)=157947018)
-17100 val loss 5.7284
-17100 val perplexity 307.4817
-17100 train 5.794311 (lr=3.0862e-06) (hash(x)=147819652)
-16700 val loss 5.6020
-16700 val perplexity 270.9568
-16700 train 5.563784 (lr=3.0898e-06) (hash(x)=146039309)
-17200 val loss 5.7297
-17200 val perplexity 307.8765
-17200 train 5.844919 (lr=3.0704e-06) (hash(x)=155307909)
-16800 val loss 5.6014
-16800 val perplexity 270.8116
-16800 train 5.567529 (lr=3.0571e-06) (hash(x)=156381445)
-17300 val loss 5.7254
-17300 val perplexity 306.5528
-17300 train 5.679380 (lr=3.0591e-06) (hash(x)=151879548)
-16900 val loss 5.6003
-16900 val perplexity 270.5111
-16900 train 5.556417 (lr=3.0287e-06) (hash(x)=146537354)
-17400 val loss 5.7246
-17400 val perplexity 306.3086
-17400 train 5.640570 (lr=3.0523e-06) (hash(x)=143641792)
-17000 val loss 5.5997
-17000 val perplexity 270.3466
-17000 train 5.523510 (lr=3.0047e-06) (hash(x)=135939444)
-17499 val loss 5.7229
-17499 val perplexity 305.7932
-17100 val loss 5.5992
-17100 val perplexity 270.2052
-17100 train 5.674840 (lr=2.9850e-06) (hash(x)=147819652)
-17200 val loss 5.5995
-17200 val perplexity 270.3027
-17200 train 5.698200 (lr=2.9697e-06) (hash(x)=155307909)
-17300 val loss 5.5965
-17300 val perplexity 269.4915
-17300 train 5.552541 (lr=2.9588e-06) (hash(x)=151879548)
-17400 val loss 5.5954
-17400 val perplexity 269.1746
-17400 train 5.514105 (lr=2.9522e-06) (hash(x)=143641792)
-17499 val loss 5.5941
-17499 val perplexity 268.8417
+0 train 11.295413 (lr=1.6250e-07) (hash(x)=150724848)
+100 val loss 9.9363
+100 val perplexity 20667.9277
+100 train 9.984186 (lr=1.6412e-05) (hash(x)=149217061)
+200 val loss 8.9109
+200 val perplexity 7412.5137
+200 train 8.928561 (lr=3.2500e-05) (hash(x)=149910534)
+300 val loss 7.9375
+300 val perplexity 2800.2134
+300 train 7.859196 (lr=3.2498e-05) (hash(x)=142185643)
+400 val loss 7.6799
+400 val perplexity 2164.3118
+400 train 7.731187 (lr=3.2490e-05) (hash(x)=148123706)
+500 val loss 7.5834
+500 val perplexity 1965.3929
+500 train 7.456921 (lr=3.2478e-05) (hash(x)=149952383)
+600 val loss 7.5402
+600 val perplexity 1882.1863
+600 train 7.357423 (lr=3.2461e-05) (hash(x)=146678221)
+700 val loss 7.4786
+700 val perplexity 1769.6854
+700 train 7.511988 (lr=3.2440e-05) (hash(x)=156180736)
+800 val loss 7.4284
+800 val perplexity 1683.0771
+800 train 7.421630 (lr=3.2413e-05) (hash(x)=151700982)
+900 val loss 7.3999
+900 val perplexity 1635.8824
+900 train 7.194687 (lr=3.2382e-05) (hash(x)=147288467)
+1000 val loss 7.3629
+1000 val perplexity 1576.4763
+1000 train 7.404816 (lr=3.2346e-05) (hash(x)=156182087)
+1100 val loss 7.3335
+1100 val perplexity 1530.6554
+1100 train 7.207426 (lr=3.2305e-05) (hash(x)=147861550)
+1200 val loss 7.3059
+1200 val perplexity 1489.0640
+1200 train 7.170646 (lr=3.2260e-05) (hash(x)=149318660)
+1300 val loss 7.2851
+1300 val perplexity 1458.4640
+1300 train 7.124465 (lr=3.2209e-05) (hash(x)=151631103)
+1400 val loss 7.2523
+1400 val perplexity 1411.4155
+1400 train 7.294272 (lr=3.2154e-05) (hash(x)=150482428)
+1500 val loss 7.2043
+1500 val perplexity 1345.2277
+1500 train 7.051772 (lr=3.2094e-05) (hash(x)=144458617)
+1600 val loss 7.1622
+1600 val perplexity 1289.7428
+1600 train 7.036353 (lr=3.2030e-05) (hash(x)=143268605)
+1700 val loss 7.1200
+1700 val perplexity 1236.4585
+1700 train 7.047909 (lr=3.1961e-05) (hash(x)=150817011)
+1800 val loss 7.0871
+1800 val perplexity 1196.4166
+1800 train 7.066686 (lr=3.1887e-05) (hash(x)=152322423)
+1900 val loss 7.0412
+1900 val perplexity 1142.7028
+1900 train 6.837695 (lr=3.1809e-05) (hash(x)=143574126)
+2000 val loss 7.0072
+2000 val perplexity 1104.5946
+2000 train 6.765584 (lr=3.1726e-05) (hash(x)=147904298)
+2100 val loss 6.9793
+2100 val perplexity 1074.1523
+2100 train 6.724745 (lr=3.1638e-05) (hash(x)=144719400)
+2200 val loss 6.9346
+2200 val perplexity 1027.2444
+2200 train 7.184346 (lr=3.1546e-05) (hash(x)=154343147)
+2300 val loss 6.8918
+2300 val perplexity 984.1760
+2300 train 6.934226 (lr=3.1449e-05) (hash(x)=154372635)
+2400 val loss 6.8595
+2400 val perplexity 952.8591
+2400 train 6.681858 (lr=3.1348e-05) (hash(x)=141843115)
+2500 val loss 6.8220
+2500 val perplexity 917.7935
+2500 train 6.727233 (lr=3.1243e-05) (hash(x)=149417679)
+2600 val loss 6.7892
+2600 val perplexity 888.1670
+2600 train 6.649594 (lr=3.1133e-05) (hash(x)=145279030)
+2700 val loss 6.7601
+2700 val perplexity 862.7621
+2700 train 6.665741 (lr=3.1019e-05) (hash(x)=146363911)
+2800 val loss 6.7510
+2800 val perplexity 854.9482
+2800 train 6.706868 (lr=3.0900e-05) (hash(x)=152507639)
+2900 val loss 6.7043
+2900 val perplexity 815.9222
+2900 train 6.495276 (lr=3.0777e-05) (hash(x)=139296191)
+3000 val loss 6.6786
+3000 val perplexity 795.1898
+3000 train 6.690280 (lr=3.0650e-05) (hash(x)=148473774)
+3100 val loss 6.6301
+3100 val perplexity 757.5317
+3100 train 6.722582 (lr=3.0518e-05) (hash(x)=154130527)
+3200 val loss 6.6033
+3200 val perplexity 737.5317
+3200 train 6.548954 (lr=3.0383e-05) (hash(x)=151117002)
+3300 val loss 6.5959
+3300 val perplexity 732.1217
+3300 train 6.740818 (lr=3.0243e-05) (hash(x)=151035383)
+3400 val loss 6.5595
+3400 val perplexity 705.9531
+3400 train 6.679100 (lr=3.0099e-05) (hash(x)=150614749)
+3500 val loss 6.5290
+3500 val perplexity 684.6951
+3500 train 6.549811 (lr=2.9952e-05) (hash(x)=151431391)
+3600 val loss 6.5081
+3600 val perplexity 670.5336
+3600 train 6.444536 (lr=2.9800e-05) (hash(x)=151140797)
+3700 val loss 6.4810
+3700 val perplexity 652.6105
+3700 train 6.683108 (lr=2.9644e-05) (hash(x)=154608340)
+3800 val loss 6.4665
+3800 val perplexity 643.2121
+3800 train 6.493763 (lr=2.9485e-05) (hash(x)=169641309)
+3900 val loss 6.4394
+3900 val perplexity 626.0336
+3900 train 6.395096 (lr=2.9321e-05) (hash(x)=150592479)
+4000 val loss 6.4139
+4000 val perplexity 610.2711
+4000 train 6.398534 (lr=2.9154e-05) (hash(x)=155495317)
+4100 val loss 6.4077
+4100 val perplexity 606.4729
+4100 train 6.611394 (lr=2.8983e-05) (hash(x)=170323970)
+4200 val loss 6.4018
+4200 val perplexity 602.9016
+4200 train 6.223932 (lr=2.8808e-05) (hash(x)=140854360)
+4300 val loss 6.3646
+4300 val perplexity 580.9186
+4300 train 6.368628 (lr=2.8630e-05) (hash(x)=148926644)
+4400 val loss 6.3470
+4400 val perplexity 570.7972
+4400 train 6.378788 (lr=2.8448e-05) (hash(x)=149997871)
+4500 val loss 6.3283
+4500 val perplexity 560.2269
+4500 train 6.274667 (lr=2.8263e-05) (hash(x)=149219289)
+4600 val loss 6.3111
+4600 val perplexity 550.6545
+4600 train 6.106242 (lr=2.8075e-05) (hash(x)=139836197)
+4700 val loss 6.2991
+4700 val perplexity 544.1017
+4700 train 6.247629 (lr=2.7883e-05) (hash(x)=150722883)
+4800 val loss 6.2893
+4800 val perplexity 538.7761
+4800 train 6.492603 (lr=2.7687e-05) (hash(x)=166864648)
+4900 val loss 6.2657
+4900 val perplexity 526.1866
+4900 train 6.368227 (lr=2.7489e-05) (hash(x)=157746743)
+5000 val loss 6.2451
+5000 val perplexity 515.5007
+5000 train 6.195556 (lr=2.7287e-05) (hash(x)=153032257)
+5100 val loss 6.2342
+5100 val perplexity 509.8943
+5100 train 6.316526 (lr=2.7082e-05) (hash(x)=151862371)
+5200 val loss 6.2151
+5200 val perplexity 500.2562
+5200 train 6.555519 (lr=2.6874e-05) (hash(x)=168862995)
+5300 val loss 6.2200
+5300 val perplexity 502.7055
+5300 train 6.186482 (lr=2.6664e-05) (hash(x)=148243567)
+5400 val loss 6.1919
+5400 val perplexity 488.7558
+5400 train 6.124026 (lr=2.6450e-05) (hash(x)=145381742)
+5500 val loss 6.1883
+5500 val perplexity 487.0385
+5500 train 6.359857 (lr=2.6233e-05) (hash(x)=155602174)
+5600 val loss 6.1593
+5600 val perplexity 473.1143
+5600 train 6.246666 (lr=2.6014e-05) (hash(x)=160637672)
+5700 val loss 6.1523
+5700 val perplexity 469.7991
+5700 train 6.077770 (lr=2.5792e-05) (hash(x)=152101131)
+5800 val loss 6.1386
+5800 val perplexity 463.4019
+5800 train 6.099304 (lr=2.5567e-05) (hash(x)=152327089)
+5900 val loss 6.1331
+5900 val perplexity 460.8697
+5900 train 6.076286 (lr=2.5340e-05) (hash(x)=151094543)
+6000 val loss 6.1258
+6000 val perplexity 457.5008
+6000 train 6.232687 (lr=2.5111e-05) (hash(x)=160233315)
+6100 val loss 6.1109
+6100 val perplexity 450.7276
+6100 train 6.216568 (lr=2.4879e-05) (hash(x)=158295337)
+6200 val loss 6.0928
+6200 val perplexity 442.6566
+6200 train 6.044076 (lr=2.4644e-05) (hash(x)=151269817)
+6300 val loss 6.0799
+6300 val perplexity 436.9921
+6300 train 6.225775 (lr=2.4408e-05) (hash(x)=156912756)
+6400 val loss 6.0694
+6400 val perplexity 432.4186
+6400 train 5.995785 (lr=2.4169e-05) (hash(x)=125779637)
+6500 val loss 6.0646
+6500 val perplexity 430.3296
+6500 train 5.961404 (lr=2.3928e-05) (hash(x)=148157661)
+6600 val loss 6.0558
+6600 val perplexity 426.5678
+6600 train 6.084833 (lr=2.3686e-05) (hash(x)=152820049)
+6700 val loss 6.0447
+6700 val perplexity 421.8885
+6700 train 5.979776 (lr=2.3441e-05) (hash(x)=155818717)
+6800 val loss 6.0267
+6800 val perplexity 414.3600
+6800 train 6.053731 (lr=2.3195e-05) (hash(x)=158833881)
+6900 val loss 6.0245
+6900 val perplexity 413.4163
+6900 train 5.945818 (lr=2.2946e-05) (hash(x)=164984909)
+7000 val loss 6.0116
+7000 val perplexity 408.1465
+7000 train 5.903196 (lr=2.2696e-05) (hash(x)=151172493)
+7100 val loss 6.0139
+7100 val perplexity 409.0786
+7100 train 6.066981 (lr=2.2445e-05) (hash(x)=154023633)
+7200 val loss 5.9966
+7200 val perplexity 402.0624
+7200 train 5.848167 (lr=2.2192e-05) (hash(x)=139351250)
+7300 val loss 5.9881
+7300 val perplexity 398.6376
+7300 train 5.944324 (lr=2.1937e-05) (hash(x)=158719458)
+7400 val loss 5.9708
+7400 val perplexity 391.8035
+7400 train 6.046550 (lr=2.1682e-05) (hash(x)=162250810)
+7500 val loss 5.9644
+7500 val perplexity 389.3349
+7500 train 5.840899 (lr=2.1425e-05) (hash(x)=142827557)
+7600 val loss 5.9555
+7600 val perplexity 385.8814
+7600 train 5.863455 (lr=2.1166e-05) (hash(x)=147830801)
+7700 val loss 5.9490
+7700 val perplexity 383.3558
+7700 train 6.045110 (lr=2.0907e-05) (hash(x)=165233314)
+7800 val loss 5.9436
+7800 val perplexity 381.3161
+7800 train 5.926939 (lr=2.0647e-05) (hash(x)=160654717)
+7900 val loss 5.9398
+7900 val perplexity 379.8509
+7900 train 5.801051 (lr=2.0386e-05) (hash(x)=140712586)
+8000 val loss 5.9262
+8000 val perplexity 374.7310
+8000 train 6.088989 (lr=2.0123e-05) (hash(x)=150309061)
+8100 val loss 5.9236
+8100 val perplexity 373.7592
+8100 train 5.924323 (lr=1.9861e-05) (hash(x)=158805466)
+8200 val loss 5.9111
+8200 val perplexity 369.1290
+8200 train 5.796899 (lr=1.9597e-05) (hash(x)=142814078)
+8300 val loss 5.9062
+8300 val perplexity 367.3261
+8300 train 5.871168 (lr=1.9333e-05) (hash(x)=148340417)
+8400 val loss 5.9016
+8400 val perplexity 365.6312
+8400 train 5.841621 (lr=1.9069e-05) (hash(x)=157132639)
+8500 val loss 5.8938
+8500 val perplexity 362.7953
+8500 train 5.823905 (lr=1.8804e-05) (hash(x)=152994240)
+8600 val loss 5.8836
+8600 val perplexity 359.0916
+8600 train 5.847437 (lr=1.8539e-05) (hash(x)=152210154)
+8700 val loss 5.8735
+8700 val perplexity 355.4971
+8700 train 5.967282 (lr=1.8273e-05) (hash(x)=149631247)
+8800 val loss 5.8664
+8800 val perplexity 352.9633
+8800 train 5.732079 (lr=1.8008e-05) (hash(x)=145617784)
+8900 val loss 5.8604
+8900 val perplexity 350.8580
+8900 train 5.963461 (lr=1.7742e-05) (hash(x)=155984970)
+9000 val loss 5.8592
+9000 val perplexity 350.4601
+9000 train 5.591073 (lr=1.7477e-05) (hash(x)=143433013)
+9100 val loss 5.8540
+9100 val perplexity 348.6308
+9100 train 5.756921 (lr=1.7211e-05) (hash(x)=152507533)
+9200 val loss 5.8445
+9200 val perplexity 345.3170
+9200 train 5.973843 (lr=1.6946e-05) (hash(x)=162400650)
+9300 val loss 5.8391
+9300 val perplexity 343.4677
+9300 train 5.885237 (lr=1.6681e-05) (hash(x)=148419068)
+9400 val loss 5.8324
+9400 val perplexity 341.1773
+9400 train 5.801330 (lr=1.6417e-05) (hash(x)=145025169)
+9500 val loss 5.8231
+9500 val perplexity 338.0238
+9500 train 5.798188 (lr=1.6153e-05) (hash(x)=145104235)
+9600 val loss 5.8215
+9600 val perplexity 337.4738
+9600 train 5.755646 (lr=1.5889e-05) (hash(x)=150499797)
+9700 val loss 5.8166
+9700 val perplexity 335.8241
+9700 train 5.619888 (lr=1.5627e-05) (hash(x)=139690022)
+9800 val loss 5.8064
+9800 val perplexity 332.4281
+9800 train 5.808864 (lr=1.5364e-05) (hash(x)=145645279)
+9900 val loss 5.7980
+9900 val perplexity 329.6533
+9900 train 5.719332 (lr=1.5103e-05) (hash(x)=150103432)
+10000 val loss 5.7952
+10000 val perplexity 328.7140
+10000 train 5.949801 (lr=1.4843e-05) (hash(x)=162309473)
+10100 val loss 5.7908
+10100 val perplexity 327.2599
+10100 train 5.681758 (lr=1.4584e-05) (hash(x)=146586772)
+10200 val loss 5.7865
+10200 val perplexity 325.8835
+10200 train 5.690024 (lr=1.4325e-05) (hash(x)=149256313)
+10300 val loss 5.7816
+10300 val perplexity 324.2705
+10300 train 5.739097 (lr=1.4068e-05) (hash(x)=151455083)
+10400 val loss 5.7808
+10400 val perplexity 324.0317
+10400 train 5.552348 (lr=1.3813e-05) (hash(x)=150210444)
+10500 val loss 5.7806
+10500 val perplexity 323.9690
+10500 train 5.548230 (lr=1.3558e-05) (hash(x)=152144428)
+10600 val loss 5.7693
+10600 val perplexity 320.3099
+10600 train 5.816607 (lr=1.3305e-05) (hash(x)=168628935)
+10700 val loss 5.7633
+10700 val perplexity 318.3990
+10700 train 5.562486 (lr=1.3054e-05) (hash(x)=164667785)
+10800 val loss 5.7554
+10800 val perplexity 315.8957
+10800 train 5.717759 (lr=1.2804e-05) (hash(x)=143621491)
+10900 val loss 5.7534
+10900 val perplexity 315.2698
+10900 train 5.649728 (lr=1.2555e-05) (hash(x)=143914826)
+11000 val loss 5.7514
+11000 val perplexity 314.6223
+11000 train 5.527834 (lr=1.2309e-05) (hash(x)=137663435)
+11100 val loss 5.7517
+11100 val perplexity 314.7245
+11100 train 5.621320 (lr=1.2064e-05) (hash(x)=149054007)
+11200 val loss 5.7488
+11200 val perplexity 313.8074
+11200 train 5.756852 (lr=1.1822e-05) (hash(x)=153473924)
+11300 val loss 5.7443
+11300 val perplexity 312.4077
+11300 train 5.535771 (lr=1.1581e-05) (hash(x)=158563275)
+11400 val loss 5.7411
+11400 val perplexity 311.4039
+11400 train 5.877691 (lr=1.1342e-05) (hash(x)=161016667)
+11500 val loss 5.7400
+11500 val perplexity 311.0557
+11500 train 5.784426 (lr=1.1106e-05) (hash(x)=157344822)
+11600 val loss 5.7276
+11600 val perplexity 307.2221
+11600 train 5.616313 (lr=1.0871e-05) (hash(x)=139977713)
+11700 val loss 5.7197
+11700 val perplexity 304.8123
+11700 train 5.714229 (lr=1.0639e-05) (hash(x)=156511738)
+11800 val loss 5.7208
+11800 val perplexity 305.1424
+11800 train 6.174135 (lr=1.0410e-05) (hash(x)=185372558)
+11900 val loss 5.7162
+11900 val perplexity 303.7593
+11900 train 5.559208 (lr=1.0183e-05) (hash(x)=138085470)
+12000 val loss 5.7170
+12000 val perplexity 303.9993
+12000 train 5.562536 (lr=9.9580e-06) (hash(x)=142928235)
+12100 val loss 5.7150
+12100 val perplexity 303.3817
+12100 train 5.564174 (lr=9.7360e-06) (hash(x)=151960396)
+12200 val loss 5.7056
+12200 val perplexity 300.5520
+12200 train 5.767758 (lr=9.5167e-06) (hash(x)=156794523)
+12300 val loss 5.6999
+12300 val perplexity 298.8499
+12300 train 5.508867 (lr=9.3001e-06) (hash(x)=138082358)
+12400 val loss 5.6970
+12400 val perplexity 297.9801
+12400 train 5.598351 (lr=9.0864e-06) (hash(x)=155255899)
+12500 val loss 5.6931
+12500 val perplexity 296.8066
+12500 train 5.658473 (lr=8.8756e-06) (hash(x)=150490528)
+12600 val loss 5.6913
+12600 val perplexity 296.2926
+12600 train 5.849111 (lr=8.6677e-06) (hash(x)=154831027)
+12700 val loss 5.6911
+12700 val perplexity 296.2101
+12700 train 5.642239 (lr=8.4629e-06) (hash(x)=147341052)
+12800 val loss 5.6896
+12800 val perplexity 295.7809
+12800 train 5.728687 (lr=8.2612e-06) (hash(x)=159333795)
+12900 val loss 5.6913
+12900 val perplexity 296.2816
+12900 train 5.534995 (lr=8.0627e-06) (hash(x)=143896867)
+13000 val loss 5.6820
+13000 val perplexity 293.5359
+13000 train 5.691289 (lr=7.8674e-06) (hash(x)=147161537)
+13100 val loss 5.6788
+13100 val perplexity 292.6069
+13100 train 5.811933 (lr=7.6754e-06) (hash(x)=147081918)
+13200 val loss 5.6773
+13200 val perplexity 292.1565
+13200 train 5.445444 (lr=7.4867e-06) (hash(x)=145079287)
+13300 val loss 5.6772
+13300 val perplexity 292.1414
+13300 train 5.475905 (lr=7.3015e-06) (hash(x)=142515041)
+13400 val loss 5.6741
+13400 val perplexity 291.2227
+13400 train 5.459397 (lr=7.1198e-06) (hash(x)=144081985)
+13500 val loss 5.6716
+13500 val perplexity 290.4957
+13500 train 5.750075 (lr=6.9416e-06) (hash(x)=149957292)
+13600 val loss 5.6662
+13600 val perplexity 288.9391
+13600 train 5.702712 (lr=6.7670e-06) (hash(x)=157123033)
+13700 val loss 5.6631
+13700 val perplexity 288.0368
+13700 train 5.641377 (lr=6.5961e-06) (hash(x)=148277247)
+13800 val loss 5.6683
+13800 val perplexity 289.5473
+13800 train 5.796854 (lr=6.4289e-06) (hash(x)=142769907)
+13900 val loss 5.6588
+13900 val perplexity 286.8043
+13900 train 5.602085 (lr=6.2655e-06) (hash(x)=144932004)
+14000 val loss 5.6571
+14000 val perplexity 286.3113
+14000 train 5.670773 (lr=6.1059e-06) (hash(x)=149532265)
+14100 val loss 5.6563
+14100 val perplexity 286.0894
+14100 train 5.469543 (lr=5.9502e-06) (hash(x)=149301374)
+14200 val loss 5.6562
+14200 val perplexity 286.0475
+14200 train 5.395635 (lr=5.7984e-06) (hash(x)=142871075)
+14300 val loss 5.6512
+14300 val perplexity 284.6200
+14300 train 5.802889 (lr=5.6506e-06) (hash(x)=166744960)
+14400 val loss 5.6476
+14400 val perplexity 283.6026
+14400 train 5.615609 (lr=5.5068e-06) (hash(x)=145950920)
+14500 val loss 5.6455
+14500 val perplexity 283.0105
+14500 train 5.639256 (lr=5.3671e-06) (hash(x)=141089155)
+14600 val loss 5.6435
+14600 val perplexity 282.4373
+14600 train 5.801688 (lr=5.2316e-06) (hash(x)=174516275)
+14700 val loss 5.6425
+14700 val perplexity 282.1671
+14700 train 5.444544 (lr=5.1002e-06) (hash(x)=137896065)
+14800 val loss 5.6417
+14800 val perplexity 281.9483
+14800 train 5.677532 (lr=4.9730e-06) (hash(x)=154396556)
+14900 val loss 5.6416
+14900 val perplexity 281.9059
+14900 train 5.487867 (lr=4.8501e-06) (hash(x)=149322394)
+15000 val loss 5.6403
+15000 val perplexity 281.5419
+15000 train 5.556224 (lr=4.7314e-06) (hash(x)=147410513)
+15100 val loss 5.6387
+15100 val perplexity 281.0874
+15100 train 5.549696 (lr=4.6171e-06) (hash(x)=145245634)
+15200 val loss 5.6342
+15200 val perplexity 279.8425
+15200 train 5.724654 (lr=4.5072e-06) (hash(x)=148341108)
+15300 val loss 5.6318
+15300 val perplexity 279.1635
+15300 train 5.440813 (lr=4.4017e-06) (hash(x)=140162437)
+15400 val loss 5.6315
+15400 val perplexity 279.0840
+15400 train 5.841183 (lr=4.3006e-06) (hash(x)=142166081)
+15500 val loss 5.6299
+15500 val perplexity 278.6382
+15500 train 5.549793 (lr=4.2040e-06) (hash(x)=141042196)
+15600 val loss 5.6295
+15600 val perplexity 278.5304
+15600 train 5.751338 (lr=4.1119e-06) (hash(x)=148672701)
+15700 val loss 5.6293
+15700 val perplexity 278.4685
+15700 train 5.535214 (lr=4.0244e-06) (hash(x)=149672385)
+15800 val loss 5.6298
+15800 val perplexity 278.6071
+15800 train 5.662232 (lr=3.9414e-06) (hash(x)=152379562)
+15900 val loss 5.6254
+15900 val perplexity 277.3748
+15900 train 5.671198 (lr=3.8630e-06) (hash(x)=154674265)
+16000 val loss 5.6235
+16000 val perplexity 276.8669
+16000 train 5.658562 (lr=3.7892e-06) (hash(x)=150368078)
+16100 val loss 5.6212
+16100 val perplexity 276.2202
+16100 train 5.732701 (lr=3.7201e-06) (hash(x)=160390561)
+16200 val loss 5.6198
+16200 val perplexity 275.8415
+16200 train 5.990137 (lr=3.6556e-06) (hash(x)=155218378)
+16300 val loss 5.6190
+16300 val perplexity 275.6219
+16300 train 5.817885 (lr=3.5959e-06) (hash(x)=160891077)
+16400 val loss 5.6182
+16400 val perplexity 275.3945
+16400 train 5.519011 (lr=3.5408e-06) (hash(x)=142789253)
+16500 val loss 5.6190
+16500 val perplexity 275.6153
+16500 train 5.362583 (lr=3.4905e-06) (hash(x)=141497841)
+16600 val loss 5.6178
+16600 val perplexity 275.2920
+16600 train 5.576753 (lr=3.4449e-06) (hash(x)=157947018)
+16700 val loss 5.6149
+16700 val perplexity 274.4941
+16700 train 5.570585 (lr=3.4041e-06) (hash(x)=146039309)
+16800 val loss 5.6135
+16800 val perplexity 274.1042
+16800 train 5.582052 (lr=3.3680e-06) (hash(x)=156381445)
+16900 val loss 5.6119
+16900 val perplexity 273.6617
+16900 train 5.573091 (lr=3.3367e-06) (hash(x)=146537354)
+17000 val loss 5.6104
+17000 val perplexity 273.2666
+17000 train 5.533173 (lr=3.3102e-06) (hash(x)=135939444)
+17100 val loss 5.6107
+17100 val perplexity 273.3477
+17100 train 5.684841 (lr=3.2886e-06) (hash(x)=147819652)
+17200 val loss 5.6112
+17200 val perplexity 273.4625
+17200 train 5.705206 (lr=3.2717e-06) (hash(x)=155307909)
+17300 val loss 5.6073
+17300 val perplexity 272.4150
+17300 train 5.563191 (lr=3.2596e-06) (hash(x)=151879548)
+17400 val loss 5.6068
+17400 val perplexity 272.2817
+17400 train 5.529061 (lr=3.2524e-06) (hash(x)=143641792)
+17499 val loss 5.6043
+17499 val perplexity 271.5879

attention_kindselective_n_heads4_seed1339/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:549de889f77b5bf64c01a1e1a9898c7a6a285b4e0ef6ca06c7839a9e7ab3ebb7
+oid sha256:f7af9fcbd1777940217e2ef7c944b0eda83764a7fd4dd1c87b97a28725052564
 size 92843394

attention_kindselective_n_heads4_seed1339/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23558819127ddbf2f275046cd652179ecfb96cdbbefa5b326070975b25199f5a
+oid sha256:dfb3e48dec60661f0ba81a025ac7df2e450c39874152bc7fab11918abe8d4445
 size 179406214