Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1343/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_30720_4_1343", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1343", "n_embd": 256}

attention_kindselective_n_heads4_seed1343/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

attention_kindselective_n_heads4_seed1343/log2.txt

@@ -1,1055 +1,402 @@
-max_steps: 17500
-0 val loss 11.3001
-0 val perplexity 80829.3594
-0 val loss 11.3001
-0 val perplexity 80829.3594
-0 train 11.304919 (lr=1.8750e-07) (hash(x)=153418542)
-0 train 11.304919 (lr=2.0000e-07) (hash(x)=153418542)
-100 val loss 9.8329
-100 val perplexity 18636.0176
-100 train 9.800076 (lr=1.8937e-05) (hash(x)=154406429)
-100 val loss 9.8212
-100 val perplexity 18420.7812
-100 train 9.789283 (lr=2.0200e-05) (hash(x)=154406429)
-200 val loss 8.5308
-200 val perplexity 5068.6626
-200 train 8.423743 (lr=3.7500e-05) (hash(x)=143722001)
-200 val loss 8.4768
-200 val perplexity 4802.1499
-200 train 8.366443 (lr=4.0000e-05) (hash(x)=143722001)
-300 val loss 7.8027
-300 val perplexity 2447.1411
-300 train 7.741059 (lr=3.7497e-05) (hash(x)=152421341)
-300 val loss 7.7235
-300 val perplexity 2260.9478
-300 train 7.657517 (lr=3.9997e-05) (hash(x)=152421341)
-400 val loss 7.6001
-400 val perplexity 1998.4463
-400 val loss 7.5567
-400 val perplexity 1913.5374
-400 train 7.495296 (lr=3.7489e-05) (hash(x)=152989689)
-400 train 7.447391 (lr=3.9988e-05) (hash(x)=152989689)
-500 val loss 7.5282
-500 val perplexity 1859.7047
-500 val loss 7.4923
-500 val perplexity 1794.2245
-500 train 7.458594 (lr=3.7475e-05) (hash(x)=145113544)
-500 train 7.418945 (lr=3.9973e-05) (hash(x)=145113544)
-600 val loss 7.4701
-600 val perplexity 1754.8171
-600 val loss 7.4354
-600 val perplexity 1694.8523
-600 train 7.463784 (lr=3.7456e-05) (hash(x)=150071018)
-600 train 7.434020 (lr=3.9953e-05) (hash(x)=150071018)
-700 val loss 7.3884
-700 val perplexity 1617.1136
-700 val loss 7.4224
-700 val perplexity 1673.0927
-700 train 7.480190 (lr=3.9926e-05) (hash(x)=156594182)
-700 train 7.505778 (lr=3.7430e-05) (hash(x)=156594182)
-800 val loss 7.3583
-800 val perplexity 1569.1648
-800 train 7.357888 (lr=3.9893e-05) (hash(x)=153358238)
-800 val loss 7.3882
-800 val perplexity 1616.8392
-800 train 7.388269 (lr=3.7400e-05) (hash(x)=153358238)
-900 val loss 7.3160
-900 val perplexity 1504.1028
-900 train 7.138016 (lr=3.9855e-05) (hash(x)=146617394)
-900 val loss 7.3554
-900 val perplexity 1564.5654
-900 train 7.184053 (lr=3.7364e-05) (hash(x)=146617394)
-1000 val loss 7.2891
-1000 val perplexity 1464.2344
-1000 train 7.218844 (lr=3.9810e-05) (hash(x)=148983354)
-1000 val loss 7.3291
-1000 val perplexity 1524.0352
-1000 train 7.271225 (lr=3.7322e-05) (hash(x)=148983354)
-1100 val loss 7.2489
-1100 val perplexity 1406.5225
-1100 train 7.274782 (lr=3.9760e-05) (hash(x)=146313352)
-1100 val loss 7.2985
-1100 val perplexity 1478.0883
-1100 train 7.326784 (lr=3.7275e-05) (hash(x)=146313352)
-1200 val loss 7.2332
-1200 val perplexity 1384.6940
-1200 train 7.190815 (lr=3.9704e-05) (hash(x)=150770333)
-1200 val loss 7.2798
-1200 val perplexity 1450.7379
-1200 train 7.232755 (lr=3.7223e-05) (hash(x)=150770333)
-1300 val loss 7.1657
-1300 val perplexity 1294.2185
-1300 train 7.110763 (lr=3.9642e-05) (hash(x)=153237933)
-1300 val loss 7.2388
-1300 val perplexity 1392.4480
-1300 train 7.181512 (lr=3.7164e-05) (hash(x)=153237933)
-1400 val loss 7.1323
-1400 val perplexity 1251.7812
-1400 train 6.970374 (lr=3.9574e-05) (hash(x)=145246201)
-1400 val loss 7.2173
-1400 val perplexity 1362.7434
-1400 train 7.056479 (lr=3.7101e-05) (hash(x)=145246201)
-1500 val loss 7.0966
-1500 val perplexity 1207.8966
-1500 train 7.092261 (lr=3.9501e-05) (hash(x)=154002540)
-1500 val loss 7.1916
-1500 val perplexity 1328.2701
-1500 train 7.191994 (lr=3.7032e-05) (hash(x)=154002540)
-1600 val loss 7.0737
-1600 val perplexity 1180.4943
-1600 train 6.847246 (lr=3.9421e-05) (hash(x)=142635842)
-1600 val loss 7.1695
-1600 val perplexity 1299.2076
-1600 train 6.949299 (lr=3.6958e-05) (hash(x)=142635842)
-1700 val loss 7.0351
-1700 val perplexity 1135.7549
-1700 train 7.076663 (lr=3.9336e-05) (hash(x)=151450555)
-1700 val loss 7.1469
-1700 val perplexity 1270.1534
-1700 train 7.183446 (lr=3.6878e-05) (hash(x)=151450555)
-1800 val loss 7.0089
-1800 val perplexity 1106.4053
-1800 train 6.922809 (lr=3.9246e-05) (hash(x)=148299832)
-1800 val loss 7.1197
-1800 val perplexity 1236.1178
-1800 train 7.035375 (lr=3.6793e-05) (hash(x)=148299832)
-1900 val loss 6.9638
-1900 val perplexity 1057.6011
-1900 train 7.122456 (lr=3.9149e-05) (hash(x)=155487476)
-1900 val loss 7.0849
-1900 val perplexity 1193.7941
-1900 train 7.249062 (lr=3.6702e-05) (hash(x)=155487476)
-2000 val loss 6.9417
-2000 val perplexity 1034.5106
-2000 train 6.923683 (lr=3.9047e-05) (hash(x)=152887721)
-2000 val loss 7.0502
-2000 val perplexity 1153.0652
-2000 train 7.028882 (lr=3.6606e-05) (hash(x)=152887721)
-2100 val loss 6.9170
-2100 val perplexity 1009.2787
-2100 train 6.620973 (lr=3.8939e-05) (hash(x)=143430603)
-2100 val loss 7.0140
-2100 val perplexity 1112.0854
-2100 train 6.726614 (lr=3.6505e-05) (hash(x)=143430603)
-2200 val loss 6.8884
-2200 val perplexity 980.8248
-2200 train 6.878071 (lr=3.8826e-05) (hash(x)=153721194)
-2200 val loss 6.9844
-2200 val perplexity 1079.6273
-2200 train 6.971331 (lr=3.6399e-05) (hash(x)=153721194)
-2300 val loss 6.8506
-2300 val perplexity 944.4535
-2300 train 6.979174 (lr=3.8707e-05) (hash(x)=148702429)
-2300 val loss 6.9476
-2300 val perplexity 1040.6342
-2300 train 7.065608 (lr=3.6288e-05) (hash(x)=148702429)
-2400 val loss 6.8137
-2400 val perplexity 910.2053
-2400 train 6.860815 (lr=3.8583e-05) (hash(x)=153047184)
-2400 val loss 6.9253
-2400 val perplexity 1017.6667
-2400 train 6.966145 (lr=3.6171e-05) (hash(x)=153047184)
-2500 val loss 6.7714
-2500 val perplexity 872.5063
-2500 train 6.653331 (lr=3.8453e-05) (hash(x)=149720123)
-2500 val loss 6.8891
-2500 val perplexity 981.5322
-2500 train 6.767345 (lr=3.6049e-05) (hash(x)=149720123)
-2600 val loss 6.7305
-2600 val perplexity 837.5394
-2600 train 6.760979 (lr=3.8317e-05) (hash(x)=152466045)
-2600 val loss 6.8655
-2600 val perplexity 958.6515
-2600 train 6.900772 (lr=3.5923e-05) (hash(x)=152466045)
-2700 val loss 6.7068
-2700 val perplexity 817.9697
-2700 train 6.640062 (lr=3.8177e-05) (hash(x)=155796305)
-2700 val loss 6.8521
-2700 val perplexity 945.9030
-2700 train 6.800490 (lr=3.5791e-05) (hash(x)=155796305)
-2800 val loss 6.6646
-2800 val perplexity 784.1478
-2800 train 6.503381 (lr=3.8031e-05) (hash(x)=151991552)
-2800 val loss 6.8246
-2800 val perplexity 920.1984
-2800 train 6.668451 (lr=3.5654e-05) (hash(x)=151991552)
-2900 val loss 6.6274
-2900 val perplexity 755.5084
-2900 train 7.209426 (lr=3.7879e-05) (hash(x)=166537989)
-2900 val loss 6.7901
-2900 val perplexity 888.9970
-2900 train 7.444793 (lr=3.5512e-05) (hash(x)=166537989)
-3000 val loss 6.5884
-3000 val perplexity 726.5895
-3000 train 6.592463 (lr=3.7723e-05) (hash(x)=147892594)
-3000 val loss 6.7554
-3000 val perplexity 858.6501
-3000 train 6.760037 (lr=3.5365e-05) (hash(x)=147892594)
-3100 val loss 6.5645
-3100 val perplexity 709.4786
-3100 train 6.522549 (lr=3.7561e-05) (hash(x)=149536927)
-3100 val loss 6.7237
-3100 val perplexity 831.8831
-3100 train 6.678670 (lr=3.5214e-05) (hash(x)=149536927)
-3200 val loss 6.5361
-3200 val perplexity 689.5991
-3200 train 6.249228 (lr=3.7394e-05) (hash(x)=135678663)
-3200 val loss 6.6918
-3200 val perplexity 805.7526
-3200 train 6.404359 (lr=3.5057e-05) (hash(x)=135678663)
-3300 val loss 6.5172
-3300 val perplexity 676.7065
-3300 train 6.580379 (lr=3.7222e-05) (hash(x)=148523646)
-3300 val loss 6.6596
-3300 val perplexity 780.2591
-3300 train 6.714272 (lr=3.4896e-05) (hash(x)=148523646)
-3400 val loss 6.4957
-3400 val perplexity 662.2582
-3400 train 6.568460 (lr=3.7045e-05) (hash(x)=150281778)
-3400 val loss 6.6333
-3400 val perplexity 760.0051
-3400 train 6.704147 (lr=3.4730e-05) (hash(x)=150281778)
-3500 val loss 6.4701
-3500 val perplexity 645.5488
-3500 train 6.417324 (lr=3.6864e-05) (hash(x)=151527953)
-3500 val loss 6.6012
-3500 val perplexity 735.9909
-3500 train 6.537759 (lr=3.4560e-05) (hash(x)=151527953)
-3600 val loss 6.4607
-3600 val perplexity 639.5080
-3600 train 6.673931 (lr=3.6677e-05) (hash(x)=141303739)
-3600 val loss 6.5851
-3600 val perplexity 724.2502
-3600 train 6.815054 (lr=3.4384e-05) (hash(x)=141303739)
-3700 val loss 6.4387
-3700 val perplexity 625.5988
-3700 train 6.471379 (lr=3.6485e-05) (hash(x)=146778597)
-3700 val loss 6.5494
-3700 val perplexity 698.8573
-3700 train 6.569457 (lr=3.4205e-05) (hash(x)=146778597)
-3800 val loss 6.4031
-3800 val perplexity 603.6975
-3800 train 6.358904 (lr=3.6289e-05) (hash(x)=146902797)
-3800 val loss 6.5228
-3800 val perplexity 680.4645
-3800 train 6.484237 (lr=3.4021e-05) (hash(x)=146902797)
-3900 val loss 6.3836
-3900 val perplexity 592.0266
-3900 train 6.312788 (lr=3.6088e-05) (hash(x)=142022555)
-3900 val loss 6.4976
-3900 val perplexity 663.5311
-3900 train 6.423472 (lr=3.3832e-05) (hash(x)=142022555)
-4000 val loss 6.3624
-4000 val perplexity 579.6483
-4000 train 6.290960 (lr=3.5882e-05) (hash(x)=151590309)
-4000 val loss 6.4783
-4000 val perplexity 650.8406
-4000 train 6.398770 (lr=3.3639e-05) (hash(x)=151590309)
-4100 val loss 6.3443
-4100 val perplexity 569.2514
-4100 train 6.188325 (lr=3.5671e-05) (hash(x)=144127100)
-4100 val loss 6.4617
-4100 val perplexity 640.1420
-4100 train 6.302608 (lr=3.3442e-05) (hash(x)=144127100)
-4200 val loss 6.3344
-4200 val perplexity 563.6245
-4200 train 6.400013 (lr=3.5457e-05) (hash(x)=149526774)
-4200 val loss 6.4405
-4200 val perplexity 626.7184
-4200 train 6.507928 (lr=3.3240e-05) (hash(x)=149526774)
-4300 val loss 6.3215
-4300 val perplexity 556.4191
-4300 train 6.272267 (lr=3.5237e-05) (hash(x)=150924362)
-4300 val loss 6.4347
-4300 val perplexity 623.0805
-4300 train 6.396029 (lr=3.3035e-05) (hash(x)=150924362)
-4400 val loss 6.2921
-4400 val perplexity 540.2718
-4400 train 6.439682 (lr=3.5014e-05) (hash(x)=155547282)
-4400 val loss 6.3944
-4400 val perplexity 598.5039
-4400 train 6.551724 (lr=3.2825e-05) (hash(x)=155547282)
-4500 val loss 6.2809
-4500 val perplexity 534.2578
-4500 train 6.262727 (lr=3.4786e-05) (hash(x)=152479203)
-4500 val loss 6.3806
-4500 val perplexity 590.2866
-4500 train 6.360958 (lr=3.2611e-05) (hash(x)=152479203)
-4600 val loss 6.2638
-4600 val perplexity 525.2311
-4600 train 6.177735 (lr=3.4553e-05) (hash(x)=146739760)
-4600 val loss 6.3688
-4600 val perplexity 583.3686
-4600 train 6.277540 (lr=3.2394e-05) (hash(x)=146739760)
-4700 val loss 6.2521
-4700 val perplexity 519.1068
-4700 train 6.473020 (lr=3.4317e-05) (hash(x)=159977388)
-4700 val loss 6.3566
-4700 val perplexity 576.2929
-4700 train 6.578290 (lr=3.2172e-05) (hash(x)=159977388)
-4800 val loss 6.2422
-4800 val perplexity 513.9703
-4800 train 6.129420 (lr=3.4077e-05) (hash(x)=142959739)
-4800 val loss 6.3445
-4800 val perplexity 569.3329
-4800 train 6.230520 (lr=3.1947e-05) (hash(x)=142959739)
-4900 val loss 6.2251
-4900 val perplexity 505.2736
-4900 train 6.180996 (lr=3.3832e-05) (hash(x)=152253034)
-4900 val loss 6.3244
-4900 val perplexity 558.0325
-4900 train 6.286006 (lr=3.1718e-05) (hash(x)=152253034)
-5000 val loss 6.2122
-5000 val perplexity 498.8215
-5000 train 6.184723 (lr=3.3584e-05) (hash(x)=140379747)
-5000 val loss 6.3155
-5000 val perplexity 553.0589
-5000 train 6.280877 (lr=3.1485e-05) (hash(x)=140379747)
-5100 val loss 6.1928
-5100 val perplexity 489.2184
-5100 train 6.213193 (lr=3.3332e-05) (hash(x)=151826392)
-5100 val loss 6.2938
-5100 val perplexity 541.1830
-5100 train 6.315104 (lr=3.1249e-05) (hash(x)=151826392)
-5200 val loss 6.1897
-5200 val perplexity 487.7077
-5200 train 6.150966 (lr=3.3076e-05) (hash(x)=152869594)
-5200 val loss 6.2842
-5200 val perplexity 536.0088
-5200 train 6.236502 (lr=3.1009e-05) (hash(x)=152869594)
-5300 val loss 6.1714
-5300 val perplexity 478.8367
-5300 train 5.955120 (lr=3.2817e-05) (hash(x)=139093709)
-5300 val loss 6.2693
-5300 val perplexity 528.1216
-5300 train 6.061146 (lr=3.0766e-05) (hash(x)=139093709)
-5400 val loss 6.1644
-5400 val perplexity 475.5219
-5400 train 6.037286 (lr=3.2554e-05) (hash(x)=145908738)
-5400 val loss 6.2641
-5400 val perplexity 525.3493
-5400 train 6.129711 (lr=3.0519e-05) (hash(x)=145908738)
-5500 val loss 6.1606
-5500 val perplexity 473.6938
-5500 train 6.016478 (lr=3.2287e-05) (hash(x)=137913426)
-5500 val loss 6.2502
-5500 val perplexity 518.0980
-5500 train 6.108302 (lr=3.0269e-05) (hash(x)=137913426)
-5600 val loss 6.1410
-5600 val perplexity 464.5248
-5600 train 5.993292 (lr=3.2017e-05) (hash(x)=140770390)
-5600 val loss 6.2283
-5600 val perplexity 506.8769
-5600 train 6.066168 (lr=3.0016e-05) (hash(x)=140770390)
-5700 val loss 6.1385
-5700 val perplexity 463.3590
-5700 train 6.046831 (lr=3.1744e-05) (hash(x)=143728578)
-5700 val loss 6.2293
-5700 val perplexity 507.3886
-5700 train 6.146498 (lr=2.9760e-05) (hash(x)=143728578)
-5800 val loss 6.1174
-5800 val perplexity 453.7044
-5800 train 6.182424 (lr=3.1468e-05) (hash(x)=154469787)
-5800 val loss 6.2102
-5800 val perplexity 497.8142
-5800 train 6.285611 (lr=2.9501e-05) (hash(x)=154469787)
-5900 val loss 6.1137
-5900 val perplexity 452.0091
-5900 train 6.078083 (lr=3.1188e-05) (hash(x)=139316559)
-5900 val loss 6.2014
-5900 val perplexity 493.4557
-5900 train 6.150805 (lr=2.9239e-05) (hash(x)=139316559)
-6000 val loss 6.1008
-6000 val perplexity 446.2053
-6000 train 6.113969 (lr=3.0905e-05) (hash(x)=152058488)
-6000 val loss 6.1958
-6000 val perplexity 490.6758
-6000 train 6.203081 (lr=2.8974e-05) (hash(x)=152058488)
-6100 val loss 6.0900
-6100 val perplexity 441.4070
-6100 train 6.101003 (lr=3.0620e-05) (hash(x)=152958689)
-6100 val loss 6.1783
-6100 val perplexity 482.1736
-6100 train 6.180849 (lr=2.8706e-05) (hash(x)=152958689)
-6200 val loss 6.0709
-6200 val perplexity 433.0719
-6200 train 6.035614 (lr=3.0332e-05) (hash(x)=150887864)
-6200 val loss 6.1637
-6200 val perplexity 475.1640
-6200 train 6.128726 (lr=2.8436e-05) (hash(x)=150887864)
-6300 val loss 6.0649
-6300 val perplexity 430.4847
-6300 train 6.011372 (lr=3.0040e-05) (hash(x)=156290469)
-6300 val loss 6.1603
-6300 val perplexity 473.5673
-6300 train 6.107643 (lr=2.8163e-05) (hash(x)=156290469)
-6400 val loss 6.0562
-6400 val perplexity 426.7660
-6400 train 6.360909 (lr=2.9747e-05) (hash(x)=159160159)
-6400 val loss 6.1479
-6400 val perplexity 467.7326
-6400 train 6.456499 (lr=2.7888e-05) (hash(x)=159160159)
-6500 val loss 6.0513
-6500 val perplexity 424.6838
-6500 train 5.965971 (lr=2.9450e-05) (hash(x)=151239121)
-6500 val loss 6.1376
-6500 val perplexity 462.9326
-6500 train 6.059669 (lr=2.7610e-05) (hash(x)=151239121)
-6600 val loss 6.0396
-6600 val perplexity 419.7285
-6600 train 6.053043 (lr=2.9152e-05) (hash(x)=151964101)
-6600 val loss 6.1316
-6600 val perplexity 460.1588
-6600 train 6.140378 (lr=2.7330e-05) (hash(x)=151964101)
-6700 val loss 6.0341
-6700 val perplexity 417.4256
-6700 train 5.943383 (lr=2.8851e-05) (hash(x)=141291229)
-6700 val loss 6.1208
-6700 val perplexity 455.2326
-6700 train 6.024530 (lr=2.7047e-05) (hash(x)=141291229)
-6800 val loss 6.0232
-6800 val perplexity 412.8877
-6800 train 5.946563 (lr=2.8547e-05) (hash(x)=160416800)
-6800 val loss 6.1158
-6800 val perplexity 452.9381
-6800 train 6.040329 (lr=2.6763e-05) (hash(x)=160416800)
-6900 val loss 6.0162
-6900 val perplexity 410.0209
-6900 train 5.912896 (lr=2.8242e-05) (hash(x)=143382691)
-6900 val loss 6.1049
-6900 val perplexity 448.0463
-6900 train 6.002789 (lr=2.6477e-05) (hash(x)=143382691)
-7000 val loss 5.9973
-7000 val perplexity 402.3608
-7000 train 5.902618 (lr=2.7934e-05) (hash(x)=147606572)
-7000 val loss 6.0921
-7000 val perplexity 442.3517
-7000 train 6.005036 (lr=2.6188e-05) (hash(x)=147606572)
-7100 val loss 5.9907
-7100 val perplexity 399.6799
-7100 train 5.814173 (lr=2.7624e-05) (hash(x)=145901538)
-7100 val loss 6.0824
-7100 val perplexity 438.0784
-7100 train 5.912265 (lr=2.5898e-05) (hash(x)=145901538)
-7200 val loss 5.9901
-7200 val perplexity 399.4538
-7200 train 5.821192 (lr=2.7313e-05) (hash(x)=139099153)
-7200 val loss 6.0794
-7200 val perplexity 436.7787
-7200 train 5.910559 (lr=2.5606e-05) (hash(x)=139099153)
-7300 val loss 5.9797
-7300 val perplexity 395.3340
-7300 train 5.948170 (lr=2.7000e-05) (hash(x)=140434004)
-7300 val loss 6.0718
-7300 val perplexity 433.4543
-7300 train 6.038386 (lr=2.5312e-05) (hash(x)=140434004)
-7400 val loss 5.9791
-7400 val perplexity 395.1002
-7400 train 6.095177 (lr=2.6685e-05) (hash(x)=155496146)
-7400 val loss 6.0672
-7400 val perplexity 431.4613
-7400 train 6.180016 (lr=2.5017e-05) (hash(x)=155496146)
-7500 val loss 5.9664
-7500 val perplexity 390.1005
-7500 train 5.892523 (lr=2.6369e-05) (hash(x)=146940054)
-7500 val loss 6.0597
-7500 val perplexity 428.2500
-7500 train 5.979121 (lr=2.4721e-05) (hash(x)=146940054)
-7600 val loss 5.9557
-7600 val perplexity 385.9278
-7600 train 6.182656 (lr=2.6051e-05) (hash(x)=165785723)
-7600 val loss 6.0449
-7600 val perplexity 421.9463
-7600 train 6.269816 (lr=2.4423e-05) (hash(x)=165785723)
-7700 val loss 5.9467
-7700 val perplexity 382.4761
-7700 train 6.093819 (lr=2.5732e-05) (hash(x)=156339008)
-7700 val loss 6.0365
-7700 val perplexity 418.4242
-7700 train 6.181146 (lr=2.4124e-05) (hash(x)=156339008)
-7800 val loss 5.9486
-7800 val perplexity 383.2071
-7800 train 5.945467 (lr=2.5411e-05) (hash(x)=148593556)
-7800 val loss 6.0319
-7800 val perplexity 416.4887
-7800 train 6.030032 (lr=2.3823e-05) (hash(x)=148593556)
-7900 val loss 5.9564
-7900 val perplexity 386.2014
-7900 train 5.891070 (lr=2.5090e-05) (hash(x)=148079742)
-7900 val loss 6.0443
-7900 val perplexity 421.7001
-7900 train 5.989381 (lr=2.3522e-05) (hash(x)=148079742)
-8000 val loss 5.9298
-8000 val perplexity 376.0685
-8000 train 5.867480 (lr=2.4767e-05) (hash(x)=159706718)
-8000 val loss 6.0159
-8000 val perplexity 409.8966
-8000 train 5.955351 (lr=2.3219e-05) (hash(x)=159706718)
-8100 val loss 5.9287
-8100 val perplexity 375.6778
-8100 train 5.920499 (lr=2.4444e-05) (hash(x)=160203225)
-8100 val loss 6.0136
-8100 val perplexity 408.9505
-8100 train 6.008880 (lr=2.2916e-05) (hash(x)=160203225)
-8200 val loss 5.9155
-8200 val perplexity 370.7519
-8200 train 5.866872 (lr=2.4120e-05) (hash(x)=146176536)
-8200 val loss 5.9986
-8200 val perplexity 402.8794
-8200 train 5.960113 (lr=2.2612e-05) (hash(x)=146176536)
-8300 val loss 5.9061
-8300 val perplexity 367.2567
-8300 train 5.982905 (lr=2.3795e-05) (hash(x)=158883824)
-8300 val loss 5.9935
-8300 val perplexity 400.8128
-8300 train 6.061623 (lr=2.2308e-05) (hash(x)=158883824)
-8400 val loss 5.8996
-8400 val perplexity 364.8946
-8400 train 5.716026 (lr=2.3469e-05) (hash(x)=137102572)
-8400 val loss 5.9840
-8400 val perplexity 397.0131
-8400 train 5.799762 (lr=2.2002e-05) (hash(x)=137102572)
-8500 val loss 5.8970
-8500 val perplexity 363.9478
-8500 train 5.928124 (lr=2.3143e-05) (hash(x)=152812797)
-8500 val loss 5.9767
-8500 val perplexity 394.1475
-8500 train 6.004527 (lr=2.1697e-05) (hash(x)=152812797)
-8600 val loss 5.8957
-8600 val perplexity 363.4700
-8600 train 5.742877 (lr=2.2817e-05) (hash(x)=148854244)
-8600 val loss 5.9755
-8600 val perplexity 393.6647
-8600 train 5.820330 (lr=2.1391e-05) (hash(x)=148854244)
-8700 val loss 5.9036
-8700 val perplexity 366.3378
-8700 train 5.714509 (lr=2.2490e-05) (hash(x)=135590023)
-8700 val loss 5.9854
-8700 val perplexity 397.5729
-8700 train 5.797290 (lr=2.1085e-05) (hash(x)=135590023)
-8800 val loss 5.8742
-8800 val perplexity 355.7392
-8800 train 5.809571 (lr=2.2163e-05) (hash(x)=144913509)
-8800 val loss 5.9599
-8800 val perplexity 387.5530
-8800 train 5.892747 (lr=2.0778e-05) (hash(x)=144913509)
-8900 val loss 5.8714
-8900 val perplexity 354.7586
-8900 train 5.992274 (lr=2.1837e-05) (hash(x)=148045669)
-8900 val loss 5.9579
-8900 val perplexity 386.7786
-8900 train 6.079168 (lr=2.0472e-05) (hash(x)=148045669)
-9000 val loss 5.8665
-9000 val perplexity 353.0065
-9000 train 5.906809 (lr=2.1510e-05) (hash(x)=146661746)
-9000 val loss 5.9502
-9000 val perplexity 383.8190
-9000 train 5.979196 (lr=2.0165e-05) (hash(x)=146661746)
-9100 val loss 5.8603
-9100 val perplexity 350.8411
-9100 train 5.797707 (lr=2.1183e-05) (hash(x)=145354881)
-9100 val loss 5.9399
-9100 val perplexity 379.8830
-9100 train 5.876511 (lr=1.9859e-05) (hash(x)=145354881)
-9200 val loss 5.8547
-9200 val perplexity 348.8842
-9200 train 5.729348 (lr=2.0857e-05) (hash(x)=150939310)
-9200 val loss 5.9363
-9200 val perplexity 378.5494
-9200 train 5.809578 (lr=1.9553e-05) (hash(x)=150939310)
-9300 val loss 5.8502
-9300 val perplexity 347.3205
-9300 train 5.692621 (lr=2.0531e-05) (hash(x)=162258309)
-9300 val loss 5.9312
-9300 val perplexity 376.6234
-9300 train 5.781178 (lr=1.9248e-05) (hash(x)=162258309)
-9400 val loss 5.8477
-9400 val perplexity 346.4448
-9400 train 5.856801 (lr=2.0205e-05) (hash(x)=154701830)
-9400 val loss 5.9250
-9400 val perplexity 374.2867
-9400 train 5.932467 (lr=1.8942e-05) (hash(x)=154701830)
-9500 val loss 5.8364
-9500 val perplexity 342.5458
-9500 train 5.730147 (lr=1.9880e-05) (hash(x)=142241404)
-9500 val loss 5.9229
-9500 val perplexity 373.5072
-9500 train 5.808607 (lr=1.8638e-05) (hash(x)=142241404)
-9600 val loss 5.8332
-9600 val perplexity 341.4551
-9600 train 5.795203 (lr=1.9556e-05) (hash(x)=148439858)
-9600 val loss 5.9120
-9600 val perplexity 369.4564
-9600 train 5.880486 (lr=1.8334e-05) (hash(x)=148439858)
-9700 val loss 5.8316
-9700 val perplexity 340.8906
-9700 train 5.828146 (lr=1.9233e-05) (hash(x)=152204410)
-9700 val loss 5.9098
-9700 val perplexity 368.6377
-9700 train 5.907160 (lr=1.8031e-05) (hash(x)=152204410)
-9800 val loss 5.8245
-9800 val perplexity 338.5030
-9800 train 5.668409 (lr=1.8910e-05) (hash(x)=141294630)
-9800 val loss 5.9046
-9800 val perplexity 366.7115
-9800 train 5.741839 (lr=1.7728e-05) (hash(x)=141294630)
-9900 val loss 5.8172
-9900 val perplexity 336.0203
-9900 train 5.789067 (lr=1.8589e-05) (hash(x)=147390033)
-9900 val loss 5.9026
-9900 val perplexity 365.9911
-9900 train 5.876742 (lr=1.7427e-05) (hash(x)=147390033)
-10000 val loss 5.8138
-10000 val perplexity 334.8972
-10000 train 5.811567 (lr=1.8268e-05) (hash(x)=144131604)
-10000 val loss 5.8944
-10000 val perplexity 363.0030
-10000 train 5.886058 (lr=1.7126e-05) (hash(x)=144131604)
-10100 val loss 5.8044
-10100 val perplexity 331.7603
-10100 train 6.006137 (lr=1.7949e-05) (hash(x)=158814818)
-10100 val loss 5.8887
-10100 val perplexity 360.9442
-10100 train 6.090841 (lr=1.6827e-05) (hash(x)=158814818)
-10200 val loss 5.8000
-10200 val perplexity 330.2987
-10200 train 5.880665 (lr=1.7631e-05) (hash(x)=153491388)
-10200 val loss 5.8807
-10200 val perplexity 358.0732
-10200 train 5.957265 (lr=1.6529e-05) (hash(x)=153491388)
-10300 val loss 5.8002
-10300 val perplexity 330.3508
-10300 train 5.778177 (lr=1.7315e-05) (hash(x)=149600707)
-10300 val loss 5.8789
-10300 val perplexity 357.4311
-10300 train 5.849335 (lr=1.6233e-05) (hash(x)=149600707)
-10400 val loss 5.7900
-10400 val perplexity 327.0261
-10400 train 5.710597 (lr=1.7000e-05) (hash(x)=152277064)
-10400 val loss 5.8727
-10400 val perplexity 355.2067
-10400 train 5.788152 (lr=1.5938e-05) (hash(x)=152277064)
-10500 val loss 5.7888
-10500 val perplexity 326.6337
-10500 train 5.694821 (lr=1.6687e-05) (hash(x)=150447091)
-10500 val loss 5.8737
-10500 val perplexity 355.5547
-10500 train 5.782475 (lr=1.5644e-05) (hash(x)=150447091)
-10600 val loss 5.7777
-10600 val perplexity 323.0083
-10600 train 5.801088 (lr=1.6376e-05) (hash(x)=151606822)
-10600 val loss 5.8607
-10600 val perplexity 350.9606
-10600 train 5.884093 (lr=1.5352e-05) (hash(x)=151606822)
-10700 val loss 5.7708
-10700 val perplexity 320.8090
-10700 train 5.758115 (lr=1.6066e-05) (hash(x)=158730240)
-10700 val loss 5.8516
-10700 val perplexity 347.7813
-10700 train 5.837807 (lr=1.5062e-05) (hash(x)=158730240)
-10800 val loss 5.7688
-10800 val perplexity 320.1381
-10800 train 5.818008 (lr=1.5758e-05) (hash(x)=163095518)
-10800 val loss 5.8491
-10800 val perplexity 346.9075
-10800 train 5.900390 (lr=1.4773e-05) (hash(x)=163095518)
-10900 val loss 5.7664
-10900 val perplexity 319.3910
-10900 train 5.526252 (lr=1.5453e-05) (hash(x)=145260395)
-10900 val loss 5.8431
-10900 val perplexity 344.8545
-10900 train 5.609083 (lr=1.4487e-05) (hash(x)=145260395)
-11000 val loss 5.7589
-11000 val perplexity 316.9950
-11000 train 5.760559 (lr=1.5149e-05) (hash(x)=155970264)
-11000 val loss 5.8394
-11000 val perplexity 343.5878
-11000 train 5.845952 (lr=1.4203e-05) (hash(x)=155970264)
-11100 val loss 5.7567
-11100 val perplexity 316.3170
-11100 train 5.850685 (lr=1.4848e-05) (hash(x)=159871097)
-11100 val loss 5.8376
-11100 val perplexity 342.9427
-11100 train 5.931939 (lr=1.3920e-05) (hash(x)=159871097)
-11200 val loss 5.7492
-11200 val perplexity 313.9388
-11200 train 5.841630 (lr=1.4550e-05) (hash(x)=161135244)
-11200 val loss 5.8295
-11200 val perplexity 340.1743
-11200 train 5.919682 (lr=1.3640e-05) (hash(x)=161135244)
-11300 val loss 5.7410
-11300 val perplexity 311.3792
-11300 train 5.780535 (lr=1.4253e-05) (hash(x)=150649336)
-11300 val loss 5.8223
-11300 val perplexity 337.7465
-11300 train 5.851455 (lr=1.3362e-05) (hash(x)=150649336)
-11400 val loss 5.7409
-11400 val perplexity 311.3582
-11400 train 5.710909 (lr=1.3960e-05) (hash(x)=152230013)
-11400 val loss 5.8216
-11400 val perplexity 337.5094
-11400 train 5.791006 (lr=1.3087e-05) (hash(x)=152230013)
-11500 val loss 5.7378
-11500 val perplexity 310.3662
-11500 train 5.758396 (lr=1.3668e-05) (hash(x)=163300613)
-11500 val loss 5.8162
-11500 val perplexity 335.6881
-11500 train 5.859279 (lr=1.2814e-05) (hash(x)=163300613)
-11600 val loss 5.7372
-11600 val perplexity 310.1859
-11600 train 5.718981 (lr=1.3380e-05) (hash(x)=151203838)
-11600 val loss 5.8142
-11600 val perplexity 335.0154
-11600 train 5.800889 (lr=1.2544e-05) (hash(x)=151203838)
-11700 val loss 5.7318
-11700 val perplexity 308.5238
-11700 train 5.752216 (lr=1.3095e-05) (hash(x)=155255111)
-11700 val loss 5.8111
-11700 val perplexity 333.9854
-11700 train 5.829021 (lr=1.2276e-05) (hash(x)=155255111)
-11800 val loss 5.7282
-11800 val perplexity 307.4052
-11800 train 5.689683 (lr=1.2812e-05) (hash(x)=139380668)
-11800 val loss 5.8103
-11800 val perplexity 333.7100
-11800 train 5.770501 (lr=1.2011e-05) (hash(x)=139380668)
-11900 val loss 5.7189
-11900 val perplexity 304.5619
-11900 train 5.653548 (lr=1.2532e-05) (hash(x)=144033405)
-11900 val loss 5.8001
-11900 val perplexity 330.3267
-11900 train 5.729938 (lr=1.1749e-05) (hash(x)=144033405)
-12000 val loss 5.7152
-12000 val perplexity 303.4598
-12000 train 5.636559 (lr=1.2256e-05) (hash(x)=148689574)
-12000 val loss 5.7968
-12000 val perplexity 329.2376
-12000 train 5.719021 (lr=1.1490e-05) (hash(x)=148689574)
-12100 val loss 5.7144
-12100 val perplexity 303.2094
-12100 train 5.659702 (lr=1.1983e-05) (hash(x)=143272239)
-12100 val loss 5.7941
-12100 val perplexity 328.3486
-12100 train 5.742701 (lr=1.1234e-05) (hash(x)=143272239)
-12200 val loss 5.7108
-12200 val perplexity 302.1156
-12200 train 5.794576 (lr=1.1713e-05) (hash(x)=157671609)
-12200 val loss 5.7898
-12200 val perplexity 326.9385
-12200 train 5.877266 (lr=1.0981e-05) (hash(x)=157671609)
-12300 val loss 5.7066
-12300 val perplexity 300.8392
-12300 train 5.723989 (lr=1.1446e-05) (hash(x)=150363247)
-12300 val loss 5.7873
-12300 val perplexity 326.1414
-12300 train 5.806932 (lr=1.0731e-05) (hash(x)=150363247)
-12400 val loss 5.7022
-12400 val perplexity 299.5182
-12400 train 5.592573 (lr=1.1183e-05) (hash(x)=146857787)
-12400 val loss 5.7808
-12400 val perplexity 324.0274
-12400 train 5.672683 (lr=1.0484e-05) (hash(x)=146857787)
-12500 val loss 5.7012
-12500 val perplexity 299.2310
-12500 train 5.667606 (lr=1.0924e-05) (hash(x)=153327320)
-12500 val loss 5.7830
-12500 val perplexity 324.7299
-12500 train 5.742136 (lr=1.0241e-05) (hash(x)=153327320)
-12600 val loss 5.6965
-12600 val perplexity 297.8202
-12600 train 5.419716 (lr=1.0668e-05) (hash(x)=143612983)
-12600 val loss 5.7757
-12600 val perplexity 322.3680
-12600 train 5.494945 (lr=1.0001e-05) (hash(x)=143612983)
-12700 val loss 5.6943
-12700 val perplexity 297.1594
-12700 train 5.625340 (lr=1.0416e-05) (hash(x)=150739132)
-12700 val loss 5.7727
-12700 val perplexity 321.4064
-12700 train 5.713191 (lr=9.7649e-06) (hash(x)=150739132)
-12800 val loss 5.6909
-12800 val perplexity 296.1632
-12800 train 5.540470 (lr=1.0168e-05) (hash(x)=147986204)
-12800 val loss 5.7704
-12800 val perplexity 320.6530
-12800 train 5.618282 (lr=9.5322e-06) (hash(x)=147986204)
-12900 val loss 5.6897
-12900 val perplexity 295.8014
-12900 train 5.856245 (lr=9.9233e-06) (hash(x)=155218833)
-12900 val loss 5.7692
-12900 val perplexity 320.2925
-12900 train 5.924139 (lr=9.3031e-06) (hash(x)=155218833)
-13000 val loss 5.6853
-13000 val perplexity 294.5142
-13000 train 5.679413 (lr=9.6829e-06) (hash(x)=148867960)
-13000 val loss 5.7643
-13000 val perplexity 318.7263
-13000 train 5.762329 (lr=9.0777e-06) (hash(x)=148867960)
-13100 val loss 5.6814
-13100 val perplexity 293.3666
-13100 train 5.713234 (lr=9.4466e-06) (hash(x)=160101478)
-13100 val loss 5.7600
-13100 val perplexity 317.3558
-13100 train 5.790013 (lr=8.8562e-06) (hash(x)=160101478)
-13200 val loss 5.6827
-13200 val perplexity 293.7467
-13200 train 5.541912 (lr=9.2144e-06) (hash(x)=151080097)
-13200 val loss 5.7604
-13200 val perplexity 317.4737
-13200 train 5.619044 (lr=8.6385e-06) (hash(x)=151080097)
-13300 val loss 5.6789
-13300 val perplexity 292.6209
-13300 train 5.679009 (lr=8.9865e-06) (hash(x)=150403039)
-13300 val loss 5.7578
-13300 val perplexity 316.6603
-13300 train 5.761934 (lr=8.4248e-06) (hash(x)=150403039)
-13400 val loss 5.6757
-13400 val perplexity 291.6800
-13400 train 5.628583 (lr=8.7628e-06) (hash(x)=129728571)
-13400 val loss 5.7552
-13400 val perplexity 315.8320
-13400 train 5.708203 (lr=8.2151e-06) (hash(x)=129728571)
-13500 val loss 5.6703
-13500 val perplexity 290.1295
-13500 train 5.593028 (lr=8.5435e-06) (hash(x)=148988625)
-13500 val loss 5.7486
-13500 val perplexity 313.7387
-13500 train 5.668513 (lr=8.0095e-06) (hash(x)=148988625)
-13600 val loss 5.6693
-13600 val perplexity 289.8436
-13600 train 5.457337 (lr=8.3286e-06) (hash(x)=149790861)
-13600 val loss 5.7477
-13600 val perplexity 313.4661
-13600 train 5.534057 (lr=7.8081e-06) (hash(x)=149790861)
-13700 val loss 5.6673
-13700 val perplexity 289.2669
-13700 train 5.596827 (lr=8.1183e-06) (hash(x)=153505062)
-13700 val loss 5.7464
-13700 val perplexity 313.0768
-13700 train 5.678047 (lr=7.6109e-06) (hash(x)=153505062)
-13800 val loss 5.6684
-13800 val perplexity 289.5752
-13800 train 5.570222 (lr=7.9125e-06) (hash(x)=143845463)
-13800 val loss 5.7460
-13800 val perplexity 312.9442
-13800 train 5.656930 (lr=7.4180e-06) (hash(x)=143845463)
-13900 val loss 5.6655
-13900 val perplexity 288.7445
-13900 train 5.464289 (lr=7.7114e-06) (hash(x)=143378076)
-13900 val loss 5.7445
-13900 val perplexity 312.4745
-13900 train 5.542448 (lr=7.2294e-06) (hash(x)=143378076)
-14000 val loss 5.6604
-14000 val perplexity 287.2565
-14000 train 5.620647 (lr=7.5149e-06) (hash(x)=148681755)
-14000 val loss 5.7382
-14000 val perplexity 310.4907
-14000 train 5.702466 (lr=7.0452e-06) (hash(x)=148681755)
-14100 val loss 5.6578
-14100 val perplexity 286.5279
-14100 train 5.536157 (lr=7.3233e-06) (hash(x)=145530465)
-14100 val loss 5.7370
-14100 val perplexity 310.1180
-14100 train 5.622849 (lr=6.8656e-06) (hash(x)=145530465)
-14200 val loss 5.6592
-14200 val perplexity 286.9243
-14200 train 5.594131 (lr=7.1365e-06) (hash(x)=162139713)
-14200 val loss 5.7349
-14200 val perplexity 309.4690
-14200 train 5.676126 (lr=6.6904e-06) (hash(x)=162139713)
-14300 val loss 5.6543
-14300 val perplexity 285.5057
-14300 train 5.672934 (lr=6.9546e-06) (hash(x)=141997596)
-14300 val loss 5.7317
-14300 val perplexity 308.5032
-14300 train 5.741300 (lr=6.5199e-06) (hash(x)=141997596)
-14400 val loss 5.6544
-14400 val perplexity 285.5538
-14400 train 5.624477 (lr=6.7776e-06) (hash(x)=146156313)
-14400 val loss 5.7329
-14400 val perplexity 308.8662
-14400 train 5.704103 (lr=6.3540e-06) (hash(x)=146156313)
-14500 val loss 5.6525
-14500 val perplexity 285.0071
-14500 train 5.428591 (lr=6.6057e-06) (hash(x)=145298178)
-14500 val loss 5.7310
-14500 val perplexity 308.2753
-14500 train 5.500850 (lr=6.1928e-06) (hash(x)=145298178)
-14600 val loss 5.6484
-14600 val perplexity 283.8466
-14600 train 5.744937 (lr=6.4389e-06) (hash(x)=148232610)
-14600 val loss 5.7246
-14600 val perplexity 306.3022
-14600 train 5.808825 (lr=6.0364e-06) (hash(x)=148232610)
-14700 val loss 5.6463
-14700 val perplexity 283.2440
-14700 train 5.532464 (lr=6.2771e-06) (hash(x)=151079115)
-14700 val loss 5.7234
-14700 val perplexity 305.9421
-14700 train 5.605141 (lr=5.8848e-06) (hash(x)=151079115)
-14800 val loss 5.6452
-14800 val perplexity 282.9186
-14800 train 5.631949 (lr=6.1206e-06) (hash(x)=151340983)
-14800 val loss 5.7220
-14800 val perplexity 305.5163
-14800 train 5.697337 (lr=5.7381e-06) (hash(x)=151340983)
-14900 val loss 5.6443
-14900 val perplexity 282.6823
-14900 train 5.854811 (lr=5.9693e-06) (hash(x)=150213803)
-14900 val loss 5.7197
-14900 val perplexity 304.8096
-14900 train 5.932733 (lr=5.5962e-06) (hash(x)=150213803)
-15000 val loss 5.6443
-15000 val perplexity 282.6677
-15000 train 5.559547 (lr=5.8233e-06) (hash(x)=157652334)
-15000 val loss 5.7195
-15000 val perplexity 304.7498
-15000 train 5.633491 (lr=5.4593e-06) (hash(x)=157652334)
-15100 val loss 5.6418
-15100 val perplexity 281.9667
-15100 train 5.371411 (lr=5.6826e-06) (hash(x)=145078577)
-15100 val loss 5.7171
-15100 val perplexity 304.0203
-15100 train 5.451268 (lr=5.3275e-06) (hash(x)=145078577)
-15200 val loss 5.6389
-15200 val perplexity 281.1660
-15200 train 5.578742 (lr=5.5473e-06) (hash(x)=155898006)
-15200 val loss 5.7147
-15200 val perplexity 303.3001
-15200 train 5.655529 (lr=5.2006e-06) (hash(x)=155898006)
-15300 val loss 5.6362
-15300 val perplexity 280.3892
-15300 train 5.665936 (lr=5.4175e-06) (hash(x)=149346671)
-15300 val loss 5.7129
-15300 val perplexity 302.7553
-15300 train 5.740127 (lr=5.0789e-06) (hash(x)=149346671)
-15400 val loss 5.6351
-15400 val perplexity 280.0972
-15400 train 5.487409 (lr=5.2931e-06) (hash(x)=145708113)
-15400 val loss 5.7106
-15400 val perplexity 302.0578
-15400 train 5.557944 (lr=4.9622e-06) (hash(x)=145708113)
-15500 val loss 5.6321
-15500 val perplexity 279.2454
-15500 train 5.809040 (lr=5.1742e-06) (hash(x)=165102970)
-15500 val loss 5.7091
-15500 val perplexity 301.5966
-15500 train 5.890470 (lr=4.8508e-06) (hash(x)=165102970)
-15600 val loss 5.6338
-15600 val perplexity 279.7115
-15600 train 5.420284 (lr=5.0608e-06) (hash(x)=139106517)
-15600 val loss 5.7103
-15600 val perplexity 301.9613
-15600 train 5.507070 (lr=4.7445e-06) (hash(x)=139106517)
-15700 val loss 5.6319
-15700 val perplexity 279.2039
-15700 train 5.497750 (lr=4.9531e-06) (hash(x)=141680838)
-15700 val loss 5.7078
-15700 val perplexity 301.2041
-15700 train 5.570164 (lr=4.6435e-06) (hash(x)=141680838)
-15800 val loss 5.6292
-15800 val perplexity 278.4494
-15800 train 5.530750 (lr=4.8509e-06) (hash(x)=136152310)
-15800 val loss 5.7050
-15800 val perplexity 300.3506
-15800 train 5.599582 (lr=4.5478e-06) (hash(x)=136152310)
-15900 val loss 5.6275
-15900 val perplexity 277.9763
-15900 train 5.688832 (lr=4.7545e-06) (hash(x)=151104640)
-15900 val loss 5.7034
-15900 val perplexity 299.8883
-15900 train 5.773046 (lr=4.4573e-06) (hash(x)=151104640)
-16000 val loss 5.6271
-16000 val perplexity 277.8607
-16000 train 5.584705 (lr=4.6637e-06) (hash(x)=153333281)
-16000 val loss 5.7023
-16000 val perplexity 299.5677
-16000 train 5.651556 (lr=4.3722e-06) (hash(x)=153333281)
-16100 val loss 5.6251
-16100 val perplexity 277.2950
-16100 train 5.577593 (lr=4.5786e-06) (hash(x)=145628059)
-16100 val loss 5.7011
-16100 val perplexity 299.1999
-16100 train 5.658373 (lr=4.2924e-06) (hash(x)=145628059)
-16200 val loss 5.6259
-16200 val perplexity 277.5304
-16200 train 5.351377 (lr=4.4993e-06) (hash(x)=137220041)
-16200 val loss 5.7009
-16200 val perplexity 299.1331
-16200 train 5.426072 (lr=4.2180e-06) (hash(x)=137220041)
-16300 val loss 5.6244
-16300 val perplexity 277.0934
-16300 train 5.800812 (lr=4.4257e-06) (hash(x)=161215791)
-16300 val loss 5.6993
-16300 val perplexity 298.6519
-16300 train 5.872560 (lr=4.1491e-06) (hash(x)=161215791)
-16400 val loss 5.6233
-16400 val perplexity 276.8119
-16400 train 5.331463 (lr=4.3579e-06) (hash(x)=136194680)
-16400 val loss 5.6981
-16400 val perplexity 298.3057
-16400 train 5.408731 (lr=4.0856e-06) (hash(x)=136194680)
-16500 val loss 5.6232
-16500 val perplexity 276.7720
-16500 train 5.408642 (lr=4.2960e-06) (hash(x)=152209568)
-16500 val loss 5.6986
-16500 val perplexity 298.4346
-16500 train 5.489052 (lr=4.0275e-06) (hash(x)=152209568)
-16600 val loss 5.6196
-16600 val perplexity 275.7748
-16600 train 5.738178 (lr=4.2399e-06) (hash(x)=157345721)
-16600 val loss 5.6951
-16600 val perplexity 297.4131
-16600 train 5.814222 (lr=3.9749e-06) (hash(x)=157345721)
-16700 val loss 5.6179
-16700 val perplexity 275.3081
-16700 train 5.744716 (lr=4.1896e-06) (hash(x)=146883642)
-16700 val loss 5.6931
-16700 val perplexity 296.8066
-16700 train 5.804691 (lr=3.9278e-06) (hash(x)=146883642)
-16800 val loss 5.6177
-16800 val perplexity 275.2513
-16800 train 5.589443 (lr=4.1452e-06) (hash(x)=153037972)
-16800 val loss 5.6927
-16800 val perplexity 296.6937
-16800 train 5.661279 (lr=3.8862e-06) (hash(x)=153037972)
-16900 val loss 5.6175
-16900 val perplexity 275.1880
-16900 train 5.389691 (lr=4.1067e-06) (hash(x)=150691367)
-16900 val loss 5.6925
-16900 val perplexity 296.6332
-16900 train 5.463097 (lr=3.8501e-06) (hash(x)=150691367)
-17000 val loss 5.6172
-17000 val perplexity 275.1071
-17000 train 5.362617 (lr=4.0741e-06) (hash(x)=148975206)
-17000 val loss 5.6918
-17000 val perplexity 296.4331
-17000 train 5.442601 (lr=3.8195e-06) (hash(x)=148975206)
-17100 val loss 5.6178
-17100 val perplexity 275.2888
-17100 train 5.347479 (lr=4.0475e-06) (hash(x)=142026221)
-17100 val loss 5.6924
-17100 val perplexity 296.6107
-17100 train 5.421805 (lr=3.7945e-06) (hash(x)=142026221)
-17200 val loss 5.6141
-17200 val perplexity 274.2535
-17200 train 5.675111 (lr=4.0267e-06) (hash(x)=138211880)
-17200 val loss 5.6894
-17200 val perplexity 295.7159
-17200 train 5.752285 (lr=3.7750e-06) (hash(x)=138211880)
-17300 val loss 5.6114
-17300 val perplexity 273.5252
-17300 train 5.478740 (lr=4.0119e-06) (hash(x)=143366666)
-17300 val loss 5.6868
-17300 val perplexity 294.9388
-17300 train 5.559046 (lr=3.7611e-06) (hash(x)=143366666)
-17400 val loss 5.6115
-17400 val perplexity 273.5565
-17400 train 5.612268 (lr=4.0030e-06) (hash(x)=150791766)
-17400 val loss 5.6855
-17400 val perplexity 294.5702
-17400 train 5.678813 (lr=3.7528e-06) (hash(x)=150791766)
-17499 val loss 5.6105
-17499 val perplexity 273.2749
-17499 val loss 5.6855
-17499 val perplexity 294.5559
+max_steps: 8750
+0 val loss 11.3014
+0 val perplexity 80933.0312
+2100 val loss 6.3224
+2100 val perplexity 556.9330
+2100 train 6.196806 (lr=3.6761e-05) (hash(x)=146254754)
+2200 val loss 6.2845
+2200 val perplexity 536.1783
+0 train 11.305981 (lr=1.0000e-07) (hash(x)=153418542)
+2200 train 6.313621 (lr=3.6358e-05) (hash(x)=150630461)
+2300 val loss 6.2546
+2300 val perplexity 520.3895
+100 val loss 9.9368
+100 val perplexity 20677.2930
+2300 train 6.399120 (lr=3.5935e-05) (hash(x)=165208942)
+100 train 9.942423 (lr=1.0100e-05) (hash(x)=143722001)
+2400 val loss 6.2406
+2400 val perplexity 513.1553
+2400 train 6.163989 (lr=3.5491e-05) (hash(x)=153448706)
+200 val loss 9.1738
+200 val perplexity 9641.2969
+200 train 9.168076 (lr=2.0100e-05) (hash(x)=152989689)
+2500 val loss 6.2074
+2500 val perplexity 496.4181
+2500 train 6.221104 (lr=3.5027e-05) (hash(x)=148505056)
+300 val loss 8.0313
+300 val perplexity 3075.7539
+300 train 8.040445 (lr=3.0100e-05) (hash(x)=150071018)
+2600 val loss 6.1837
+2600 val perplexity 484.7639
+2600 train 6.135411 (lr=3.4545e-05) (hash(x)=142911960)
+400 val loss 7.5503
+400 val perplexity 1901.2579
+400 train 7.585745 (lr=4.0100e-05) (hash(x)=153358238)
+2700 val loss 6.1688
+2700 val perplexity 477.6137
+2700 train 6.176059 (lr=3.4044e-05) (hash(x)=149951660)
+500 val loss 7.4193
+500 val perplexity 1667.9182
+500 train 7.292376 (lr=5.0000e-05) (hash(x)=148983354)
+2800 val loss 6.1399
+2800 val perplexity 464.0015
+2800 train 6.131063 (lr=3.3526e-05) (hash(x)=152956713)
+600 val loss 7.3269
+600 val perplexity 1520.5901
+600 train 7.215260 (lr=4.9984e-05) (hash(x)=150770333)
+2900 val loss 6.1245
+2900 val perplexity 456.9059
+2900 train 6.038101 (lr=3.2992e-05) (hash(x)=147247056)
+700 val loss 7.2152
+700 val perplexity 1359.8782
+700 train 7.133345 (lr=4.9935e-05) (hash(x)=145246201)
+3000 val loss 6.1072
+3000 val perplexity 449.0740
+3000 train 5.915527 (lr=3.2441e-05) (hash(x)=146911716)
+800 val loss 7.1187
+800 val perplexity 1234.8700
+800 train 6.945539 (lr=4.9853e-05) (hash(x)=142635842)
+3100 val loss 6.0823
+3100 val perplexity 438.0373
+3100 train 6.100656 (lr=3.1875e-05) (hash(x)=153282809)
+900 val loss 7.0203
+900 val perplexity 1119.0918
+900 train 7.039608 (lr=4.9739e-05) (hash(x)=148299832)
+3200 val loss 6.0701
+3200 val perplexity 432.7231
+3200 train 5.970005 (lr=3.1295e-05) (hash(x)=152009984)
+1000 val loss 6.9440
+1000 val perplexity 1036.9138
+1000 train 6.905874 (lr=4.9593e-05) (hash(x)=152887721)
+3300 val loss 6.0525
+3300 val perplexity 425.1593
+3300 train 5.965965 (lr=3.0702e-05) (hash(x)=150012952)
+1100 val loss 6.8704
+1100 val perplexity 963.2864
+1100 train 6.759805 (lr=4.9415e-05) (hash(x)=153721194)
+3400 val loss 6.0426
+3400 val perplexity 420.9893
+3400 train 5.870891 (lr=3.0095e-05) (hash(x)=146217477)
+1200 val loss 6.7894
+1200 val perplexity 888.3673
+1200 train 6.780546 (lr=4.9205e-05) (hash(x)=153047184)
+3500 val loss 6.0158
+3500 val perplexity 409.8569
+3500 train 6.183609 (lr=2.9477e-05) (hash(x)=180156144)
+1300 val loss 6.7008
+1300 val perplexity 813.0482
+1300 train 6.604268 (lr=4.8964e-05) (hash(x)=152466045)
+3600 val loss 6.0103
+3600 val perplexity 407.6101
+3600 train 6.041433 (lr=2.8849e-05) (hash(x)=154243319)
+1400 val loss 6.6371
+1400 val perplexity 762.8847
+1400 train 6.432257 (lr=4.8691e-05) (hash(x)=151991552)
+3700 val loss 5.9959
+3700 val perplexity 401.7838
+3700 train 5.900227 (lr=2.8210e-05) (hash(x)=154330476)
+1500 val loss 6.5722
+1500 val perplexity 714.9579
+1500 train 6.518350 (lr=4.8388e-05) (hash(x)=147892594)
+3800 val loss 5.9745
+3800 val perplexity 393.2638
+3800 train 5.912947 (lr=2.7562e-05) (hash(x)=145988858)
+1600 val loss 6.5054
+1600 val perplexity 668.7148
+1600 train 6.312996 (lr=4.8055e-05) (hash(x)=135678663)
+3900 val loss 5.9611
+3900 val perplexity 388.0560
+3900 train 5.967092 (lr=2.6907e-05) (hash(x)=149648609)
+1700 val loss 6.4774
+1700 val perplexity 650.2755
+1700 train 6.497375 (lr=4.7691e-05) (hash(x)=143909276)
+4000 val loss 5.9522
+4000 val perplexity 384.5835
+4000 train 5.925228 (lr=2.6244e-05) (hash(x)=154149272)
+1800 val loss 6.4415
+1800 val perplexity 627.3212
+1800 train 6.596472 (lr=4.7299e-05) (hash(x)=155393468)
+4100 val loss 5.9394
+4100 val perplexity 379.7084
+4100 train 5.833369 (lr=2.5575e-05) (hash(x)=132608538)
+1900 val loss 6.3915
+1900 val perplexity 596.7308
+1900 train 6.438015 (lr=4.6878e-05) (hash(x)=150957388)
+4200 val loss 5.9195
+4200 val perplexity 372.2381
+4200 train 5.868464 (lr=2.4900e-05) (hash(x)=140443636)
+2000 val loss 6.3512
+2000 val perplexity 573.1802
+2000 train 6.613527 (lr=4.6428e-05) (hash(x)=144030938)
+4300 val loss 5.9172
+4300 val perplexity 371.3643
+4300 train 5.843295 (lr=2.4222e-05) (hash(x)=138919540)
+2100 val loss 6.3281
+2100 val perplexity 560.1060
+2100 train 6.201053 (lr=4.5951e-05) (hash(x)=146254754)
+4400 val loss 5.9048
+4400 val perplexity 366.7889
+4400 train 5.880773 (lr=2.3540e-05) (hash(x)=153594684)
+2200 val loss 6.2873
+2200 val perplexity 537.7227
+2200 train 6.317572 (lr=4.5448e-05) (hash(x)=150630461)
+4500 val loss 5.8904
+4500 val perplexity 361.5636
+4500 train 5.839415 (lr=2.2856e-05) (hash(x)=144084750)
+2300 val loss 6.2593
+2300 val perplexity 522.8305
+2300 train 6.396164 (lr=4.4918e-05) (hash(x)=165208942)
+4600 val loss 5.8777
+4600 val perplexity 356.9865
+4600 train 5.835000 (lr=2.2171e-05) (hash(x)=147423675)
+2400 val loss 6.2430
+2400 val perplexity 514.4112
+2400 train 6.165377 (lr=4.4363e-05) (hash(x)=153448706)
+4700 val loss 5.8663
+4700 val perplexity 352.9321
+4700 train 5.934175 (lr=2.1486e-05) (hash(x)=162157039)
+2500 val loss 6.2069
+2500 val perplexity 496.1466
+2500 train 6.221903 (lr=4.3784e-05) (hash(x)=148505056)
+4800 val loss 5.8548
+4800 val perplexity 348.8984
+4800 train 5.754379 (lr=2.0801e-05) (hash(x)=134970942)
+2600 val loss 6.1822
+2600 val perplexity 484.0474
+2600 train 6.129616 (lr=4.3181e-05) (hash(x)=142911960)
+4900 val loss 5.8505
+4900 val perplexity 347.4015
+4900 train 5.765707 (lr=2.0118e-05) (hash(x)=160093370)
+2700 val loss 6.1644
+2700 val perplexity 475.5042
+2700 train 6.167647 (lr=4.2555e-05) (hash(x)=149951660)
+5000 val loss 5.8379
+5000 val perplexity 343.0621
+5000 train 5.732998 (lr=1.9438e-05) (hash(x)=154986299)
+2800 val loss 6.1327
+2800 val perplexity 460.6697
+2800 train 6.113997 (lr=4.1908e-05) (hash(x)=152956713)
+5100 val loss 5.8256
+5100 val perplexity 338.8670
+5100 train 5.676945 (lr=1.8762e-05) (hash(x)=145406582)
+2900 val loss 6.1145
+2900 val perplexity 452.3685
+2900 train 6.028293 (lr=4.1240e-05) (hash(x)=147247056)
+5200 val loss 5.8219
+5200 val perplexity 337.6047
+5200 train 5.873819 (lr=1.8090e-05) (hash(x)=148029261)
+3000 val loss 6.0926
+3000 val perplexity 442.5560
+3000 train 5.884184 (lr=4.0551e-05) (hash(x)=146911716)
+5300 val loss 5.8068
+5300 val perplexity 332.5408
+5300 train 5.803960 (lr=1.7424e-05) (hash(x)=157909233)
+3100 val loss 6.0708
+3100 val perplexity 433.0459
+3100 train 6.087581 (lr=3.9844e-05) (hash(x)=153282809)
+5400 val loss 5.7990
+5400 val perplexity 329.9706
+5400 train 5.673873 (lr=1.6765e-05) (hash(x)=142250860)
+3200 val loss 6.0537
+3200 val perplexity 425.6975
+3200 train 5.953282 (lr=3.9119e-05) (hash(x)=152009984)
+5500 val loss 5.7910
+5500 val perplexity 327.3319
+5500 train 5.669307 (lr=1.6113e-05) (hash(x)=145694703)
+3300 val loss 6.0375
+3300 val perplexity 418.8448
+3300 train 5.941113 (lr=3.8377e-05) (hash(x)=150012952)
+5600 val loss 5.7805
+5600 val perplexity 323.9262
+5600 train 5.709619 (lr=1.5469e-05) (hash(x)=141788252)
+3400 val loss 6.0252
+3400 val perplexity 413.7304
+3400 train 5.861111 (lr=3.7619e-05) (hash(x)=146217477)
+5700 val loss 5.7769
+5700 val perplexity 322.7441
+5700 train 5.676227 (lr=1.4836e-05) (hash(x)=150886692)
+3500 val loss 5.9987
+3500 val perplexity 402.9040
+3500 train 6.165073 (lr=3.6847e-05) (hash(x)=180156144)
+5800 val loss 5.7699
+5800 val perplexity 320.5020
+5800 train 5.680188 (lr=1.4212e-05) (hash(x)=151814419)
+3600 val loss 5.9841
+3600 val perplexity 397.0735
+3600 train 6.020372 (lr=3.6061e-05) (hash(x)=154243319)
+5900 val loss 5.7615
+5900 val perplexity 317.8253
+5900 train 5.777674 (lr=1.3600e-05) (hash(x)=146972539)
+3700 val loss 5.9718
+3700 val perplexity 392.2184
+3700 train 5.873587 (lr=3.5263e-05) (hash(x)=154330476)
+6000 val loss 5.7513
+6000 val perplexity 314.6142
+6000 train 5.636987 (lr=1.3000e-05) (hash(x)=150219954)
+3800 val loss 5.9490
+3800 val perplexity 383.3524
+3800 train 5.881207 (lr=3.4453e-05) (hash(x)=145988858)
+6100 val loss 5.7502
+6100 val perplexity 314.2680
+6100 train 5.747046 (lr=1.2413e-05) (hash(x)=161131435)
+3900 val loss 5.9333
+3900 val perplexity 377.3957
+3900 train 5.938285 (lr=3.3633e-05) (hash(x)=149648609)
+6200 val loss 5.7391
+6200 val perplexity 310.7870
+6200 train 5.753590 (lr=1.1840e-05) (hash(x)=150226893)
+4000 val loss 5.9220
+4000 val perplexity 373.1478
+4000 train 5.897403 (lr=3.2805e-05) (hash(x)=154149272)
+6300 val loss 5.7344
+6300 val perplexity 309.3321
+6300 train 5.564880 (lr=1.1282e-05) (hash(x)=144389552)
+4100 val loss 5.9014
+4100 val perplexity 365.5590
+4100 train 5.797142 (lr=3.1968e-05) (hash(x)=132608538)
+6400 val loss 5.7310
+6400 val perplexity 308.2737
+6400 train 5.513679 (lr=1.0739e-05) (hash(x)=147757943)
+4200 val loss 5.8845
+4200 val perplexity 359.4143
+4200 train 5.833921 (lr=3.1126e-05) (hash(x)=140443636)
+6500 val loss 5.7216
+6500 val perplexity 305.4032
+6500 train 5.706445 (lr=1.0213e-05) (hash(x)=144616373)
+4300 val loss 5.8805
+4300 val perplexity 358.0006
+4300 train 5.813493 (lr=3.0277e-05) (hash(x)=138919540)
+6600 val loss 5.7210
+6600 val perplexity 305.2003
+6600 train 5.544097 (lr=9.7032e-06) (hash(x)=145266452)
+4400 val loss 5.8668
+4400 val perplexity 353.1261
+4400 train 5.851140 (lr=2.9425e-05) (hash(x)=153594684)
+6700 val loss 5.7158
+6700 val perplexity 303.6178
+6700 train 5.709424 (lr=9.2116e-06) (hash(x)=149952261)
+4500 val loss 5.8488
+4500 val perplexity 346.8007
+4500 train 5.799411 (lr=2.8571e-05) (hash(x)=144084750)
+6800 val loss 5.7100
+6800 val perplexity 301.8742
+6800 train 5.594436 (lr=8.7387e-06) (hash(x)=143351199)
+4600 val loss 5.8407
+4600 val perplexity 344.0316
+4600 train 5.799302 (lr=2.7714e-05) (hash(x)=147423675)
+6900 val loss 5.7100
+6900 val perplexity 301.8849
+6900 train 5.672192 (lr=8.2849e-06) (hash(x)=156055618)
+4700 val loss 5.8252
+4700 val perplexity 338.7215
+4700 train 5.892113 (lr=2.6857e-05) (hash(x)=162157039)
+7000 val loss 5.7022
+7000 val perplexity 299.5153
+7000 train 5.774765 (lr=7.8510e-06) (hash(x)=164290908)
+4800 val loss 5.8125
+4800 val perplexity 334.4376
+4800 train 5.717739 (lr=2.6002e-05) (hash(x)=134970942)
+7100 val loss 5.6998
+7100 val perplexity 298.7951
+7100 train 5.805682 (lr=7.4377e-06) (hash(x)=150263832)
+4900 val loss 5.8041
+4900 val perplexity 331.6604
+4900 train 5.717675 (lr=2.5148e-05) (hash(x)=160093370)
+7200 val loss 5.6960
+7200 val perplexity 297.6701
+7200 train 5.380092 (lr=7.0455e-06) (hash(x)=139219680)
+5000 val loss 5.7910
+5000 val perplexity 327.3266
+5000 train 5.678819 (lr=2.4298e-05) (hash(x)=154986299)
+7300 val loss 5.6911
+7300 val perplexity 296.2145
+7300 train 5.636185 (lr=6.6749e-06) (hash(x)=150662994)
+5100 val loss 5.7785
+5100 val perplexity 323.2635
+5100 train 5.629937 (lr=2.3452e-05) (hash(x)=145406582)
+7400 val loss 5.6884
+7400 val perplexity 295.4274
+7400 train 5.530379 (lr=6.3266e-06) (hash(x)=139007967)
+5200 val loss 5.7709
+5200 val perplexity 320.8177
+5200 train 5.830273 (lr=2.2613e-05) (hash(x)=148029261)
+7500 val loss 5.6865
+7500 val perplexity 294.8652
+7500 train 5.512374 (lr=6.0010e-06) (hash(x)=138142461)
+5300 val loss 5.7569
+5300 val perplexity 316.3814
+5300 train 5.755656 (lr=2.1780e-05) (hash(x)=157909233)
+7600 val loss 5.6826
+7600 val perplexity 293.7216
+7600 train 5.603275 (lr=5.6986e-06) (hash(x)=150023998)
+5400 val loss 5.7469
+5400 val perplexity 313.2240
+5400 train 5.614817 (lr=2.0956e-05) (hash(x)=142250860)
+7700 val loss 5.6784
+7700 val perplexity 292.4821
+7700 train 5.571332 (lr=5.4198e-06) (hash(x)=143393355)
+5500 val loss 5.7398
+5500 val perplexity 310.9943
+5500 train 5.616222 (lr=2.0141e-05) (hash(x)=145694703)
+7800 val loss 5.6787
+7800 val perplexity 292.5575
+7800 train 5.549252 (lr=5.1650e-06) (hash(x)=144916472)
+5600 val loss 5.7284
+5600 val perplexity 307.4781
+5600 train 5.665291 (lr=1.9337e-05) (hash(x)=141788252)
+7900 val loss 5.6739
+7900 val perplexity 291.1699
+7900 train 5.630789 (lr=4.9347e-06) (hash(x)=150236934)
+5700 val loss 5.7235
+5700 val perplexity 305.9635
+5700 train 5.621311 (lr=1.8545e-05) (hash(x)=150886692)
+8000 val loss 5.6710
+8000 val perplexity 290.3376
+8000 train 5.667097 (lr=4.7291e-06) (hash(x)=146536422)
+5800 val loss 5.7146
+5800 val perplexity 303.2628
+5800 train 5.627031 (lr=1.7765e-05) (hash(x)=151814419)
+8100 val loss 5.6706
+8100 val perplexity 290.2078
+8100 train 5.504349 (lr=4.5486e-06) (hash(x)=151300857)
+5900 val loss 5.7072
+5900 val perplexity 301.0217
+5900 train 5.720672 (lr=1.7000e-05) (hash(x)=146972539)
+8200 val loss 5.6692
+8200 val perplexity 289.8167
+8200 train 5.704734 (lr=4.3933e-06) (hash(x)=160351956)
+6000 val loss 5.6969
+6000 val perplexity 297.9379
+6000 train 5.586535 (lr=1.6250e-05) (hash(x)=150219954)
+8300 val loss 5.6657
+8300 val perplexity 288.7936
+8300 train 5.714310 (lr=4.2636e-06) (hash(x)=150514540)
+6100 val loss 5.6921
+6100 val perplexity 296.5178
+6100 train 5.682611 (lr=1.5516e-05) (hash(x)=161131435)
+8400 val loss 5.6639
+8400 val perplexity 288.2700
+8400 train 5.613809 (lr=4.1596e-06) (hash(x)=155904762)
+6200 val loss 5.6831
+6200 val perplexity 293.8677
+6200 train 5.696242 (lr=1.4800e-05) (hash(x)=150226893)
+8500 val loss 5.6650
+8500 val perplexity 288.5907
+8500 train 5.469418 (lr=4.0815e-06) (hash(x)=146923196)
+6300 val loss 5.6761
+6300 val perplexity 291.8135
+6300 train 5.511406 (lr=1.4102e-05) (hash(x)=144389552)
+8600 val loss 5.6607
+8600 val perplexity 287.3493
+8600 train 5.639455 (lr=4.0294e-06) (hash(x)=138977080)
+6400 val loss 5.6739
+6400 val perplexity 291.1631
+6400 train 5.455398 (lr=1.3424e-05) (hash(x)=147757943)
+8700 val loss 5.6583
+8700 val perplexity 286.6700
+8700 train 5.469374 (lr=4.0033e-06) (hash(x)=146442792)
+6500 val loss 5.6628
+6500 val perplexity 287.9458
+6500 train 5.650785 (lr=1.2766e-05) (hash(x)=144616373)
+8749 val loss 5.6585
+8749 val perplexity 286.7288

attention_kindselective_n_heads4_seed1343/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ee53b4486c4cb41c99be5888f5595be96f01a47f3d6d1d579a99796a3ee0edb
+size 92843394

attention_kindselective_n_heads4_seed1343/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d575091f4789d6077dd6bfd3d412c5f5e486d4a56e8ef7e5786a8c8edf62788c
+size 179406214