Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_8/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_8", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.05e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.05e-5_30720_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1340", "n_embd": 256}

attention_kindselective_n_heads4_seed1340/log2.txt

@@ -1,1056 +1,528 @@
 max_steps: 17500
-max_steps: 17500
-0 val loss 11.2714
-0 val perplexity 78545.6094
 0 val loss 11.2714
 0 val perplexity 78545.6094
-0 train 11.276529 (lr=1.4750e-07) (hash(x)=164406924)
-0 train 11.276529 (lr=1.5250e-07) (hash(x)=164406924)
-100 val loss 9.9612
-100 val perplexity 21188.6855
-100 train 10.006220 (lr=1.4897e-05) (hash(x)=177407419)
-100 val loss 9.9558
-100 val perplexity 21073.3965
-100 train 10.003653 (lr=1.5403e-05) (hash(x)=177407419)
-200 val loss 9.2026
-200 val perplexity 9923.2881
-200 train 9.172215 (lr=2.9500e-05) (hash(x)=144903932)
-200 val loss 9.1251
-200 val perplexity 9182.9336
-200 train 9.093577 (lr=3.0500e-05) (hash(x)=144903932)
-300 val loss 8.0609
-300 val perplexity 3168.1726
-300 train 8.293051 (lr=2.9498e-05) (hash(x)=173839165)
-300 val loss 8.0313
-300 val perplexity 3075.7188
-300 train 8.270664 (lr=3.0498e-05) (hash(x)=173839165)
-400 val loss 7.6644
-400 val perplexity 2131.1240
-400 train 7.737237 (lr=2.9491e-05) (hash(x)=167734596)
-400 val loss 7.6533
-400 val perplexity 2107.5828
-400 train 7.728591 (lr=3.0491e-05) (hash(x)=167734596)
-500 val loss 7.5344
-500 val perplexity 1871.2731
-500 train 7.543181 (lr=2.9480e-05) (hash(x)=153224076)
-500 val loss 7.5275
-500 val perplexity 1858.4547
-500 train 7.543041 (lr=3.0480e-05) (hash(x)=153224076)
-600 val loss 7.4460
-600 val perplexity 1712.9829
-600 train 7.355884 (lr=2.9465e-05) (hash(x)=149619098)
-600 val loss 7.4438
-600 val perplexity 1709.2794
-600 train 7.351353 (lr=3.0464e-05) (hash(x)=149619098)
-700 val loss 7.3917
-700 val perplexity 1622.5197
-700 train 7.316007 (lr=2.9445e-05) (hash(x)=146539909)
-700 val loss 7.3963
-700 val perplexity 1629.8839
-700 train 7.320476 (lr=3.0443e-05) (hash(x)=146539909)
-800 val loss 7.3327
-800 val perplexity 1529.4814
-800 train 7.291888 (lr=2.9421e-05) (hash(x)=153710890)
-800 val loss 7.3376
-800 val perplexity 1537.0096
-800 train 7.298015 (lr=3.0419e-05) (hash(x)=153710890)
-900 val loss 7.2943
-900 val perplexity 1471.8456
-900 train 7.229879 (lr=2.9393e-05) (hash(x)=155873620)
-900 val loss 7.2932
-900 val perplexity 1470.3263
-900 train 7.232080 (lr=3.0389e-05) (hash(x)=155873620)
-1000 val loss 7.2531
-1000 val perplexity 1412.4503
-1000 train 7.169968 (lr=2.9360e-05) (hash(x)=145450636)
-1000 val loss 7.2468
-1000 val perplexity 1403.5846
-1000 train 7.168369 (lr=3.0355e-05) (hash(x)=145450636)
-1100 val loss 7.2089
-1100 val perplexity 1351.3810
-1100 train 7.291048 (lr=2.9323e-05) (hash(x)=154123388)
-1100 val loss 7.1965
-1100 val perplexity 1334.7189
-1100 train 7.278819 (lr=3.0317e-05) (hash(x)=154123388)
-1200 val loss 7.1646
-1200 val perplexity 1292.8929
-1200 train 7.036717 (lr=2.9282e-05) (hash(x)=145249251)
-1200 val loss 7.1517
-1200 val perplexity 1276.2872
-1200 train 7.016351 (lr=3.0274e-05) (hash(x)=145249251)
-1300 val loss 7.1247
-1300 val perplexity 1242.2773
-1300 train 7.016234 (lr=2.9236e-05) (hash(x)=148937127)
-1300 val loss 7.1006
-1300 val perplexity 1212.6757
-1300 train 6.996818 (lr=3.0227e-05) (hash(x)=148937127)
-1400 val loss 7.0852
-1400 val perplexity 1194.1539
-1400 train 7.204118 (lr=2.9186e-05) (hash(x)=150475545)
-1400 val loss 7.0536
-1400 val perplexity 1157.0510
-1400 train 7.171196 (lr=3.0175e-05) (hash(x)=150475545)
-1500 val loss 7.0447
-1500 val perplexity 1146.7710
-1500 train 6.967178 (lr=2.9132e-05) (hash(x)=154653428)
-1500 val loss 7.0131
-1500 val perplexity 1111.0433
-1500 train 6.936914 (lr=3.0119e-05) (hash(x)=154653428)
-1600 val loss 7.0077
-1600 val perplexity 1105.1024
-1600 train 6.988377 (lr=2.9073e-05) (hash(x)=144483776)
-1600 val loss 6.9723
-1600 val perplexity 1066.6764
-1600 train 6.953465 (lr=3.0059e-05) (hash(x)=144483776)
-1700 val loss 6.9640
-1700 val perplexity 1057.8845
-1700 train 7.115364 (lr=2.9011e-05) (hash(x)=157395496)
-1700 val loss 6.9215
-1700 val perplexity 1013.8137
-1700 train 7.072005 (lr=2.9994e-05) (hash(x)=157395496)
-1800 val loss 6.9364
-1800 val perplexity 1029.0996
-1800 train 6.959817 (lr=2.8944e-05) (hash(x)=157916369)
-1800 val loss 6.8781
-1800 val perplexity 970.8041
-1800 train 6.901133 (lr=2.9925e-05) (hash(x)=157916369)
-1900 val loss 6.9014
-1900 val perplexity 993.6882
-1900 train 7.113924 (lr=2.8872e-05) (hash(x)=166073923)
-1900 val loss 6.8391
-1900 val perplexity 933.6599
-1900 train 7.033895 (lr=2.9851e-05) (hash(x)=166073923)
-2000 val loss 6.8831
-2000 val perplexity 975.6500
-2000 train 6.949834 (lr=2.8797e-05) (hash(x)=154856891)
-2000 val loss 6.8018
-2000 val perplexity 899.4428
-2000 train 6.876286 (lr=2.9773e-05) (hash(x)=154856891)
-2100 val loss 6.8510
-2100 val perplexity 944.8170
-2100 train 6.844026 (lr=2.8718e-05) (hash(x)=151925203)
-2100 val loss 6.7696
-2100 val perplexity 870.9310
-2100 train 6.761881 (lr=2.9691e-05) (hash(x)=151925203)
-2200 val loss 6.8167
-2200 val perplexity 912.9668
-2200 train 6.528778 (lr=2.8634e-05) (hash(x)=136191502)
-2200 val loss 6.7244
-2200 val perplexity 832.5077
-2200 train 6.428940 (lr=2.9605e-05) (hash(x)=136191502)
-2300 val loss 6.7885
-2300 val perplexity 887.5883
-2300 train 6.960556 (lr=2.8546e-05) (hash(x)=153273362)
-2300 val loss 6.6809
-2300 val perplexity 797.0347
-2300 train 6.863890 (lr=2.9514e-05) (hash(x)=153273362)
-2400 val loss 6.7598
-2400 val perplexity 862.4758
-2400 train 6.706277 (lr=2.8455e-05) (hash(x)=148021541)
-2400 val loss 6.6474
-2400 val perplexity 770.8004
-2400 train 6.601905 (lr=2.9419e-05) (hash(x)=148021541)
-2500 val loss 6.7341
-2500 val perplexity 840.6030
-2500 train 6.681586 (lr=2.8359e-05) (hash(x)=141356608)
-2500 val loss 6.6245
-2500 val perplexity 753.3402
-2500 train 6.575332 (lr=2.9320e-05) (hash(x)=141356608)
-2600 val loss 6.7021
-2600 val perplexity 814.1523
-2600 train 6.645393 (lr=2.8259e-05) (hash(x)=146005217)
-2600 val loss 6.5878
-2600 val perplexity 726.2054
-2600 train 6.530656 (lr=2.9217e-05) (hash(x)=146005217)
-2700 val loss 6.6751
-2700 val perplexity 792.4149
-2700 train 6.515198 (lr=2.8155e-05) (hash(x)=144511718)
-2700 val loss 6.5667
-2700 val perplexity 711.0460
-2700 train 6.407398 (lr=2.9110e-05) (hash(x)=144511718)
-2800 val loss 6.6421
-2800 val perplexity 766.7025
-2800 train 6.523296 (lr=2.8048e-05) (hash(x)=146019502)
-2800 val loss 6.5385
-2800 val perplexity 691.2452
-2800 train 6.414044 (lr=2.8998e-05) (hash(x)=146019502)
-2900 val loss 6.6151
-2900 val perplexity 746.3038
-2900 train 6.535785 (lr=2.7936e-05) (hash(x)=146496200)
-2900 val loss 6.5143
-2900 val perplexity 674.7101
-2900 train 6.425395 (lr=2.8883e-05) (hash(x)=146496200)
-3000 val loss 6.5938
-3000 val perplexity 730.5236
-3000 train 6.537404 (lr=2.7821e-05) (hash(x)=150127281)
-3000 val loss 6.5004
-3000 val perplexity 665.4392
-3000 train 6.449020 (lr=2.8764e-05) (hash(x)=150127281)
-3100 val loss 6.5534
-3100 val perplexity 701.6056
-3100 train 6.532422 (lr=2.7701e-05) (hash(x)=142022255)
-3100 val loss 6.4610
-3100 val perplexity 639.6718
-3100 train 6.444718 (lr=2.8640e-05) (hash(x)=142022255)
-3200 val loss 6.5303
-3200 val perplexity 685.6353
-3200 train 6.586569 (lr=2.7578e-05) (hash(x)=154120875)
-3200 val loss 6.4382
-3200 val perplexity 625.3065
-3200 train 6.493232 (lr=2.8513e-05) (hash(x)=154120875)
-3300 val loss 6.5133
-3300 val perplexity 674.0442
-3300 train 6.572422 (lr=2.7452e-05) (hash(x)=153999717)
-3400 val loss 6.4945
-3400 val perplexity 661.4954
-3400 train 6.261442 (lr=2.7321e-05) (hash(x)=139694097)
-3300 val loss 6.4252
-3300 val perplexity 617.1780
-3300 train 6.495522 (lr=2.8382e-05) (hash(x)=153999717)
-3500 val loss 6.4641
-3500 val perplexity 641.7006
-3500 train 6.660486 (lr=2.7187e-05) (hash(x)=162992732)
-3400 val loss 6.4077
-3400 val perplexity 606.4818
-3400 train 6.175876 (lr=2.8247e-05) (hash(x)=139694097)
-3600 val loss 6.4390
-3600 val perplexity 625.7778
-3600 train 6.411188 (lr=2.7049e-05) (hash(x)=147574101)
-3500 val loss 6.3812
-3500 val perplexity 590.6281
-3500 train 6.569015 (lr=2.8108e-05) (hash(x)=162992732)
-3700 val loss 6.4230
-3700 val perplexity 615.8516
-3700 train 6.538586 (lr=2.6908e-05) (hash(x)=157763099)
-3600 val loss 6.3560
-3600 val perplexity 575.9402
-3600 train 6.329640 (lr=2.7966e-05) (hash(x)=147574101)
-3800 val loss 6.3991
-3800 val perplexity 601.2900
-3800 train 6.568602 (lr=2.6763e-05) (hash(x)=170800034)
-3700 val loss 6.3427
-3700 val perplexity 568.3350
-3700 train 6.453629 (lr=2.7820e-05) (hash(x)=157763099)
-3900 val loss 6.3828
-3900 val perplexity 591.5856
-3900 train 6.461458 (lr=2.6615e-05) (hash(x)=164984528)
-3800 val loss 6.3177
-3800 val perplexity 554.2958
-3800 train 6.477601 (lr=2.7670e-05) (hash(x)=170800034)
-4000 val loss 6.3664
-4000 val perplexity 581.9594
-4000 train 6.224706 (lr=2.6463e-05) (hash(x)=141743323)
-3900 val loss 6.3055
-3900 val perplexity 547.5947
-3900 train 6.378394 (lr=2.7517e-05) (hash(x)=164984528)
-4100 val loss 6.3640
-4100 val perplexity 580.5658
-4100 train 6.380234 (lr=2.6308e-05) (hash(x)=153392872)
-4000 val loss 6.2930
-4000 val perplexity 540.7996
-4000 train 6.150142 (lr=2.7360e-05) (hash(x)=141743323)
-4200 val loss 6.3509
-4200 val perplexity 573.0050
-4200 train 6.207468 (lr=2.6149e-05) (hash(x)=149074933)
-4100 val loss 6.2880
-4100 val perplexity 538.0890
-4100 train 6.308994 (lr=2.7199e-05) (hash(x)=153392872)
-4300 val loss 6.3258
-4300 val perplexity 558.8289
-4300 train 6.732800 (lr=2.5987e-05) (hash(x)=167823423)
-4200 val loss 6.2769
-4200 val perplexity 532.1280
-4200 train 6.140339 (lr=2.7036e-05) (hash(x)=149074933)
-4400 val loss 6.3022
-4400 val perplexity 545.7892
-4400 train 6.065737 (lr=2.5822e-05) (hash(x)=141203114)
-4300 val loss 6.2568
-4300 val perplexity 521.5397
-4300 train 6.652555 (lr=2.6868e-05) (hash(x)=167823423)
-4500 val loss 6.2880
-4500 val perplexity 538.0849
-4500 train 6.280446 (lr=2.5654e-05) (hash(x)=146284780)
-4400 val loss 6.2329
-4400 val perplexity 509.2309
-4400 train 5.999859 (lr=2.6698e-05) (hash(x)=141203114)
-4600 val loss 6.2694
-4600 val perplexity 528.1438
-4600 train 6.071446 (lr=2.5483e-05) (hash(x)=141126464)
-4500 val loss 6.2196
-4500 val perplexity 502.4771
-4500 train 6.213103 (lr=2.6524e-05) (hash(x)=146284780)
-4700 val loss 6.2613
-4700 val perplexity 523.8964
-4700 train 6.287362 (lr=2.5309e-05) (hash(x)=154751926)
-4600 val loss 6.2020
-4600 val perplexity 493.7245
-4600 train 6.001839 (lr=2.6347e-05) (hash(x)=141126464)
-4800 val loss 6.2422
-4800 val perplexity 514.0108
-4800 train 6.328922 (lr=2.5132e-05) (hash(x)=154793198)
-4700 val loss 6.1925
-4700 val perplexity 489.0862
-4700 train 6.219493 (lr=2.6167e-05) (hash(x)=154751926)
-4900 val loss 6.2359
-4900 val perplexity 510.7386
-4900 train 5.951726 (lr=2.4951e-05) (hash(x)=139406392)
-4800 val loss 6.1818
-4800 val perplexity 483.8584
-4800 train 6.263108 (lr=2.5983e-05) (hash(x)=154793198)
-5000 val loss 6.2496
-5000 val perplexity 517.8170
-5000 train 5.963733 (lr=2.4768e-05) (hash(x)=153548741)
-4900 val loss 6.1752
-4900 val perplexity 480.6560
-4900 train 5.892793 (lr=2.5797e-05) (hash(x)=139406392)
-5100 val loss 6.2215
-5100 val perplexity 503.4706
-5100 train 6.301985 (lr=2.4582e-05) (hash(x)=160488568)
-5000 val loss 6.1842
-5000 val perplexity 485.0205
-5000 train 5.899323 (lr=2.5608e-05) (hash(x)=153548741)
-5200 val loss 6.1934
-5200 val perplexity 489.5265
-5200 train 6.208883 (lr=2.4394e-05) (hash(x)=149645053)
-5100 val loss 6.1545
-5100 val perplexity 470.8249
-5100 train 6.236096 (lr=2.5416e-05) (hash(x)=160488568)
-5300 val loss 6.1741
-5300 val perplexity 480.1360
-5300 train 6.268353 (lr=2.4202e-05) (hash(x)=155820556)
-5200 val loss 6.1354
-5200 val perplexity 461.9279
-5200 train 6.151593 (lr=2.5221e-05) (hash(x)=149645053)
-5400 val loss 6.1655
-5400 val perplexity 476.0362
-5400 train 6.107147 (lr=2.4008e-05) (hash(x)=147538134)
-5300 val loss 6.1214
-5300 val perplexity 455.5058
-5300 train 6.220849 (lr=2.5023e-05) (hash(x)=155820556)
-5500 val loss 6.1542
-5500 val perplexity 470.6933
-5500 train 6.365412 (lr=2.3812e-05) (hash(x)=166889307)
-5400 val loss 6.1098
-5400 val perplexity 450.2610
-5400 train 6.053759 (lr=2.4822e-05) (hash(x)=147538134)
-5600 val loss 6.1425
-5600 val perplexity 465.2056
-5600 train 5.843464 (lr=2.3613e-05) (hash(x)=139516699)
-5500 val loss 6.0992
-5500 val perplexity 445.4850
-5500 train 6.309348 (lr=2.4619e-05) (hash(x)=166889307)
-5700 val loss 6.1412
-5700 val perplexity 464.6161
-5700 train 5.821039 (lr=2.3411e-05) (hash(x)=140453511)
-5600 val loss 6.0895
-5600 val perplexity 441.1877
-5600 train 5.789936 (lr=2.4413e-05) (hash(x)=139516699)
-5800 val loss 6.1338
-5800 val perplexity 461.1799
-5800 train 6.074871 (lr=2.3207e-05) (hash(x)=162964847)
-5700 val loss 6.0868
-5700 val perplexity 439.9948
-5700 train 5.766156 (lr=2.4205e-05) (hash(x)=140453511)
-5900 val loss 6.1149
-5900 val perplexity 452.5323
-5900 train 6.124761 (lr=2.3001e-05) (hash(x)=150606634)
-5800 val loss 6.0816
-5800 val perplexity 437.7437
-5800 train 6.022608 (lr=2.3994e-05) (hash(x)=162964847)
-6000 val loss 6.0988
-6000 val perplexity 445.3229
-6000 train 6.196362 (lr=2.2793e-05) (hash(x)=149890857)
-5900 val loss 6.0711
-5900 val perplexity 433.1378
-5900 train 6.078366 (lr=2.3781e-05) (hash(x)=150606634)
-6100 val loss 6.0935
-6100 val perplexity 442.9765
-6100 train 6.199735 (lr=2.2582e-05) (hash(x)=173884145)
-6000 val loss 6.0504
-6000 val perplexity 424.2790
-6000 train 6.140145 (lr=2.3565e-05) (hash(x)=149890857)
-6200 val loss 6.0823
-6200 val perplexity 438.0464
-6200 train 6.139063 (lr=2.2370e-05) (hash(x)=151987098)
-6100 val loss 6.0425
-6100 val perplexity 420.9497
-6100 train 6.145662 (lr=2.3348e-05) (hash(x)=173884145)
-6300 val loss 6.0696
-6300 val perplexity 432.5190
-6300 train 6.020157 (lr=2.2155e-05) (hash(x)=148853562)
-6200 val loss 6.0331
-6200 val perplexity 416.9969
-6200 train 6.082293 (lr=2.3128e-05) (hash(x)=151987098)
-6400 val loss 6.0591
-6400 val perplexity 427.9744
-6400 train 5.805988 (lr=2.1938e-05) (hash(x)=141530101)
-6300 val loss 6.0234
-6300 val perplexity 412.9968
-6300 train 5.979120 (lr=2.2906e-05) (hash(x)=148853562)
-6500 val loss 6.0577
-6500 val perplexity 427.3907
-6500 train 5.916256 (lr=2.1720e-05) (hash(x)=142297809)
-6400 val loss 6.0141
-6400 val perplexity 409.1740
-6400 train 5.763428 (lr=2.2682e-05) (hash(x)=141530101)
-6600 val loss 6.0428
-6600 val perplexity 421.0878
-6600 train 5.931437 (lr=2.1499e-05) (hash(x)=142447782)
-6500 val loss 6.0080
-6500 val perplexity 406.6629
-6500 train 5.870059 (lr=2.2456e-05) (hash(x)=142297809)
-6700 val loss 6.0333
-6700 val perplexity 417.0826
-6700 train 6.019310 (lr=2.1277e-05) (hash(x)=147004686)
-6800 val loss 6.0276
-6800 val perplexity 414.7079
-6800 train 5.771273 (lr=2.1054e-05) (hash(x)=133438702)
-6600 val loss 5.9967
-6600 val perplexity 402.1040
-6600 train 5.886486 (lr=2.2228e-05) (hash(x)=142447782)
-6900 val loss 6.0200
-6900 val perplexity 411.5841
-6900 train 6.039604 (lr=2.0828e-05) (hash(x)=157085143)
-6700 val loss 5.9911
-6700 val perplexity 399.8509
-6700 train 5.985365 (lr=2.1999e-05) (hash(x)=147004686)
-7000 val loss 6.0151
-7000 val perplexity 409.5512
-7000 train 5.892037 (lr=2.0601e-05) (hash(x)=139437666)
-6800 val loss 5.9823
-6800 val perplexity 396.3528
-6800 train 5.729127 (lr=2.1767e-05) (hash(x)=133438702)
-7100 val loss 6.0100
-7100 val perplexity 407.4838
-7100 train 6.032113 (lr=2.0373e-05) (hash(x)=159792986)
-6900 val loss 5.9791
-6900 val perplexity 395.0695
-6900 train 6.009018 (lr=2.1534e-05) (hash(x)=157085143)
-7200 val loss 5.9949
-7200 val perplexity 401.3906
-7200 train 5.973925 (lr=2.0143e-05) (hash(x)=144930687)
-7000 val loss 5.9687
-7000 val perplexity 390.9916
-7000 train 5.842317 (lr=2.1300e-05) (hash(x)=139437666)
-7300 val loss 5.9862
-7300 val perplexity 397.8849
-7300 train 6.036131 (lr=1.9912e-05) (hash(x)=156242690)
-7100 val loss 5.9667
-7100 val perplexity 390.2237
-7100 train 5.990251 (lr=2.1064e-05) (hash(x)=159792986)
-7400 val loss 5.9789
-7400 val perplexity 394.9920
-7400 train 5.817974 (lr=1.9680e-05) (hash(x)=148183719)
-7200 val loss 5.9492
-7200 val perplexity 383.4538
-7200 train 5.930403 (lr=2.0826e-05) (hash(x)=144930687)
-7500 val loss 5.9766
-7500 val perplexity 394.1110
-7500 train 6.009599 (lr=1.9447e-05) (hash(x)=152494758)
-7300 val loss 5.9448
-7300 val perplexity 381.7592
-7300 train 5.994674 (lr=2.0587e-05) (hash(x)=156242690)
-7600 val loss 5.9707
-7600 val perplexity 391.7646
-7600 train 5.660017 (lr=1.9213e-05) (hash(x)=142485027)
-7400 val loss 5.9358
-7400 val perplexity 378.3456
-7400 train 5.770916 (lr=2.0347e-05) (hash(x)=148183719)
-7700 val loss 5.9690
-7700 val perplexity 391.1080
-7700 train 5.841330 (lr=1.8977e-05) (hash(x)=147512165)
-7500 val loss 5.9322
-7500 val perplexity 376.9727
-7500 train 5.978520 (lr=2.0106e-05) (hash(x)=152494758)
-7800 val loss 5.9483
-7800 val perplexity 383.0841
-7800 train 5.834394 (lr=1.8741e-05) (hash(x)=160346994)
-7600 val loss 5.9272
-7600 val perplexity 375.1145
-7600 train 5.623278 (lr=1.9864e-05) (hash(x)=142485027)
-7900 val loss 5.9439
-7900 val perplexity 381.4062
-7900 train 5.812826 (lr=1.8504e-05) (hash(x)=144488254)
-7700 val loss 5.9254
-7700 val perplexity 374.4411
-7700 train 5.796915 (lr=1.9620e-05) (hash(x)=147512165)
-8000 val loss 5.9390
-8000 val perplexity 379.5373
-8000 train 5.753357 (lr=1.8266e-05) (hash(x)=147637019)
-7800 val loss 5.9027
-7800 val perplexity 366.0223
-7800 train 5.785273 (lr=1.9376e-05) (hash(x)=160346994)
-8100 val loss 5.9349
-8100 val perplexity 378.0092
-8100 train 5.791764 (lr=1.8027e-05) (hash(x)=147340534)
-7900 val loss 5.9005
-7900 val perplexity 365.2079
-7900 train 5.766904 (lr=1.9131e-05) (hash(x)=144488254)
-8200 val loss 5.9269
-8200 val perplexity 375.0050
-8200 train 5.984520 (lr=1.7788e-05) (hash(x)=151630665)
-8000 val loss 5.8916
-8000 val perplexity 361.9913
-8000 train 5.701701 (lr=1.8885e-05) (hash(x)=147637019)
-8300 val loss 5.9208
-8300 val perplexity 372.6976
-8300 train 6.020539 (lr=1.7549e-05) (hash(x)=149747064)
-8100 val loss 5.8897
-8100 val perplexity 361.2820
-8100 train 5.744904 (lr=1.8639e-05) (hash(x)=147340534)
-8400 val loss 5.9112
-8400 val perplexity 369.1445
-8400 train 6.101376 (lr=1.7309e-05) (hash(x)=154245770)
-8200 val loss 5.8844
-8200 val perplexity 359.3797
-8200 train 5.940163 (lr=1.8391e-05) (hash(x)=151630665)
-8500 val loss 5.9057
-8500 val perplexity 367.1293
-8500 train 5.791136 (lr=1.7068e-05) (hash(x)=152559100)
-8300 val loss 5.8750
-8300 val perplexity 356.0187
-8300 train 5.979764 (lr=1.8144e-05) (hash(x)=149747064)
-8600 val loss 5.9003
-8600 val perplexity 365.1461
-8600 train 6.306458 (lr=1.6827e-05) (hash(x)=181365926)
-8400 val loss 5.8646
-8400 val perplexity 352.3515
-8400 train 6.064944 (lr=1.7895e-05) (hash(x)=154245770)
-8700 val loss 5.8952
-8700 val perplexity 363.2717
-8700 train 5.731752 (lr=1.6587e-05) (hash(x)=154405991)
-8500 val loss 5.8618
-8500 val perplexity 351.3398
-8500 train 5.747521 (lr=1.7647e-05) (hash(x)=152559100)
-8800 val loss 5.8902
-8800 val perplexity 361.4599
-8800 train 5.836451 (lr=1.6346e-05) (hash(x)=153755904)
-8600 val loss 5.8546
-8600 val perplexity 348.8516
-8600 train 6.279002 (lr=1.7398e-05) (hash(x)=181365926)
-8900 val loss 5.8853
-8900 val perplexity 359.7249
-8900 train 5.799029 (lr=1.6104e-05) (hash(x)=152120568)
-8700 val loss 5.8515
-8700 val perplexity 347.7646
-8700 train 5.685025 (lr=1.7149e-05) (hash(x)=154405991)
-9000 val loss 5.8697
-9000 val perplexity 354.1284
-9000 train 5.660941 (lr=1.5863e-05) (hash(x)=142797279)
-8800 val loss 5.8433
-8800 val perplexity 344.9015
-8800 train 5.790849 (lr=1.6900e-05) (hash(x)=153755904)
-9100 val loss 5.8662
-9100 val perplexity 352.8938
-9100 train 5.730277 (lr=1.5623e-05) (hash(x)=143037503)
-8900 val loss 5.8412
-8900 val perplexity 344.2032
-8900 train 5.750267 (lr=1.6650e-05) (hash(x)=152120568)
-9200 val loss 5.8677
-9200 val perplexity 353.4283
-9200 train 5.714442 (lr=1.5382e-05) (hash(x)=113690273)
-9000 val loss 5.8249
-9000 val perplexity 338.6162
-9000 train 5.621056 (lr=1.6401e-05) (hash(x)=142797279)
-9300 val loss 5.8614
-9300 val perplexity 351.2326
-9300 train 5.799366 (lr=1.5141e-05) (hash(x)=158025077)
-9100 val loss 5.8196
-9100 val perplexity 336.8413
-9100 train 5.684803 (lr=1.6152e-05) (hash(x)=143037503)
-9400 val loss 5.8511
-9400 val perplexity 347.6318
-9400 train 5.929654 (lr=1.4901e-05) (hash(x)=158251718)
-9200 val loss 5.8194
-9200 val perplexity 336.7675
-9200 train 5.681998 (lr=1.5903e-05) (hash(x)=113690273)
-9500 val loss 5.8466
-9500 val perplexity 346.0451
-9500 train 5.887828 (lr=1.4662e-05) (hash(x)=154752610)
-9300 val loss 5.8163
-9300 val perplexity 335.7299
-9300 train 5.755792 (lr=1.5655e-05) (hash(x)=158025077)
-9600 val loss 5.8436
-9600 val perplexity 345.0094
-9600 train 5.716836 (lr=1.4423e-05) (hash(x)=146889093)
-9400 val loss 5.8058
-9400 val perplexity 332.2243
-9400 train 5.883069 (lr=1.5406e-05) (hash(x)=158251718)
-9700 val loss 5.8362
-9700 val perplexity 342.4679
-9700 train 5.837679 (lr=1.4184e-05) (hash(x)=156906516)
-9500 val loss 5.8029
-9500 val perplexity 331.2690
-9500 train 5.845038 (lr=1.5159e-05) (hash(x)=154752610)
-9800 val loss 5.8320
-9800 val perplexity 341.0322
-9800 train 5.661427 (lr=1.3946e-05) (hash(x)=153841927)
-9600 val loss 5.7953
-9600 val perplexity 328.7651
-9600 train 5.674147 (lr=1.4911e-05) (hash(x)=146889093)
-9900 val loss 5.8267
-9900 val perplexity 339.2269
-9900 train 6.035855 (lr=1.3709e-05) (hash(x)=163514334)
-9700 val loss 5.7869
-9700 val perplexity 325.9968
-9700 train 5.789225 (lr=1.4665e-05) (hash(x)=156906516)
-10000 val loss 5.8212
-10000 val perplexity 337.3610
-10000 train 5.808736 (lr=1.3473e-05) (hash(x)=144745561)
-9800 val loss 5.7854
-9800 val perplexity 325.5192
-9800 train 5.612669 (lr=1.4419e-05) (hash(x)=153841927)
-10100 val loss 5.8184
-10100 val perplexity 336.4173
-10100 train 6.062419 (lr=1.3237e-05) (hash(x)=161651149)
-9900 val loss 5.7797
-9900 val perplexity 323.6472
-9900 train 5.995339 (lr=1.4174e-05) (hash(x)=163514334)
-10200 val loss 5.8097
-10200 val perplexity 333.5081
-10200 train 5.753011 (lr=1.3003e-05) (hash(x)=143667037)
-10000 val loss 5.7740
-10000 val perplexity 321.8253
-10000 train 5.772288 (lr=1.3930e-05) (hash(x)=144745561)
-10300 val loss 5.8036
-10300 val perplexity 331.4956
-10300 train 5.699328 (lr=1.2770e-05) (hash(x)=151687262)
-10100 val loss 5.7695
-10100 val perplexity 320.3718
-10100 train 6.002267 (lr=1.3686e-05) (hash(x)=161651149)
-10400 val loss 5.8024
-10400 val perplexity 331.0900
-10400 train 5.663408 (lr=1.2538e-05) (hash(x)=153694955)
-10200 val loss 5.7626
-10200 val perplexity 318.1747
-10200 train 5.704512 (lr=1.3444e-05) (hash(x)=143667037)
-10500 val loss 5.8022
-10500 val perplexity 331.0374
-10500 train 5.745658 (lr=1.2307e-05) (hash(x)=150218029)
-10300 val loss 5.7589
-10300 val perplexity 317.0058
-10300 train 5.654809 (lr=1.3203e-05) (hash(x)=151687262)
-10600 val loss 5.7928
-10600 val perplexity 327.9349
-10600 train 5.884135 (lr=1.2077e-05) (hash(x)=155093716)
-10400 val loss 5.7559
-10400 val perplexity 316.0611
-10400 train 5.609686 (lr=1.2963e-05) (hash(x)=153694955)
-10700 val loss 5.7890
-10700 val perplexity 326.6857
-10700 train 5.832874 (lr=1.1849e-05) (hash(x)=151631831)
-10800 val loss 5.7814
-10800 val perplexity 324.2130
-10800 train 5.630476 (lr=1.1622e-05) (hash(x)=136465175)
-10500 val loss 5.7543
-10500 val perplexity 315.5335
-10500 train 5.699656 (lr=1.2724e-05) (hash(x)=150218029)
-10900 val loss 5.7802
-10900 val perplexity 323.8365
-10900 train 5.713594 (lr=1.1396e-05) (hash(x)=154461514)
-10600 val loss 5.7471
-10600 val perplexity 313.2661
-10600 train 5.829342 (lr=1.2486e-05) (hash(x)=155093716)
-11000 val loss 5.7771
-11000 val perplexity 322.8137
-11000 train 5.770298 (lr=1.1173e-05) (hash(x)=141877432)
-10700 val loss 5.7432
-10700 val perplexity 312.0654
-10700 train 5.784804 (lr=1.2250e-05) (hash(x)=151631831)
-11100 val loss 5.7741
-11100 val perplexity 321.8510
-11100 train 5.568636 (lr=1.0951e-05) (hash(x)=139572894)
-10800 val loss 5.7346
-10800 val perplexity 309.3985
-10800 train 5.592227 (lr=1.2016e-05) (hash(x)=136465175)
-11200 val loss 5.7670
-11200 val perplexity 319.5854
-11200 train 5.819455 (lr=1.0730e-05) (hash(x)=155591443)
-10900 val loss 5.7335
-10900 val perplexity 309.0493
-10900 train 5.677105 (lr=1.1783e-05) (hash(x)=154461514)
-11300 val loss 5.7609
-11300 val perplexity 317.6336
-11300 train 5.775189 (lr=1.0512e-05) (hash(x)=153349136)
-11000 val loss 5.7313
-11000 val perplexity 308.3673
-11000 train 5.726075 (lr=1.1551e-05) (hash(x)=141877432)
-11400 val loss 5.7602
-11400 val perplexity 317.3967
-11400 train 5.713534 (lr=1.0295e-05) (hash(x)=148109395)
-11100 val loss 5.7299
-11100 val perplexity 307.9398
-11100 train 5.529337 (lr=1.1322e-05) (hash(x)=139572894)
-11500 val loss 5.7564
-11500 val perplexity 316.2190
-11500 train 5.742272 (lr=1.0080e-05) (hash(x)=153276620)
-11200 val loss 5.7218
-11200 val perplexity 305.4484
-11200 train 5.770621 (lr=1.1094e-05) (hash(x)=155591443)
-11600 val loss 5.7552
-11600 val perplexity 315.8361
-11600 train 5.616879 (lr=9.8678e-06) (hash(x)=151416388)
-11300 val loss 5.7162
-11300 val perplexity 303.7396
-11300 train 5.737414 (lr=1.0868e-05) (hash(x)=153349136)
-11700 val loss 5.7527
-11700 val perplexity 315.0398
-11700 train 5.625453 (lr=9.6572e-06) (hash(x)=143420927)
-11400 val loss 5.7141
-11400 val perplexity 303.1199
-11400 train 5.666860 (lr=1.0644e-05) (hash(x)=148109395)
-11800 val loss 5.7483
-11800 val perplexity 313.6524
-11800 train 5.818071 (lr=9.4488e-06) (hash(x)=165023996)
-11500 val loss 5.7104
-11500 val perplexity 301.9946
-11500 train 5.702409 (lr=1.0422e-05) (hash(x)=153276620)
-11900 val loss 5.7417
-11900 val perplexity 311.5804
-11900 train 5.652960 (lr=9.2426e-06) (hash(x)=144465076)
-11600 val loss 5.7081
-11600 val perplexity 301.3068
-11600 train 5.569135 (lr=1.0202e-05) (hash(x)=151416388)
-12000 val loss 5.7392
-12000 val perplexity 310.8163
-12000 train 5.741705 (lr=9.0388e-06) (hash(x)=155991892)
-11700 val loss 5.7050
-11700 val perplexity 300.3701
-11700 train 5.576050 (lr=9.9846e-06) (hash(x)=143420927)
-12100 val loss 5.7372
-12100 val perplexity 310.1921
-12100 train 5.762617 (lr=8.8373e-06) (hash(x)=150748844)
-11800 val loss 5.7007
-11800 val perplexity 299.0769
-11800 train 5.769990 (lr=9.7691e-06) (hash(x)=165023996)
-12200 val loss 5.7339
-12200 val perplexity 309.1668
-12200 train 5.691323 (lr=8.6382e-06) (hash(x)=149712638)
-11900 val loss 5.6943
-11900 val perplexity 297.1792
-11900 train 5.603235 (lr=9.5560e-06) (hash(x)=144465076)
-12300 val loss 5.7318
-12300 val perplexity 308.5097
-12300 train 5.547514 (lr=8.4417e-06) (hash(x)=140428230)
-12000 val loss 5.6930
-12000 val perplexity 296.7691
-12000 train 5.705177 (lr=9.3452e-06) (hash(x)=155991892)
-12400 val loss 5.7297
-12400 val perplexity 307.8627
-12400 train 5.912132 (lr=8.2477e-06) (hash(x)=163614308)
-12100 val loss 5.6917
-12100 val perplexity 296.3846
-12100 train 5.737043 (lr=9.1369e-06) (hash(x)=150748844)
-12500 val loss 5.7229
-12500 val perplexity 305.7929
-12500 train 5.686202 (lr=8.0563e-06) (hash(x)=153366153)
-12200 val loss 5.6864
-12200 val perplexity 294.8292
-12200 train 5.652375 (lr=8.9310e-06) (hash(x)=149712638)
-12600 val loss 5.7210
-12600 val perplexity 305.2249
-12600 train 5.610923 (lr=7.8676e-06) (hash(x)=146832066)
-12300 val loss 5.6849
-12300 val perplexity 294.3959
-12300 train 5.509468 (lr=8.7278e-06) (hash(x)=140428230)
-12700 val loss 5.7181
-12700 val perplexity 304.3285
-12700 train 5.893554 (lr=7.6817e-06) (hash(x)=154387414)
-12400 val loss 5.6830
-12400 val perplexity 293.8289
-12400 train 5.865954 (lr=8.5273e-06) (hash(x)=163614308)
-12800 val loss 5.7176
-12800 val perplexity 304.1666
-12800 train 5.512416 (lr=7.4986e-06) (hash(x)=139465199)
-12500 val loss 5.6773
-12500 val perplexity 292.1711
-12500 train 5.646071 (lr=8.3294e-06) (hash(x)=153366153)
-12900 val loss 5.7196
-12900 val perplexity 304.7740
-12900 train 5.425612 (lr=7.3184e-06) (hash(x)=143326374)
-12600 val loss 5.6756
-12600 val perplexity 291.6678
-12600 train 5.576900 (lr=8.1343e-06) (hash(x)=146832066)
-13000 val loss 5.7133
-13000 val perplexity 302.8570
-13000 train 5.862164 (lr=7.1412e-06) (hash(x)=158817835)
-12700 val loss 5.6706
-12700 val perplexity 290.2032
-12700 train 5.843725 (lr=7.9421e-06) (hash(x)=154387414)
-13100 val loss 5.7101
-13100 val perplexity 301.8954
-13100 train 5.583322 (lr=6.9669e-06) (hash(x)=142027874)
-12800 val loss 5.6699
-12800 val perplexity 289.9976
-12800 train 5.468815 (lr=7.7528e-06) (hash(x)=139465199)
-13200 val loss 5.7104
-13200 val perplexity 302.0002
-13200 train 5.691601 (lr=6.7956e-06) (hash(x)=148497730)
-12900 val loss 5.6711
-12900 val perplexity 290.3456
-12900 train 5.379877 (lr=7.5665e-06) (hash(x)=143326374)
-13300 val loss 5.7070
-13300 val perplexity 300.9532
-13300 train 5.610484 (lr=6.6275e-06) (hash(x)=148094633)
-13000 val loss 5.6661
-13000 val perplexity 288.9040
-13000 train 5.814001 (lr=7.3832e-06) (hash(x)=158817835)
-13400 val loss 5.7056
-13400 val perplexity 300.5562
-13400 train 5.584362 (lr=6.4626e-06) (hash(x)=150548214)
-13100 val loss 5.6625
-13100 val perplexity 287.8592
-13100 train 5.548450 (lr=7.2030e-06) (hash(x)=142027874)
-13500 val loss 5.6977
-13500 val perplexity 298.1949
-13500 train 5.812096 (lr=6.3008e-06) (hash(x)=152789725)
-13200 val loss 5.6639
-13200 val perplexity 288.2632
-13200 train 5.640491 (lr=7.0260e-06) (hash(x)=148497730)
-13600 val loss 5.6964
-13600 val perplexity 297.7827
-13600 train 5.741010 (lr=6.1424e-06) (hash(x)=153220230)
-13300 val loss 5.6616
-13300 val perplexity 287.6039
-13300 train 5.561835 (lr=6.8522e-06) (hash(x)=148094633)
-13700 val loss 5.6945
-13700 val perplexity 297.2277
-13700 train 5.709183 (lr=5.9872e-06) (hash(x)=147805699)
-13400 val loss 5.6577
-13400 val perplexity 286.4991
-13400 train 5.527104 (lr=6.6816e-06) (hash(x)=150548214)
-13800 val loss 5.6934
-13800 val perplexity 296.9066
-13800 train 5.508570 (lr=5.8355e-06) (hash(x)=146267341)
-13500 val loss 5.6513
-13500 val perplexity 284.6556
-13500 train 5.760323 (lr=6.5144e-06) (hash(x)=152789725)
-13900 val loss 5.6940
-13900 val perplexity 297.0901
-13900 train 5.511325 (lr=5.6871e-06) (hash(x)=139467501)
-13600 val loss 5.6496
-13600 val perplexity 284.1806
-13600 train 5.690665 (lr=6.3506e-06) (hash(x)=153220230)
-14000 val loss 5.6916
-14000 val perplexity 296.3788
-14000 train 5.703028 (lr=5.5423e-06) (hash(x)=152185436)
-13700 val loss 5.6465
-13700 val perplexity 283.2844
-13700 train 5.669954 (lr=6.1902e-06) (hash(x)=147805699)
-14100 val loss 5.6867
-14100 val perplexity 294.9062
-14100 train 5.816646 (lr=5.4009e-06) (hash(x)=176479139)
-13800 val loss 5.6473
-13800 val perplexity 283.5201
-13800 train 5.465186 (lr=6.0333e-06) (hash(x)=146267341)
-14200 val loss 5.6826
-14200 val perplexity 293.7156
-14200 train 5.764902 (lr=5.2632e-06) (hash(x)=151052040)
-14300 val loss 5.6824
-14300 val perplexity 293.6393
-14300 train 5.622111 (lr=5.1290e-06) (hash(x)=149668097)
-13900 val loss 5.6469
-13900 val perplexity 283.4095
-13900 train 5.472127 (lr=5.8799e-06) (hash(x)=139467501)
-14400 val loss 5.6799
-14400 val perplexity 292.9124
-14400 train 5.691532 (lr=4.9985e-06) (hash(x)=151761941)
-14000 val loss 5.6432
-14000 val perplexity 282.3772
-14000 train 5.656591 (lr=5.7301e-06) (hash(x)=152185436)
-14500 val loss 5.6789
-14500 val perplexity 292.6418
-14500 train 5.625138 (lr=4.8717e-06) (hash(x)=149412009)
-14100 val loss 5.6388
-14100 val perplexity 281.1252
-14100 train 5.771299 (lr=5.5840e-06) (hash(x)=176479139)
-14600 val loss 5.6778
-14600 val perplexity 292.2943
-14600 train 5.543564 (lr=4.7487e-06) (hash(x)=145048020)
-14200 val loss 5.6357
-14200 val perplexity 280.2610
-14200 train 5.723955 (lr=5.4416e-06) (hash(x)=151052040)
-14700 val loss 5.6761
-14700 val perplexity 291.8094
-14700 train 5.634588 (lr=4.6294e-06) (hash(x)=141771666)
-14300 val loss 5.6349
-14300 val perplexity 280.0309
-14300 train 5.577355 (lr=5.3029e-06) (hash(x)=149668097)
-14800 val loss 5.6725
-14800 val perplexity 290.7669
-14800 train 5.603262 (lr=4.5139e-06) (hash(x)=148759740)
-14400 val loss 5.6322
-14400 val perplexity 279.2820
-14400 train 5.639875 (lr=5.1679e-06) (hash(x)=151761941)
-14900 val loss 5.6704
-14900 val perplexity 290.1640
-14900 train 5.591803 (lr=4.4024e-06) (hash(x)=146287032)
-14500 val loss 5.6323
-14500 val perplexity 279.2992
-14500 train 5.582808 (lr=5.0368e-06) (hash(x)=149412009)
-15000 val loss 5.6689
-15000 val perplexity 289.7048
-15000 train 5.614189 (lr=4.2947e-06) (hash(x)=143519156)
-14600 val loss 5.6302
-14600 val perplexity 278.7218
-14600 train 5.506608 (lr=4.9096e-06) (hash(x)=145048020)
-15100 val loss 5.6712
-15100 val perplexity 290.3797
-15100 train 5.652482 (lr=4.1909e-06) (hash(x)=151330688)
-14700 val loss 5.6287
-14700 val perplexity 278.3051
-14700 train 5.582683 (lr=4.7863e-06) (hash(x)=141771666)
-15200 val loss 5.6724
-15200 val perplexity 290.7222
-15200 train 5.465379 (lr=4.0912e-06) (hash(x)=140380090)
-14800 val loss 5.6246
-14800 val perplexity 277.1598
-14800 train 5.554821 (lr=4.6670e-06) (hash(x)=148759740)
-15300 val loss 5.6673
-15300 val perplexity 289.2567
-15300 train 5.571069 (lr=3.9954e-06) (hash(x)=152598160)
-14900 val loss 5.6228
-14900 val perplexity 276.6518
-14900 train 5.546693 (lr=4.5516e-06) (hash(x)=146287032)
-15400 val loss 5.6630
-15400 val perplexity 288.0186
-15400 train 5.924234 (lr=3.9036e-06) (hash(x)=164136966)
-15000 val loss 5.6212
-15000 val perplexity 276.2336
-15000 train 5.560616 (lr=4.4403e-06) (hash(x)=143519156)
-15500 val loss 5.6637
-15500 val perplexity 288.1993
-15500 train 5.864338 (lr=3.8160e-06) (hash(x)=158733990)
-15100 val loss 5.6242
-15100 val perplexity 277.0419
-15100 train 5.603961 (lr=4.3330e-06) (hash(x)=151330688)
-15600 val loss 5.6607
-15600 val perplexity 287.3360
-15600 train 5.696318 (lr=3.7324e-06) (hash(x)=149149390)
-15200 val loss 5.6244
-15200 val perplexity 277.0932
-15200 train 5.417127 (lr=4.2298e-06) (hash(x)=140380090)
-15700 val loss 5.6599
-15700 val perplexity 287.1277
-15700 train 5.618332 (lr=3.6529e-06) (hash(x)=146746394)
-15300 val loss 5.6200
-15300 val perplexity 275.9015
-15300 train 5.521589 (lr=4.1308e-06) (hash(x)=152598160)
-15800 val loss 5.6593
-15800 val perplexity 286.9609
-15800 train 5.686823 (lr=3.5776e-06) (hash(x)=159405219)
-15400 val loss 5.6160
-15400 val perplexity 274.7924
-15400 train 5.854386 (lr=4.0360e-06) (hash(x)=164136966)
-15900 val loss 5.6585
-15900 val perplexity 286.7202
-15900 train 5.689947 (lr=3.5064e-06) (hash(x)=154034029)
-15500 val loss 5.6155
-15500 val perplexity 274.6567
-15500 train 5.812257 (lr=3.9453e-06) (hash(x)=158733990)
-16000 val loss 5.6553
-16000 val perplexity 285.8087
-16000 train 5.656010 (lr=3.4394e-06) (hash(x)=153634210)
-15600 val loss 5.6131
-15600 val perplexity 273.9823
-15600 train 5.653943 (lr=3.8589e-06) (hash(x)=149149390)
-16100 val loss 5.6539
-16100 val perplexity 285.3910
-16100 train 5.586763 (lr=3.3767e-06) (hash(x)=155646721)
-15700 val loss 5.6127
-15700 val perplexity 273.8733
-15700 train 5.564451 (lr=3.7767e-06) (hash(x)=146746394)
-16200 val loss 5.6525
-16200 val perplexity 285.0045
-16200 train 5.558064 (lr=3.3182e-06) (hash(x)=142461643)
-15800 val loss 5.6125
-15800 val perplexity 273.8196
-15800 train 5.630822 (lr=3.6988e-06) (hash(x)=159405219)
-16300 val loss 5.6528
-16300 val perplexity 285.0961
-16300 train 5.547411 (lr=3.2639e-06) (hash(x)=141080038)
-15900 val loss 5.6109
-15900 val perplexity 273.3778
-15900 train 5.639421 (lr=3.6253e-06) (hash(x)=154034029)
-16400 val loss 5.6520
-16400 val perplexity 284.8572
-16400 train 5.673695 (lr=3.2140e-06) (hash(x)=153744166)
-16000 val loss 5.6082
-16000 val perplexity 272.6521
-16000 train 5.607837 (lr=3.5560e-06) (hash(x)=153634210)
-16500 val loss 5.6554
-16500 val perplexity 285.8272
-16500 train 5.462168 (lr=3.1683e-06) (hash(x)=149783517)
-16100 val loss 5.6061
-16100 val perplexity 272.0915
-16100 train 5.536754 (lr=3.4912e-06) (hash(x)=155646721)
-16600 val loss 5.6564
-16600 val perplexity 286.1058
-16600 train 5.490465 (lr=3.1269e-06) (hash(x)=142659127)
-16200 val loss 5.6064
-16200 val perplexity 272.1524
-16200 train 5.513966 (lr=3.4307e-06) (hash(x)=142461643)
-16700 val loss 5.6499
-16700 val perplexity 284.2628
-16700 train 5.717015 (lr=3.0898e-06) (hash(x)=149725921)
-16300 val loss 5.6049
-16300 val perplexity 271.7546
-16300 train 5.501796 (lr=3.3746e-06) (hash(x)=141080038)
-16800 val loss 5.6471
-16800 val perplexity 283.4809
-16800 train 5.496078 (lr=3.0571e-06) (hash(x)=140678087)
-16400 val loss 5.6047
-16400 val perplexity 271.7025
-16400 train 5.614680 (lr=3.3229e-06) (hash(x)=153744166)
-16900 val loss 5.6469
-16900 val perplexity 283.4187
-16900 train 5.660885 (lr=3.0287e-06) (hash(x)=146935413)
-16500 val loss 5.6076
-16500 val perplexity 272.4845
-16500 train 5.423594 (lr=3.2757e-06) (hash(x)=149783517)
-17000 val loss 5.6443
-17000 val perplexity 282.6658
-17000 train 5.652937 (lr=3.0047e-06) (hash(x)=144408437)
-16600 val loss 5.6088
-16600 val perplexity 272.8082
-16600 train 5.444359 (lr=3.2329e-06) (hash(x)=142659127)
-17100 val loss 5.6438
-17100 val perplexity 282.5326
-17100 train 5.498270 (lr=2.9850e-06) (hash(x)=149229905)
-16700 val loss 5.6026
-16700 val perplexity 271.1376
-16700 train 5.672741 (lr=3.1946e-06) (hash(x)=149725921)
-17200 val loss 5.6441
-17200 val perplexity 282.6191
-17200 train 5.593342 (lr=2.9697e-06) (hash(x)=150471865)
-16800 val loss 5.5997
-16800 val perplexity 270.3323
-16800 train 5.452441 (lr=3.1607e-06) (hash(x)=140678087)
-17300 val loss 5.6467
-17300 val perplexity 283.3627
-17300 train 5.537985 (lr=2.9588e-06) (hash(x)=155784916)
-16900 val loss 5.5990
-16900 val perplexity 270.1491
-16900 train 5.616210 (lr=3.1314e-06) (hash(x)=146935413)
-17400 val loss 5.6487
-17400 val perplexity 283.9193
-17400 train 5.422133 (lr=2.9522e-06) (hash(x)=148482413)
-17000 val loss 5.5976
-17000 val perplexity 269.7783
-17000 train 5.607773 (lr=3.1065e-06) (hash(x)=144408437)
-17499 val loss 5.6416
-17499 val perplexity 281.9253
-17100 val loss 5.5968
-17100 val perplexity 269.5699
-17100 train 5.439863 (lr=3.0862e-06) (hash(x)=149229905)
-17200 val loss 5.5971
-17200 val perplexity 269.6393
-17200 train 5.554598 (lr=3.0704e-06) (hash(x)=150471865)
-17300 val loss 5.5988
-17300 val perplexity 270.0900
-17300 train 5.485799 (lr=3.0591e-06) (hash(x)=155784916)
-17400 val loss 5.6014
-17400 val perplexity 270.8054
-17400 train 5.375536 (lr=3.0523e-06) (hash(x)=148482413)
-17499 val loss 5.5943
-17499 val perplexity 268.8979
+0 train 11.276529 (lr=1.3750e-07) (hash(x)=164406924)
+100 val loss 9.9731
+100 val perplexity 21442.4805
+100 train 10.017424 (lr=1.3888e-05) (hash(x)=177407419)
+200 val loss 9.2801
+200 val perplexity 10722.6357
+200 train 9.251415 (lr=2.7500e-05) (hash(x)=144903932)
+300 val loss 8.2441
+300 val perplexity 3805.2205
+300 train 8.448895 (lr=2.7498e-05) (hash(x)=173839165)
+400 val loss 7.7675
+400 val perplexity 2362.5933
+400 train 7.834853 (lr=2.7492e-05) (hash(x)=167734596)
+500 val loss 7.6153
+500 val perplexity 2029.0551
+500 train 7.632237 (lr=2.7482e-05) (hash(x)=153224076)
+600 val loss 7.5465
+600 val perplexity 1894.1356
+600 train 7.458223 (lr=2.7467e-05) (hash(x)=149619098)
+700 val loss 7.4934
+700 val perplexity 1796.2216
+700 train 7.430860 (lr=2.7449e-05) (hash(x)=146539909)
+800 val loss 7.4395
+800 val perplexity 1701.8477
+800 train 7.405320 (lr=2.7427e-05) (hash(x)=153710890)
+900 val loss 7.4022
+900 val perplexity 1639.6630
+900 train 7.351761 (lr=2.7400e-05) (hash(x)=155873620)
+1000 val loss 7.3731
+1000 val perplexity 1592.5757
+1000 train 7.293677 (lr=2.7370e-05) (hash(x)=145450636)
+1100 val loss 7.3420
+1100 val perplexity 1543.8702
+1100 train 7.411211 (lr=2.7335e-05) (hash(x)=154123388)
+1200 val loss 7.3175
+1200 val perplexity 1506.4254
+1200 train 7.192797 (lr=2.7297e-05) (hash(x)=145249251)
+1300 val loss 7.2937
+1300 val perplexity 1470.9861
+1300 train 7.192117 (lr=2.7254e-05) (hash(x)=148937127)
+1400 val loss 7.2611
+1400 val perplexity 1423.7788
+1400 train 7.375382 (lr=2.7207e-05) (hash(x)=150475545)
+1500 val loss 7.2285
+1500 val perplexity 1378.1627
+1500 train 7.159981 (lr=2.7157e-05) (hash(x)=154653428)
+1600 val loss 7.1952
+1600 val perplexity 1332.9622
+1600 train 7.168732 (lr=2.7102e-05) (hash(x)=144483776)
+1700 val loss 7.1576
+1700 val perplexity 1283.8523
+1700 train 7.323493 (lr=2.7044e-05) (hash(x)=157395496)
+1800 val loss 7.1242
+1800 val perplexity 1241.6288
+1800 train 7.143543 (lr=2.6981e-05) (hash(x)=157916369)
+1900 val loss 7.0900
+1900 val perplexity 1199.8519
+1900 train 7.300702 (lr=2.6915e-05) (hash(x)=166073923)
+2000 val loss 7.0677
+2000 val perplexity 1173.4039
+2000 train 7.154405 (lr=2.6845e-05) (hash(x)=154856891)
+2100 val loss 7.0385
+2100 val perplexity 1139.6696
+2100 train 7.025952 (lr=2.6771e-05) (hash(x)=151925203)
+2200 val loss 7.0078
+2200 val perplexity 1105.1830
+2200 train 6.739420 (lr=2.6693e-05) (hash(x)=136191502)
+2300 val loss 6.9791
+2300 val perplexity 1073.9137
+2300 train 7.118790 (lr=2.6611e-05) (hash(x)=153273362)
+2400 val loss 6.9431
+2400 val perplexity 1036.0015
+2400 train 6.893271 (lr=2.6525e-05) (hash(x)=148021541)
+2500 val loss 6.9181
+2500 val perplexity 1010.3660
+2500 train 6.859591 (lr=2.6436e-05) (hash(x)=141356608)
+2600 val loss 6.8876
+2600 val perplexity 980.0590
+2600 train 6.832819 (lr=2.6343e-05) (hash(x)=146005217)
+2700 val loss 6.8686
+2700 val perplexity 961.5852
+2700 train 6.702522 (lr=2.6246e-05) (hash(x)=144511718)
+2800 val loss 6.8366
+2800 val perplexity 931.3602
+2800 train 6.717809 (lr=2.6146e-05) (hash(x)=146019502)
+2900 val loss 6.8097
+2900 val perplexity 906.5668
+2900 train 6.722950 (lr=2.6042e-05) (hash(x)=146496200)
+3000 val loss 6.7843
+3000 val perplexity 883.8278
+3000 train 6.737381 (lr=2.5934e-05) (hash(x)=150127281)
+3100 val loss 6.7480
+3100 val perplexity 852.3773
+3100 train 6.713565 (lr=2.5823e-05) (hash(x)=142022255)
+3200 val loss 6.7136
+3200 val perplexity 823.5388
+3200 train 6.768692 (lr=2.5709e-05) (hash(x)=154120875)
+3300 val loss 6.6878
+3300 val perplexity 802.5413
+3300 train 6.725103 (lr=2.5590e-05) (hash(x)=153999717)
+3400 val loss 6.6609
+3400 val perplexity 781.2561
+3400 train 6.438278 (lr=2.5469e-05) (hash(x)=139694097)
+3500 val loss 6.6217
+3500 val perplexity 751.2528
+3500 train 6.818465 (lr=2.5344e-05) (hash(x)=162992732)
+3600 val loss 6.5921
+3600 val perplexity 729.2863
+3600 train 6.571229 (lr=2.5215e-05) (hash(x)=147574101)
+3700 val loss 6.5713
+3700 val perplexity 714.2859
+3700 train 6.682472 (lr=2.5083e-05) (hash(x)=157763099)
+3800 val loss 6.5394
+3800 val perplexity 691.8955
+3800 train 6.709700 (lr=2.4948e-05) (hash(x)=170800034)
+3900 val loss 6.5194
+3900 val perplexity 678.1909
+3900 train 6.590747 (lr=2.4810e-05) (hash(x)=164984528)
+4000 val loss 6.4997
+4000 val perplexity 664.9139
+4000 train 6.365149 (lr=2.4669e-05) (hash(x)=141743323)
+4100 val loss 6.4864
+4100 val perplexity 656.1646
+4100 train 6.497841 (lr=2.4524e-05) (hash(x)=153392872)
+4200 val loss 6.4768
+4200 val perplexity 649.9053
+4200 train 6.337422 (lr=2.4376e-05) (hash(x)=149074933)
+4300 val loss 6.4497
+4300 val perplexity 632.5142
+4300 train 6.847894 (lr=2.4226e-05) (hash(x)=167823423)
+4400 val loss 6.4251
+4400 val perplexity 617.1310
+4400 train 6.184004 (lr=2.4072e-05) (hash(x)=141203114)
+4500 val loss 6.4056
+4500 val perplexity 605.2375
+4500 train 6.401709 (lr=2.3915e-05) (hash(x)=146284780)
+4600 val loss 6.3853
+4600 val perplexity 593.0421
+4600 train 6.187861 (lr=2.3755e-05) (hash(x)=141126464)
+4700 val loss 6.3758
+4700 val perplexity 587.4626
+4700 train 6.397047 (lr=2.3593e-05) (hash(x)=154751926)
+4800 val loss 6.3579
+4800 val perplexity 577.0293
+4800 train 6.442084 (lr=2.3428e-05) (hash(x)=154793198)
+4900 val loss 6.3522
+4900 val perplexity 573.7279
+4900 train 6.075529 (lr=2.3260e-05) (hash(x)=139406392)
+5000 val loss 6.3531
+5000 val perplexity 574.2896
+5000 train 6.091638 (lr=2.3089e-05) (hash(x)=153548741)
+5100 val loss 6.3227
+5100 val perplexity 557.0525
+5100 train 6.400088 (lr=2.2916e-05) (hash(x)=160488568)
+5200 val loss 6.2955
+5200 val perplexity 542.1231
+5200 train 6.306382 (lr=2.2740e-05) (hash(x)=149645053)
+5300 val loss 6.2791
+5300 val perplexity 533.3140
+5300 train 6.368320 (lr=2.2561e-05) (hash(x)=155820556)
+5400 val loss 6.2695
+5400 val perplexity 528.2314
+5400 train 6.208383 (lr=2.2381e-05) (hash(x)=147538134)
+5500 val loss 6.2563
+5500 val perplexity 521.2764
+5500 train 6.459201 (lr=2.2197e-05) (hash(x)=166889307)
+5600 val loss 6.2452
+5600 val perplexity 515.5353
+5600 train 5.949936 (lr=2.2012e-05) (hash(x)=139516699)
+5700 val loss 6.2415
+5700 val perplexity 513.6536
+5700 train 5.917127 (lr=2.1824e-05) (hash(x)=140453511)
+5800 val loss 6.2322
+5800 val perplexity 508.8809
+5800 train 6.170446 (lr=2.1634e-05) (hash(x)=162964847)
+5900 val loss 6.2160
+5900 val perplexity 500.7015
+5900 train 6.217047 (lr=2.1442e-05) (hash(x)=150606634)
+6000 val loss 6.1975
+6000 val perplexity 491.5151
+6000 train 6.297403 (lr=2.1248e-05) (hash(x)=149890857)
+6100 val loss 6.1889
+6100 val perplexity 487.3126
+6100 train 6.297916 (lr=2.1051e-05) (hash(x)=173884145)
+6200 val loss 6.1787
+6200 val perplexity 482.3753
+6200 train 6.234455 (lr=2.0853e-05) (hash(x)=151987098)
+6300 val loss 6.1627
+6300 val perplexity 474.7043
+6300 train 6.114050 (lr=2.0653e-05) (hash(x)=148853562)
+6400 val loss 6.1547
+6400 val perplexity 470.9389
+6400 train 5.904671 (lr=2.0451e-05) (hash(x)=141530101)
+6500 val loss 6.1497
+6500 val perplexity 468.5626
+6500 train 6.001200 (lr=2.0247e-05) (hash(x)=142297809)
+6600 val loss 6.1329
+6600 val perplexity 460.7879
+6600 train 6.009923 (lr=2.0042e-05) (hash(x)=142447782)
+6700 val loss 6.1284
+6700 val perplexity 458.7029
+6700 train 6.097614 (lr=1.9835e-05) (hash(x)=147004686)
+6800 val loss 6.1164
+6800 val perplexity 453.2365
+6800 train 5.853673 (lr=1.9626e-05) (hash(x)=133438702)
+6900 val loss 6.1146
+6900 val perplexity 452.4294
+6900 train 6.133292 (lr=1.9416e-05) (hash(x)=157085143)
+7000 val loss 6.1045
+7000 val perplexity 447.8570
+7000 train 5.982401 (lr=1.9205e-05) (hash(x)=139437666)
+7100 val loss 6.0977
+7100 val perplexity 444.8145
+7100 train 6.124189 (lr=1.8992e-05) (hash(x)=159792986)
+7200 val loss 6.0787
+7200 val perplexity 436.4404
+7200 train 6.051832 (lr=1.8778e-05) (hash(x)=144930687)
+7300 val loss 6.0730
+7300 val perplexity 433.9602
+7300 train 6.145269 (lr=1.8562e-05) (hash(x)=156242690)
+7400 val loss 6.0645
+7400 val perplexity 430.3212
+7400 train 5.903626 (lr=1.8346e-05) (hash(x)=148183719)
+7500 val loss 6.0597
+7500 val perplexity 428.2467
+7500 train 6.097398 (lr=1.8128e-05) (hash(x)=152494758)
+7600 val loss 6.0553
+7600 val perplexity 426.3716
+7600 train 5.742872 (lr=1.7910e-05) (hash(x)=142485027)
+7700 val loss 6.0525
+7700 val perplexity 425.1640
+7700 train 5.918979 (lr=1.7691e-05) (hash(x)=147512165)
+7800 val loss 6.0337
+7800 val perplexity 417.2719
+7800 train 5.932111 (lr=1.7470e-05) (hash(x)=160346994)
+7900 val loss 6.0295
+7900 val perplexity 415.5274
+7900 train 5.900640 (lr=1.7249e-05) (hash(x)=144488254)
+8000 val loss 6.0187
+8000 val perplexity 411.0602
+8000 train 5.831340 (lr=1.7028e-05) (hash(x)=147637019)
+8100 val loss 6.0147
+8100 val perplexity 409.4227
+8100 train 5.867096 (lr=1.6805e-05) (hash(x)=147340534)
+8200 val loss 6.0159
+8200 val perplexity 409.9099
+8200 train 6.063663 (lr=1.6582e-05) (hash(x)=151630665)
+8300 val loss 5.9982
+8300 val perplexity 402.7017
+8300 train 6.094231 (lr=1.6359e-05) (hash(x)=149747064)
+8400 val loss 5.9914
+8400 val perplexity 399.9586
+8400 train 6.179846 (lr=1.6135e-05) (hash(x)=154245770)
+8500 val loss 5.9852
+8500 val perplexity 397.5050
+8500 train 5.869342 (lr=1.5911e-05) (hash(x)=152559100)
+8600 val loss 5.9830
+8600 val perplexity 396.6477
+8600 train 6.408178 (lr=1.5687e-05) (hash(x)=181365926)
+8700 val loss 5.9769
+8700 val perplexity 394.2012
+8700 train 5.815026 (lr=1.5462e-05) (hash(x)=154405991)
+8800 val loss 5.9675
+8800 val perplexity 390.5453
+8800 train 5.918090 (lr=1.5237e-05) (hash(x)=153755904)
+8900 val loss 5.9610
+8900 val perplexity 387.9935
+8900 train 5.875622 (lr=1.5013e-05) (hash(x)=152120568)
+9000 val loss 5.9489
+9000 val perplexity 383.3480
+9000 train 5.740575 (lr=1.4788e-05) (hash(x)=142797279)
+9100 val loss 5.9462
+9100 val perplexity 382.3155
+9100 train 5.807010 (lr=1.4563e-05) (hash(x)=143037503)
+9200 val loss 5.9478
+9200 val perplexity 382.9004
+9200 train 5.797348 (lr=1.4339e-05) (hash(x)=113690273)
+9300 val loss 5.9384
+9300 val perplexity 379.3094
+9300 train 5.884220 (lr=1.4115e-05) (hash(x)=158025077)
+9400 val loss 5.9316
+9400 val perplexity 376.7456
+9400 train 6.010771 (lr=1.3891e-05) (hash(x)=158251718)
+9500 val loss 5.9264
+9500 val perplexity 374.7893
+9500 train 5.963029 (lr=1.3668e-05) (hash(x)=154752610)
+9600 val loss 5.9210
+9600 val perplexity 372.7921
+9600 train 5.796573 (lr=1.3445e-05) (hash(x)=146889093)
+9700 val loss 5.9120
+9700 val perplexity 369.4364
+9700 train 5.912082 (lr=1.3222e-05) (hash(x)=156906516)
+9800 val loss 5.9093
+9800 val perplexity 368.4616
+9800 train 5.740803 (lr=1.3001e-05) (hash(x)=153841927)
+9900 val loss 5.9022
+9900 val perplexity 365.8515
+9900 train 6.114119 (lr=1.2780e-05) (hash(x)=163514334)
+10000 val loss 5.8963
+10000 val perplexity 363.7026
+10000 train 5.880864 (lr=1.2559e-05) (hash(x)=144745561)
+10100 val loss 5.8931
+10100 val perplexity 362.5178
+10100 train 6.153583 (lr=1.2340e-05) (hash(x)=161651149)
+10200 val loss 5.8885
+10200 val perplexity 360.8580
+10200 train 5.830371 (lr=1.2122e-05) (hash(x)=143667037)
+10300 val loss 5.8789
+10300 val perplexity 357.4200
+10300 train 5.773458 (lr=1.1904e-05) (hash(x)=151687262)
+10400 val loss 5.8772
+10400 val perplexity 356.8200
+10400 train 5.737283 (lr=1.1688e-05) (hash(x)=153694955)
+10500 val loss 5.8772
+10500 val perplexity 356.8165
+10500 train 5.813218 (lr=1.1472e-05) (hash(x)=150218029)
+10600 val loss 5.8695
+10600 val perplexity 354.0586
+10600 train 5.960885 (lr=1.1258e-05) (hash(x)=155093716)
+10700 val loss 5.8643
+10700 val perplexity 352.2238
+10700 train 5.894807 (lr=1.1045e-05) (hash(x)=151631831)
+10800 val loss 5.8555
+10800 val perplexity 349.1389
+10800 train 5.700920 (lr=1.0834e-05) (hash(x)=136465175)
+10900 val loss 5.8544
+10900 val perplexity 348.7563
+10900 train 5.792243 (lr=1.0624e-05) (hash(x)=154461514)
+11000 val loss 5.8521
+11000 val perplexity 347.9656
+11000 train 5.833532 (lr=1.0415e-05) (hash(x)=141877432)
+11100 val loss 5.8500
+11100 val perplexity 347.2448
+11100 train 5.651769 (lr=1.0208e-05) (hash(x)=139572894)
+11200 val loss 5.8433
+11200 val perplexity 344.9298
+11200 train 5.893256 (lr=1.0003e-05) (hash(x)=155591443)
+11300 val loss 5.8366
+11300 val perplexity 342.6000
+11300 train 5.848417 (lr=9.7991e-06) (hash(x)=153349136)
+11400 val loss 5.8334
+11400 val perplexity 341.5110
+11400 train 5.779039 (lr=9.5972e-06) (hash(x)=148109395)
+11500 val loss 5.8305
+11500 val perplexity 340.5268
+11500 train 5.811443 (lr=9.3970e-06) (hash(x)=153276620)
+11600 val loss 5.8272
+11600 val perplexity 339.4094
+11600 train 5.692456 (lr=9.1988e-06) (hash(x)=151416388)
+11700 val loss 5.8240
+11700 val perplexity 338.3086
+11700 train 5.703767 (lr=9.0025e-06) (hash(x)=143420927)
+11800 val loss 5.8212
+11800 val perplexity 337.3615
+11800 train 5.891651 (lr=8.8082e-06) (hash(x)=165023996)
+11900 val loss 5.8153
+11900 val perplexity 335.3907
+11900 train 5.725571 (lr=8.6160e-06) (hash(x)=144465076)
+12000 val loss 5.8122
+12000 val perplexity 334.3644
+12000 train 5.813470 (lr=8.4260e-06) (hash(x)=155991892)
+12100 val loss 5.8106
+12100 val perplexity 333.8300
+12100 train 5.835740 (lr=8.2381e-06) (hash(x)=150748844)
+12200 val loss 5.8070
+12200 val perplexity 332.6322
+12200 train 5.762444 (lr=8.0526e-06) (hash(x)=149712638)
+12300 val loss 5.8057
+12300 val perplexity 332.1779
+12300 train 5.616854 (lr=7.8694e-06) (hash(x)=140428230)
+12400 val loss 5.8017
+12400 val perplexity 330.8764
+12400 train 5.978186 (lr=7.6885e-06) (hash(x)=163614308)
+12500 val loss 5.7967
+12500 val perplexity 329.2067
+12500 train 5.755881 (lr=7.5101e-06) (hash(x)=153366153)
+12600 val loss 5.7936
+12600 val perplexity 328.1790
+12600 train 5.680386 (lr=7.3342e-06) (hash(x)=146832066)
+12700 val loss 5.7905
+12700 val perplexity 327.1713
+12700 train 5.961281 (lr=7.1609e-06) (hash(x)=154387414)
+12800 val loss 5.7895
+12800 val perplexity 326.8367
+12800 train 5.580901 (lr=6.9903e-06) (hash(x)=139465199)
+12900 val loss 5.7915
+12900 val perplexity 327.5130
+12900 train 5.499762 (lr=6.8223e-06) (hash(x)=143326374)
+13000 val loss 5.7844
+13000 val perplexity 325.1768
+13000 train 5.929033 (lr=6.6570e-06) (hash(x)=158817835)
+13100 val loss 5.7811
+13100 val perplexity 324.1093
+13100 train 5.653330 (lr=6.4945e-06) (hash(x)=142027874)
+13200 val loss 5.7819
+13200 val perplexity 324.3774
+13200 train 5.751643 (lr=6.3349e-06) (hash(x)=148497730)
+13300 val loss 5.7788
+13300 val perplexity 323.3740
+13300 train 5.683223 (lr=6.1782e-06) (hash(x)=148094633)
+13400 val loss 5.7768
+13400 val perplexity 322.7143
+13400 train 5.658297 (lr=6.0244e-06) (hash(x)=150548214)
+13500 val loss 5.7698
+13500 val perplexity 320.4637
+13500 train 5.875854 (lr=5.8737e-06) (hash(x)=152789725)
+13600 val loss 5.7679
+13600 val perplexity 319.8513
+13600 train 5.810717 (lr=5.7259e-06) (hash(x)=153220230)
+13700 val loss 5.7654
+13700 val perplexity 319.0808
+13700 train 5.776061 (lr=5.5813e-06) (hash(x)=147805699)
+13800 val loss 5.7657
+13800 val perplexity 319.1692
+13800 train 5.588718 (lr=5.4398e-06) (hash(x)=146267341)
+13900 val loss 5.7647
+13900 val perplexity 318.8525
+13900 train 5.575336 (lr=5.3016e-06) (hash(x)=139467501)
+14000 val loss 5.7620
+14000 val perplexity 317.9841
+14000 train 5.771669 (lr=5.1665e-06) (hash(x)=152185436)
+14100 val loss 5.7571
+14100 val perplexity 316.4215
+14100 train 5.881826 (lr=5.0348e-06) (hash(x)=176479139)
+14200 val loss 5.7532
+14200 val perplexity 315.1906
+14200 train 5.844451 (lr=4.9063e-06) (hash(x)=151052040)
+14300 val loss 5.7521
+14300 val perplexity 314.8360
+14300 train 5.689722 (lr=4.7813e-06) (hash(x)=149668097)
+14400 val loss 5.7507
+14400 val perplexity 314.4009
+14400 train 5.753808 (lr=4.6596e-06) (hash(x)=151761941)
+14500 val loss 5.7489
+14500 val perplexity 313.8326
+14500 train 5.693327 (lr=4.5414e-06) (hash(x)=149412009)
+14600 val loss 5.7481
+14600 val perplexity 313.5924
+14600 train 5.620662 (lr=4.4267e-06) (hash(x)=145048020)
+14700 val loss 5.7455
+14700 val perplexity 312.7794
+14700 train 5.693942 (lr=4.3155e-06) (hash(x)=141771666)
+14800 val loss 5.7421
+14800 val perplexity 311.7278
+14800 train 5.678078 (lr=4.2079e-06) (hash(x)=148759740)
+14900 val loss 5.7406
+14900 val perplexity 311.2366
+14900 train 5.661452 (lr=4.1039e-06) (hash(x)=146287032)
+15000 val loss 5.7379
+15000 val perplexity 310.4199
+15000 train 5.682189 (lr=4.0035e-06) (hash(x)=143519156)
+15100 val loss 5.7409
+15100 val perplexity 311.3491
+15100 train 5.713648 (lr=3.9068e-06) (hash(x)=151330688)
+15200 val loss 5.7432
+15200 val perplexity 312.0658
+15200 train 5.534642 (lr=3.8138e-06) (hash(x)=140380090)
+15300 val loss 5.7370
+15300 val perplexity 310.1329
+15300 train 5.646035 (lr=3.7245e-06) (hash(x)=152598160)
+15400 val loss 5.7328
+15400 val perplexity 308.8225
+15400 train 5.975647 (lr=3.6390e-06) (hash(x)=164136966)
+15500 val loss 5.7329
+15500 val perplexity 308.8720
+15500 train 5.923656 (lr=3.5572e-06) (hash(x)=158733990)
+15600 val loss 5.7302
+15600 val perplexity 308.0399
+15600 train 5.764136 (lr=3.4793e-06) (hash(x)=149149390)
+15700 val loss 5.7287
+15700 val perplexity 307.5585
+15700 train 5.683558 (lr=3.4052e-06) (hash(x)=146746394)
+15800 val loss 5.7283
+15800 val perplexity 307.4477
+15800 train 5.752231 (lr=3.3350e-06) (hash(x)=159405219)
+15900 val loss 5.7270
+15900 val perplexity 307.0499
+15900 train 5.754789 (lr=3.2687e-06) (hash(x)=154034029)
+16000 val loss 5.7246
+16000 val perplexity 306.3018
+16000 train 5.720493 (lr=3.2063e-06) (hash(x)=153634210)
+16100 val loss 5.7229
+16100 val perplexity 305.7868
+16100 train 5.652885 (lr=3.1478e-06) (hash(x)=155646721)
+16200 val loss 5.7215
+16200 val perplexity 305.3638
+16200 train 5.616024 (lr=3.0932e-06) (hash(x)=142461643)
+16300 val loss 5.7221
+16300 val perplexity 305.5579
+16300 train 5.613182 (lr=3.0427e-06) (hash(x)=141080038)
+16400 val loss 5.7206
+16400 val perplexity 305.0727
+16400 train 5.736214 (lr=2.9961e-06) (hash(x)=153744166)
+16500 val loss 5.7237
+16500 val perplexity 306.0345
+16500 train 5.528856 (lr=2.9535e-06) (hash(x)=149783517)
+16600 val loss 5.7239
+16600 val perplexity 306.0968
+16600 train 5.559315 (lr=2.9149e-06) (hash(x)=142659127)
+16700 val loss 5.7176
+16700 val perplexity 304.1617
+16700 train 5.785442 (lr=2.8804e-06) (hash(x)=149725921)
+16800 val loss 5.7161
+16800 val perplexity 303.7324
+16800 train 5.558208 (lr=2.8498e-06) (hash(x)=140678087)
+16900 val loss 5.7151
+16900 val perplexity 303.4019
+16900 train 5.719929 (lr=2.8234e-06) (hash(x)=146935413)
+17000 val loss 5.7129
+17000 val perplexity 302.7523
+17000 train 5.719940 (lr=2.8010e-06) (hash(x)=144408437)
+17100 val loss 5.7124
+17100 val perplexity 302.5996
+17100 train 5.563908 (lr=2.7826e-06) (hash(x)=149229905)
+17200 val loss 5.7120
+17200 val perplexity 302.4892
+17200 train 5.655513 (lr=2.7684e-06) (hash(x)=150471865)
+17300 val loss 5.7140
+17300 val perplexity 303.0953
+17300 train 5.612057 (lr=2.7582e-06) (hash(x)=155784916)
+17400 val loss 5.7160
+17400 val perplexity 303.6875
+17400 train 5.491227 (lr=2.7520e-06) (hash(x)=148482413)
+17499 val loss 5.7103
+17499 val perplexity 301.9635

attention_kindselective_n_heads4_seed1340/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41282f5b0c60ace5dfaf1816e20260600b338e452b440e186e82787f342dabf3
+oid sha256:3e54be31ee083fbf686a04344b3dc90c0ae2aed84d00d52ed48fda0df48e4332
 size 92843394

attention_kindselective_n_heads4_seed1340/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05126194a9f4acf0d9e67f380d612f3904ca5c94f9a38a1332dd14252a8e1fb7
+oid sha256:0215abcdc306b9d2d39c059ac45fb925f56d1720cf099a52b8d414551ce242e8
 size 179406214