Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_8/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_8", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.95e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.95e-5_30720_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,1055 +1,528 @@
 max_steps: 17500
 0 val loss 11.2663
 0 val perplexity 78147.0000
-0 val loss 11.2663
-0 val perplexity 78147.0000
-0 train 11.263341 (lr=1.4750e-07) (hash(x)=150327452)
-0 train 11.263341 (lr=1.5250e-07) (hash(x)=150327452)
-100 val loss 9.9076
-100 val perplexity 20083.0977
-100 train 9.918514 (lr=1.4897e-05) (hash(x)=166441190)
-100 val loss 9.9071
-100 val perplexity 20071.5703
-100 train 9.916539 (lr=1.5403e-05) (hash(x)=166441190)
-200 val loss 9.0154
-200 val perplexity 8229.1504
-200 train 9.126813 (lr=2.9500e-05) (hash(x)=166780046)
-200 val loss 8.9599
-200 val perplexity 7784.6006
-200 train 9.076227 (lr=3.0500e-05) (hash(x)=166780046)
-300 val loss 8.1193
-300 val perplexity 3358.5723
-300 train 8.069273 (lr=2.9498e-05) (hash(x)=159835303)
-300 val loss 8.0310
-300 val perplexity 3074.8564
-300 train 7.975445 (lr=3.0498e-05) (hash(x)=159835303)
-400 val loss 7.7935
-400 val perplexity 2424.8623
-400 train 7.599485 (lr=2.9491e-05) (hash(x)=155040610)
-400 val loss 7.6952
-400 val perplexity 2197.8633
-400 train 7.491219 (lr=3.0491e-05) (hash(x)=155040610)
-500 val loss 7.6344
-500 val perplexity 2068.2085
-500 train 7.394850 (lr=2.9480e-05) (hash(x)=130190460)
-500 val loss 7.5836
-500 val perplexity 1965.6169
-500 train 7.331265 (lr=3.0480e-05) (hash(x)=130190460)
-600 val loss 7.5263
-600 val perplexity 1856.1548
-600 train 7.550632 (lr=2.9465e-05) (hash(x)=155504036)
-600 val loss 7.5003
-600 val perplexity 1808.5304
-600 train 7.518277 (lr=3.0464e-05) (hash(x)=155504036)
-700 val loss 7.4587
-700 val perplexity 1734.9453
-700 train 7.313744 (lr=2.9445e-05) (hash(x)=137347213)
-700 val loss 7.4316
-700 val perplexity 1688.5563
-700 train 7.283154 (lr=3.0443e-05) (hash(x)=137347213)
-800 val loss 7.4114
-800 val perplexity 1654.7174
-800 train 7.266835 (lr=2.9421e-05) (hash(x)=143823248)
-800 val loss 7.3838
-800 val perplexity 1609.6575
-800 train 7.244073 (lr=3.0419e-05) (hash(x)=143823248)
-900 val loss 7.3673
-900 val perplexity 1583.4285
-900 train 7.476106 (lr=2.9393e-05) (hash(x)=156260416)
-900 val loss 7.3501
-900 val perplexity 1556.3418
-900 train 7.461842 (lr=3.0389e-05) (hash(x)=156260416)
-1000 val loss 7.3283
-1000 val perplexity 1522.7211
-1000 train 7.297588 (lr=2.9360e-05) (hash(x)=143734685)
-1000 val loss 7.3181
-1000 val perplexity 1507.3265
-1000 train 7.285740 (lr=3.0355e-05) (hash(x)=143734685)
-1100 val loss 7.2969
-1100 val perplexity 1475.7740
-1100 train 7.232541 (lr=2.9323e-05) (hash(x)=160013925)
-1100 val loss 7.2863
-1100 val perplexity 1460.1194
-1100 train 7.221999 (lr=3.0317e-05) (hash(x)=160013925)
-1200 val loss 7.2601
-1200 val perplexity 1422.4136
-1200 train 7.000230 (lr=2.9282e-05) (hash(x)=150678249)
-1200 val loss 7.2559
-1200 val perplexity 1416.3729
-1200 train 7.000189 (lr=3.0274e-05) (hash(x)=150678249)
-1300 val loss 7.2196
-1300 val perplexity 1365.8862
-1300 train 7.000329 (lr=2.9236e-05) (hash(x)=149073315)
-1300 val loss 7.2200
-1300 val perplexity 1366.4353
-1300 train 7.003819 (lr=3.0227e-05) (hash(x)=149073315)
-1400 val loss 7.1563
-1400 val perplexity 1282.1772
-1400 train 7.479748 (lr=2.9186e-05) (hash(x)=175802021)
-1400 val loss 7.1615
-1400 val perplexity 1288.8317
-1400 train 7.499377 (lr=3.0175e-05) (hash(x)=175802021)
-1500 val loss 7.1010
-1500 val perplexity 1213.2402
-1500 train 7.362651 (lr=2.9132e-05) (hash(x)=171034639)
-1500 val loss 7.1156
-1500 val perplexity 1231.0304
-1500 train 7.370233 (lr=3.0119e-05) (hash(x)=171034639)
-1600 val loss 7.0507
-1600 val perplexity 1153.6410
-1600 train 7.256294 (lr=2.9073e-05) (hash(x)=158681215)
-1600 val loss 7.0744
-1600 val perplexity 1181.3113
-1600 train 7.280097 (lr=3.0059e-05) (hash(x)=158681215)
-1700 val loss 7.0107
-1700 val perplexity 1108.4774
-1700 train 6.948504 (lr=2.9011e-05) (hash(x)=152116061)
-1700 val loss 7.0318
-1700 val perplexity 1132.1229
-1700 train 6.966795 (lr=2.9994e-05) (hash(x)=152116061)
-1800 val loss 6.9811
-1800 val perplexity 1076.1487
-1800 train 6.916253 (lr=2.8944e-05) (hash(x)=146108145)
-1800 val loss 7.0014
-1800 val perplexity 1098.1465
-1800 train 6.935902 (lr=2.9925e-05) (hash(x)=146108145)
-1900 val loss 6.9431
-1900 val perplexity 1035.9658
-1900 train 6.775837 (lr=2.8872e-05) (hash(x)=147598108)
-1900 val loss 6.9806
-1900 val perplexity 1075.5316
-1900 train 6.815755 (lr=2.9851e-05) (hash(x)=147598108)
-2000 val loss 6.9203
-2000 val perplexity 1012.6552
-2000 train 6.787498 (lr=2.8797e-05) (hash(x)=154996086)
-2000 val loss 6.9555
-2000 val perplexity 1048.9139
-2000 train 6.821686 (lr=2.9773e-05) (hash(x)=154996086)
-2100 val loss 6.8941
-2100 val perplexity 986.4349
-2100 train 6.683231 (lr=2.8718e-05) (hash(x)=153396183)
-2100 val loss 6.9347
-2100 val perplexity 1027.2611
-2100 train 6.722253 (lr=2.9691e-05) (hash(x)=153396183)
-2200 val loss 6.8412
-2200 val perplexity 935.6088
-2200 train 6.829289 (lr=2.8634e-05) (hash(x)=153885445)
-2200 val loss 6.8864
-2200 val perplexity 978.8835
-2200 train 6.871447 (lr=2.9605e-05) (hash(x)=153885445)
-2300 val loss 6.7972
-2300 val perplexity 895.3289
-2300 train 6.799411 (lr=2.8546e-05) (hash(x)=159666385)
-2300 val loss 6.8379
-2300 val perplexity 932.5040
-2300 train 6.839467 (lr=2.9514e-05) (hash(x)=159666385)
-2400 val loss 6.7551
-2400 val perplexity 858.4303
-2400 train 6.732621 (lr=2.8455e-05) (hash(x)=142353087)
-2400 val loss 6.7998
-2400 val perplexity 897.6368
-2400 train 6.777557 (lr=2.9419e-05) (hash(x)=142353087)
-2500 val loss 6.7215
-2500 val perplexity 830.0513
-2500 train 6.742167 (lr=2.8359e-05) (hash(x)=146491718)
-2500 val loss 6.7711
-2500 val perplexity 872.3088
-2500 train 6.782372 (lr=2.9320e-05) (hash(x)=146491718)
-2600 val loss 6.6859
-2600 val perplexity 801.0070
-2600 train 6.610720 (lr=2.8259e-05) (hash(x)=150750353)
-2600 val loss 6.7397
-2600 val perplexity 845.3187
-2600 train 6.663429 (lr=2.9217e-05) (hash(x)=150750353)
-2700 val loss 6.6470
-2700 val perplexity 770.4656
-2700 train 6.366074 (lr=2.8155e-05) (hash(x)=129849193)
-2700 val loss 6.7030
-2700 val perplexity 814.8619
-2700 train 6.420519 (lr=2.9110e-05) (hash(x)=129849193)
-2800 val loss 6.6245
-2800 val perplexity 753.3086
-2800 train 6.468822 (lr=2.8048e-05) (hash(x)=152767913)
-2800 val loss 6.6867
-2800 val perplexity 801.6673
-2800 train 6.541181 (lr=2.8998e-05) (hash(x)=152767913)
-2900 val loss 6.6024
-2900 val perplexity 736.8776
-2900 train 6.373911 (lr=2.7936e-05) (hash(x)=146531140)
-2900 val loss 6.6610
-2900 val perplexity 781.3325
-2900 train 6.440368 (lr=2.8883e-05) (hash(x)=146531140)
-3000 val loss 6.5567
-3000 val perplexity 703.9352
-3000 train 6.641496 (lr=2.7821e-05) (hash(x)=151562048)
-3000 val loss 6.6254
-3000 val perplexity 754.0115
-3000 train 6.702426 (lr=2.8764e-05) (hash(x)=151562048)
-3100 val loss 6.5190
-3100 val perplexity 677.9296
-3100 train 6.488099 (lr=2.7701e-05) (hash(x)=146001424)
-3100 val loss 6.5921
-3100 val perplexity 729.3231
-3100 train 6.565769 (lr=2.8640e-05) (hash(x)=146001424)
-3200 val loss 6.4975
-3200 val perplexity 663.4761
-3200 train 6.557775 (lr=2.7578e-05) (hash(x)=166486165)
-3200 val loss 6.5761
-3200 val perplexity 717.7675
-3200 train 6.637276 (lr=2.8513e-05) (hash(x)=166486165)
-3300 val loss 6.4619
-3300 val perplexity 640.2613
-3300 train 6.387141 (lr=2.7452e-05) (hash(x)=150866680)
-3300 val loss 6.5417
-3300 val perplexity 693.4370
-3300 train 6.477298 (lr=2.8382e-05) (hash(x)=150866680)
-3400 val loss 6.4415
-3400 val perplexity 627.3589
-3400 train 6.334854 (lr=2.7321e-05) (hash(x)=143900419)
-3400 val loss 6.5273
-3400 val perplexity 683.5585
-3400 train 6.421918 (lr=2.8247e-05) (hash(x)=143900419)
-3500 val loss 6.4255
-3500 val perplexity 617.4097
-3500 train 6.239334 (lr=2.7187e-05) (hash(x)=148845794)
-3500 val loss 6.5001
-3500 val perplexity 665.2279
-3500 train 6.318225 (lr=2.8108e-05) (hash(x)=148845794)
-3600 val loss 6.4074
-3600 val perplexity 606.2994
-3600 train 6.242413 (lr=2.7049e-05) (hash(x)=145667796)
-3600 val loss 6.4816
-3600 val perplexity 653.0432
-3600 train 6.315657 (lr=2.7966e-05) (hash(x)=145667796)
-3700 val loss 6.3926
-3700 val perplexity 597.4324
-3700 train 6.409256 (lr=2.6908e-05) (hash(x)=163563851)
-3700 val loss 6.4696
-3700 val perplexity 645.2346
-3700 train 6.490496 (lr=2.7820e-05) (hash(x)=163563851)
-3800 val loss 6.3783
-3800 val perplexity 588.9238
-3800 train 6.319212 (lr=2.6763e-05) (hash(x)=147488689)
-3800 val loss 6.4534
-3800 val perplexity 634.8288
-3800 train 6.395573 (lr=2.7670e-05) (hash(x)=147488689)
-3900 val loss 6.3478
-3900 val perplexity 571.2350
-3900 train 6.351666 (lr=2.6615e-05) (hash(x)=148186608)
-4000 val loss 6.3288
-4000 val perplexity 560.5073
-4000 train 6.200518 (lr=2.6463e-05) (hash(x)=142970187)
-3900 val loss 6.4252
-3900 val perplexity 617.1830
-3900 train 6.424561 (lr=2.7517e-05) (hash(x)=148186608)
-4100 val loss 6.3036
-4100 val perplexity 546.5350
-4100 train 6.401168 (lr=2.6308e-05) (hash(x)=141584883)
-4000 val loss 6.4059
-4000 val perplexity 605.4064
-4000 train 6.278652 (lr=2.7360e-05) (hash(x)=142970187)
-4200 val loss 6.2910
-4200 val perplexity 539.7009
-4200 train 6.088043 (lr=2.6149e-05) (hash(x)=145664585)
-4100 val loss 6.3843
-4100 val perplexity 592.4446
-4100 train 6.457413 (lr=2.7199e-05) (hash(x)=141584883)
-4300 val loss 6.2746
-4300 val perplexity 530.9036
-4300 train 6.143443 (lr=2.5987e-05) (hash(x)=143736499)
-4200 val loss 6.3707
-4200 val perplexity 584.4645
-4200 train 6.167514 (lr=2.7036e-05) (hash(x)=145664585)
-4400 val loss 6.2691
-4400 val perplexity 527.9794
-4400 train 6.089228 (lr=2.5822e-05) (hash(x)=151883322)
-4300 val loss 6.3551
-4300 val perplexity 575.4345
-4300 train 6.215100 (lr=2.6868e-05) (hash(x)=143736499)
-4500 val loss 6.2599
-4500 val perplexity 523.1622
-4500 train 6.141528 (lr=2.5654e-05) (hash(x)=153904871)
-4400 val loss 6.3513
-4400 val perplexity 573.2176
-4400 train 6.176329 (lr=2.6698e-05) (hash(x)=151883322)
-4600 val loss 6.2443
-4600 val perplexity 515.0596
-4600 train 6.332364 (lr=2.5483e-05) (hash(x)=154893521)
-4500 val loss 6.3352
-4500 val perplexity 564.1061
-4500 train 6.219083 (lr=2.6524e-05) (hash(x)=153904871)
-4700 val loss 6.2282
-4700 val perplexity 506.8488
-4700 train 6.280324 (lr=2.5309e-05) (hash(x)=152323949)
-4600 val loss 6.3186
-4600 val perplexity 554.8040
-4600 train 6.403486 (lr=2.6347e-05) (hash(x)=154893521)
-4800 val loss 6.2034
-4800 val perplexity 494.4304
-4800 train 6.150989 (lr=2.5132e-05) (hash(x)=154104619)
-4700 val loss 6.2953
-4700 val perplexity 542.0172
-4700 train 6.351778 (lr=2.6167e-05) (hash(x)=152323949)
-4900 val loss 6.1953
-4900 val perplexity 490.4498
-4900 train 6.312655 (lr=2.4951e-05) (hash(x)=146311426)
-4800 val loss 6.2802
-4800 val perplexity 533.8707
-4800 train 6.225656 (lr=2.5983e-05) (hash(x)=154104619)
-5000 val loss 6.1825
-5000 val perplexity 484.1859
-5000 train 6.173968 (lr=2.4768e-05) (hash(x)=156741847)
-4900 val loss 6.2652
-4900 val perplexity 525.9521
-4900 train 6.378856 (lr=2.5797e-05) (hash(x)=146311426)
-5100 val loss 6.1725
-5100 val perplexity 479.3932
-5100 train 5.950968 (lr=2.4582e-05) (hash(x)=142086346)
-5000 val loss 6.2484
-5000 val perplexity 517.1873
-5000 train 6.246004 (lr=2.5608e-05) (hash(x)=156741847)
-5200 val loss 6.1659
-5200 val perplexity 476.2065
-5200 train 5.933645 (lr=2.4394e-05) (hash(x)=150265428)
-5100 val loss 6.2422
-5100 val perplexity 513.9789
-5100 train 6.019436 (lr=2.5416e-05) (hash(x)=142086346)
-5300 val loss 6.1615
-5300 val perplexity 474.1213
-5300 train 6.040591 (lr=2.4202e-05) (hash(x)=151339108)
-5200 val loss 6.2376
-5200 val perplexity 511.6395
-5200 train 6.012814 (lr=2.5221e-05) (hash(x)=150265428)
-5400 val loss 6.1453
-5400 val perplexity 466.5169
-5400 train 6.124017 (lr=2.4008e-05) (hash(x)=154654372)
-5300 val loss 6.2278
-5300 val perplexity 506.6393
-5300 train 6.114930 (lr=2.5023e-05) (hash(x)=151339108)
-5500 val loss 6.1322
-5500 val perplexity 460.4604
-5500 train 6.141157 (lr=2.3812e-05) (hash(x)=150575051)
-5400 val loss 6.2154
-5400 val perplexity 500.3874
-5400 train 6.200837 (lr=2.4822e-05) (hash(x)=154654372)
-5600 val loss 6.1195
-5600 val perplexity 454.6242
-5600 train 6.028224 (lr=2.3613e-05) (hash(x)=140396423)
-5500 val loss 6.2017
-5500 val perplexity 493.5960
-5500 train 6.214929 (lr=2.4619e-05) (hash(x)=150575051)
-5700 val loss 6.1076
-5700 val perplexity 449.2407
-5700 train 5.963513 (lr=2.3411e-05) (hash(x)=144678758)
-5600 val loss 6.1827
-5600 val perplexity 484.2859
-5600 train 6.086453 (lr=2.4413e-05) (hash(x)=140396423)
-5800 val loss 6.1000
-5800 val perplexity 445.8460
-5800 train 6.158457 (lr=2.3207e-05) (hash(x)=151992743)
-5700 val loss 6.1684
-5700 val perplexity 477.4142
-5700 train 6.030063 (lr=2.4205e-05) (hash(x)=144678758)
-5900 val loss 6.0906
-5900 val perplexity 441.6646
-5900 train 5.879590 (lr=2.3001e-05) (hash(x)=144396927)
-5800 val loss 6.1578
-5800 val perplexity 472.3819
-5800 train 6.227654 (lr=2.3994e-05) (hash(x)=151992743)
-6000 val loss 6.0806
-6000 val perplexity 437.3022
-6000 train 6.039478 (lr=2.2793e-05) (hash(x)=165478625)
-5900 val loss 6.1479
-5900 val perplexity 467.7183
-5900 train 5.937796 (lr=2.3781e-05) (hash(x)=144396927)
-6100 val loss 6.0842
-6100 val perplexity 438.8777
-6100 train 5.815226 (lr=2.2582e-05) (hash(x)=147088621)
-6200 val loss 6.0710
-6200 val perplexity 433.1289
-6200 train 5.835066 (lr=2.2370e-05) (hash(x)=140794994)
-6000 val loss 6.1377
-6000 val perplexity 462.9787
-6000 train 6.104814 (lr=2.3565e-05) (hash(x)=165478625)
-6300 val loss 6.0525
-6300 val perplexity 425.1672
-6300 train 5.985744 (lr=2.2155e-05) (hash(x)=134780906)
-6100 val loss 6.1424
-6100 val perplexity 465.1563
-6100 train 5.878911 (lr=2.3348e-05) (hash(x)=147088621)
-6400 val loss 6.0476
-6400 val perplexity 423.0864
-6400 train 6.030969 (lr=2.1938e-05) (hash(x)=149023655)
-6200 val loss 6.1332
-6200 val perplexity 460.9255
-6200 train 5.896582 (lr=2.3128e-05) (hash(x)=140794994)
-6500 val loss 6.0291
-6500 val perplexity 415.3430
-6500 train 5.883899 (lr=2.1720e-05) (hash(x)=147497796)
-6300 val loss 6.1138
-6300 val perplexity 452.0466
-6300 train 6.047512 (lr=2.2906e-05) (hash(x)=134780906)
-6600 val loss 6.0303
-6600 val perplexity 415.8454
-6600 train 5.884706 (lr=2.1499e-05) (hash(x)=152902689)
-6400 val loss 6.0998
-6400 val perplexity 445.7763
-6400 train 6.086147 (lr=2.2682e-05) (hash(x)=149023655)
-6700 val loss 6.0302
-6700 val perplexity 415.8018
-6700 train 6.064247 (lr=2.1277e-05) (hash(x)=153846046)
-6500 val loss 6.0877
-6500 val perplexity 440.3858
-6500 train 5.945824 (lr=2.2456e-05) (hash(x)=147497796)
-6800 val loss 6.0090
-6800 val perplexity 407.0601
-6800 train 6.146492 (lr=2.1054e-05) (hash(x)=158512738)
-6600 val loss 6.0846
-6600 val perplexity 439.0240
-6600 train 5.940582 (lr=2.2228e-05) (hash(x)=152902689)
-6900 val loss 6.0014
-6900 val perplexity 403.9747
-6900 train 6.530525 (lr=2.0828e-05) (hash(x)=156849968)
-6700 val loss 6.0808
-6700 val perplexity 437.3650
-6700 train 6.123950 (lr=2.1999e-05) (hash(x)=153846046)
-7000 val loss 5.9989
-7000 val perplexity 402.9937
-7000 train 5.848539 (lr=2.0601e-05) (hash(x)=142395855)
-6800 val loss 6.0607
-6800 val perplexity 428.6592
-6800 train 6.189629 (lr=2.1767e-05) (hash(x)=158512738)
-7100 val loss 5.9881
-7100 val perplexity 398.6460
-7100 train 5.835137 (lr=2.0373e-05) (hash(x)=147114884)
-6900 val loss 6.0502
-6900 val perplexity 424.2151
-6900 train 6.573297 (lr=2.1534e-05) (hash(x)=156849968)
-7200 val loss 5.9794
-7200 val perplexity 395.2083
-7200 train 6.012267 (lr=2.0143e-05) (hash(x)=156979839)
-7000 val loss 6.0424
-7000 val perplexity 420.8963
-7000 train 5.893548 (lr=2.1300e-05) (hash(x)=142395855)
-7300 val loss 5.9750
-7300 val perplexity 393.4776
-7300 train 5.766932 (lr=1.9912e-05) (hash(x)=145584373)
-7100 val loss 6.0434
-7100 val perplexity 421.3395
-7100 train 5.894209 (lr=2.1064e-05) (hash(x)=147114884)
-7400 val loss 5.9685
-7400 val perplexity 390.9178
-7400 train 5.718100 (lr=1.9680e-05) (hash(x)=141508204)
-7200 val loss 6.0260
-7200 val perplexity 414.0711
-7200 train 6.056330 (lr=2.0826e-05) (hash(x)=156979839)
-7500 val loss 5.9593
-7500 val perplexity 387.3212
-7500 train 5.996938 (lr=1.9447e-05) (hash(x)=148803965)
-7300 val loss 6.0239
-7300 val perplexity 413.2011
-7300 train 5.810219 (lr=2.0587e-05) (hash(x)=145584373)
-7600 val loss 5.9563
-7600 val perplexity 386.1667
-7600 train 6.017087 (lr=1.9213e-05) (hash(x)=151019676)
-7400 val loss 6.0167
-7400 val perplexity 410.2079
-7400 train 5.765154 (lr=2.0347e-05) (hash(x)=141508204)
-7700 val loss 5.9395
-7700 val perplexity 379.7386
-7700 train 6.016877 (lr=1.8977e-05) (hash(x)=143155750)
-7500 val loss 6.0038
-7500 val perplexity 404.9713
-7500 train 6.038814 (lr=2.0106e-05) (hash(x)=148803965)
-7800 val loss 5.9326
-7800 val perplexity 377.1354
-7800 train 5.957977 (lr=1.8741e-05) (hash(x)=152569653)
-7600 val loss 6.0006
-7600 val perplexity 403.6666
-7600 train 6.062526 (lr=1.9864e-05) (hash(x)=151019676)
-7900 val loss 5.9255
-7900 val perplexity 374.4493
-7900 train 5.795002 (lr=1.8504e-05) (hash(x)=143519455)
-7700 val loss 5.9872
-7700 val perplexity 398.2939
-7700 train 6.054319 (lr=1.9620e-05) (hash(x)=143155750)
-8000 val loss 5.9216
-8000 val perplexity 372.9991
-8000 train 6.032913 (lr=1.8266e-05) (hash(x)=161180944)
-7800 val loss 5.9770
-7800 val perplexity 394.2439
-7800 train 6.000058 (lr=1.9376e-05) (hash(x)=152569653)
-8100 val loss 5.9175
-8100 val perplexity 371.4865
-8100 train 5.939320 (lr=1.8027e-05) (hash(x)=154107345)
-7900 val loss 5.9733
-7900 val perplexity 392.8031
-7900 train 5.845306 (lr=1.9131e-05) (hash(x)=143519455)
-8200 val loss 5.9133
-8200 val perplexity 369.9321
-8200 train 5.934331 (lr=1.7788e-05) (hash(x)=152486517)
-8300 val loss 5.9036
-8300 val perplexity 366.3696
-8300 train 5.916872 (lr=1.7549e-05) (hash(x)=156167749)
-8000 val loss 5.9651
-8000 val perplexity 389.5776
-8000 train 6.081837 (lr=1.8885e-05) (hash(x)=161180944)
-8400 val loss 5.8961
-8400 val perplexity 363.6209
-8400 train 5.985699 (lr=1.7309e-05) (hash(x)=149155006)
-8100 val loss 5.9620
-8100 val perplexity 388.4050
-8100 train 5.993131 (lr=1.8639e-05) (hash(x)=154107345)
-8500 val loss 5.8856
-8500 val perplexity 359.8330
-8500 train 5.839111 (lr=1.7068e-05) (hash(x)=147844390)
-8200 val loss 5.9571
-8200 val perplexity 386.4959
-8200 train 5.984252 (lr=1.8391e-05) (hash(x)=152486517)
-8600 val loss 5.8826
-8600 val perplexity 358.7306
-8600 train 6.137978 (lr=1.6827e-05) (hash(x)=165753320)
-8300 val loss 5.9479
-8300 val perplexity 382.9322
-8300 train 5.955806 (lr=1.8144e-05) (hash(x)=156167749)
-8700 val loss 5.8751
-8700 val perplexity 356.0525
-8700 train 5.656183 (lr=1.6587e-05) (hash(x)=146079979)
-8400 val loss 5.9382
-8400 val perplexity 379.2699
-8400 train 6.020640 (lr=1.7895e-05) (hash(x)=149155006)
-8800 val loss 5.8752
-8800 val perplexity 356.0977
-8800 train 6.067141 (lr=1.6346e-05) (hash(x)=172259509)
-8500 val loss 5.9279
-8500 val perplexity 375.3781
-8500 train 5.881935 (lr=1.7647e-05) (hash(x)=147844390)
-8900 val loss 5.8686
-8900 val perplexity 353.7440
-8900 train 5.545335 (lr=1.6104e-05) (hash(x)=145148314)
-8600 val loss 5.9222
-8600 val perplexity 373.2485
-8600 train 6.178904 (lr=1.7398e-05) (hash(x)=165753320)
-9000 val loss 5.8650
-9000 val perplexity 352.4770
-9000 train 5.784966 (lr=1.5863e-05) (hash(x)=144250633)
-8700 val loss 5.9185
-8700 val perplexity 371.8361
-8700 train 5.701425 (lr=1.7149e-05) (hash(x)=146079979)
-9100 val loss 5.8589
-9100 val perplexity 350.3411
-9100 train 5.957079 (lr=1.5623e-05) (hash(x)=157219797)
-8800 val loss 5.9200
-8800 val perplexity 372.4270
-8800 train 6.115524 (lr=1.6900e-05) (hash(x)=172259509)
-9200 val loss 5.8462
-9200 val perplexity 345.9230
-9200 train 5.889227 (lr=1.5382e-05) (hash(x)=142743778)
-8900 val loss 5.9069
-8900 val perplexity 367.5737
-8900 train 5.585574 (lr=1.6650e-05) (hash(x)=145148314)
-9300 val loss 5.8425
-9300 val perplexity 344.6294
-9300 train 5.760115 (lr=1.5141e-05) (hash(x)=139669771)
-9000 val loss 5.9026
-9000 val perplexity 366.0031
-9000 train 5.818547 (lr=1.6401e-05) (hash(x)=144250633)
-9400 val loss 5.8364
-9400 val perplexity 342.5404
-9400 train 5.873798 (lr=1.4901e-05) (hash(x)=145916843)
-9100 val loss 5.8958
-9100 val perplexity 363.4954
-9100 train 5.994218 (lr=1.6152e-05) (hash(x)=157219797)
-9500 val loss 5.8345
-9500 val perplexity 341.9007
-9500 train 5.751485 (lr=1.4662e-05) (hash(x)=150196125)
-9200 val loss 5.8853
-9200 val perplexity 359.6972
-9200 train 5.912038 (lr=1.5903e-05) (hash(x)=142743778)
-9600 val loss 5.8270
-9600 val perplexity 339.3380
-9600 train 6.134868 (lr=1.4423e-05) (hash(x)=160041419)
-9300 val loss 5.8791
-9300 val perplexity 357.4924
-9300 train 5.794174 (lr=1.5655e-05) (hash(x)=139669771)
-9700 val loss 5.8290
-9700 val perplexity 340.0282
-9700 train 5.608489 (lr=1.4184e-05) (hash(x)=139931627)
-9400 val loss 5.8709
-9400 val perplexity 354.5629
-9400 train 5.899670 (lr=1.5406e-05) (hash(x)=145916843)
-9800 val loss 5.8202
-9800 val perplexity 337.0427
-9800 train 5.799592 (lr=1.3946e-05) (hash(x)=150370792)
-9500 val loss 5.8681
-9500 val perplexity 353.5906
-9500 train 5.789189 (lr=1.5159e-05) (hash(x)=150196125)
-9900 val loss 5.8131
-9900 val perplexity 334.6431
-9900 train 5.659364 (lr=1.3709e-05) (hash(x)=153014886)
-9600 val loss 5.8616
-9600 val perplexity 351.2767
-9600 train 6.168010 (lr=1.4911e-05) (hash(x)=160041419)
-10000 val loss 5.8098
-10000 val perplexity 333.5407
-10000 train 5.712080 (lr=1.3473e-05) (hash(x)=141915265)
-9700 val loss 5.8592
-9700 val perplexity 350.4358
-9700 train 5.638835 (lr=1.4665e-05) (hash(x)=139931627)
-10100 val loss 5.8029
-10100 val perplexity 331.2668
-10100 train 6.318121 (lr=1.3237e-05) (hash(x)=197770952)
-9800 val loss 5.8548
-9800 val perplexity 348.8911
-9800 train 5.835329 (lr=1.4419e-05) (hash(x)=150370792)
-10200 val loss 5.7977
-10200 val perplexity 329.5564
-10200 train 5.787279 (lr=1.3003e-05) (hash(x)=156806005)
-9900 val loss 5.8479
-9900 val perplexity 346.5043
-9900 train 5.693243 (lr=1.4174e-05) (hash(x)=153014886)
-10300 val loss 5.7939
-10300 val perplexity 328.2841
-10300 train 5.600894 (lr=1.2770e-05) (hash(x)=145378216)
-10400 val loss 5.7886
-10400 val perplexity 326.5699
-10400 train 5.692913 (lr=1.2538e-05) (hash(x)=145125407)
-10000 val loss 5.8431
-10000 val perplexity 344.8638
-10000 train 5.754855 (lr=1.3930e-05) (hash(x)=141915265)
-10500 val loss 5.7872
-10500 val perplexity 326.1052
-10500 train 5.593261 (lr=1.2307e-05) (hash(x)=147553745)
-10100 val loss 5.8385
-10100 val perplexity 343.2745
-10100 train 6.350883 (lr=1.3686e-05) (hash(x)=197770952)
-10600 val loss 5.7829
-10600 val perplexity 324.7039
-10600 train 5.421621 (lr=1.2077e-05) (hash(x)=134894889)
-10200 val loss 5.8312
-10200 val perplexity 340.7835
-10200 train 5.818911 (lr=1.3444e-05) (hash(x)=156806005)
-10700 val loss 5.7771
-10700 val perplexity 322.8188
-10700 train 5.701269 (lr=1.1849e-05) (hash(x)=151041040)
-10300 val loss 5.8268
-10300 val perplexity 339.2614
-10300 train 5.626854 (lr=1.3203e-05) (hash(x)=145378216)
-10800 val loss 5.7702
-10800 val perplexity 320.6035
-10800 train 5.707076 (lr=1.1622e-05) (hash(x)=145685829)
-10400 val loss 5.8192
-10400 val perplexity 336.7047
-10400 train 5.720162 (lr=1.2963e-05) (hash(x)=145125407)
-10900 val loss 5.7673
-10900 val perplexity 319.6614
-10900 train 5.848281 (lr=1.1396e-05) (hash(x)=156000716)
-10500 val loss 5.8183
-10500 val perplexity 336.4149
-10500 train 5.627436 (lr=1.2724e-05) (hash(x)=147553745)
-11000 val loss 5.7645
-11000 val perplexity 318.7768
-11000 train 5.766081 (lr=1.1173e-05) (hash(x)=149858163)
-10600 val loss 5.8124
-10600 val perplexity 334.4131
-10600 train 5.445238 (lr=1.2486e-05) (hash(x)=134894889)
-11100 val loss 5.7624
-11100 val perplexity 318.1149
-11100 train 6.040233 (lr=1.0951e-05) (hash(x)=164873575)
-10700 val loss 5.8066
-10700 val perplexity 332.5002
-10700 train 5.730758 (lr=1.2250e-05) (hash(x)=151041040)
-11200 val loss 5.7598
-11200 val perplexity 317.2926
-11200 train 5.644976 (lr=1.0730e-05) (hash(x)=156370069)
-10800 val loss 5.7980
-10800 val perplexity 329.6276
-10800 train 5.733639 (lr=1.2016e-05) (hash(x)=145685829)
-11300 val loss 5.7567
-11300 val perplexity 316.2876
-11300 train 5.855590 (lr=1.0512e-05) (hash(x)=160356892)
-10900 val loss 5.7956
-10900 val perplexity 328.8546
-10900 train 5.870820 (lr=1.1783e-05) (hash(x)=156000716)
-11400 val loss 5.7475
-11400 val perplexity 313.3970
-11400 train 5.826313 (lr=1.0295e-05) (hash(x)=158955430)
-11000 val loss 5.7929
-11000 val perplexity 327.9536
-11000 train 5.798615 (lr=1.1551e-05) (hash(x)=149858163)
-11500 val loss 5.7444
-11500 val perplexity 312.4479
-11500 train 5.711458 (lr=1.0080e-05) (hash(x)=146598377)
-11100 val loss 5.7905
-11100 val perplexity 327.1687
-11100 train 6.069217 (lr=1.1322e-05) (hash(x)=164873575)
-11600 val loss 5.7588
-11600 val perplexity 316.9520
-11600 train 5.903492 (lr=9.8678e-06) (hash(x)=156045795)
-11200 val loss 5.7901
-11200 val perplexity 327.0437
-11200 train 5.669724 (lr=1.1094e-05) (hash(x)=156370069)
-11700 val loss 5.7414
-11700 val perplexity 311.4857
-11700 train 5.606212 (lr=9.6572e-06) (hash(x)=143963334)
-11300 val loss 5.7868
-11300 val perplexity 325.9556
-11300 train 5.881620 (lr=1.0868e-05) (hash(x)=160356892)
-11800 val loss 5.7406
-11800 val perplexity 311.2511
-11800 train 5.576191 (lr=9.4488e-06) (hash(x)=145831579)
-11400 val loss 5.7745
-11400 val perplexity 321.9794
-11400 train 5.857344 (lr=1.0644e-05) (hash(x)=158955430)
-11900 val loss 5.7341
-11900 val perplexity 309.2360
-11900 train 5.903021 (lr=9.2426e-06) (hash(x)=165154141)
-11500 val loss 5.7715
-11500 val perplexity 321.0229
-11500 train 5.729584 (lr=1.0422e-05) (hash(x)=146598377)
-12000 val loss 5.7290
-12000 val perplexity 307.6619
-12000 train 5.805437 (lr=9.0388e-06) (hash(x)=160373794)
-11600 val loss 5.7839
-11600 val perplexity 325.0162
-11600 train 5.927474 (lr=1.0202e-05) (hash(x)=156045795)
-12100 val loss 5.7248
-12100 val perplexity 306.3609
-12100 train 5.511147 (lr=8.8373e-06) (hash(x)=141476346)
-11700 val loss 5.7663
-11700 val perplexity 319.3398
-11700 train 5.623090 (lr=9.9846e-06) (hash(x)=143963334)
-12200 val loss 5.7239
-12200 val perplexity 306.0979
-12200 train 5.986765 (lr=8.6382e-06) (hash(x)=164508058)
-11800 val loss 5.7636
-11800 val perplexity 318.5077
-11800 train 5.601118 (lr=9.7691e-06) (hash(x)=145831579)
-12300 val loss 5.7208
-12300 val perplexity 305.1534
-12300 train 5.689639 (lr=8.4417e-06) (hash(x)=140407664)
-11900 val loss 5.7583
-11900 val perplexity 316.7961
-12400 val loss 5.7197
-12400 val perplexity 304.7993
-11900 train 5.922744 (lr=9.5560e-06) (hash(x)=165154141)
-12400 train 5.575074 (lr=8.2477e-06) (hash(x)=149526189)
-12500 val loss 5.7144
-12500 val perplexity 303.2103
-12500 train 5.557684 (lr=8.0563e-06) (hash(x)=142060916)
-12000 val loss 5.7519
-12000 val perplexity 314.7938
-12000 train 5.830003 (lr=9.3452e-06) (hash(x)=160373794)
-12600 val loss 5.7087
-12600 val perplexity 301.4715
-12600 train 5.898150 (lr=7.8676e-06) (hash(x)=168650848)
-12100 val loss 5.7498
-12100 val perplexity 314.1295
-12100 train 5.532583 (lr=9.1369e-06) (hash(x)=141476346)
-12700 val loss 5.7080
-12700 val perplexity 301.2578
-12700 train 5.730024 (lr=7.6817e-06) (hash(x)=154353018)
-12200 val loss 5.7461
-12200 val perplexity 312.9735
-12200 train 5.997993 (lr=8.9310e-06) (hash(x)=164508058)
-12800 val loss 5.7091
-12800 val perplexity 301.6128
-12800 train 5.862825 (lr=7.4986e-06) (hash(x)=156115265)
-12300 val loss 5.7443
-12300 val perplexity 312.3968
-12300 train 5.706179 (lr=8.7278e-06) (hash(x)=140407664)
-12900 val loss 5.7049
-12900 val perplexity 300.3245
-12900 train 5.630099 (lr=7.3184e-06) (hash(x)=143378175)
-12400 val loss 5.7422
-12400 val perplexity 311.7594
-12400 train 5.592043 (lr=8.5273e-06) (hash(x)=149526189)
-13000 val loss 5.7035
-13000 val perplexity 299.9164
-13000 train 5.840638 (lr=7.1412e-06) (hash(x)=166124864)
-12500 val loss 5.7368
-12500 val perplexity 310.0856
-12500 train 5.580209 (lr=8.3294e-06) (hash(x)=142060916)
-13100 val loss 5.7011
-13100 val perplexity 299.1846
-13100 train 5.571644 (lr=6.9669e-06) (hash(x)=146615385)
-12600 val loss 5.7315
-12600 val perplexity 308.4184
-12600 train 5.919592 (lr=8.1343e-06) (hash(x)=168650848)
-13200 val loss 5.7003
-13200 val perplexity 298.9580
-13200 train 5.600441 (lr=6.7956e-06) (hash(x)=149236047)
-12700 val loss 5.7301
-12700 val perplexity 307.9875
-12700 train 5.751512 (lr=7.9421e-06) (hash(x)=154353018)
-13300 val loss 5.6947
-13300 val perplexity 297.2975
-13300 train 5.630510 (lr=6.6275e-06) (hash(x)=151346972)
-12800 val loss 5.7310
-12800 val perplexity 308.2858
-12800 train 5.885005 (lr=7.7528e-06) (hash(x)=156115265)
-13400 val loss 5.6925
-13400 val perplexity 296.6468
-13400 train 5.779521 (lr=6.4626e-06) (hash(x)=159167093)
-12900 val loss 5.7263
-12900 val perplexity 306.8454
-12900 train 5.652340 (lr=7.5665e-06) (hash(x)=143378175)
-13500 val loss 5.6929
-13500 val perplexity 296.7407
-13500 train 5.411228 (lr=6.3008e-06) (hash(x)=135054650)
-13000 val loss 5.7231
-13000 val perplexity 305.8369
-13000 train 5.852026 (lr=7.3832e-06) (hash(x)=166124864)
-13600 val loss 5.6906
-13600 val perplexity 296.0607
-13600 train 5.575928 (lr=6.1424e-06) (hash(x)=151033927)
-13100 val loss 5.7233
-13100 val perplexity 305.9273
-13100 train 5.595488 (lr=7.2030e-06) (hash(x)=146615385)
-13700 val loss 5.6895
-13700 val perplexity 295.7369
-13700 train 5.389245 (lr=5.9872e-06) (hash(x)=141119172)
-13200 val loss 5.7203
-13200 val perplexity 305.0067
-13200 train 5.623034 (lr=7.0260e-06) (hash(x)=149236047)
-13800 val loss 5.6886
-13800 val perplexity 295.4740
-13800 train 5.404707 (lr=5.8355e-06) (hash(x)=144197766)
-13300 val loss 5.7154
-13300 val perplexity 303.5121
-13300 train 5.648562 (lr=6.8522e-06) (hash(x)=151346972)
-13900 val loss 5.6838
-13900 val perplexity 294.0775
-13900 train 5.628905 (lr=5.6871e-06) (hash(x)=145148158)
-13400 val loss 5.7141
-13400 val perplexity 303.1176
-13400 train 5.799570 (lr=6.6816e-06) (hash(x)=159167093)
-14000 val loss 5.6818
-14000 val perplexity 293.4707
-14000 train 6.193847 (lr=5.5423e-06) (hash(x)=179068751)
-13500 val loss 5.7132
-13500 val perplexity 302.8383
-13500 train 5.431733 (lr=6.5144e-06) (hash(x)=135054650)
-14100 val loss 5.6794
-14100 val perplexity 292.7716
-14100 train 5.544680 (lr=5.4009e-06) (hash(x)=144988373)
-13600 val loss 5.7099
-13600 val perplexity 301.8558
-13600 train 5.590559 (lr=6.3506e-06) (hash(x)=151033927)
-14200 val loss 5.6789
-14200 val perplexity 292.6157
-14200 train 5.607419 (lr=5.2632e-06) (hash(x)=133379696)
-13700 val loss 5.7093
-13700 val perplexity 301.6660
-13700 train 5.405561 (lr=6.1902e-06) (hash(x)=141119172)
-14300 val loss 5.6779
-14300 val perplexity 292.3262
-14300 train 5.559547 (lr=5.1290e-06) (hash(x)=149883422)
-13800 val loss 5.7080
-13800 val perplexity 301.2714
-13800 train 5.430277 (lr=6.0333e-06) (hash(x)=144197766)
-14400 val loss 5.6791
-14400 val perplexity 292.6717
-14400 train 5.549683 (lr=4.9985e-06) (hash(x)=145709198)
-14500 val loss 5.6729
-14500 val perplexity 290.8757
-14500 train 5.697035 (lr=4.8717e-06) (hash(x)=148016579)
-13900 val loss 5.7034
-13900 val perplexity 299.8968
-13900 train 5.648022 (lr=5.8799e-06) (hash(x)=145148158)
-14600 val loss 5.6713
-14600 val perplexity 290.4252
-14600 train 5.640500 (lr=4.7487e-06) (hash(x)=150218222)
-14000 val loss 5.6998
-14000 val perplexity 298.8001
-14000 train 6.204247 (lr=5.7301e-06) (hash(x)=179068751)
-14700 val loss 5.6712
-14700 val perplexity 290.3797
-14700 train 5.592179 (lr=4.6294e-06) (hash(x)=142258737)
-14100 val loss 5.6985
-14100 val perplexity 298.4277
-14100 train 5.562762 (lr=5.5840e-06) (hash(x)=144988373)
-14800 val loss 5.6683
-14800 val perplexity 289.5556
-14800 train 5.561450 (lr=4.5139e-06) (hash(x)=147628003)
-14200 val loss 5.6977
-14200 val perplexity 298.1909
-14200 train 5.620621 (lr=5.4416e-06) (hash(x)=133379696)
-14900 val loss 5.6688
-14900 val perplexity 289.6768
-14900 train 5.516444 (lr=4.4024e-06) (hash(x)=149144036)
-14300 val loss 5.6972
-14300 val perplexity 298.0186
-14300 train 5.574907 (lr=5.3029e-06) (hash(x)=149883422)
-15000 val loss 5.6689
-15000 val perplexity 289.7125
-15000 train 5.602131 (lr=4.2947e-06) (hash(x)=145925409)
-14400 val loss 5.6980
-14400 val perplexity 298.2687
-14400 train 5.573201 (lr=5.1679e-06) (hash(x)=145709198)
-15100 val loss 5.6645
-15100 val perplexity 288.4543
-15100 train 5.543977 (lr=4.1909e-06) (hash(x)=144555751)
-14500 val loss 5.6919
-14500 val perplexity 296.4501
-14500 train 5.713484 (lr=5.0368e-06) (hash(x)=148016579)
-15200 val loss 5.6625
-15200 val perplexity 287.8590
-15200 train 5.475387 (lr=4.0912e-06) (hash(x)=143448852)
-14600 val loss 5.6904
-14600 val perplexity 296.0243
-14600 train 5.654492 (lr=4.9096e-06) (hash(x)=150218222)
-15300 val loss 5.6612
-15300 val perplexity 287.4874
-15300 train 5.663271 (lr=3.9954e-06) (hash(x)=156347253)
-14700 val loss 5.6895
-14700 val perplexity 295.7311
-14700 train 5.608206 (lr=4.7863e-06) (hash(x)=142258737)
-15400 val loss 5.6600
-15400 val perplexity 287.1386
-15400 train 5.610167 (lr=3.9036e-06) (hash(x)=153010179)
-14800 val loss 5.6856
-14800 val perplexity 294.6054
-14800 train 5.578854 (lr=4.6670e-06) (hash(x)=147628003)
-15500 val loss 5.6603
-15500 val perplexity 287.2272
-15500 train 5.445764 (lr=3.8160e-06) (hash(x)=144066445)
-14900 val loss 5.6871
-14900 val perplexity 295.0244
-14900 train 5.536798 (lr=4.5516e-06) (hash(x)=149144036)
-15600 val loss 5.6596
-15600 val perplexity 287.0202
-15600 train 5.400588 (lr=3.7324e-06) (hash(x)=146107722)
-15000 val loss 5.6857
-15000 val perplexity 294.6293
-15000 train 5.619944 (lr=4.4403e-06) (hash(x)=145925409)
-15700 val loss 5.6560
-15700 val perplexity 286.0013
-15700 train 5.793683 (lr=3.6529e-06) (hash(x)=152533799)
-15100 val loss 5.6821
-15100 val perplexity 293.5591
-15100 train 5.559801 (lr=4.3330e-06) (hash(x)=144555751)
-15800 val loss 5.6538
-15800 val perplexity 285.3648
-15800 train 5.709746 (lr=3.5776e-06) (hash(x)=151756366)
-15200 val loss 5.6804
-15200 val perplexity 293.0698
-15200 train 5.491567 (lr=4.2298e-06) (hash(x)=143448852)
-15900 val loss 5.6537
-15900 val perplexity 285.3371
-15900 train 5.636307 (lr=3.5064e-06) (hash(x)=151735039)
-15300 val loss 5.6786
-15300 val perplexity 292.5335
-15300 train 5.678647 (lr=4.1308e-06) (hash(x)=156347253)
-16000 val loss 5.6531
-16000 val perplexity 285.1819
-16000 train 5.683552 (lr=3.4394e-06) (hash(x)=154633226)
-15400 val loss 5.6774
-15400 val perplexity 292.2015
-15400 train 5.630104 (lr=4.0360e-06) (hash(x)=153010179)
-16100 val loss 5.6539
-16100 val perplexity 285.4001
-16100 train 5.504101 (lr=3.3767e-06) (hash(x)=158094333)
-15500 val loss 5.6769
-15500 val perplexity 292.0530
-15500 train 5.459423 (lr=3.9453e-06) (hash(x)=144066445)
-16200 val loss 5.6527
-16200 val perplexity 285.0681
-16200 train 5.413481 (lr=3.3182e-06) (hash(x)=142772927)
-15600 val loss 5.6769
-15600 val perplexity 292.0411
-15600 train 5.411953 (lr=3.8589e-06) (hash(x)=146107722)
-16300 val loss 5.6518
-16300 val perplexity 284.7901
-16300 train 5.545006 (lr=3.2639e-06) (hash(x)=148042896)
-15700 val loss 5.6730
-15700 val perplexity 290.9038
-15700 train 5.807902 (lr=3.7767e-06) (hash(x)=152533799)
-16400 val loss 5.6486
-16400 val perplexity 283.8896
-16400 train 6.150700 (lr=3.2140e-06) (hash(x)=170306399)
-15800 val loss 5.6700
-15800 val perplexity 290.0362
-15800 train 5.724797 (lr=3.6988e-06) (hash(x)=151756366)
-16500 val loss 5.6480
-16500 val perplexity 283.7330
-16500 train 5.661663 (lr=3.1683e-06) (hash(x)=141049320)
-15900 val loss 5.6691
-15900 val perplexity 289.7662
-16600 val loss 5.6485
-16600 val perplexity 283.8764
-15900 train 5.654049 (lr=3.6253e-06) (hash(x)=151735039)
-16600 train 5.508330 (lr=3.1269e-06) (hash(x)=150512136)
-16700 val loss 5.6461
-16700 val perplexity 283.1884
-16700 train 5.655158 (lr=3.0898e-06) (hash(x)=151843645)
-16000 val loss 5.6689
-16000 val perplexity 289.7253
-16000 train 5.699039 (lr=3.5560e-06) (hash(x)=154633226)
-16800 val loss 5.6445
-16800 val perplexity 282.7275
-16800 train 5.706742 (lr=3.0571e-06) (hash(x)=140073659)
-16100 val loss 5.6694
-16100 val perplexity 289.8476
-16100 train 5.528376 (lr=3.4912e-06) (hash(x)=158094333)
-16900 val loss 5.6425
-16900 val perplexity 282.1565
-16900 train 5.840839 (lr=3.0287e-06) (hash(x)=171157212)
-16200 val loss 5.6685
-16200 val perplexity 289.5962
-16200 train 5.432844 (lr=3.4307e-06) (hash(x)=142772927)
-17000 val loss 5.6418
-17000 val perplexity 281.9566
-17000 train 5.567492 (lr=3.0047e-06) (hash(x)=144553276)
-16300 val loss 5.6684
-16300 val perplexity 289.5571
-16300 train 5.564351 (lr=3.3746e-06) (hash(x)=148042896)
-17100 val loss 5.6420
-17100 val perplexity 282.0205
-17100 train 5.638415 (lr=2.9850e-06) (hash(x)=157717565)
-16400 val loss 5.6647
-16400 val perplexity 288.4952
-16400 train 6.147547 (lr=3.3229e-06) (hash(x)=170306399)
-17200 val loss 5.6419
-17200 val perplexity 282.0104
-17200 train 5.514342 (lr=2.9697e-06) (hash(x)=149843860)
-16500 val loss 5.6637
-16500 val perplexity 288.2268
-16500 train 5.678766 (lr=3.2757e-06) (hash(x)=141049320)
-17300 val loss 5.6395
-17300 val perplexity 281.3223
-17300 train 5.704100 (lr=2.9588e-06) (hash(x)=162171979)
-16600 val loss 5.6634
-16600 val perplexity 288.1192
-16600 train 5.522741 (lr=3.2329e-06) (hash(x)=150512136)
-17400 val loss 5.6381
-17400 val perplexity 280.9318
-17400 train 5.611559 (lr=2.9522e-06) (hash(x)=148400702)
-16700 val loss 5.6617
-16700 val perplexity 287.6277
-16700 train 5.674246 (lr=3.1946e-06) (hash(x)=151843645)
-17499 val loss 5.6373
-17499 val perplexity 280.6932
-16800 val loss 5.6597
-16800 val perplexity 287.0615
-16800 train 5.709817 (lr=3.1607e-06) (hash(x)=140073659)
-16900 val loss 5.6578
-16900 val perplexity 286.5085
-16900 train 5.845744 (lr=3.1314e-06) (hash(x)=171157212)
-17000 val loss 5.6571
-17000 val perplexity 286.3196
-17000 train 5.585116 (lr=3.1065e-06) (hash(x)=144553276)
-17100 val loss 5.6568
-17100 val perplexity 286.2260
-17100 train 5.643761 (lr=3.0862e-06) (hash(x)=157717565)
-17200 val loss 5.6569
-17200 val perplexity 286.2656
-17200 train 5.526436 (lr=3.0704e-06) (hash(x)=149843860)
-17300 val loss 5.6542
-17300 val perplexity 285.4991
-17300 train 5.712016 (lr=3.0591e-06) (hash(x)=162171979)
-17400 val loss 5.6526
-17400 val perplexity 285.0189
-17400 train 5.627303 (lr=3.0523e-06) (hash(x)=148400702)
-17499 val loss 5.6522
-17499 val perplexity 284.9042
+0 train 11.263341 (lr=1.5000e-07) (hash(x)=150327452)
+100 val loss 9.9014
+100 val perplexity 19958.0000
+100 train 9.912344 (lr=1.5150e-05) (hash(x)=166441190)
+200 val loss 8.8366
+200 val perplexity 6881.3281
+200 train 8.955378 (lr=3.0000e-05) (hash(x)=166780046)
+300 val loss 7.8593
+300 val perplexity 2589.7876
+300 train 7.803794 (lr=2.9998e-05) (hash(x)=159835303)
+400 val loss 7.5790
+400 val perplexity 1956.7469
+400 train 7.361750 (lr=2.9991e-05) (hash(x)=155040610)
+500 val loss 7.4912
+500 val perplexity 1792.2339
+500 train 7.228088 (lr=2.9980e-05) (hash(x)=130190460)
+600 val loss 7.4113
+600 val perplexity 1654.6433
+600 train 7.433746 (lr=2.9964e-05) (hash(x)=155504036)
+700 val loss 7.3336
+700 val perplexity 1530.9218
+700 train 7.190465 (lr=2.9944e-05) (hash(x)=137347213)
+800 val loss 7.2757
+800 val perplexity 1444.7686
+800 train 7.128286 (lr=2.9920e-05) (hash(x)=143823248)
+900 val loss 7.2234
+900 val perplexity 1371.0818
+900 train 7.338418 (lr=2.9891e-05) (hash(x)=156260416)
+1000 val loss 7.1780
+1000 val perplexity 1310.3419
+1000 train 7.143191 (lr=2.9858e-05) (hash(x)=143734685)
+1100 val loss 7.1197
+1100 val perplexity 1236.0919
+1100 train 7.042824 (lr=2.9820e-05) (hash(x)=160013925)
+1200 val loss 7.0802
+1200 val perplexity 1188.1973
+1200 train 6.814084 (lr=2.9778e-05) (hash(x)=150678249)
+1300 val loss 7.0305
+1300 val perplexity 1130.6367
+1300 train 6.803394 (lr=2.9732e-05) (hash(x)=149073315)
+1400 val loss 6.9647
+1400 val perplexity 1058.5557
+1400 train 7.277344 (lr=2.9681e-05) (hash(x)=175802021)
+1500 val loss 6.9098
+1500 val perplexity 1002.0621
+1500 train 7.176047 (lr=2.9626e-05) (hash(x)=171034639)
+1600 val loss 6.8487
+1600 val perplexity 942.6979
+1600 train 7.057546 (lr=2.9566e-05) (hash(x)=158681215)
+1700 val loss 6.8005
+1700 val perplexity 898.2830
+1700 train 6.742936 (lr=2.9502e-05) (hash(x)=152116061)
+1800 val loss 6.7597
+1800 val perplexity 862.3545
+1800 train 6.696222 (lr=2.9434e-05) (hash(x)=146108145)
+1900 val loss 6.7213
+1900 val perplexity 829.9302
+1900 train 6.552172 (lr=2.9362e-05) (hash(x)=147598108)
+2000 val loss 6.6885
+2000 val perplexity 803.0875
+2000 train 6.533982 (lr=2.9285e-05) (hash(x)=154996086)
+2100 val loss 6.6541
+2100 val perplexity 775.9614
+2100 train 6.430016 (lr=2.9204e-05) (hash(x)=153396183)
+2200 val loss 6.5975
+2200 val perplexity 733.2267
+2200 train 6.589885 (lr=2.9119e-05) (hash(x)=153885445)
+2300 val loss 6.5654
+2300 val perplexity 710.1285
+2300 train 6.570909 (lr=2.9030e-05) (hash(x)=159666385)
+2400 val loss 6.5211
+2400 val perplexity 679.3126
+2400 train 6.510075 (lr=2.8937e-05) (hash(x)=142353087)
+2500 val loss 6.4907
+2500 val perplexity 658.9900
+2500 train 6.512077 (lr=2.8840e-05) (hash(x)=146491718)
+2600 val loss 6.4671
+2600 val perplexity 643.6051
+2600 train 6.386323 (lr=2.8738e-05) (hash(x)=150750353)
+2700 val loss 6.4439
+2700 val perplexity 628.8747
+2700 train 6.166628 (lr=2.8633e-05) (hash(x)=129849193)
+2800 val loss 6.4297
+2800 val perplexity 619.9781
+2800 train 6.259184 (lr=2.8523e-05) (hash(x)=152767913)
+2900 val loss 6.4164
+2900 val perplexity 611.7715
+2900 train 6.186160 (lr=2.8410e-05) (hash(x)=146531140)
+3000 val loss 6.3808
+3000 val perplexity 590.4099
+3000 train 6.471205 (lr=2.8292e-05) (hash(x)=151562048)
+3100 val loss 6.3554
+3100 val perplexity 575.6011
+3100 train 6.312563 (lr=2.8171e-05) (hash(x)=146001424)
+3200 val loss 6.3392
+3200 val perplexity 566.3698
+3200 train 6.420431 (lr=2.8046e-05) (hash(x)=166486165)
+3300 val loss 6.3088
+3300 val perplexity 549.3705
+3300 train 6.239015 (lr=2.7917e-05) (hash(x)=150866680)
+3400 val loss 6.3019
+3400 val perplexity 545.5984
+3400 train 6.192265 (lr=2.7784e-05) (hash(x)=143900419)
+3500 val loss 6.2890
+3500 val perplexity 538.6371
+3500 train 6.110422 (lr=2.7648e-05) (hash(x)=148845794)
+3600 val loss 6.2716
+3600 val perplexity 529.3439
+3600 train 6.100612 (lr=2.7508e-05) (hash(x)=145667796)
+3700 val loss 6.2611
+3700 val perplexity 523.8117
+3700 train 6.278327 (lr=2.7364e-05) (hash(x)=163563851)
+3800 val loss 6.2502
+3800 val perplexity 518.0978
+3800 train 6.200144 (lr=2.7216e-05) (hash(x)=147488689)
+3900 val loss 6.2217
+3900 val perplexity 503.5543
+3900 train 6.232891 (lr=2.7066e-05) (hash(x)=148186608)
+4000 val loss 6.2090
+4000 val perplexity 497.1790
+4000 train 6.097112 (lr=2.6911e-05) (hash(x)=142970187)
+4100 val loss 6.1900
+4100 val perplexity 487.8564
+4100 train 6.318220 (lr=2.6754e-05) (hash(x)=141584883)
+4200 val loss 6.1778
+4200 val perplexity 481.9508
+4200 train 5.972452 (lr=2.6592e-05) (hash(x)=145664585)
+4300 val loss 6.1629
+4300 val perplexity 474.8204
+4300 train 6.035709 (lr=2.6428e-05) (hash(x)=143736499)
+4400 val loss 6.1599
+4400 val perplexity 473.3930
+4400 train 5.981257 (lr=2.6260e-05) (hash(x)=151883322)
+4500 val loss 6.1525
+4500 val perplexity 469.8898
+4500 train 6.029921 (lr=2.6089e-05) (hash(x)=153904871)
+4600 val loss 6.1340
+4600 val perplexity 461.2766
+4600 train 6.236507 (lr=2.5915e-05) (hash(x)=154893521)
+4700 val loss 6.1143
+4700 val perplexity 452.2870
+4700 train 6.159433 (lr=2.5738e-05) (hash(x)=152323949)
+4800 val loss 6.0991
+4800 val perplexity 445.4421
+4800 train 6.051072 (lr=2.5558e-05) (hash(x)=154104619)
+4900 val loss 6.0881
+4900 val perplexity 440.5904
+4900 train 6.224781 (lr=2.5374e-05) (hash(x)=146311426)
+5000 val loss 6.0798
+5000 val perplexity 436.9629
+5000 train 6.060164 (lr=2.5188e-05) (hash(x)=156741847)
+5100 val loss 6.0701
+5100 val perplexity 432.7412
+5100 train 5.850953 (lr=2.4999e-05) (hash(x)=142086346)
+5200 val loss 6.0631
+5200 val perplexity 429.6880
+5200 train 5.839406 (lr=2.4807e-05) (hash(x)=150265428)
+5300 val loss 6.0595
+5300 val perplexity 428.1700
+5300 train 5.935390 (lr=2.4613e-05) (hash(x)=151339108)
+5400 val loss 6.0461
+5400 val perplexity 422.4570
+5400 train 6.015918 (lr=2.4415e-05) (hash(x)=154654372)
+5500 val loss 6.0314
+5500 val perplexity 416.3066
+5500 train 6.041653 (lr=2.4215e-05) (hash(x)=150575051)
+5600 val loss 6.0226
+5600 val perplexity 412.6557
+5600 train 5.942938 (lr=2.4013e-05) (hash(x)=140396423)
+5700 val loss 6.0114
+5700 val perplexity 408.0675
+5700 train 5.867752 (lr=2.3808e-05) (hash(x)=144678758)
+5800 val loss 6.0055
+5800 val perplexity 405.6359
+5800 train 6.061187 (lr=2.3601e-05) (hash(x)=151992743)
+5900 val loss 5.9950
+5900 val perplexity 401.4122
+5900 train 5.792812 (lr=2.3391e-05) (hash(x)=144396927)
+6000 val loss 5.9863
+6000 val perplexity 397.9262
+6000 train 5.942162 (lr=2.3179e-05) (hash(x)=165478625)
+6100 val loss 5.9891
+6100 val perplexity 399.0456
+6100 train 5.721336 (lr=2.2965e-05) (hash(x)=147088621)
+6200 val loss 5.9814
+6200 val perplexity 395.9857
+6200 train 5.744236 (lr=2.2749e-05) (hash(x)=140794994)
+6300 val loss 5.9617
+6300 val perplexity 388.2704
+6300 train 5.908691 (lr=2.2530e-05) (hash(x)=134780906)
+6400 val loss 5.9512
+6400 val perplexity 384.2211
+6400 train 5.939652 (lr=2.2310e-05) (hash(x)=149023655)
+6500 val loss 5.9371
+6500 val perplexity 378.8390
+6500 train 5.787305 (lr=2.2088e-05) (hash(x)=147497796)
+6600 val loss 5.9364
+6600 val perplexity 378.5837
+6600 train 5.789012 (lr=2.1864e-05) (hash(x)=152902689)
+6700 val loss 5.9391
+6700 val perplexity 379.5893
+6700 train 5.982219 (lr=2.1638e-05) (hash(x)=153846046)
+6800 val loss 5.9154
+6800 val perplexity 370.7135
+6800 train 6.043589 (lr=2.1410e-05) (hash(x)=158512738)
+6900 val loss 5.9053
+6900 val perplexity 366.9849
+6900 train 6.442078 (lr=2.1181e-05) (hash(x)=156849968)
+7000 val loss 5.8981
+7000 val perplexity 364.3483
+7000 train 5.747692 (lr=2.0951e-05) (hash(x)=142395855)
+7100 val loss 5.8983
+7100 val perplexity 364.4187
+7100 train 5.745164 (lr=2.0718e-05) (hash(x)=147114884)
+7200 val loss 5.8833
+7200 val perplexity 358.9946
+7200 train 5.913739 (lr=2.0485e-05) (hash(x)=156979839)
+7300 val loss 5.8794
+7300 val perplexity 357.5988
+7300 train 5.672119 (lr=2.0250e-05) (hash(x)=145584373)
+7400 val loss 5.8717
+7400 val perplexity 354.8532
+7400 train 5.615767 (lr=2.0014e-05) (hash(x)=141508204)
+7500 val loss 5.8635
+7500 val perplexity 351.9554
+7500 train 5.898952 (lr=1.9777e-05) (hash(x)=148803965)
+7600 val loss 5.8569
+7600 val perplexity 349.6490
+7600 train 5.917202 (lr=1.9538e-05) (hash(x)=151019676)
+7700 val loss 5.8453
+7700 val perplexity 345.6068
+7700 train 5.947480 (lr=1.9299e-05) (hash(x)=143155750)
+7800 val loss 5.8386
+7800 val perplexity 343.2999
+7800 train 5.866199 (lr=1.9059e-05) (hash(x)=152569653)
+7900 val loss 5.8312
+7900 val perplexity 340.7828
+7900 train 5.705254 (lr=1.8817e-05) (hash(x)=143519455)
+8000 val loss 5.8251
+8000 val perplexity 338.6843
+8000 train 5.929979 (lr=1.8576e-05) (hash(x)=161180944)
+8100 val loss 5.8268
+8100 val perplexity 339.2856
+8100 train 5.857555 (lr=1.8333e-05) (hash(x)=154107345)
+8200 val loss 5.8214
+8200 val perplexity 337.4545
+8200 train 5.856858 (lr=1.8090e-05) (hash(x)=152486517)
+8300 val loss 5.8096
+8300 val perplexity 333.4917
+8300 train 5.808879 (lr=1.7846e-05) (hash(x)=156167749)
+8400 val loss 5.8043
+8400 val perplexity 331.7203
+8400 train 5.892135 (lr=1.7602e-05) (hash(x)=149155006)
+8500 val loss 5.7925
+8500 val perplexity 327.8286
+8500 train 5.742015 (lr=1.7357e-05) (hash(x)=147844390)
+8600 val loss 5.7897
+8600 val perplexity 326.9220
+8600 train 6.050057 (lr=1.7113e-05) (hash(x)=165753320)
+8700 val loss 5.7839
+8700 val perplexity 325.0404
+8700 train 5.560269 (lr=1.6868e-05) (hash(x)=146079979)
+8800 val loss 5.7823
+8800 val perplexity 324.5202
+8800 train 5.968102 (lr=1.6623e-05) (hash(x)=172259509)
+8900 val loss 5.7766
+8900 val perplexity 322.6667
+8900 train 5.455958 (lr=1.6377e-05) (hash(x)=145148314)
+9000 val loss 5.7730
+9000 val perplexity 321.4933
+9000 train 5.693444 (lr=1.6132e-05) (hash(x)=144250633)
+9100 val loss 5.7631
+9100 val perplexity 318.3478
+9100 train 5.865222 (lr=1.5887e-05) (hash(x)=157219797)
+9200 val loss 5.7563
+9200 val perplexity 316.1644
+9200 train 5.801858 (lr=1.5643e-05) (hash(x)=142743778)
+9300 val loss 5.7519
+9300 val perplexity 314.7833
+9300 train 5.675515 (lr=1.5398e-05) (hash(x)=139669771)
+9400 val loss 5.7446
+9400 val perplexity 312.5020
+9400 train 5.790522 (lr=1.5154e-05) (hash(x)=145916843)
+9500 val loss 5.7433
+9500 val perplexity 312.0899
+9500 train 5.667969 (lr=1.4910e-05) (hash(x)=150196125)
+9600 val loss 5.7371
+9600 val perplexity 310.1722
+9600 train 6.052320 (lr=1.4667e-05) (hash(x)=160041419)
+9700 val loss 5.7378
+9700 val perplexity 310.3841
+9700 train 5.515315 (lr=1.4424e-05) (hash(x)=139931627)
+9800 val loss 5.7303
+9800 val perplexity 308.0637
+9800 train 5.713668 (lr=1.4183e-05) (hash(x)=150370792)
+9900 val loss 5.7241
+9900 val perplexity 306.1474
+9900 train 5.560071 (lr=1.3941e-05) (hash(x)=153014886)
+10000 val loss 5.7211
+10000 val perplexity 305.2354
+10000 train 5.627098 (lr=1.3701e-05) (hash(x)=141915265)
+10100 val loss 5.7186
+10100 val perplexity 304.4876
+10100 train 6.229914 (lr=1.3462e-05) (hash(x)=197770952)
+10200 val loss 5.7086
+10200 val perplexity 301.4522
+10200 train 5.686818 (lr=1.3223e-05) (hash(x)=156806005)
+10300 val loss 5.7046
+10300 val perplexity 300.2360
+10300 train 5.515134 (lr=1.2986e-05) (hash(x)=145378216)
+10400 val loss 5.7010
+10400 val perplexity 299.1595
+10400 train 5.611872 (lr=1.2750e-05) (hash(x)=145125407)
+10500 val loss 5.6972
+10500 val perplexity 298.0323
+10500 train 5.516371 (lr=1.2515e-05) (hash(x)=147553745)
+10600 val loss 5.6944
+10600 val perplexity 297.2043
+10600 train 5.339860 (lr=1.2282e-05) (hash(x)=134894889)
+10700 val loss 5.6894
+10700 val perplexity 295.7046
+10700 train 5.614913 (lr=1.2049e-05) (hash(x)=151041040)
+10800 val loss 5.6831
+10800 val perplexity 293.8572
+10800 train 5.614328 (lr=1.1819e-05) (hash(x)=145685829)
+10900 val loss 5.6789
+10900 val perplexity 292.6139
+10900 train 5.759324 (lr=1.1590e-05) (hash(x)=156000716)
+11000 val loss 5.6758
+11000 val perplexity 291.7285
+11000 train 5.686039 (lr=1.1362e-05) (hash(x)=149858163)
+11100 val loss 5.6741
+11100 val perplexity 291.2157
+11100 train 5.953506 (lr=1.1136e-05) (hash(x)=164873575)
+11200 val loss 5.6732
+11200 val perplexity 290.9572
+11200 train 5.551642 (lr=1.0912e-05) (hash(x)=156370069)
+11300 val loss 5.6717
+11300 val perplexity 290.5277
+11300 train 5.766811 (lr=1.0690e-05) (hash(x)=160356892)
+11400 val loss 5.6607
+11400 val perplexity 287.3395
+11400 train 5.736993 (lr=1.0470e-05) (hash(x)=158955430)
+11500 val loss 5.6569
+11500 val perplexity 286.2544
+11500 train 5.627035 (lr=1.0251e-05) (hash(x)=146598377)
+11600 val loss 5.6774
+11600 val perplexity 292.1937
+11600 train 5.819471 (lr=1.0035e-05) (hash(x)=156045795)
+11700 val loss 5.6552
+11700 val perplexity 285.7665
+11700 train 5.507897 (lr=9.8209e-06) (hash(x)=143963334)
+11800 val loss 5.6532
+11800 val perplexity 285.1887
+11800 train 5.492560 (lr=9.6090e-06) (hash(x)=145831579)
+11900 val loss 5.6478
+11900 val perplexity 283.6758
+11900 train 5.812625 (lr=9.3993e-06) (hash(x)=165154141)
+12000 val loss 5.6428
+12000 val perplexity 282.2388
+12000 train 5.726239 (lr=9.1920e-06) (hash(x)=160373794)
+12100 val loss 5.6393
+12100 val perplexity 281.2672
+12100 train 5.430665 (lr=8.9871e-06) (hash(x)=141476346)
+12200 val loss 5.6392
+12200 val perplexity 281.2291
+12200 train 5.895460 (lr=8.7846e-06) (hash(x)=164508058)
+12300 val loss 5.6363
+12300 val perplexity 280.4270
+12300 train 5.623287 (lr=8.5847e-06) (hash(x)=140407664)
+12400 val loss 5.6351
+12400 val perplexity 280.0981
+12400 train 5.487605 (lr=8.3875e-06) (hash(x)=149526189)
+12500 val loss 5.6286
+12500 val perplexity 278.2783
+12500 train 5.479519 (lr=8.1929e-06) (hash(x)=142060916)
+12600 val loss 5.6244
+12600 val perplexity 277.1057
+12600 train 5.806100 (lr=8.0010e-06) (hash(x)=168650848)
+12700 val loss 5.6219
+12700 val perplexity 276.4150
+12700 train 5.641911 (lr=7.8119e-06) (hash(x)=154353018)
+12800 val loss 5.6236
+12800 val perplexity 276.8924
+12800 train 5.778793 (lr=7.6257e-06) (hash(x)=156115265)
+12900 val loss 5.6207
+12900 val perplexity 276.0897
+12900 train 5.549165 (lr=7.4425e-06) (hash(x)=143378175)
+13000 val loss 5.6184
+13000 val perplexity 275.4428
+13000 train 5.725389 (lr=7.2622e-06) (hash(x)=166124864)
+13100 val loss 5.6173
+13100 val perplexity 275.1555
+13100 train 5.492951 (lr=7.0850e-06) (hash(x)=146615385)
+13200 val loss 5.6163
+13200 val perplexity 274.8670
+13200 train 5.517329 (lr=6.9108e-06) (hash(x)=149236047)
+13300 val loss 5.6108
+13300 val perplexity 273.3650
+13300 train 5.535242 (lr=6.7398e-06) (hash(x)=151346972)
+13400 val loss 5.6094
+13400 val perplexity 272.9731
+13400 train 5.699604 (lr=6.5721e-06) (hash(x)=159167093)
+13500 val loss 5.6083
+13500 val perplexity 272.6931
+13500 train 5.333128 (lr=6.4076e-06) (hash(x)=135054650)
+13600 val loss 5.6057
+13600 val perplexity 271.9643
+13600 train 5.492490 (lr=6.2465e-06) (hash(x)=151033927)
+13700 val loss 5.6058
+13700 val perplexity 271.9900
+13700 train 5.300904 (lr=6.0887e-06) (hash(x)=141119172)
+13800 val loss 5.6043
+13800 val perplexity 271.5937
+13800 train 5.319691 (lr=5.9344e-06) (hash(x)=144197766)
+13900 val loss 5.6004
+13900 val perplexity 270.5355
+13900 train 5.540504 (lr=5.7835e-06) (hash(x)=145148158)
+14000 val loss 5.5963
+14000 val perplexity 269.4165
+14000 train 6.091290 (lr=5.6362e-06) (hash(x)=179068751)
+14100 val loss 5.5959
+14100 val perplexity 269.3181
+14100 train 5.462370 (lr=5.4925e-06) (hash(x)=144988373)
+14200 val loss 5.5943
+14200 val perplexity 268.8805
+14200 train 5.537189 (lr=5.3524e-06) (hash(x)=133379696)
+14300 val loss 5.5946
+14300 val perplexity 268.9815
+14300 train 5.475476 (lr=5.2159e-06) (hash(x)=149883422)
+14400 val loss 5.5950
+14400 val perplexity 269.0731
+14400 train 5.464013 (lr=5.0832e-06) (hash(x)=145709198)
+14500 val loss 5.5900
+14500 val perplexity 267.7359
+14500 train 5.611162 (lr=4.9543e-06) (hash(x)=148016579)
+14600 val loss 5.5875
+14600 val perplexity 267.0565
+14600 train 5.556258 (lr=4.8291e-06) (hash(x)=150218222)
+14700 val loss 5.5881
+14700 val perplexity 267.2294
+14700 train 5.514695 (lr=4.7079e-06) (hash(x)=142258737)
+14800 val loss 5.5846
+14800 val perplexity 266.2829
+14800 train 5.476633 (lr=4.5905e-06) (hash(x)=147628003)
+14900 val loss 5.5862
+14900 val perplexity 266.7187
+14900 train 5.434494 (lr=4.4770e-06) (hash(x)=149144036)
+15000 val loss 5.5857
+15000 val perplexity 266.5893
+15000 train 5.518596 (lr=4.3675e-06) (hash(x)=145925409)
+15100 val loss 5.5817
+15100 val perplexity 265.5213
+15100 train 5.467402 (lr=4.2620e-06) (hash(x)=144555751)
+15200 val loss 5.5800
+15200 val perplexity 265.0629
+15200 train 5.394983 (lr=4.1605e-06) (hash(x)=143448852)
+15300 val loss 5.5793
+15300 val perplexity 264.8942
+15300 train 5.581673 (lr=4.0631e-06) (hash(x)=156347253)
+15400 val loss 5.5771
+15400 val perplexity 264.2980
+15400 train 5.532283 (lr=3.9698e-06) (hash(x)=153010179)
+15500 val loss 5.5772
+15500 val perplexity 264.3268
+15500 train 5.355952 (lr=3.8806e-06) (hash(x)=144066445)
+15600 val loss 5.5775
+15600 val perplexity 264.4116
+15600 train 5.308009 (lr=3.7956e-06) (hash(x)=146107722)
+15700 val loss 5.5734
+15700 val perplexity 263.3213
+15700 train 5.720090 (lr=3.7148e-06) (hash(x)=152533799)
+15800 val loss 5.5723
+15800 val perplexity 263.0380
+15800 train 5.625428 (lr=3.6382e-06) (hash(x)=151756366)
+15900 val loss 5.5707
+15900 val perplexity 262.6128
+15900 train 5.554205 (lr=3.5658e-06) (hash(x)=151735039)
+16000 val loss 5.5702
+16000 val perplexity 262.4980
+16000 train 5.604529 (lr=3.4977e-06) (hash(x)=154633226)
+16100 val loss 5.5719
+16100 val perplexity 262.9393
+16100 train 5.413339 (lr=3.4339e-06) (hash(x)=158094333)
+16200 val loss 5.5713
+16200 val perplexity 262.7858
+16200 train 5.327019 (lr=3.3744e-06) (hash(x)=142772927)
+16300 val loss 5.5702
+16300 val perplexity 262.4890
+16300 train 5.470050 (lr=3.3193e-06) (hash(x)=148042896)
+16400 val loss 5.5667
+16400 val perplexity 261.5669
+16400 train 6.053113 (lr=3.2684e-06) (hash(x)=170306399)
+16500 val loss 5.5664
+16500 val perplexity 261.4984
+16500 train 5.586644 (lr=3.2220e-06) (hash(x)=141049320)
+16600 val loss 5.5668
+16600 val perplexity 261.6037
+16600 train 5.419602 (lr=3.1799e-06) (hash(x)=150512136)
+16700 val loss 5.5640
+16700 val perplexity 260.8613
+16700 train 5.573275 (lr=3.1422e-06) (hash(x)=151843645)
+16800 val loss 5.5622
+16800 val perplexity 260.3896
+16800 train 5.632764 (lr=3.1089e-06) (hash(x)=140073659)
+16900 val loss 5.5605
+16900 val perplexity 259.9638
+16900 train 5.725865 (lr=3.0801e-06) (hash(x)=171157212)
+17000 val loss 5.5600
+17000 val perplexity 259.8117
+17000 train 5.495543 (lr=3.0556e-06) (hash(x)=144553276)
+17100 val loss 5.5607
+17100 val perplexity 260.0120
+17100 train 5.550316 (lr=3.0356e-06) (hash(x)=157717565)
+17200 val loss 5.5605
+17200 val perplexity 259.9615
+17200 train 5.423789 (lr=3.0200e-06) (hash(x)=149843860)
+17300 val loss 5.5578
+17300 val perplexity 259.2461
+17300 train 5.617573 (lr=3.0089e-06) (hash(x)=162171979)
+17400 val loss 5.5566
+17400 val perplexity 258.9522
+17400 train 5.530491 (lr=3.0022e-06) (hash(x)=148400702)
+17499 val loss 5.5565
+17499 val perplexity 258.9096

attention_kindselective_n_heads4_seed1338/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c3f48ba49d9ea6c6f39ae19e3c9c3acc00a69b4d6968b90cfd18f1910d14edf7
+oid sha256:f5db00d51e9a60a9ec51cc45865e30b30d959eba2e4420d76385259452d3336c
 size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6935002545e2a43807dc04db1c487cced3d3af38a975da6e94f88aec28724ddc
+oid sha256:131f98c60e821509e44529eec0de5685d6511c9b5b4badf98603ac9f496120d2
 size 179406214