Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1344/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1344", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1344", "n_embd": 256}

attention_kindselective_n_heads4_seed1344/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2848
 0 val perplexity 79602.4766
-0 train 11.283560 (lr=1.5000e-07) (hash(x)=145830960)
-100 val loss 9.9692
-100 val perplexity 21358.1699
-100 train 9.986609 (lr=1.5150e-05) (hash(x)=156335112)
-200 val loss 9.1055
-200 val perplexity 9005.0107
-200 train 9.164769 (lr=3.0000e-05) (hash(x)=144157453)
-300 val loss 8.1325
-300 val perplexity 3403.3740
-300 train 8.135631 (lr=2.9998e-05) (hash(x)=149245414)
-400 val loss 7.6795
-400 val perplexity 2163.4895
-400 train 7.722869 (lr=2.9991e-05) (hash(x)=146764602)
-500 val loss 7.5390
-500 val perplexity 1879.9814
-500 train 7.497005 (lr=2.9980e-05) (hash(x)=152540261)
-600 val loss 7.4717
-600 val perplexity 1757.6008
-600 train 7.247765 (lr=2.9964e-05) (hash(x)=140559124)
-700 val loss 7.4065
-700 val perplexity 1646.7147
-700 train 7.458422 (lr=2.9944e-05) (hash(x)=156003556)
-800 val loss 7.3485
-800 val perplexity 1553.9318
-800 train 7.892264 (lr=2.9920e-05) (hash(x)=166024176)
-900 val loss 7.3074
-900 val perplexity 1491.2965
-900 train 7.358513 (lr=2.9891e-05) (hash(x)=156196976)
-1000 val loss 7.2559
-1000 val perplexity 1416.4850
-1000 train 7.332314 (lr=2.9858e-05) (hash(x)=161040668)
-1100 val loss 7.2075
-1100 val perplexity 1349.4982
-1100 train 7.274787 (lr=2.9820e-05) (hash(x)=175899671)
-1200 val loss 7.1561
-1200 val perplexity 1281.8519
-1200 train 7.172896 (lr=2.9778e-05) (hash(x)=148878990)
-1300 val loss 7.0955
-1300 val perplexity 1206.5381
-1300 train 7.090349 (lr=2.9732e-05) (hash(x)=147253797)
-1400 val loss 7.0572
-1400 val perplexity 1161.1394
-1400 train 7.078465 (lr=2.9681e-05) (hash(x)=150420695)
-1500 val loss 6.9883
-1500 val perplexity 1083.8916
-1500 train 6.935720 (lr=2.9626e-05) (hash(x)=146933283)
-1600 val loss 6.9395
-1600 val perplexity 1032.2332
-1600 train 6.901734 (lr=2.9566e-05) (hash(x)=151995229)
-1700 val loss 6.9046
-1700 val perplexity 996.8332
-1700 train 7.139846 (lr=2.9502e-05) (hash(x)=154468626)
-1800 val loss 6.8615
-1800 val perplexity 954.8381
-1800 train 6.561424 (lr=2.9434e-05) (hash(x)=139853932)
-1900 val loss 6.8130
-1900 val perplexity 909.5905
-1900 train 6.602960 (lr=2.9362e-05) (hash(x)=142492588)
-2000 val loss 6.7646
-2000 val perplexity 866.5991
-2000 train 7.098177 (lr=2.9285e-05) (hash(x)=174104207)
-2100 val loss 6.7188
-2100 val perplexity 827.8433
-2100 train 6.714899 (lr=2.9204e-05) (hash(x)=151046997)
-2200 val loss 6.6761
-2200 val perplexity 793.2269
-2200 train 6.536441 (lr=2.9119e-05) (hash(x)=146275038)
-2300 val loss 6.6381
-2300 val perplexity 763.6530
-2300 train 6.574053 (lr=2.9030e-05) (hash(x)=147779512)
-2400 val loss 6.6098
-2400 val perplexity 742.3422
-2400 train 6.482061 (lr=2.8937e-05) (hash(x)=141403655)
-2500 val loss 6.5681
-2500 val perplexity 712.0099
-2500 train 6.546371 (lr=2.8840e-05) (hash(x)=160143436)
-2600 val loss 6.5341
-2600 val perplexity 688.2089
-2600 train 6.783262 (lr=2.8738e-05) (hash(x)=173675040)
-2700 val loss 6.5074
-2700 val perplexity 670.1088
-2700 train 6.542911 (lr=2.8633e-05) (hash(x)=150124772)
-2800 val loss 6.4810
-2800 val perplexity 652.6323
-2800 train 6.611236 (lr=2.8523e-05) (hash(x)=162475906)
-2900 val loss 6.4627
-2900 val perplexity 640.7689
-2900 train 6.393878 (lr=2.8410e-05) (hash(x)=152789231)
-3000 val loss 6.4435
-3000 val perplexity 628.6244
-3000 train 6.368320 (lr=2.8292e-05) (hash(x)=152785612)
-3100 val loss 6.4100
-3100 val perplexity 607.8701
-3100 train 6.428837 (lr=2.8171e-05) (hash(x)=156140235)
-3200 val loss 6.3834
-3200 val perplexity 591.9541
-3200 train 6.398758 (lr=2.8046e-05) (hash(x)=151987383)
-3300 val loss 6.3576
-3300 val perplexity 576.8705
-3300 train 6.542221 (lr=2.7917e-05) (hash(x)=176259299)
-3400 val loss 6.3385
-3400 val perplexity 565.9324
-3400 train 6.146057 (lr=2.7784e-05) (hash(x)=144540187)
-3500 val loss 6.3221
-3500 val perplexity 556.7604
-3500 train 6.537164 (lr=2.7648e-05) (hash(x)=156773596)
-3600 val loss 6.3115
-3600 val perplexity 550.8785
-3600 train 6.537225 (lr=2.7508e-05) (hash(x)=160934110)
-3700 val loss 6.2907
-3700 val perplexity 539.5314
-3700 train 6.405750 (lr=2.7364e-05) (hash(x)=156100357)
-3800 val loss 6.2763
-3800 val perplexity 531.8310
-3800 train 6.380016 (lr=2.7216e-05) (hash(x)=158048867)
-3900 val loss 6.2723
-3900 val perplexity 529.6952
-3900 train 6.104307 (lr=2.7066e-05) (hash(x)=148195960)
-4000 val loss 6.2419
-4000 val perplexity 513.8405
-4000 train 6.238558 (lr=2.6911e-05) (hash(x)=151976963)
-4100 val loss 6.2256
-4100 val perplexity 505.5300
-4100 train 6.186838 (lr=2.6754e-05) (hash(x)=157598041)
-4200 val loss 6.2065
-4200 val perplexity 495.9598
-4200 train 6.071241 (lr=2.6592e-05) (hash(x)=139121993)
-4300 val loss 6.1931
-4300 val perplexity 489.3787
-4300 train 6.278595 (lr=2.6428e-05) (hash(x)=154546719)
-4400 val loss 6.1838
-4400 val perplexity 484.8242
-4400 train 6.113948 (lr=2.6260e-05) (hash(x)=148310119)
-4500 val loss 6.1640
-4500 val perplexity 475.3333
-4500 train 6.048776 (lr=2.6089e-05) (hash(x)=147070971)
-4600 val loss 6.1615
-4600 val perplexity 474.1256
-4600 train 7.687497 (lr=2.5915e-05) (hash(x)=232298831)
-4700 val loss 6.1528
-4700 val perplexity 470.0258
-4700 train 6.114225 (lr=2.5738e-05) (hash(x)=149549680)
-4800 val loss 6.1339
-4800 val perplexity 461.2135
-4800 train 6.275864 (lr=2.5558e-05) (hash(x)=155703970)
-4900 val loss 6.1186
-4900 val perplexity 454.2499
-4900 train 6.110617 (lr=2.5374e-05) (hash(x)=158725785)
-5000 val loss 6.1065
-5000 val perplexity 448.7472
-5000 train 6.052335 (lr=2.5188e-05) (hash(x)=146983088)
-5100 val loss 6.0916
-5100 val perplexity 442.1179
-5100 train 6.025856 (lr=2.4999e-05) (hash(x)=144882522)
-5200 val loss 6.0840
-5200 val perplexity 438.7811
-5200 train 5.971626 (lr=2.4807e-05) (hash(x)=145880132)
-5300 val loss 6.0749
-5300 val perplexity 434.8229
-5300 train 5.903040 (lr=2.4613e-05) (hash(x)=140482543)
-5400 val loss 6.0653
-5400 val perplexity 430.6625
-5400 train 5.909417 (lr=2.4415e-05) (hash(x)=148902645)
-5500 val loss 6.0568
-5500 val perplexity 427.0222
-5500 train 5.798812 (lr=2.4215e-05) (hash(x)=149411471)
-5600 val loss 6.0457
-5600 val perplexity 422.3114
-5600 train 5.997655 (lr=2.4013e-05) (hash(x)=148673383)
-5700 val loss 6.0285
-5700 val perplexity 415.1040
-5700 train 6.085971 (lr=2.3808e-05) (hash(x)=157502794)
-5800 val loss 6.0220
-5800 val perplexity 412.3956
-5800 train 5.868658 (lr=2.3601e-05) (hash(x)=148057218)
-5900 val loss 6.0099
-5900 val perplexity 407.4564
-5900 train 5.854727 (lr=2.3391e-05) (hash(x)=141239696)
-6000 val loss 6.0013
-6000 val perplexity 403.9641
-6000 train 6.088947 (lr=2.3179e-05) (hash(x)=152338728)
-6100 val loss 5.9970
-6100 val perplexity 402.2112
-6100 train 5.684302 (lr=2.2965e-05) (hash(x)=143749713)
-6200 val loss 5.9905
-6200 val perplexity 399.6147
-6200 train 6.051906 (lr=2.2749e-05) (hash(x)=152128518)
-6300 val loss 5.9858
-6300 val perplexity 397.7259
-6300 train 5.860623 (lr=2.2530e-05) (hash(x)=152992195)
-6400 val loss 5.9688
-6400 val perplexity 391.0414
-6400 train 5.868921 (lr=2.2310e-05) (hash(x)=143799591)
-6500 val loss 5.9531
-6500 val perplexity 384.9623
-6500 train 6.056067 (lr=2.2088e-05) (hash(x)=144266888)
-6600 val loss 5.9429
-6600 val perplexity 381.0557
-6600 train 6.216046 (lr=2.1864e-05) (hash(x)=167374028)
-6700 val loss 5.9368
-6700 val perplexity 378.7207
-6700 train 5.912586 (lr=2.1638e-05) (hash(x)=151301787)
-6800 val loss 5.9301
-6800 val perplexity 376.1754
-6800 train 5.752751 (lr=2.1410e-05) (hash(x)=143204898)
-6900 val loss 5.9215
-6900 val perplexity 372.9644
-6900 train 6.009975 (lr=2.1181e-05) (hash(x)=148058660)
-7000 val loss 5.9250
-7000 val perplexity 374.2801
-7000 train 5.896644 (lr=2.0951e-05) (hash(x)=157509461)
-7100 val loss 5.9075
-7100 val perplexity 367.7890
-7100 train 6.227095 (lr=2.0718e-05) (hash(x)=171579342)
-7200 val loss 5.8980
-7200 val perplexity 364.3020
-7200 train 5.892464 (lr=2.0485e-05) (hash(x)=158180787)
-7300 val loss 5.8878
-7300 val perplexity 360.5948
-7300 train 5.758845 (lr=2.0250e-05) (hash(x)=150122338)
-7400 val loss 5.8823
-7400 val perplexity 358.6264
-7400 train 5.820298 (lr=2.0014e-05) (hash(x)=147561852)
-7500 val loss 5.8777
-7500 val perplexity 356.9875
-7500 train 5.868729 (lr=1.9777e-05) (hash(x)=149353924)
-7600 val loss 5.8703
-7600 val perplexity 354.3709
-7600 train 5.894732 (lr=1.9538e-05) (hash(x)=144545768)
-7700 val loss 5.8606
-7700 val perplexity 350.9321
-7700 train 5.883697 (lr=1.9299e-05) (hash(x)=142691770)
-7800 val loss 5.8543
-7800 val perplexity 348.7289
-7800 train 5.985229 (lr=1.9059e-05) (hash(x)=158378295)
-7900 val loss 5.8550
-7900 val perplexity 348.9877
-7900 train 5.711920 (lr=1.8817e-05) (hash(x)=150691918)
-8000 val loss 5.8351
-8000 val perplexity 342.1070
-8000 train 5.726991 (lr=1.8576e-05) (hash(x)=147837896)
-8100 val loss 5.8347
-8100 val perplexity 341.9752
-8100 train 5.806161 (lr=1.8333e-05) (hash(x)=144751226)
-8200 val loss 5.8267
-8200 val perplexity 339.2332
-8200 train 5.600180 (lr=1.8090e-05) (hash(x)=139168726)
-8300 val loss 5.8215
-8300 val perplexity 337.4666
-8300 train 5.742377 (lr=1.7846e-05) (hash(x)=154279434)
-8400 val loss 5.8104
-8400 val perplexity 333.7566
-8400 train 5.684167 (lr=1.7602e-05) (hash(x)=140111678)
-8500 val loss 5.8033
-8500 val perplexity 331.4041
-8500 train 5.792879 (lr=1.7357e-05) (hash(x)=152124088)
-8600 val loss 5.8023
-8600 val perplexity 331.0701
-8600 train 5.577775 (lr=1.7113e-05) (hash(x)=142869347)
-8700 val loss 5.7923
-8700 val perplexity 327.7723
-8700 train 5.696801 (lr=1.6868e-05) (hash(x)=145697606)
-8800 val loss 5.7945
-8800 val perplexity 328.4869
-8800 train 5.763039 (lr=1.6623e-05) (hash(x)=157314231)
-8900 val loss 5.7809
-8900 val perplexity 324.0471
-8900 train 5.758891 (lr=1.6377e-05) (hash(x)=149357509)
-9000 val loss 5.7770
-9000 val perplexity 322.7733
-9000 train 5.710581 (lr=1.6132e-05) (hash(x)=140434750)
-9100 val loss 5.7688
-9100 val perplexity 320.1650
-9100 train 5.773588 (lr=1.5887e-05) (hash(x)=144954543)
-9200 val loss 5.7651
-9200 val perplexity 318.9758
-9200 train 5.689574 (lr=1.5643e-05) (hash(x)=156329199)
-9300 val loss 5.7582
-9300 val perplexity 316.7898
-9300 train 5.556169 (lr=1.5398e-05) (hash(x)=137925654)
-9400 val loss 5.7588
-9400 val perplexity 316.9700
-9400 train 5.634879 (lr=1.5154e-05) (hash(x)=146385919)
-9500 val loss 5.7472
-9500 val perplexity 313.3140
-9500 train 5.686854 (lr=1.4910e-05) (hash(x)=144400019)
-9600 val loss 5.7486
-9600 val perplexity 313.7517
-9600 train 5.701641 (lr=1.4667e-05) (hash(x)=130562204)
-9700 val loss 5.7357
-9700 val perplexity 309.7330
-9700 train 5.676045 (lr=1.4424e-05) (hash(x)=152732805)
-9800 val loss 5.7323
-9800 val perplexity 308.6738
-9800 train 5.795771 (lr=1.4183e-05) (hash(x)=151860383)
-9900 val loss 5.7268
-9900 val perplexity 306.9808
-9900 train 5.828072 (lr=1.3941e-05) (hash(x)=156732259)
-10000 val loss 5.7220
-10000 val perplexity 305.5055
-10000 train 5.650548 (lr=1.3701e-05) (hash(x)=139943350)
-10100 val loss 5.7186
-10100 val perplexity 304.4856
-10100 train 5.526080 (lr=1.3462e-05) (hash(x)=145593924)
-10200 val loss 5.7150
-10200 val perplexity 303.3745
-10200 train 5.586910 (lr=1.3223e-05) (hash(x)=153996667)
-10300 val loss 5.7164
-10300 val perplexity 303.8144
-10300 train 5.620425 (lr=1.2986e-05) (hash(x)=154200987)
-10400 val loss 5.7155
-10400 val perplexity 303.5305
-10400 train 5.528822 (lr=1.2750e-05) (hash(x)=156872686)
-10500 val loss 5.7024
-10500 val perplexity 299.5930
-10500 train 5.794130 (lr=1.2515e-05) (hash(x)=155008723)
-10600 val loss 5.6952
-10600 val perplexity 297.4219
-10600 train 5.627241 (lr=1.2282e-05) (hash(x)=152715441)
-10700 val loss 5.6918
-10700 val perplexity 296.4389
-10700 train 5.847895 (lr=1.2049e-05) (hash(x)=166668700)
-10800 val loss 5.6873
-10800 val perplexity 295.1098
-10800 train 5.695483 (lr=1.1819e-05) (hash(x)=150698847)
-10900 val loss 5.6859
-10900 val perplexity 294.6873
-10900 train 5.507511 (lr=1.1590e-05) (hash(x)=137094574)
-11000 val loss 5.6800
-11000 val perplexity 292.9599
-11000 train 5.522943 (lr=1.1362e-05) (hash(x)=144225378)
-11100 val loss 5.6841
-11100 val perplexity 294.1578
-11100 train 5.603326 (lr=1.1136e-05) (hash(x)=151666772)
-11200 val loss 5.6811
-11200 val perplexity 293.2774
-11200 train 5.681242 (lr=1.0912e-05) (hash(x)=157843445)
-11300 val loss 5.6706
-11300 val perplexity 290.1950
-11300 train 5.572571 (lr=1.0690e-05) (hash(x)=140305735)
-11400 val loss 5.6664
-11400 val perplexity 288.9919
-11400 train 5.594697 (lr=1.0470e-05) (hash(x)=143435929)
-11500 val loss 5.6605
-11500 val perplexity 287.3021
-11500 train 5.694790 (lr=1.0251e-05) (hash(x)=150858606)
-11600 val loss 5.6594
-11600 val perplexity 286.9900
-11600 train 5.477522 (lr=1.0035e-05) (hash(x)=141643887)
-11700 val loss 5.6571
-11700 val perplexity 286.3209
-11700 train 5.519454 (lr=9.8209e-06) (hash(x)=151097313)
-11800 val loss 5.6553
-11800 val perplexity 285.7881
-11800 train 5.460770 (lr=9.6090e-06) (hash(x)=150639616)
-11900 val loss 5.6575
-11900 val perplexity 286.4227
-11900 train 5.337720 (lr=9.3993e-06) (hash(x)=154557688)
-12000 val loss 5.6531
-12000 val perplexity 285.1815
-12000 train 5.787493 (lr=9.1920e-06) (hash(x)=159324159)
-12100 val loss 5.6444
-12100 val perplexity 282.7010
-12100 train 5.700291 (lr=8.9871e-06) (hash(x)=151185357)
-12200 val loss 5.6409
-12200 val perplexity 281.7193
-12200 train 5.533353 (lr=8.7846e-06) (hash(x)=137457899)
-12300 val loss 5.6375
-12300 val perplexity 280.7644
-12300 train 5.693140 (lr=8.5847e-06) (hash(x)=146370056)
-12400 val loss 5.6356
-12400 val perplexity 280.2367
-12400 train 5.485180 (lr=8.3875e-06) (hash(x)=148352898)
-12500 val loss 5.6327
-12500 val perplexity 279.4252
-12500 train 5.597754 (lr=8.1929e-06) (hash(x)=161514681)
-12600 val loss 5.6327
-12600 val perplexity 279.4115
-12600 train 5.494642 (lr=8.0010e-06) (hash(x)=146207280)
-12700 val loss 5.6343
-12700 val perplexity 279.8726
-12700 train 5.465546 (lr=7.8119e-06) (hash(x)=142615619)
-12800 val loss 5.6293
-12800 val perplexity 278.4552
-12800 train 5.659070 (lr=7.6257e-06) (hash(x)=144604802)
-12900 val loss 5.6242
-12900 val perplexity 277.0586
-12900 train 5.622487 (lr=7.4425e-06) (hash(x)=143212941)
-13000 val loss 5.6172
-13000 val perplexity 275.1291
-13000 train 5.696371 (lr=7.2622e-06) (hash(x)=155987866)
-13100 val loss 5.6161
-13100 val perplexity 274.8131
-13100 train 5.900623 (lr=7.0850e-06) (hash(x)=155920713)
-13200 val loss 5.6141
-13200 val perplexity 274.2628
-13200 train 5.793137 (lr=6.9108e-06) (hash(x)=152145878)
-13300 val loss 5.6115
-13300 val perplexity 273.5495
-13300 train 5.438443 (lr=6.7398e-06) (hash(x)=134141709)
-13400 val loss 5.6135
-13400 val perplexity 274.1128
-13400 train 5.531283 (lr=6.5721e-06) (hash(x)=146096299)
-13500 val loss 5.6123
-13500 val perplexity 273.7640
-13500 train 5.370831 (lr=6.4076e-06) (hash(x)=144011431)
-13600 val loss 5.6047
-13600 val perplexity 271.6991
-13600 train 5.862495 (lr=6.2465e-06) (hash(x)=163950932)
-13700 val loss 5.6026
-13700 val perplexity 271.1429
-13700 train 5.737296 (lr=6.0887e-06) (hash(x)=156652406)
-13800 val loss 5.5982
-13800 val perplexity 269.9393
-13800 train 5.509688 (lr=5.9344e-06) (hash(x)=145753166)
-13900 val loss 5.5959
-13900 val perplexity 269.3103
-13900 train 5.459958 (lr=5.7835e-06) (hash(x)=147065551)
-14000 val loss 5.5955
-14000 val perplexity 269.1991
-14000 train 6.079162 (lr=5.6362e-06) (hash(x)=179077542)
-14100 val loss 5.5942
-14100 val perplexity 268.8653
-14100 train 5.684788 (lr=5.4925e-06) (hash(x)=171962802)
-14200 val loss 5.5939
-14200 val perplexity 268.7884
-14200 train 5.663789 (lr=5.3524e-06) (hash(x)=153299913)
-14300 val loss 5.5935
-14300 val perplexity 268.6757
-14300 train 5.508217 (lr=5.2159e-06) (hash(x)=154750565)
-14400 val loss 5.5911
-14400 val perplexity 268.0197
-14400 train 5.859001 (lr=5.0832e-06) (hash(x)=154536694)
-14500 val loss 5.5876
-14500 val perplexity 267.0922
-14500 train 5.642450 (lr=4.9543e-06) (hash(x)=151236383)
-14600 val loss 5.5843
-14600 val perplexity 266.2125
-14600 train 5.612250 (lr=4.8291e-06) (hash(x)=152584272)
-14700 val loss 5.5824
-14700 val perplexity 265.7056
-14700 train 5.555599 (lr=4.7079e-06) (hash(x)=175301826)
-14800 val loss 5.5823
-14800 val perplexity 265.6867
-14800 train 5.505548 (lr=4.5905e-06) (hash(x)=148729408)
-14900 val loss 5.5806
-14900 val perplexity 265.2391
-14900 train 5.777581 (lr=4.4770e-06) (hash(x)=150978283)
-15000 val loss 5.5811
-15000 val perplexity 265.3698
-15000 train 5.508034 (lr=4.3675e-06) (hash(x)=153621389)
-15100 val loss 5.5811
-15100 val perplexity 265.3546
-15100 train 5.494666 (lr=4.2620e-06) (hash(x)=150653793)
-15200 val loss 5.5787
-15200 val perplexity 264.7404
-15200 train 5.763510 (lr=4.1605e-06) (hash(x)=140802934)
-15300 val loss 5.5737
-15300 val perplexity 263.4139
-15300 train 5.551564 (lr=4.0631e-06) (hash(x)=152007556)
-15400 val loss 5.5736
-15400 val perplexity 263.3871
-15400 train 5.518318 (lr=3.9698e-06) (hash(x)=149057446)
-15500 val loss 5.5717
-15500 val perplexity 262.8695
-15500 train 5.481091 (lr=3.8806e-06) (hash(x)=147521774)
-15600 val loss 5.5706
-15600 val perplexity 262.5835
-15600 train 5.547856 (lr=3.7956e-06) (hash(x)=148944405)
-15700 val loss 5.5698
-15700 val perplexity 262.3817
-15700 train 5.518208 (lr=3.7148e-06) (hash(x)=148472532)
-15800 val loss 5.5692
-15800 val perplexity 262.2169
-15800 train 5.329184 (lr=3.6382e-06) (hash(x)=149048780)
-15900 val loss 5.5697
-15900 val perplexity 262.3451
-15900 train 5.534845 (lr=3.5658e-06) (hash(x)=150175778)
-16000 val loss 5.5708
-16000 val perplexity 262.6354
-16000 train 5.352664 (lr=3.4977e-06) (hash(x)=145943831)
-16100 val loss 5.5642
-16100 val perplexity 260.9034
-16100 train 5.749448 (lr=3.4339e-06) (hash(x)=134314842)
-16200 val loss 5.5623
-16200 val perplexity 260.4116
-16200 train 5.599839 (lr=3.3744e-06) (hash(x)=153755135)
-16300 val loss 5.5628
-16300 val perplexity 260.5445
-16300 train 5.281954 (lr=3.3193e-06) (hash(x)=140070802)
-16400 val loss 5.5612
-16400 val perplexity 260.1248
-16400 train 5.618970 (lr=3.2684e-06) (hash(x)=151132329)
-16500 val loss 5.5586
-16500 val perplexity 259.4483
-16500 train 5.575374 (lr=3.2220e-06) (hash(x)=151585550)
-16600 val loss 5.5569
-16600 val perplexity 259.0252
-16600 train 5.718242 (lr=3.1799e-06) (hash(x)=164389325)
-16700 val loss 5.5559
-16700 val perplexity 258.7569
-16700 train 5.469742 (lr=3.1422e-06) (hash(x)=146016298)
-16800 val loss 5.5566
-16800 val perplexity 258.9420
-16800 train 5.464295 (lr=3.1089e-06) (hash(x)=147668192)
-16900 val loss 5.5556
-16900 val perplexity 258.6944
-16900 train 5.519043 (lr=3.0801e-06) (hash(x)=155692530)
-17000 val loss 5.5567
-17000 val perplexity 258.9579
-17000 train 5.459873 (lr=3.0556e-06) (hash(x)=156598351)
-17100 val loss 5.5573
-17100 val perplexity 259.1112
-17100 train 5.451344 (lr=3.0356e-06) (hash(x)=157344236)
-17200 val loss 5.5540
-17200 val perplexity 258.2800
-17200 train 5.380807 (lr=3.0200e-06) (hash(x)=126288656)
-17300 val loss 5.5513
-17300 val perplexity 257.5620
-17300 train 5.597789 (lr=3.0089e-06) (hash(x)=146365959)
-17400 val loss 5.5494
-17400 val perplexity 257.0894
-17400 train 5.534318 (lr=3.0022e-06) (hash(x)=152729388)
-17499 val loss 5.5481
-17499 val perplexity 256.7398
+0 train 11.283560 (lr=1.6250e-07) (hash(x)=145830960)
+100 val loss 9.9599
+100 val perplexity 21160.5156
+100 train 9.980744 (lr=1.6412e-05) (hash(x)=156335112)
+200 val loss 9.0912
+200 val perplexity 8876.4326
+200 train 9.151538 (lr=3.2500e-05) (hash(x)=144157453)
+300 val loss 8.1860
+300 val perplexity 3590.2598
+300 train 8.190341 (lr=3.2498e-05) (hash(x)=149245414)
+400 val loss 7.8564
+400 val perplexity 2582.1475
+400 train 7.900827 (lr=3.2490e-05) (hash(x)=146764602)
+500 val loss 7.6707
+500 val perplexity 2144.5127
+500 train 7.630148 (lr=3.2478e-05) (hash(x)=152540261)
+600 val loss 7.5701
+600 val perplexity 1939.3589
+600 train 7.349800 (lr=3.2461e-05) (hash(x)=140559124)
+700 val loss 7.4952
+700 val perplexity 1799.3420
+700 train 7.542197 (lr=3.2440e-05) (hash(x)=156003556)
+800 val loss 7.4500
+800 val perplexity 1719.9104
+800 train 7.963747 (lr=3.2413e-05) (hash(x)=166024176)
+900 val loss 7.4207
+900 val perplexity 1670.2000
+900 train 7.463220 (lr=3.2382e-05) (hash(x)=156196976)
+1000 val loss 7.3957
+1000 val perplexity 1628.8840
+1000 train 7.477394 (lr=3.2346e-05) (hash(x)=161040668)
+1100 val loss 7.3753
+1100 val perplexity 1596.0294
+1100 train 7.463724 (lr=3.2305e-05) (hash(x)=175899671)
+1200 val loss 7.3571
+1200 val perplexity 1567.2632
+1200 train 7.373929 (lr=3.2260e-05) (hash(x)=148878990)
+1300 val loss 7.3335
+1300 val perplexity 1530.7035
+1300 train 7.324839 (lr=3.2209e-05) (hash(x)=147253797)
+1400 val loss 7.3084
+1400 val perplexity 1492.7358
+1400 train 7.329314 (lr=3.2154e-05) (hash(x)=150420695)
+1500 val loss 7.2696
+1500 val perplexity 1435.9690
+1500 train 7.223187 (lr=3.2094e-05) (hash(x)=146933283)
+1600 val loss 7.2424
+1600 val perplexity 1397.3754
+1600 train 7.211983 (lr=3.2030e-05) (hash(x)=151995229)
+1700 val loss 7.2171
+1700 val perplexity 1362.4900
+1700 train 7.459997 (lr=3.1961e-05) (hash(x)=154468626)
+1800 val loss 7.2010
+1800 val perplexity 1340.7111
+1800 train 6.917479 (lr=3.1887e-05) (hash(x)=139853932)
+1900 val loss 7.1688
+1900 val perplexity 1298.2341
+1900 train 6.966431 (lr=3.1809e-05) (hash(x)=142492588)
+2000 val loss 7.1404
+2000 val perplexity 1261.8771
+2000 train 7.475838 (lr=3.1726e-05) (hash(x)=174104207)
+2100 val loss 7.1090
+2100 val perplexity 1222.9773
+2100 train 7.088420 (lr=3.1638e-05) (hash(x)=151046997)
+2200 val loss 7.0901
+2200 val perplexity 1200.0785
+2200 train 6.951235 (lr=3.1546e-05) (hash(x)=146275038)
+2300 val loss 7.0700
+2300 val perplexity 1176.1006
+2300 train 6.987461 (lr=3.1449e-05) (hash(x)=147779512)
+2400 val loss 7.0425
+2400 val perplexity 1144.2594
+2400 train 6.903728 (lr=3.1348e-05) (hash(x)=141403655)
+2500 val loss 7.0185
+2500 val perplexity 1117.1313
+2500 train 6.996836 (lr=3.1243e-05) (hash(x)=160143436)
+2600 val loss 6.9944
+2600 val perplexity 1090.4697
+2600 train 7.262850 (lr=3.1133e-05) (hash(x)=173675040)
+2700 val loss 6.9657
+2700 val perplexity 1059.6196
+2700 train 6.981391 (lr=3.1019e-05) (hash(x)=150124772)
+2800 val loss 6.9322
+2800 val perplexity 1024.7571
+2800 train 7.053523 (lr=3.0900e-05) (hash(x)=162475906)
+2900 val loss 6.9104
+2900 val perplexity 1002.6725
+2900 train 6.838933 (lr=3.0777e-05) (hash(x)=152789231)
+3000 val loss 6.8856
+3000 val perplexity 978.1113
+3000 train 6.811411 (lr=3.0650e-05) (hash(x)=152785612)
+3100 val loss 6.8575
+3100 val perplexity 950.9545
+3100 train 6.902193 (lr=3.0518e-05) (hash(x)=156140235)
+3200 val loss 6.8196
+3200 val perplexity 915.5900
+3200 train 6.835784 (lr=3.0383e-05) (hash(x)=151987383)
+3300 val loss 6.7851
+3300 val perplexity 884.5807
+3300 train 7.016383 (lr=3.0243e-05) (hash(x)=176259299)
+3400 val loss 6.7537
+3400 val perplexity 857.2028
+3400 train 6.583331 (lr=3.0099e-05) (hash(x)=144540187)
+3500 val loss 6.7176
+3500 val perplexity 826.8558
+3500 train 6.904707 (lr=2.9952e-05) (hash(x)=156773596)
+3600 val loss 6.6901
+3600 val perplexity 804.3963
+3600 train 6.913805 (lr=2.9800e-05) (hash(x)=160934110)
+3700 val loss 6.6640
+3700 val perplexity 783.6652
+3700 train 6.768859 (lr=2.9644e-05) (hash(x)=156100357)
+3800 val loss 6.6411
+3800 val perplexity 765.9611
+3800 train 6.768399 (lr=2.9485e-05) (hash(x)=158048867)
+3900 val loss 6.6292
+3900 val perplexity 756.8519
+3900 train 6.455472 (lr=2.9321e-05) (hash(x)=148195960)
+4000 val loss 6.5926
+4000 val perplexity 729.6400
+4000 train 6.582702 (lr=2.9154e-05) (hash(x)=151976963)
+4100 val loss 6.5624
+4100 val perplexity 707.9741
+4100 train 6.539586 (lr=2.8983e-05) (hash(x)=157598041)
+4200 val loss 6.5374
+4200 val perplexity 690.4905
+4200 train 6.390263 (lr=2.8808e-05) (hash(x)=139121993)
+4300 val loss 6.5188
+4300 val perplexity 677.7638
+4300 train 6.616024 (lr=2.8630e-05) (hash(x)=154546719)
+4400 val loss 6.4994
+4400 val perplexity 664.7399
+4400 train 6.433190 (lr=2.8448e-05) (hash(x)=148310119)
+4500 val loss 6.4796
+4500 val perplexity 651.6804
+4500 train 6.363911 (lr=2.8263e-05) (hash(x)=147070971)
+4600 val loss 6.4720
+4600 val perplexity 646.7939
+4600 train 7.847555 (lr=2.8075e-05) (hash(x)=232298831)
+4700 val loss 6.4593
+4700 val perplexity 638.5972
+4700 train 6.411119 (lr=2.7883e-05) (hash(x)=149549680)
+4800 val loss 6.4289
+4800 val perplexity 619.5186
+4800 train 6.571551 (lr=2.7687e-05) (hash(x)=155703970)
+4900 val loss 6.4157
+4900 val perplexity 611.3965
+4900 train 6.413851 (lr=2.7489e-05) (hash(x)=158725785)
+5000 val loss 6.3958
+5000 val perplexity 599.3246
+5000 train 6.331246 (lr=2.7287e-05) (hash(x)=146983088)
+5100 val loss 6.3799
+5100 val perplexity 589.8960
+5100 train 6.302820 (lr=2.7082e-05) (hash(x)=144882522)
+5200 val loss 6.3708
+5200 val perplexity 584.5114
+5200 train 6.247755 (lr=2.6874e-05) (hash(x)=145880132)
+5300 val loss 6.3553
+5300 val perplexity 575.5067
+5300 train 6.157270 (lr=2.6664e-05) (hash(x)=140482543)
+5400 val loss 6.3421
+5400 val perplexity 567.9766
+5400 train 6.185915 (lr=2.6450e-05) (hash(x)=148902645)
+5500 val loss 6.3351
+5500 val perplexity 564.0076
+5500 train 6.093838 (lr=2.6233e-05) (hash(x)=149411471)
+5600 val loss 6.3083
+5600 val perplexity 549.0921
+5600 train 6.244935 (lr=2.6014e-05) (hash(x)=148673383)
+5700 val loss 6.2897
+5700 val perplexity 538.9991
+5700 train 6.356031 (lr=2.5792e-05) (hash(x)=157502794)
+5800 val loss 6.2787
+5800 val perplexity 533.0735
+5800 train 6.129450 (lr=2.5567e-05) (hash(x)=148057218)
+5900 val loss 6.2668
+5900 val perplexity 526.7706
+5900 train 6.101112 (lr=2.5340e-05) (hash(x)=141239696)
+6000 val loss 6.2553
+6000 val perplexity 520.7495
+6000 train 6.325564 (lr=2.5111e-05) (hash(x)=152338728)
+6100 val loss 6.2476
+6100 val perplexity 516.7849
+6100 train 5.917364 (lr=2.4879e-05) (hash(x)=143749713)
+6200 val loss 6.2363
+6200 val perplexity 510.9547
+6200 train 6.289093 (lr=2.4644e-05) (hash(x)=152128518)
+6300 val loss 6.2393
+6300 val perplexity 512.4878
+6300 train 6.130268 (lr=2.4408e-05) (hash(x)=152992195)
+6400 val loss 6.2214
+6400 val perplexity 503.4134
+6400 train 6.114389 (lr=2.4169e-05) (hash(x)=143799591)
+6500 val loss 6.1948
+6500 val perplexity 490.1777
+6500 train 6.273989 (lr=2.3928e-05) (hash(x)=144266888)
+6600 val loss 6.1813
+6600 val perplexity 483.6074
+6600 train 6.451832 (lr=2.3686e-05) (hash(x)=167374028)
+6700 val loss 6.1793
+6700 val perplexity 482.6491
+6700 train 6.151701 (lr=2.3441e-05) (hash(x)=151301787)
+6800 val loss 6.1644
+6800 val perplexity 475.5119
+6800 train 5.995218 (lr=2.3195e-05) (hash(x)=143204898)
+6900 val loss 6.1564
+6900 val perplexity 471.7352
+6900 train 6.231202 (lr=2.2946e-05) (hash(x)=148058660)
+7000 val loss 6.1538
+7000 val perplexity 470.5068
+7000 train 6.126843 (lr=2.2696e-05) (hash(x)=157509461)
+7100 val loss 6.1411
+7100 val perplexity 464.5543
+7100 train 6.477844 (lr=2.2445e-05) (hash(x)=171579342)
+7200 val loss 6.1258
+7200 val perplexity 457.5132
+7200 train 6.116988 (lr=2.2192e-05) (hash(x)=158180787)
+7300 val loss 6.1153
+7300 val perplexity 452.7334
+7300 train 5.979165 (lr=2.1937e-05) (hash(x)=150122338)
+7400 val loss 6.1110
+7400 val perplexity 450.7910
+7400 train 6.041814 (lr=2.1682e-05) (hash(x)=147561852)
+7500 val loss 6.1040
+7500 val perplexity 447.6480
+7500 train 6.087430 (lr=2.1425e-05) (hash(x)=149353924)
+7600 val loss 6.0976
+7600 val perplexity 444.7703
+7600 train 6.114073 (lr=2.1166e-05) (hash(x)=144545768)
+7700 val loss 6.0831
+7700 val perplexity 438.3668
+7700 train 6.091139 (lr=2.0907e-05) (hash(x)=142691770)
+7800 val loss 6.0767
+7800 val perplexity 435.5718
+7800 train 6.184749 (lr=2.0647e-05) (hash(x)=158378295)
+7900 val loss 6.0706
+7900 val perplexity 432.9561
+7900 train 5.934933 (lr=2.0386e-05) (hash(x)=150691918)
+8000 val loss 6.0554
+8000 val perplexity 426.4263
+8000 train 5.936950 (lr=2.0123e-05) (hash(x)=147837896)
+8100 val loss 6.0555
+8100 val perplexity 426.4411
+8100 train 6.014183 (lr=1.9861e-05) (hash(x)=144751226)
+8200 val loss 6.0455
+8200 val perplexity 422.1995
+8200 train 5.816062 (lr=1.9597e-05) (hash(x)=139168726)
+8300 val loss 6.0436
+8300 val perplexity 421.4195
+8300 train 5.974075 (lr=1.9333e-05) (hash(x)=154279434)
+8400 val loss 6.0303
+8400 val perplexity 415.8287
+8400 train 5.884507 (lr=1.9069e-05) (hash(x)=140111678)
+8500 val loss 6.0229
+8500 val perplexity 412.7881
+8500 train 6.001132 (lr=1.8804e-05) (hash(x)=152124088)
+8600 val loss 6.0159
+8600 val perplexity 409.8954
+8600 train 5.797539 (lr=1.8539e-05) (hash(x)=142869347)
+8700 val loss 6.0099
+8700 val perplexity 407.4459
+8700 train 5.903512 (lr=1.8273e-05) (hash(x)=145697606)
+8800 val loss 6.0079
+8800 val perplexity 406.6162
+8800 train 5.987753 (lr=1.8008e-05) (hash(x)=157314231)
+8900 val loss 5.9946
+8900 val perplexity 401.2674
+8900 train 5.984644 (lr=1.7742e-05) (hash(x)=149357509)
+9000 val loss 5.9892
+9000 val perplexity 399.1118
+9000 train 5.898522 (lr=1.7477e-05) (hash(x)=140434750)
+9100 val loss 5.9785
+9100 val perplexity 394.8529
+9100 train 5.972139 (lr=1.7211e-05) (hash(x)=144954543)
+9200 val loss 5.9733
+9200 val perplexity 392.7979
+9200 train 5.902909 (lr=1.6946e-05) (hash(x)=156329199)
+9300 val loss 5.9674
+9300 val perplexity 390.4979
+9300 train 5.757013 (lr=1.6681e-05) (hash(x)=137925654)
+9400 val loss 5.9667
+9400 val perplexity 390.2034
+9400 train 5.850411 (lr=1.6417e-05) (hash(x)=146385919)
+9500 val loss 5.9551
+9500 val perplexity 385.7179
+9500 train 5.880592 (lr=1.6153e-05) (hash(x)=144400019)
+9600 val loss 5.9563
+9600 val perplexity 386.1725
+9600 train 5.890789 (lr=1.5889e-05) (hash(x)=130562204)
+9700 val loss 5.9439
+9700 val perplexity 381.4185
+9700 train 5.884833 (lr=1.5627e-05) (hash(x)=152732805)
+9800 val loss 5.9368
+9800 val perplexity 378.7395
+9800 train 6.004889 (lr=1.5364e-05) (hash(x)=151860383)
+9900 val loss 5.9330
+9900 val perplexity 377.3029
+9900 train 6.009057 (lr=1.5103e-05) (hash(x)=156732259)
+10000 val loss 5.9277
+10000 val perplexity 375.2834
+10000 train 5.854128 (lr=1.4843e-05) (hash(x)=139943350)
+10100 val loss 5.9212
+10100 val perplexity 372.8618
+10100 train 5.743155 (lr=1.4584e-05) (hash(x)=145593924)
+10200 val loss 5.9188
+10200 val perplexity 371.9498
+10200 train 5.780659 (lr=1.4325e-05) (hash(x)=153996667)
+10300 val loss 5.9219
+10300 val perplexity 373.1076
+10300 train 5.832670 (lr=1.4068e-05) (hash(x)=154200987)
+10400 val loss 5.9178
+10400 val perplexity 371.5873
+10400 train 5.739534 (lr=1.3813e-05) (hash(x)=156872686)
+10500 val loss 5.9038
+10500 val perplexity 366.4414
+10500 train 5.989748 (lr=1.3558e-05) (hash(x)=155008723)
+10600 val loss 5.8966
+10600 val perplexity 363.7883
+10600 train 5.831800 (lr=1.3305e-05) (hash(x)=152715441)
+10700 val loss 5.8915
+10700 val perplexity 361.9330
+10700 train 6.048138 (lr=1.3054e-05) (hash(x)=166668700)
+10800 val loss 5.8887
+10800 val perplexity 360.9201
+10800 train 5.889436 (lr=1.2804e-05) (hash(x)=150698847)
+10900 val loss 5.8845
+10900 val perplexity 359.4253
+10900 train 5.701465 (lr=1.2555e-05) (hash(x)=137094574)
+11000 val loss 5.8794
+11000 val perplexity 357.5838
+11000 train 5.731179 (lr=1.2309e-05) (hash(x)=144225378)
+11100 val loss 5.8834
+11100 val perplexity 359.0289
+11100 train 5.797372 (lr=1.2064e-05) (hash(x)=151666772)
+11200 val loss 5.8800
+11200 val perplexity 357.8239
+11200 train 5.896419 (lr=1.1822e-05) (hash(x)=157843445)
+11300 val loss 5.8663
+11300 val perplexity 352.9310
+11300 train 5.771248 (lr=1.1581e-05) (hash(x)=140305735)
+11400 val loss 5.8649
+11400 val perplexity 352.4369
+11400 train 5.781058 (lr=1.1342e-05) (hash(x)=143435929)
+11500 val loss 5.8583
+11500 val perplexity 350.1213
+11500 train 5.890773 (lr=1.1106e-05) (hash(x)=150858606)
+11600 val loss 5.8555
+11600 val perplexity 349.1357
+11600 train 5.671831 (lr=1.0871e-05) (hash(x)=141643887)
+11700 val loss 5.8525
+11700 val perplexity 348.1206
+11700 train 5.714521 (lr=1.0639e-05) (hash(x)=151097313)
+11800 val loss 5.8506
+11800 val perplexity 347.4531
+11800 train 5.658457 (lr=1.0410e-05) (hash(x)=150639616)
+11900 val loss 5.8537
+11900 val perplexity 348.5312
+11900 train 5.559586 (lr=1.0183e-05) (hash(x)=154557688)
+12000 val loss 5.8478
+12000 val perplexity 346.4668
+12000 train 5.983258 (lr=9.9580e-06) (hash(x)=159324159)
+12100 val loss 5.8373
+12100 val perplexity 342.8491
+12100 train 5.883004 (lr=9.7360e-06) (hash(x)=151185357)
+12200 val loss 5.8367
+12200 val perplexity 342.6588
+12200 train 5.716962 (lr=9.5167e-06) (hash(x)=137457899)
+12300 val loss 5.8313
+12300 val perplexity 340.8114
+12300 train 5.890007 (lr=9.3001e-06) (hash(x)=146370056)
+12400 val loss 5.8278
+12400 val perplexity 339.5993
+12400 train 5.673313 (lr=9.0864e-06) (hash(x)=148352898)
+12500 val loss 5.8252
+12500 val perplexity 338.7279
+12500 train 5.817701 (lr=8.8756e-06) (hash(x)=161514681)
+12600 val loss 5.8246
+12600 val perplexity 338.5426
+12600 train 5.698085 (lr=8.6677e-06) (hash(x)=146207280)
+12700 val loss 5.8231
+12700 val perplexity 338.0049
+12700 train 5.654124 (lr=8.4629e-06) (hash(x)=142615619)
+12800 val loss 5.8199
+12800 val perplexity 336.9413
+12800 train 5.843594 (lr=8.2612e-06) (hash(x)=144604802)
+12900 val loss 5.8149
+12900 val perplexity 335.2452
+12900 train 5.804498 (lr=8.0627e-06) (hash(x)=143212941)
+13000 val loss 5.8070
+13000 val perplexity 332.6330
+13000 train 5.882431 (lr=7.8674e-06) (hash(x)=155987866)
+13100 val loss 5.8062
+13100 val perplexity 332.3383
+13100 train 6.088308 (lr=7.6754e-06) (hash(x)=155920713)
+13200 val loss 5.8028
+13200 val perplexity 331.2296
+13200 train 5.985959 (lr=7.4867e-06) (hash(x)=152145878)
+13300 val loss 5.8005
+13300 val perplexity 330.4608
+13300 train 5.606394 (lr=7.3015e-06) (hash(x)=134141709)
+13400 val loss 5.8019
+13400 val perplexity 330.9319
+13400 train 5.717030 (lr=7.1198e-06) (hash(x)=146096299)
+13500 val loss 5.8000
+13500 val perplexity 330.3149
+13500 train 5.557479 (lr=6.9416e-06) (hash(x)=144011431)
+13600 val loss 5.7930
+13600 val perplexity 328.0098
+13600 train 6.033184 (lr=6.7670e-06) (hash(x)=163950932)
+13700 val loss 5.7892
+13700 val perplexity 326.7588
+13700 train 5.933305 (lr=6.5961e-06) (hash(x)=156652406)
+13800 val loss 5.7860
+13800 val perplexity 325.7170
+13800 train 5.694222 (lr=6.4289e-06) (hash(x)=145753166)
+13900 val loss 5.7844
+13900 val perplexity 325.1796
+13900 train 5.654222 (lr=6.2655e-06) (hash(x)=147065551)
+14000 val loss 5.7817
+14000 val perplexity 324.3047
+14000 train 6.244597 (lr=6.1059e-06) (hash(x)=179077542)
+14100 val loss 5.7802
+14100 val perplexity 323.8351
+14100 train 5.867098 (lr=5.9502e-06) (hash(x)=171962802)
+14200 val loss 5.7802
+14200 val perplexity 323.8290
+14200 train 5.847116 (lr=5.7984e-06) (hash(x)=153299913)
+14300 val loss 5.7801
+14300 val perplexity 323.8044
+14300 train 5.706236 (lr=5.6506e-06) (hash(x)=154750565)
+14400 val loss 5.7769
+14400 val perplexity 322.7437
+14400 train 6.051046 (lr=5.5068e-06) (hash(x)=154536694)
+14500 val loss 5.7720
+14500 val perplexity 321.1904
+14500 train 5.821687 (lr=5.3671e-06) (hash(x)=151236383)
+14600 val loss 5.7703
+14600 val perplexity 320.6415
+14600 train 5.790136 (lr=5.2316e-06) (hash(x)=152584272)
+14700 val loss 5.7687
+14700 val perplexity 320.1265
+14700 train 5.789655 (lr=5.1002e-06) (hash(x)=175301826)
+14800 val loss 5.7669
+14800 val perplexity 319.5508
+14800 train 5.700191 (lr=4.9730e-06) (hash(x)=148729408)
+14900 val loss 5.7655
+14900 val perplexity 319.1001
+14900 train 5.943477 (lr=4.8501e-06) (hash(x)=150978283)
+15000 val loss 5.7656
+15000 val perplexity 319.1203
+15000 train 5.704052 (lr=4.7314e-06) (hash(x)=153621389)
+15100 val loss 5.7652
+15100 val perplexity 319.0132
+15100 train 5.685680 (lr=4.6171e-06) (hash(x)=150653793)
+15200 val loss 5.7633
+15200 val perplexity 318.4034
+15200 train 5.915984 (lr=4.5072e-06) (hash(x)=140802934)
+15300 val loss 5.7578
+15300 val perplexity 316.6652
+15300 train 5.736080 (lr=4.4017e-06) (hash(x)=152007556)
+15400 val loss 5.7567
+15400 val perplexity 316.2986
+15400 train 5.709007 (lr=4.3006e-06) (hash(x)=149057446)
+15500 val loss 5.7558
+15500 val perplexity 316.0076
+15500 train 5.668916 (lr=4.2040e-06) (hash(x)=147521774)
+15600 val loss 5.7535
+15600 val perplexity 315.2928
+15600 train 5.722573 (lr=4.1119e-06) (hash(x)=148944405)
+15700 val loss 5.7537
+15700 val perplexity 315.3688
+15700 train 5.707011 (lr=4.0244e-06) (hash(x)=148472532)
+15800 val loss 5.7524
+15800 val perplexity 314.9471
+15800 train 5.531003 (lr=3.9414e-06) (hash(x)=149048780)
+15900 val loss 5.7525
+15900 val perplexity 314.9665
+15900 train 5.719010 (lr=3.8630e-06) (hash(x)=150175778)
+16000 val loss 5.7528
+16000 val perplexity 315.0829
+16000 train 5.550600 (lr=3.7892e-06) (hash(x)=145943831)
+16100 val loss 5.7465
+16100 val perplexity 313.0834
+16100 train 5.913585 (lr=3.7201e-06) (hash(x)=134314842)
+16200 val loss 5.7448
+16200 val perplexity 312.5483
+16200 train 5.767522 (lr=3.6556e-06) (hash(x)=153755135)
+16300 val loss 5.7441
+16300 val perplexity 312.3481
+16300 train 5.476809 (lr=3.5959e-06) (hash(x)=140070802)
+16400 val loss 5.7436
+16400 val perplexity 312.1762
+16400 train 5.759050 (lr=3.5408e-06) (hash(x)=151132329)
+16500 val loss 5.7412
+16500 val perplexity 311.4370
+16500 train 5.777473 (lr=3.4905e-06) (hash(x)=151585550)
+16600 val loss 5.7391
+16600 val perplexity 310.7890
+16600 train 5.895504 (lr=3.4449e-06) (hash(x)=164389325)
+16700 val loss 5.7378
+16700 val perplexity 310.3712
+16700 train 5.643941 (lr=3.4041e-06) (hash(x)=146016298)
+16800 val loss 5.7378
+16800 val perplexity 310.3949
+16800 train 5.648126 (lr=3.3680e-06) (hash(x)=147668192)
+16900 val loss 5.7370
+16900 val perplexity 310.1412
+16900 train 5.715910 (lr=3.3367e-06) (hash(x)=155692530)
+17000 val loss 5.7375
+17000 val perplexity 310.2915
+17000 train 5.663022 (lr=3.3102e-06) (hash(x)=156598351)
+17100 val loss 5.7380
+17100 val perplexity 310.4377
+17100 train 5.648726 (lr=3.2886e-06) (hash(x)=157344236)
+17200 val loss 5.7349
+17200 val perplexity 309.4906
+17200 train 5.538077 (lr=3.2717e-06) (hash(x)=126288656)
+17300 val loss 5.7323
+17300 val perplexity 308.6758
+17300 train 5.773144 (lr=3.2596e-06) (hash(x)=146365959)
+17400 val loss 5.7304
+17400 val perplexity 308.0987
+17400 train 5.714909 (lr=3.2524e-06) (hash(x)=152729388)
+17499 val loss 5.7284
+17499 val perplexity 307.4802

attention_kindselective_n_heads4_seed1344/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87806558a4cf546cf3180d16eda42f953540aad9c91c4a39b6d87c196a91d101
+oid sha256:f1b343ac4384c6e6dce1c9b65490a03b915bcd842d344c17e51207b80f2a41e8
 size 92843394

attention_kindselective_n_heads4_seed1344/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c7ed780c8b3f89184cf2d18cf5f0126733793e07acad9e400c342d53dbf8fdc
+oid sha256:82f52a9e837518df1666833e3cb6f8e76aa3c06bb81e76837bee3913c88adac0
 size 179406214