Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1341", "n_embd": 256}

attention_kindselective_n_heads4_seed1341/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.3107
 0 val perplexity 81693.5234
-0 train 11.308768 (lr=1.3750e-07) (hash(x)=145079536)
-100 val loss 9.9993
-100 val perplexity 22010.1719
-100 train 9.939669 (lr=1.3888e-05) (hash(x)=154745873)
-200 val loss 9.1864
-200 val perplexity 9763.2383
-200 train 9.216236 (lr=2.7500e-05) (hash(x)=155800595)
-300 val loss 8.0336
-300 val perplexity 3082.9048
-300 train 8.000475 (lr=2.7498e-05) (hash(x)=148595389)
-400 val loss 7.6898
-400 val perplexity 2185.9587
-400 train 7.606212 (lr=2.7492e-05) (hash(x)=145606733)
-500 val loss 7.5730
-500 val perplexity 1944.9144
-500 train 7.413569 (lr=2.7482e-05) (hash(x)=138221231)
-600 val loss 7.5082
-600 val perplexity 1822.9774
-600 train 7.518361 (lr=2.7467e-05) (hash(x)=150367139)
-700 val loss 7.4621
-700 val perplexity 1740.8705
-700 train 7.613705 (lr=2.7449e-05) (hash(x)=155579314)
-800 val loss 7.4197
-800 val perplexity 1668.4584
-800 train 7.498192 (lr=2.7427e-05) (hash(x)=155747374)
-900 val loss 7.3766
-900 val perplexity 1598.2067
-900 train 7.581522 (lr=2.7400e-05) (hash(x)=159334575)
-1000 val loss 7.3366
-1000 val perplexity 1535.4816
-1000 train 7.210344 (lr=2.7370e-05) (hash(x)=140604760)
-1100 val loss 7.3040
-1100 val perplexity 1486.2832
-1100 train 7.140453 (lr=2.7335e-05) (hash(x)=146208052)
-1200 val loss 7.2680
-1200 val perplexity 1433.7283
-1200 train 7.167972 (lr=2.7297e-05) (hash(x)=148404734)
-1300 val loss 7.2364
-1300 val perplexity 1389.1176
-1300 train 7.170488 (lr=2.7254e-05) (hash(x)=155681970)
-1400 val loss 7.2065
-1400 val perplexity 1348.1545
-1400 train 7.154507 (lr=2.7207e-05) (hash(x)=148115934)
-1500 val loss 7.1673
-1500 val perplexity 1296.3444
-1500 train 7.319304 (lr=2.7157e-05) (hash(x)=157074034)
-1600 val loss 7.1201
-1600 val perplexity 1236.6083
-1600 train 6.882675 (lr=2.7102e-05) (hash(x)=137464699)
-1700 val loss 7.0777
-1700 val perplexity 1185.2509
-1700 train 7.243387 (lr=2.7044e-05) (hash(x)=166955614)
-1800 val loss 7.0408
-1800 val perplexity 1142.3480
-1800 train 6.867510 (lr=2.6981e-05) (hash(x)=143886042)
-1900 val loss 7.0111
-1900 val perplexity 1108.8199
-1900 train 7.622176 (lr=2.6915e-05) (hash(x)=193610391)
-2000 val loss 6.9824
-2000 val perplexity 1077.5485
-2000 train 7.160104 (lr=2.6845e-05) (hash(x)=163799796)
-2100 val loss 6.9365
-2100 val perplexity 1029.1138
-2100 train 6.915351 (lr=2.6771e-05) (hash(x)=154107339)
-2200 val loss 6.9076
-2200 val perplexity 999.8671
-2200 train 6.908937 (lr=2.6693e-05) (hash(x)=144592844)
-2300 val loss 6.8779
-2300 val perplexity 970.5944
-2300 train 6.958189 (lr=2.6611e-05) (hash(x)=154535861)
-2400 val loss 6.8486
-2400 val perplexity 942.5140
-2400 train 7.593205 (lr=2.6525e-05) (hash(x)=204706354)
-2500 val loss 6.8168
-2500 val perplexity 913.0473
-2500 train 6.615187 (lr=2.6436e-05) (hash(x)=143783202)
-2600 val loss 6.7786
-2600 val perplexity 878.8146
-2600 train 6.676283 (lr=2.6343e-05) (hash(x)=150862210)
-2700 val loss 6.7375
-2700 val perplexity 843.4412
-2700 train 6.790683 (lr=2.6246e-05) (hash(x)=150753426)
-2800 val loss 6.7151
-2800 val perplexity 824.7335
-2800 train 6.720888 (lr=2.6146e-05) (hash(x)=147766811)
-2900 val loss 6.6649
-2900 val perplexity 784.4006
-2900 train 6.601873 (lr=2.6042e-05) (hash(x)=151177814)
-3000 val loss 6.6305
-3000 val perplexity 757.8395
-3000 train 6.486729 (lr=2.5934e-05) (hash(x)=135925327)
-3100 val loss 6.6070
-3100 val perplexity 740.2267
-3100 train 6.250475 (lr=2.5823e-05) (hash(x)=124842353)
-3200 val loss 6.5818
-3200 val perplexity 721.8633
-3200 train 6.691461 (lr=2.5709e-05) (hash(x)=160440642)
-3300 val loss 6.5474
-3300 val perplexity 697.4006
-3300 train 6.458886 (lr=2.5590e-05) (hash(x)=143017131)
-3400 val loss 6.5191
-3400 val perplexity 677.9949
-3400 train 6.827440 (lr=2.5469e-05) (hash(x)=148143474)
-3500 val loss 6.4999
-3500 val perplexity 665.0741
-3500 train 6.319547 (lr=2.5344e-05) (hash(x)=148732639)
-3600 val loss 6.4765
-3600 val perplexity 649.7057
-3600 train 6.477383 (lr=2.5215e-05) (hash(x)=146333803)
-3700 val loss 6.4615
-3700 val perplexity 640.0327
-3700 train 6.259657 (lr=2.5083e-05) (hash(x)=143611445)
-3800 val loss 6.4465
-3800 val perplexity 630.4874
-3800 train 6.230040 (lr=2.4948e-05) (hash(x)=141272115)
-3900 val loss 6.4254
-3900 val perplexity 617.3164
-3900 train 6.358279 (lr=2.4810e-05) (hash(x)=149161848)
-4000 val loss 6.3938
-4000 val perplexity 598.1159
-4000 train 6.475472 (lr=2.4669e-05) (hash(x)=152036955)
-4100 val loss 6.3766
-4100 val perplexity 587.9465
-4100 train 6.431635 (lr=2.4524e-05) (hash(x)=153260968)
-4200 val loss 6.3591
-4200 val perplexity 577.7065
-4200 train 6.244727 (lr=2.4376e-05) (hash(x)=147629478)
-4300 val loss 6.3432
-4300 val perplexity 568.6074
-4300 train 6.326876 (lr=2.4226e-05) (hash(x)=161701080)
-4400 val loss 6.3301
-4400 val perplexity 561.1898
-4400 train 6.374499 (lr=2.4072e-05) (hash(x)=155290524)
-4500 val loss 6.3202
-4500 val perplexity 555.6777
-4500 train 6.102327 (lr=2.3915e-05) (hash(x)=144726593)
-4600 val loss 6.3096
-4600 val perplexity 549.8514
-4600 train 6.254610 (lr=2.3755e-05) (hash(x)=149517202)
-4700 val loss 6.2907
-4700 val perplexity 539.5439
-4700 train 6.293215 (lr=2.3593e-05) (hash(x)=139403008)
-4800 val loss 6.2714
-4800 val perplexity 529.2137
-4800 train 6.360953 (lr=2.3428e-05) (hash(x)=153387238)
-4900 val loss 6.2676
-4900 val perplexity 527.2242
-4900 train 6.016084 (lr=2.3260e-05) (hash(x)=139881569)
-5000 val loss 6.2394
-5000 val perplexity 512.5274
-5000 train 6.236556 (lr=2.3089e-05) (hash(x)=152602580)
-5100 val loss 6.2321
-5100 val perplexity 508.8243
-5100 train 6.281746 (lr=2.2916e-05) (hash(x)=150598460)
-5200 val loss 6.2226
-5200 val perplexity 503.9968
-5200 train 6.106723 (lr=2.2740e-05) (hash(x)=147660496)
-5300 val loss 6.2120
-5300 val perplexity 498.7130
-5300 train 5.953794 (lr=2.2561e-05) (hash(x)=147693848)
-5400 val loss 6.2089
-5400 val perplexity 497.1427
-5400 train 5.962179 (lr=2.2381e-05) (hash(x)=136048145)
-5500 val loss 6.1849
-5500 val perplexity 485.3567
-5500 train 6.196786 (lr=2.2197e-05) (hash(x)=140313123)
-5600 val loss 6.1751
-5600 val perplexity 480.6118
-5600 train 6.196830 (lr=2.2012e-05) (hash(x)=156997993)
-5700 val loss 6.1596
-5700 val perplexity 473.2623
-5700 train 6.171885 (lr=2.1824e-05) (hash(x)=143513880)
-5800 val loss 6.1521
-5800 val perplexity 469.7099
-5800 train 6.271136 (lr=2.1634e-05) (hash(x)=153736628)
-5900 val loss 6.1446
-5900 val perplexity 466.1858
-5900 train 6.286342 (lr=2.1442e-05) (hash(x)=166385417)
-6000 val loss 6.1329
-6000 val perplexity 460.7750
-6000 train 6.367750 (lr=2.1248e-05) (hash(x)=153216517)
-6100 val loss 6.1302
-6100 val perplexity 459.5361
-6100 train 6.014255 (lr=2.1051e-05) (hash(x)=145387545)
-6200 val loss 6.1215
-6200 val perplexity 455.5325
-6200 train 6.245870 (lr=2.0853e-05) (hash(x)=156306460)
-6300 val loss 6.1079
-6300 val perplexity 449.4084
-6300 train 6.042010 (lr=2.0653e-05) (hash(x)=148610447)
-6400 val loss 6.0925
-6400 val perplexity 442.5284
-6400 train 6.320738 (lr=2.0451e-05) (hash(x)=158343535)
-6500 val loss 6.0817
-6500 val perplexity 437.7669
-6500 train 6.248390 (lr=2.0247e-05) (hash(x)=157742557)
-6600 val loss 6.0811
-6600 val perplexity 437.5079
-6600 train 5.983360 (lr=2.0042e-05) (hash(x)=150457308)
-6700 val loss 6.0731
-6700 val perplexity 434.0280
-6700 train 5.936596 (lr=1.9835e-05) (hash(x)=142426205)
-6800 val loss 6.0693
-6800 val perplexity 432.3716
-6800 train 5.861384 (lr=1.9626e-05) (hash(x)=151522525)
-6900 val loss 6.0550
-6900 val perplexity 426.2327
-6900 train 6.003469 (lr=1.9416e-05) (hash(x)=152029630)
-7000 val loss 6.0406
-7000 val perplexity 420.1532
-7000 train 5.885346 (lr=1.9205e-05) (hash(x)=142978954)
-7100 val loss 6.0321
-7100 val perplexity 416.6035
-7100 train 5.914243 (lr=1.8992e-05) (hash(x)=148064359)
-7200 val loss 6.0269
-7200 val perplexity 414.4176
-7200 train 5.927231 (lr=1.8778e-05) (hash(x)=155771845)
-7300 val loss 6.0232
-7300 val perplexity 412.8852
-7300 train 5.828673 (lr=1.8562e-05) (hash(x)=141639844)
-7400 val loss 6.0229
-7400 val perplexity 412.7574
-7400 train 6.078304 (lr=1.8346e-05) (hash(x)=153466021)
-7500 val loss 6.0097
-7500 val perplexity 407.3585
-7500 train 6.001205 (lr=1.8128e-05) (hash(x)=148843587)
-7600 val loss 5.9989
-7600 val perplexity 402.9885
-7600 train 6.119777 (lr=1.7910e-05) (hash(x)=152053933)
-7700 val loss 5.9926
-7700 val perplexity 400.4386
-7700 train 5.853600 (lr=1.7691e-05) (hash(x)=141564003)
-7800 val loss 5.9839
-7800 val perplexity 396.9968
-7800 train 5.855370 (lr=1.7470e-05) (hash(x)=148617843)
-7900 val loss 5.9810
-7900 val perplexity 395.8490
-7900 train 5.949779 (lr=1.7249e-05) (hash(x)=146417668)
-8000 val loss 5.9751
-8000 val perplexity 393.5120
-8000 train 5.772148 (lr=1.7028e-05) (hash(x)=150165488)
-8100 val loss 5.9658
-8100 val perplexity 389.8839
-8100 train 5.963167 (lr=1.6805e-05) (hash(x)=144818855)
-8200 val loss 5.9597
-8200 val perplexity 387.4996
-8200 train 6.048904 (lr=1.6582e-05) (hash(x)=143074954)
-8300 val loss 5.9525
-8300 val perplexity 384.7203
-8300 train 5.833498 (lr=1.6359e-05) (hash(x)=145172494)
-8400 val loss 5.9454
-8400 val perplexity 381.9796
-8400 train 5.982568 (lr=1.6135e-05) (hash(x)=152284393)
-8500 val loss 5.9396
-8500 val perplexity 379.7978
-8500 train 5.799961 (lr=1.5911e-05) (hash(x)=147044705)
-8600 val loss 5.9406
-8600 val perplexity 380.1599
-8600 train 5.820881 (lr=1.5687e-05) (hash(x)=147762608)
-8700 val loss 5.9296
-8700 val perplexity 375.9991
-8700 train 5.935435 (lr=1.5462e-05) (hash(x)=141994567)
-8800 val loss 5.9242
-8800 val perplexity 373.9842
-8800 train 6.012501 (lr=1.5237e-05) (hash(x)=152552896)
-8900 val loss 5.9175
-8900 val perplexity 371.4999
-8900 train 5.755026 (lr=1.5013e-05) (hash(x)=139714454)
-9000 val loss 5.9095
-9000 val perplexity 368.5270
-9000 train 5.943279 (lr=1.4788e-05) (hash(x)=152274419)
-9100 val loss 5.9073
-9100 val perplexity 367.7229
-9100 train 5.999683 (lr=1.4563e-05) (hash(x)=160536885)
-9200 val loss 5.9038
-9200 val perplexity 366.4267
-9200 train 5.710702 (lr=1.4339e-05) (hash(x)=138386899)
-9300 val loss 5.8969
-9300 val perplexity 363.8964
-9300 train 5.944336 (lr=1.4115e-05) (hash(x)=149957457)
-9400 val loss 5.8869
-9400 val perplexity 360.2753
-9400 train 5.909703 (lr=1.3891e-05) (hash(x)=161671548)
-9500 val loss 5.8875
-9500 val perplexity 360.5039
-9500 train 6.022851 (lr=1.3668e-05) (hash(x)=171857400)
-9600 val loss 5.8794
-9600 val perplexity 357.5925
-9600 train 5.638875 (lr=1.3445e-05) (hash(x)=135378471)
-9700 val loss 5.8734
-9700 val perplexity 355.4576
-9700 train 5.805750 (lr=1.3222e-05) (hash(x)=149514591)
-9800 val loss 5.8676
-9800 val perplexity 353.3880
-9800 train 5.845808 (lr=1.3001e-05) (hash(x)=152679072)
-9900 val loss 5.8620
-9900 val perplexity 351.4184
-9900 train 5.809219 (lr=1.2780e-05) (hash(x)=150415193)
-10000 val loss 5.8582
-10000 val perplexity 350.1083
-10000 train 5.701046 (lr=1.2559e-05) (hash(x)=146910087)
-10100 val loss 5.8528
-10100 val perplexity 348.1940
-10100 train 5.768382 (lr=1.2340e-05) (hash(x)=152091050)
-10200 val loss 5.8537
-10200 val perplexity 348.5076
-10200 train 5.697745 (lr=1.2122e-05) (hash(x)=143101306)
-10300 val loss 5.8457
-10300 val perplexity 345.7297
-10300 train 5.859143 (lr=1.1904e-05) (hash(x)=150803985)
-10400 val loss 5.8446
-10400 val perplexity 345.3572
-10400 train 5.820405 (lr=1.1688e-05) (hash(x)=148671575)
-10500 val loss 5.8366
-10500 val perplexity 342.6211
-10500 train 5.637980 (lr=1.1472e-05) (hash(x)=146312794)
-10600 val loss 5.8366
-10600 val perplexity 342.6121
-10600 train 5.661836 (lr=1.1258e-05) (hash(x)=137878985)
-10700 val loss 5.8311
-10700 val perplexity 340.7318
-10700 train 5.640622 (lr=1.1045e-05) (hash(x)=148897486)
-10800 val loss 5.8273
-10800 val perplexity 339.4301
-10800 train 5.929305 (lr=1.0834e-05) (hash(x)=147219580)
-10900 val loss 5.8305
-10900 val perplexity 340.5177
-10900 train 5.855745 (lr=1.0624e-05) (hash(x)=152048929)
-11000 val loss 5.8134
-11000 val perplexity 334.7551
-11000 train 5.701807 (lr=1.0415e-05) (hash(x)=146417066)
-11100 val loss 5.8115
-11100 val perplexity 334.1171
-11100 train 5.671261 (lr=1.0208e-05) (hash(x)=148784436)
-11200 val loss 5.8088
-11200 val perplexity 333.2312
-11200 train 5.778788 (lr=1.0003e-05) (hash(x)=144721870)
-11300 val loss 5.8068
-11300 val perplexity 332.5641
-11300 train 5.729158 (lr=9.7991e-06) (hash(x)=149442981)
-11400 val loss 5.8033
-11400 val perplexity 331.3796
-11400 train 5.692762 (lr=9.5972e-06) (hash(x)=154481228)
-11500 val loss 5.7970
-11500 val perplexity 329.2994
-11500 train 5.479829 (lr=9.3970e-06) (hash(x)=115164581)
-11600 val loss 5.7922
-11600 val perplexity 327.7303
-11600 train 6.014553 (lr=9.1988e-06) (hash(x)=155048082)
-11700 val loss 5.7885
-11700 val perplexity 326.5278
-11700 train 5.846733 (lr=9.0025e-06) (hash(x)=152123770)
-11800 val loss 5.7866
-11800 val perplexity 325.8922
-11800 train 5.729887 (lr=8.8082e-06) (hash(x)=152705857)
-11900 val loss 5.7835
-11900 val perplexity 324.8806
-11900 train 5.503841 (lr=8.6160e-06) (hash(x)=137392362)
-12000 val loss 5.7836
-12000 val perplexity 324.9294
-12000 train 5.593571 (lr=8.4260e-06) (hash(x)=143492562)
-12100 val loss 5.7785
-12100 val perplexity 323.2764
-12100 train 5.755323 (lr=8.2381e-06) (hash(x)=141825431)
-12200 val loss 5.7731
-12200 val perplexity 321.5305
-12200 train 5.674329 (lr=8.0526e-06) (hash(x)=141774370)
-12300 val loss 5.7699
-12300 val perplexity 320.4902
-12300 train 5.789728 (lr=7.8694e-06) (hash(x)=152995173)
-12400 val loss 5.7699
-12400 val perplexity 320.5191
-12400 train 5.592089 (lr=7.6885e-06) (hash(x)=144703471)
-12500 val loss 5.7671
-12500 val perplexity 319.6013
-12500 train 5.667903 (lr=7.5101e-06) (hash(x)=144866179)
-12600 val loss 5.7646
-12600 val perplexity 318.8022
-12600 train 5.770620 (lr=7.3342e-06) (hash(x)=157019663)
-12700 val loss 5.7630
-12700 val perplexity 318.2904
-12700 train 5.726803 (lr=7.1609e-06) (hash(x)=152872020)
-12800 val loss 5.7580
-12800 val perplexity 316.7254
-12800 train 5.795256 (lr=6.9903e-06) (hash(x)=158975636)
-12900 val loss 5.7524
-12900 val perplexity 314.9327
-12900 train 5.794280 (lr=6.8223e-06) (hash(x)=143495701)
-13000 val loss 5.7524
-13000 val perplexity 314.9420
-13000 train 5.674680 (lr=6.6570e-06) (hash(x)=151940005)
-13100 val loss 5.7502
-13100 val perplexity 314.2474
-13100 train 6.036286 (lr=6.4945e-06) (hash(x)=155920713)
-13200 val loss 5.7482
-13200 val perplexity 313.6359
-13200 train 5.937761 (lr=6.3349e-06) (hash(x)=152145878)
-13300 val loss 5.7452
-13300 val perplexity 312.6820
-13300 train 5.543325 (lr=6.1782e-06) (hash(x)=134141709)
-13400 val loss 5.7481
-13400 val perplexity 313.5864
-13400 train 5.669959 (lr=6.0244e-06) (hash(x)=146096299)
-13500 val loss 5.7450
-13500 val perplexity 312.6364
-13500 train 5.501187 (lr=5.8737e-06) (hash(x)=144011431)
-13600 val loss 5.7382
-13600 val perplexity 310.4945
-13600 train 5.982531 (lr=5.7259e-06) (hash(x)=163950932)
-13700 val loss 5.7341
-13700 val perplexity 309.2354
-13700 train 5.881189 (lr=5.5813e-06) (hash(x)=156652406)
-13800 val loss 5.7304
-13800 val perplexity 308.0932
-13800 train 5.646419 (lr=5.4398e-06) (hash(x)=145753166)
-13900 val loss 5.7296
-13900 val perplexity 307.8456
-13900 train 5.600583 (lr=5.3016e-06) (hash(x)=147065551)
-14000 val loss 5.7269
-14000 val perplexity 307.0269
-14000 train 6.192235 (lr=5.1665e-06) (hash(x)=179077542)
-14100 val loss 5.7259
-14100 val perplexity 306.7160
-14100 train 5.803774 (lr=5.0348e-06) (hash(x)=171962802)
-14200 val loss 5.7267
-14200 val perplexity 306.9601
-14200 train 5.801540 (lr=4.9063e-06) (hash(x)=153299913)
-14300 val loss 5.7263
-14300 val perplexity 306.8460
-14300 train 5.652976 (lr=4.7813e-06) (hash(x)=154750565)
-14400 val loss 5.7242
-14400 val perplexity 306.1801
-14400 train 6.001330 (lr=4.6596e-06) (hash(x)=154536694)
-14500 val loss 5.7192
-14500 val perplexity 304.6466
-14500 train 5.773301 (lr=4.5414e-06) (hash(x)=151236383)
-14600 val loss 5.7168
-14600 val perplexity 303.9378
-14600 train 5.725638 (lr=4.4267e-06) (hash(x)=152584272)
-14700 val loss 5.7140
-14700 val perplexity 303.0956
-14700 train 5.713636 (lr=4.3155e-06) (hash(x)=175301826)
-14800 val loss 5.7137
-14800 val perplexity 302.9854
-14800 train 5.628159 (lr=4.2079e-06) (hash(x)=148729408)
-14900 val loss 5.7130
-14900 val perplexity 302.7845
-14900 train 5.891885 (lr=4.1039e-06) (hash(x)=150978283)
-15000 val loss 5.7144
-15000 val perplexity 303.2153
-15000 train 5.661406 (lr=4.0035e-06) (hash(x)=153621389)
-15100 val loss 5.7129
-15100 val perplexity 302.7448
-15100 train 5.637873 (lr=3.9068e-06) (hash(x)=150653793)
-15200 val loss 5.7099
-15200 val perplexity 301.8424
-15200 train 5.894018 (lr=3.8138e-06) (hash(x)=140802934)
-15300 val loss 5.7049
-15300 val perplexity 300.3208
-15300 train 5.675510 (lr=3.7245e-06) (hash(x)=152007556)
-15400 val loss 5.7041
-15400 val perplexity 300.0896
-15400 train 5.662685 (lr=3.6390e-06) (hash(x)=149057446)
-15500 val loss 5.7029
-15500 val perplexity 299.7485
-15500 train 5.633412 (lr=3.5572e-06) (hash(x)=147521774)
-15600 val loss 5.7019
-15600 val perplexity 299.4391
-15600 train 5.676160 (lr=3.4793e-06) (hash(x)=148944405)
-15700 val loss 5.7012
-15700 val perplexity 299.2253
-15700 train 5.650658 (lr=3.4052e-06) (hash(x)=148472532)
-15800 val loss 5.7003
-15800 val perplexity 298.9456
-15800 train 5.472708 (lr=3.3350e-06) (hash(x)=149048780)
-15900 val loss 5.7025
-15900 val perplexity 299.6013
-15900 train 5.669323 (lr=3.2687e-06) (hash(x)=150175778)
-16000 val loss 5.7021
-16000 val perplexity 299.4836
-16000 train 5.492558 (lr=3.2063e-06) (hash(x)=145943831)
-16100 val loss 5.6955
-16100 val perplexity 297.5219
-16100 train 5.860049 (lr=3.1478e-06) (hash(x)=134314842)
-16200 val loss 5.6942
-16200 val perplexity 297.1338
-16200 train 5.722423 (lr=3.0932e-06) (hash(x)=153755135)
-16300 val loss 5.6933
-16300 val perplexity 296.8820
-16300 train 5.815263 (lr=3.0427e-06) (hash(x)=146916346)
-16400 val loss 5.6944
-16400 val perplexity 297.2124
-16400 train 5.611971 (lr=2.9961e-06) (hash(x)=155023643)
-16500 val loss 5.6925
-16500 val perplexity 296.6199
-16500 train 5.773914 (lr=2.9535e-06) (hash(x)=148910191)
-16600 val loss 5.6919
-16600 val perplexity 296.4496
-16600 train 5.614566 (lr=2.9149e-06) (hash(x)=150107746)
-16700 val loss 5.6883
-16700 val perplexity 295.3911
-16700 train 5.811312 (lr=2.8804e-06) (hash(x)=159943976)
-16800 val loss 5.6868
-16800 val perplexity 294.9484
-16800 train 5.690961 (lr=2.8498e-06) (hash(x)=160255226)
-16900 val loss 5.6852
-16900 val perplexity 294.4747
-16900 train 5.677330 (lr=2.8234e-06) (hash(x)=157121826)
-17000 val loss 5.6848
-17000 val perplexity 294.3450
-17000 train 5.815355 (lr=2.8010e-06) (hash(x)=159298029)
-17100 val loss 5.6850
-17100 val perplexity 294.4066
-17100 train 5.642745 (lr=2.7826e-06) (hash(x)=149355903)
-17200 val loss 5.6852
-17200 val perplexity 294.4778
-17200 train 5.502587 (lr=2.7684e-06) (hash(x)=144467909)
-17300 val loss 5.6827
-17300 val perplexity 293.7423
-17300 train 5.851804 (lr=2.7582e-06) (hash(x)=157150350)
-17400 val loss 5.6817
-17400 val perplexity 293.4457
-17400 train 5.723725 (lr=2.7520e-06) (hash(x)=154825212)
-17499 val loss 5.6803
-17499 val perplexity 293.0353
+0 train 11.308768 (lr=1.6250e-07) (hash(x)=145079536)
+100 val loss 9.9739
+100 val perplexity 21458.1914
+100 train 9.913584 (lr=1.6412e-05) (hash(x)=154745873)
+200 val loss 9.1007
+200 val perplexity 8961.2510
+200 train 9.134552 (lr=3.2500e-05) (hash(x)=155800595)
+300 val loss 8.1021
+300 val perplexity 3301.3257
+300 train 8.068833 (lr=3.2498e-05) (hash(x)=148595389)
+400 val loss 7.8385
+400 val perplexity 2536.2754
+400 train 7.767007 (lr=3.2490e-05) (hash(x)=145606733)
+500 val loss 7.7050
+500 val perplexity 2219.4236
+500 train 7.567992 (lr=3.2478e-05) (hash(x)=138221231)
+600 val loss 7.6210
+600 val perplexity 2040.6199
+600 train 7.637340 (lr=3.2461e-05) (hash(x)=150367139)
+700 val loss 7.5714
+700 val perplexity 1941.8647
+700 train 7.718476 (lr=3.2440e-05) (hash(x)=155579314)
+800 val loss 7.5225
+800 val perplexity 1849.2454
+800 train 7.599151 (lr=3.2413e-05) (hash(x)=155747374)
+900 val loss 7.4832
+900 val perplexity 1777.8491
+900 train 7.677075 (lr=3.2382e-05) (hash(x)=159334575)
+1000 val loss 7.4560
+1000 val perplexity 1730.2222
+1000 train 7.342261 (lr=3.2346e-05) (hash(x)=140604760)
+1100 val loss 7.4345
+1100 val perplexity 1693.3530
+1100 train 7.285876 (lr=3.2305e-05) (hash(x)=146208052)
+1200 val loss 7.4151
+1200 val perplexity 1660.9293
+1200 train 7.338825 (lr=3.2260e-05) (hash(x)=148404734)
+1300 val loss 7.4038
+1300 val perplexity 1642.2231
+1300 train 7.350505 (lr=3.2209e-05) (hash(x)=155681970)
+1400 val loss 7.3806
+1400 val perplexity 1604.5261
+1400 train 7.327487 (lr=3.2154e-05) (hash(x)=148115934)
+1500 val loss 7.3603
+1500 val perplexity 1572.3585
+1500 train 7.493609 (lr=3.2094e-05) (hash(x)=157074034)
+1600 val loss 7.3380
+1600 val perplexity 1537.5741
+1600 train 7.120191 (lr=3.2030e-05) (hash(x)=137464699)
+1700 val loss 7.3213
+1700 val perplexity 1512.1512
+1700 train 7.518990 (lr=3.1961e-05) (hash(x)=166955614)
+1800 val loss 7.3032
+1800 val perplexity 1485.0101
+1800 train 7.143173 (lr=3.1887e-05) (hash(x)=143886042)
+1900 val loss 7.2869
+1900 val perplexity 1461.0220
+1900 train 7.935486 (lr=3.1809e-05) (hash(x)=193610391)
+2000 val loss 7.2771
+2000 val perplexity 1446.7656
+2000 train 7.443820 (lr=3.1726e-05) (hash(x)=163799796)
+2100 val loss 7.2386
+2100 val perplexity 1392.1798
+2100 train 7.208788 (lr=3.1638e-05) (hash(x)=154107339)
+2200 val loss 7.2311
+2200 val perplexity 1381.6909
+2200 train 7.212371 (lr=3.1546e-05) (hash(x)=144592844)
+2300 val loss 7.2114
+2300 val perplexity 1354.8523
+2300 train 7.292489 (lr=3.1449e-05) (hash(x)=154535861)
+2400 val loss 7.1925
+2400 val perplexity 1329.3987
+2400 train 7.940537 (lr=3.1348e-05) (hash(x)=204706354)
+2500 val loss 7.1954
+2500 val perplexity 1333.2507
+2500 train 7.010689 (lr=3.1243e-05) (hash(x)=143783202)
+2600 val loss 7.1631
+2600 val perplexity 1290.9364
+2600 train 7.057330 (lr=3.1133e-05) (hash(x)=150862210)
+2700 val loss 7.1397
+2700 val perplexity 1260.9904
+2700 train 7.215629 (lr=3.1019e-05) (hash(x)=150753426)
+2800 val loss 7.1199
+2800 val perplexity 1236.3182
+2800 train 7.096500 (lr=3.0900e-05) (hash(x)=147766811)
+2900 val loss 7.0915
+2900 val perplexity 1201.7019
+2900 train 7.022207 (lr=3.0777e-05) (hash(x)=151177814)
+3000 val loss 7.0706
+3000 val perplexity 1176.8248
+3000 train 6.900554 (lr=3.0650e-05) (hash(x)=135925327)
+3100 val loss 7.0520
+3100 val perplexity 1155.1228
+3100 train 6.746537 (lr=3.0518e-05) (hash(x)=124842353)
+3200 val loss 7.0296
+3200 val perplexity 1129.6343
+3200 train 7.138424 (lr=3.0383e-05) (hash(x)=160440642)
+3300 val loss 7.0038
+3300 val perplexity 1100.7715
+3300 train 6.907925 (lr=3.0243e-05) (hash(x)=143017131)
+3400 val loss 6.9814
+3400 val perplexity 1076.4725
+3400 train 7.221414 (lr=3.0099e-05) (hash(x)=148143474)
+3500 val loss 6.9657
+3500 val perplexity 1059.6819
+3500 train 6.782108 (lr=2.9952e-05) (hash(x)=148732639)
+3600 val loss 6.9393
+3600 val perplexity 1032.0707
+3600 train 6.931870 (lr=2.9800e-05) (hash(x)=146333803)
+3700 val loss 6.9195
+3700 val perplexity 1011.7950
+3700 train 6.725532 (lr=2.9644e-05) (hash(x)=143611445)
+3800 val loss 6.8994
+3800 val perplexity 991.6656
+3800 train 6.701790 (lr=2.9485e-05) (hash(x)=141272115)
+3900 val loss 6.8669
+3900 val perplexity 959.9904
+3900 train 6.793236 (lr=2.9321e-05) (hash(x)=149161848)
+4000 val loss 6.8334
+4000 val perplexity 928.3252
+4000 train 6.906411 (lr=2.9154e-05) (hash(x)=152036955)
+4100 val loss 6.8104
+4100 val perplexity 907.1899
+4100 train 6.867271 (lr=2.8983e-05) (hash(x)=153260968)
+4200 val loss 6.7852
+4200 val perplexity 884.6647
+4200 train 6.657335 (lr=2.8808e-05) (hash(x)=147629478)
+4300 val loss 6.7604
+4300 val perplexity 862.9633
+4300 train 6.778302 (lr=2.8630e-05) (hash(x)=161701080)
+4400 val loss 6.7326
+4400 val perplexity 839.3168
+4400 train 6.780152 (lr=2.8448e-05) (hash(x)=155290524)
+4500 val loss 6.7115
+4500 val perplexity 821.7939
+4500 train 6.514741 (lr=2.8263e-05) (hash(x)=144726593)
+4600 val loss 6.6935
+4600 val perplexity 807.1732
+4600 train 6.629334 (lr=2.8075e-05) (hash(x)=149517202)
+4700 val loss 6.6669
+4700 val perplexity 785.9705
+4700 train 6.673309 (lr=2.7883e-05) (hash(x)=139403008)
+4800 val loss 6.6345
+4800 val perplexity 760.9359
+4800 train 6.706923 (lr=2.7687e-05) (hash(x)=153387238)
+4900 val loss 6.6114
+4900 val perplexity 743.5439
+4900 train 6.372381 (lr=2.7489e-05) (hash(x)=139881569)
+5000 val loss 6.5846
+5000 val perplexity 723.8486
+5000 train 6.562537 (lr=2.7287e-05) (hash(x)=152602580)
+5100 val loss 6.5696
+5100 val perplexity 713.1135
+5100 train 6.626162 (lr=2.7082e-05) (hash(x)=150598460)
+5200 val loss 6.5469
+5200 val perplexity 697.1136
+5200 train 6.444059 (lr=2.6874e-05) (hash(x)=147660496)
+5300 val loss 6.5266
+5300 val perplexity 683.1017
+5300 train 6.272954 (lr=2.6664e-05) (hash(x)=147693848)
+5400 val loss 6.5189
+5400 val perplexity 677.8152
+5400 train 6.264269 (lr=2.6450e-05) (hash(x)=136048145)
+5500 val loss 6.4788
+5500 val perplexity 651.1843
+5500 train 6.460757 (lr=2.6233e-05) (hash(x)=140313123)
+5600 val loss 6.4612
+5600 val perplexity 639.7990
+5600 train 6.481189 (lr=2.6014e-05) (hash(x)=156997993)
+5700 val loss 6.4430
+5700 val perplexity 628.2891
+5700 train 6.447996 (lr=2.5792e-05) (hash(x)=143513880)
+5800 val loss 6.4244
+5800 val perplexity 616.6926
+5800 train 6.570982 (lr=2.5567e-05) (hash(x)=153736628)
+5900 val loss 6.4183
+5900 val perplexity 612.9313
+5900 train 6.596351 (lr=2.5340e-05) (hash(x)=166385417)
+6000 val loss 6.3909
+6000 val perplexity 596.3846
+6000 train 6.613896 (lr=2.5111e-05) (hash(x)=153216517)
+6100 val loss 6.3874
+6100 val perplexity 594.3225
+6100 train 6.256468 (lr=2.4879e-05) (hash(x)=145387545)
+6200 val loss 6.3841
+6200 val perplexity 592.3782
+6200 train 6.484996 (lr=2.4644e-05) (hash(x)=156306460)
+6300 val loss 6.3470
+6300 val perplexity 570.7662
+6300 train 6.255859 (lr=2.4408e-05) (hash(x)=148610447)
+6400 val loss 6.3294
+6400 val perplexity 560.8345
+6400 train 6.544854 (lr=2.4169e-05) (hash(x)=158343535)
+6500 val loss 6.3157
+6500 val perplexity 553.2084
+6500 train 6.476159 (lr=2.3928e-05) (hash(x)=157742557)
+6600 val loss 6.3104
+6600 val perplexity 550.2731
+6600 train 6.226171 (lr=2.3686e-05) (hash(x)=150457308)
+6700 val loss 6.2964
+6700 val perplexity 542.6016
+6700 train 6.168110 (lr=2.3441e-05) (hash(x)=142426205)
+6800 val loss 6.2923
+6800 val perplexity 540.4186
+6800 train 6.098007 (lr=2.3195e-05) (hash(x)=151522525)
+6900 val loss 6.2760
+6900 val perplexity 531.6586
+6900 train 6.213217 (lr=2.2946e-05) (hash(x)=152029630)
+7000 val loss 6.2544
+7000 val perplexity 520.2831
+7000 train 6.107107 (lr=2.2696e-05) (hash(x)=142978954)
+7100 val loss 6.2466
+7100 val perplexity 516.2451
+7100 train 6.122114 (lr=2.2445e-05) (hash(x)=148064359)
+7200 val loss 6.2384
+7200 val perplexity 512.0293
+7200 train 6.143100 (lr=2.2192e-05) (hash(x)=155771845)
+7300 val loss 6.2305
+7300 val perplexity 508.0258
+7300 train 6.031136 (lr=2.1937e-05) (hash(x)=141639844)
+7400 val loss 6.2285
+7400 val perplexity 507.0009
+7400 train 6.278035 (lr=2.1682e-05) (hash(x)=153466021)
+7500 val loss 6.2116
+7500 val perplexity 498.5069
+7500 train 6.201186 (lr=2.1425e-05) (hash(x)=148843587)
+7600 val loss 6.1974
+7600 val perplexity 491.4469
+7600 train 6.312661 (lr=2.1166e-05) (hash(x)=152053933)
+7700 val loss 6.1902
+7700 val perplexity 487.9364
+7700 train 6.026810 (lr=2.0907e-05) (hash(x)=141564003)
+7800 val loss 6.1822
+7800 val perplexity 484.0695
+7800 train 6.056354 (lr=2.0647e-05) (hash(x)=148617843)
+7900 val loss 6.1779
+7900 val perplexity 481.9819
+7900 train 6.136011 (lr=2.0386e-05) (hash(x)=146417668)
+8000 val loss 6.1723
+8000 val perplexity 479.2695
+8000 train 5.964910 (lr=2.0123e-05) (hash(x)=150165488)
+8100 val loss 6.1573
+8100 val perplexity 472.1736
+8100 train 6.143494 (lr=1.9861e-05) (hash(x)=144818855)
+8200 val loss 6.1473
+8200 val perplexity 467.4410
+8200 train 6.219149 (lr=1.9597e-05) (hash(x)=143074954)
+8300 val loss 6.1385
+8300 val perplexity 463.3588
+8300 train 6.022402 (lr=1.9333e-05) (hash(x)=145172494)
+8400 val loss 6.1339
+8400 val perplexity 461.2326
+8400 train 6.162875 (lr=1.9069e-05) (hash(x)=152284393)
+8500 val loss 6.1236
+8500 val perplexity 456.4986
+8500 train 5.980865 (lr=1.8804e-05) (hash(x)=147044705)
+8600 val loss 6.1191
+8600 val perplexity 454.4670
+8600 train 6.005243 (lr=1.8539e-05) (hash(x)=147762608)
+8700 val loss 6.1099
+8700 val perplexity 450.3132
+8700 train 6.105801 (lr=1.8273e-05) (hash(x)=141994567)
+8800 val loss 6.1019
+8800 val perplexity 446.7247
+8800 train 6.182958 (lr=1.8008e-05) (hash(x)=152552896)
+8900 val loss 6.0918
+8900 val perplexity 442.2117
+8900 train 5.924414 (lr=1.7742e-05) (hash(x)=139714454)
+9000 val loss 6.0852
+9000 val perplexity 439.2988
+9000 train 6.105251 (lr=1.7477e-05) (hash(x)=152274419)
+9100 val loss 6.0780
+9100 val perplexity 436.1500
+9100 train 6.168942 (lr=1.7211e-05) (hash(x)=160536885)
+9200 val loss 6.0731
+9200 val perplexity 434.0107
+9200 train 5.885173 (lr=1.6946e-05) (hash(x)=138386899)
+9300 val loss 6.0633
+9300 val perplexity 429.7767
+9300 train 6.108029 (lr=1.6681e-05) (hash(x)=149957457)
+9400 val loss 6.0550
+9400 val perplexity 426.2285
+9400 train 6.107460 (lr=1.6417e-05) (hash(x)=161671548)
+9500 val loss 6.0511
+9500 val perplexity 424.5732
+9500 train 6.202872 (lr=1.6153e-05) (hash(x)=171857400)
+9600 val loss 6.0415
+9600 val perplexity 420.5262
+9600 train 5.794603 (lr=1.5889e-05) (hash(x)=135378471)
+9700 val loss 6.0354
+9700 val perplexity 417.9745
+9700 train 5.960491 (lr=1.5627e-05) (hash(x)=149514591)
+9800 val loss 6.0305
+9800 val perplexity 415.9110
+9800 train 6.013536 (lr=1.5364e-05) (hash(x)=152679072)
+9900 val loss 6.0199
+9900 val perplexity 411.5178
+9900 train 5.961017 (lr=1.5103e-05) (hash(x)=150415193)
+10000 val loss 6.0163
+10000 val perplexity 410.0448
+10000 train 5.852025 (lr=1.4843e-05) (hash(x)=146910087)
+10100 val loss 6.0116
+10100 val perplexity 408.1298
+10100 train 5.924100 (lr=1.4584e-05) (hash(x)=152091050)
+10200 val loss 6.0145
+10200 val perplexity 409.3116
+10200 train 5.860289 (lr=1.4325e-05) (hash(x)=143101306)
+10300 val loss 6.0003
+10300 val perplexity 403.5591
+10300 train 6.010289 (lr=1.4068e-05) (hash(x)=150803985)
+10400 val loss 6.0047
+10400 val perplexity 405.3331
+10400 train 5.995926 (lr=1.3813e-05) (hash(x)=148671575)
+10500 val loss 5.9919
+10500 val perplexity 400.1769
+10500 train 5.794977 (lr=1.3558e-05) (hash(x)=146312794)
+10600 val loss 5.9875
+10600 val perplexity 398.4159
+10600 train 5.810483 (lr=1.3305e-05) (hash(x)=137878985)
+10700 val loss 5.9827
+10700 val perplexity 396.5163
+10700 train 5.780869 (lr=1.3054e-05) (hash(x)=148897486)
+10800 val loss 5.9783
+10800 val perplexity 394.7668
+10800 train 6.067101 (lr=1.2804e-05) (hash(x)=147219580)
+10900 val loss 5.9864
+10900 val perplexity 397.9805
+10900 train 6.017478 (lr=1.2555e-05) (hash(x)=152048929)
+11000 val loss 5.9631
+11000 val perplexity 388.8121
+11000 train 5.854570 (lr=1.2309e-05) (hash(x)=146417066)
+11100 val loss 5.9600
+11100 val perplexity 387.6020
+11100 train 5.825752 (lr=1.2064e-05) (hash(x)=148784436)
+11200 val loss 5.9561
+11200 val perplexity 386.0916
+11200 train 5.908297 (lr=1.1822e-05) (hash(x)=144721870)
+11300 val loss 5.9498
+11300 val perplexity 383.6834
+11300 train 5.868901 (lr=1.1581e-05) (hash(x)=149442981)
+11400 val loss 5.9500
+11400 val perplexity 383.7384
+11400 train 5.857729 (lr=1.1342e-05) (hash(x)=154481228)
+11500 val loss 5.9434
+11500 val perplexity 381.2192
+11500 train 5.623575 (lr=1.1106e-05) (hash(x)=115164581)
+11600 val loss 5.9383
+11600 val perplexity 379.2775
+11600 train 6.173423 (lr=1.0871e-05) (hash(x)=155048082)
+11700 val loss 5.9342
+11700 val perplexity 377.7426
+11700 train 5.989237 (lr=1.0639e-05) (hash(x)=152123770)
+11800 val loss 5.9311
+11800 val perplexity 376.5839
+11800 train 5.877898 (lr=1.0410e-05) (hash(x)=152705857)
+11900 val loss 5.9275
+11900 val perplexity 375.2242
+11900 train 5.648075 (lr=1.0183e-05) (hash(x)=137392362)
+12000 val loss 5.9263
+12000 val perplexity 374.7537
+12000 train 5.739065 (lr=9.9580e-06) (hash(x)=143492562)
+12100 val loss 5.9204
+12100 val perplexity 372.5483
+12100 train 5.895456 (lr=9.7360e-06) (hash(x)=141825431)
+12200 val loss 5.9131
+12200 val perplexity 369.8425
+12200 train 5.817440 (lr=9.5167e-06) (hash(x)=141774370)
+12300 val loss 5.9095
+12300 val perplexity 368.5233
+12300 train 5.927475 (lr=9.3001e-06) (hash(x)=152995173)
+12400 val loss 5.9080
+12400 val perplexity 367.9576
+12400 train 5.728159 (lr=9.0864e-06) (hash(x)=144703471)
+12500 val loss 5.9056
+12500 val perplexity 367.0979
+12500 train 5.791191 (lr=8.8756e-06) (hash(x)=144866179)
+12600 val loss 5.9030
+12600 val perplexity 366.1330
+12600 train 5.901381 (lr=8.6677e-06) (hash(x)=157019663)
+12700 val loss 5.9021
+12700 val perplexity 365.7953
+12700 train 5.874472 (lr=8.4629e-06) (hash(x)=152872020)
+12800 val loss 5.8946
+12800 val perplexity 363.0851
+12800 train 5.937847 (lr=8.2612e-06) (hash(x)=158975636)
+12900 val loss 5.8881
+12900 val perplexity 360.7154
+12900 train 5.927866 (lr=8.0627e-06) (hash(x)=143495701)
+13000 val loss 5.8866
+13000 val perplexity 360.1819
+13000 train 5.805252 (lr=7.8674e-06) (hash(x)=151940005)
+13100 val loss 5.8831
+13100 val perplexity 358.9209
+13100 train 6.148221 (lr=7.6754e-06) (hash(x)=155920713)
+13200 val loss 5.8811
+13200 val perplexity 358.2096
+13200 train 6.047833 (lr=7.4867e-06) (hash(x)=152145878)
+13300 val loss 5.8787
+13300 val perplexity 357.3310
+13300 train 5.667600 (lr=7.3015e-06) (hash(x)=134141709)
+13400 val loss 5.8799
+13400 val perplexity 357.7690
+13400 train 5.797377 (lr=7.1198e-06) (hash(x)=146096299)
+13500 val loss 5.8794
+13500 val perplexity 357.5978
+13500 train 5.634058 (lr=6.9416e-06) (hash(x)=144011431)
+13600 val loss 5.8686
+13600 val perplexity 353.7581
+13600 train 6.117513 (lr=6.7670e-06) (hash(x)=163950932)
+13700 val loss 5.8664
+13700 val perplexity 352.9912
+13700 train 6.019842 (lr=6.5961e-06) (hash(x)=156652406)
+13800 val loss 5.8622
+13800 val perplexity 351.4952
+13800 train 5.771851 (lr=6.4289e-06) (hash(x)=145753166)
+13900 val loss 5.8598
+13900 val perplexity 350.6560
+13900 train 5.734013 (lr=6.2655e-06) (hash(x)=147065551)
+14000 val loss 5.8566
+14000 val perplexity 349.5451
+14000 train 6.318323 (lr=6.1059e-06) (hash(x)=179077542)
+14100 val loss 5.8558
+14100 val perplexity 349.2574
+14100 train 5.933926 (lr=5.9502e-06) (hash(x)=171962802)
+14200 val loss 5.8544
+14200 val perplexity 348.7620
+14200 train 5.915273 (lr=5.7984e-06) (hash(x)=153299913)
+14300 val loss 5.8551
+14300 val perplexity 349.0220
+14300 train 5.784843 (lr=5.6506e-06) (hash(x)=154750565)
+14400 val loss 5.8523
+14400 val perplexity 348.0174
+14400 train 6.138902 (lr=5.5068e-06) (hash(x)=154536694)
+14500 val loss 5.8460
+14500 val perplexity 345.8539
+14500 train 5.895782 (lr=5.3671e-06) (hash(x)=151236383)
+14600 val loss 5.8431
+14600 val perplexity 344.8444
+14600 train 5.847693 (lr=5.2316e-06) (hash(x)=152584272)
+14700 val loss 5.8414
+14700 val perplexity 344.2503
+14700 train 5.875105 (lr=5.1002e-06) (hash(x)=175301826)
+14800 val loss 5.8404
+14800 val perplexity 343.9079
+14800 train 5.767904 (lr=4.9730e-06) (hash(x)=148729408)
+14900 val loss 5.8386
+14900 val perplexity 343.3042
+14900 train 6.004839 (lr=4.8501e-06) (hash(x)=150978283)
+15000 val loss 5.8388
+15000 val perplexity 343.3620
+15000 train 5.784351 (lr=4.7314e-06) (hash(x)=153621389)
+15100 val loss 5.8389
+15100 val perplexity 343.3981
+15100 train 5.754816 (lr=4.6171e-06) (hash(x)=150653793)
+15200 val loss 5.8352
+15200 val perplexity 342.1168
+15200 train 6.008870 (lr=4.5072e-06) (hash(x)=140802934)
+15300 val loss 5.8295
+15300 val perplexity 340.1902
+15300 train 5.792707 (lr=4.4017e-06) (hash(x)=152007556)
+15400 val loss 5.8288
+15400 val perplexity 339.9514
+15400 train 5.779943 (lr=4.3006e-06) (hash(x)=149057446)
+15500 val loss 5.8278
+15500 val perplexity 339.6246
+15500 train 5.757745 (lr=4.2040e-06) (hash(x)=147521774)
+15600 val loss 5.8260
+15600 val perplexity 338.9973
+15600 train 5.794466 (lr=4.1119e-06) (hash(x)=148944405)
+15700 val loss 5.8244
+15700 val perplexity 338.4497
+15700 train 5.771770 (lr=4.0244e-06) (hash(x)=148472532)
+15800 val loss 5.8235
+15800 val perplexity 338.1595
+15800 train 5.611291 (lr=3.9414e-06) (hash(x)=149048780)
+15900 val loss 5.8251
+15900 val perplexity 338.7098
+15900 train 5.776637 (lr=3.8630e-06) (hash(x)=150175778)
+16000 val loss 5.8256
+16000 val perplexity 338.8643
+16000 train 5.623484 (lr=3.7892e-06) (hash(x)=145943831)
+16100 val loss 5.8186
+16100 val perplexity 336.4933
+16100 train 5.962903 (lr=3.7201e-06) (hash(x)=134314842)
+16200 val loss 5.8169
+16200 val perplexity 335.9197
+16200 train 5.832172 (lr=3.6556e-06) (hash(x)=153755135)
+16300 val loss 5.8167
+16300 val perplexity 335.8620
+16300 train 5.944980 (lr=3.5959e-06) (hash(x)=146916346)
+16400 val loss 5.8162
+16400 val perplexity 335.6877
+16400 train 5.738881 (lr=3.5408e-06) (hash(x)=155023643)
+16500 val loss 5.8135
+16500 val perplexity 334.7997
+16500 train 5.884605 (lr=3.4905e-06) (hash(x)=148910191)
+16600 val loss 5.8129
+16600 val perplexity 334.5828
+16600 train 5.725785 (lr=3.4449e-06) (hash(x)=150107746)
+16700 val loss 5.8094
+16700 val perplexity 333.4117
+16700 train 5.943489 (lr=3.4041e-06) (hash(x)=159943976)
+16800 val loss 5.8079
+16800 val perplexity 332.9350
+16800 train 5.812941 (lr=3.3680e-06) (hash(x)=160255226)
+16900 val loss 5.8058
+16900 val perplexity 332.2311
+16900 train 5.790132 (lr=3.3367e-06) (hash(x)=157121826)
+17000 val loss 5.8052
+17000 val perplexity 332.0128
+17000 train 5.929333 (lr=3.3102e-06) (hash(x)=159298029)
+17100 val loss 5.8048
+17100 val perplexity 331.8996
+17100 train 5.756987 (lr=3.2886e-06) (hash(x)=149355903)
+17200 val loss 5.8050
+17200 val perplexity 331.9643
+17200 train 5.621988 (lr=3.2717e-06) (hash(x)=144467909)
+17300 val loss 5.8022
+17300 val perplexity 331.0107
+17300 train 5.966762 (lr=3.2596e-06) (hash(x)=157150350)
+17400 val loss 5.7999
+17400 val perplexity 330.2749
+17400 train 5.854440 (lr=3.2524e-06) (hash(x)=154825212)
+17499 val loss 5.7990
+17499 val perplexity 329.9840

attention_kindselective_n_heads4_seed1341/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c7cd902b46cbc421e2c48ee77f0e0aa4b5befdcf357d1af96e2d5e9ccd48c45
+oid sha256:d70158dff7c77ab9d959dd129b842afbfd51eb0b1e97217f34de668a673d9987
 size 92843394

attention_kindselective_n_heads4_seed1341/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a0cb68dbad563d91d661387bf39ace20ba219489270a1c9461a784234115623
+oid sha256:d94c5126d81ac2e77872dca0b06969c284caea2576d8b5117b39e6cda2322556
 size 179406214