Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 7e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "7e-5_10240_2_1340", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "15e-5_10240_2_1340", "n_embd": 128}

attention_kindselective_n_heads2_seed1340/log2.txt

@@ -1,603 +1,423 @@
 max_steps: 10000
+3000 val loss 6.8497
+3000 val perplexity 943.5969
+3000 train 6.557367 (lr=8.3057e-05) (hash(x)=44336167)
 0 val loss 11.8201
 0 val perplexity 135955.4688
-0 val loss 11.8201
-0 val perplexity 135955.4688
-0 train 11.815376 (lr=2.5000e-07) (hash(x)=57791809)
-0 train 11.815376 (lr=3.5000e-07) (hash(x)=57791809)
-100 val loss 10.1415
-100 val perplexity 25373.9316
-100 train 10.167628 (lr=2.5250e-05) (hash(x)=48211824)
-100 val loss 9.9846
-100 val perplexity 21690.5820
-100 train 10.007619 (lr=3.5350e-05) (hash(x)=48211824)
-200 val loss 8.9560
-200 val perplexity 7754.3325
-200 train 8.916373 (lr=5.0000e-05) (hash(x)=50375849)
-200 val loss 8.3972
-200 val perplexity 4434.5674
-200 train 8.351953 (lr=7.0000e-05) (hash(x)=50375849)
-300 val loss 8.1359
-300 val perplexity 3414.7334
-300 train 8.400735 (lr=4.9988e-05) (hash(x)=57250808)
-300 val loss 7.9232
-300 val perplexity 2760.7124
-300 train 8.212564 (lr=6.9984e-05) (hash(x)=57250808)
-400 val loss 7.8720
-400 val perplexity 2622.8079
-400 train 8.402547 (lr=4.9954e-05) (hash(x)=62519858)
-400 val loss 7.7554
-400 val perplexity 2334.1978
-400 train 8.325225 (lr=6.9935e-05) (hash(x)=62519858)
-500 val loss 7.7049
-500 val perplexity 2219.0955
-500 train 7.604552 (lr=4.9896e-05) (hash(x)=47226806)
-500 val loss 7.6750
-500 val perplexity 2153.8015
-500 train 7.583454 (lr=6.9854e-05) (hash(x)=47226806)
-600 val loss 7.6373
-600 val perplexity 2074.1904
-600 train 7.663011 (lr=4.9815e-05) (hash(x)=51149322)
-600 val loss 7.6327
-600 val perplexity 2064.5627
-600 train 7.648909 (lr=6.9741e-05) (hash(x)=51149322)
-700 val loss 7.6224
-700 val perplexity 2043.4777
-700 train 7.632802 (lr=4.9712e-05) (hash(x)=51564551)
-700 val loss 7.6209
-700 val perplexity 2040.4641
-700 train 7.626678 (lr=6.9596e-05) (hash(x)=51564551)
-800 val loss 7.6059
-800 val perplexity 2010.0964
-800 train 7.368664 (lr=4.9585e-05) (hash(x)=45093459)
-800 val loss 7.6051
-800 val perplexity 2008.5107
-800 train 7.361159 (lr=6.9419e-05) (hash(x)=45093459)
-900 val loss 7.6009
-900 val perplexity 1999.9039
-900 train 7.899217 (lr=4.9436e-05) (hash(x)=54988361)
-900 val loss 7.5850
-900 val perplexity 1968.4664
-900 train 7.888000 (lr=6.9210e-05) (hash(x)=54988361)
-1000 val loss 7.5910
-1000 val perplexity 1980.3516
-1000 train 7.476419 (lr=4.9264e-05) (hash(x)=47588648)
-1000 val loss 7.5780
-1000 val perplexity 1954.8005
-1000 train 7.459035 (lr=6.8970e-05) (hash(x)=47588648)
-1100 val loss 7.5702
-1100 val perplexity 1939.6012
-1100 train 7.172775 (lr=4.9070e-05) (hash(x)=37984588)
-1100 val loss 7.5466
-1100 val perplexity 1894.2449
-1100 train 7.153616 (lr=6.8698e-05) (hash(x)=37984588)
-1200 val loss 7.5385
-1200 val perplexity 1878.9194
-1200 train 7.705000 (lr=4.8854e-05) (hash(x)=56333817)
-1300 val loss 7.5312
-1300 val perplexity 1865.3256
-1200 val loss 7.5242
-1200 val perplexity 1852.2689
-1300 train 7.666130 (lr=4.8616e-05) (hash(x)=53454056)
-1200 train 7.697269 (lr=6.8395e-05) (hash(x)=56333817)
-1400 val loss 7.4924
-1400 val perplexity 1794.3845
-1400 train 7.599366 (lr=4.8356e-05) (hash(x)=55284163)
-1300 val loss 7.5314
-1300 val perplexity 1865.7615
-1300 train 7.681477 (lr=6.8062e-05) (hash(x)=53454056)
-1500 val loss 7.4758
-1500 val perplexity 1764.8745
-1500 train 7.364212 (lr=4.8074e-05) (hash(x)=48162598)
-1400 val loss 7.4858
-1400 val perplexity 1782.5961
-1400 train 7.581354 (lr=6.7698e-05) (hash(x)=55284163)
-1600 val loss 7.4609
-1600 val perplexity 1738.7384
-1600 train 7.523167 (lr=4.7772e-05) (hash(x)=54214535)
-1500 val loss 7.4677
-1500 val perplexity 1750.5398
-1500 train 7.366483 (lr=6.7304e-05) (hash(x)=48162598)
-1700 val loss 7.4489
-1700 val perplexity 1718.0170
-1700 train 7.658994 (lr=4.7448e-05) (hash(x)=53525003)
-1600 val loss 7.4239
-1600 val perplexity 1675.5126
-1600 train 7.474539 (lr=6.6881e-05) (hash(x)=54214535)
-1800 val loss 7.4343
-1800 val perplexity 1693.1310
-1800 train 7.507501 (lr=4.7105e-05) (hash(x)=51848994)
-1700 val loss 7.3955
-1700 val perplexity 1628.6207
-1700 train 7.592460 (lr=6.6428e-05) (hash(x)=53525003)
-1900 val loss 7.4327
-1900 val perplexity 1690.2996
-1900 train 7.251468 (lr=4.6741e-05) (hash(x)=48405987)
-1800 val loss 7.3557
-1800 val perplexity 1565.1370
-1800 train 7.437174 (lr=6.5947e-05) (hash(x)=51848994)
-2000 val loss 7.4107
-2000 val perplexity 1653.5981
-2000 train 7.784187 (lr=4.6357e-05) (hash(x)=58592291)
-1900 val loss 7.3525
-1900 val perplexity 1560.1082
-1900 train 7.168375 (lr=6.5437e-05) (hash(x)=48405987)
-2100 val loss 7.4144
-2100 val perplexity 1659.6460
-2100 train 7.489458 (lr=4.5954e-05) (hash(x)=51167081)
-2000 val loss 7.3089
-2000 val perplexity 1493.5795
-2000 train 7.667870 (lr=6.4900e-05) (hash(x)=58592291)
-2200 val loss 7.3779
-2200 val perplexity 1600.2672
-2200 train 7.367453 (lr=4.5532e-05) (hash(x)=47994988)
-2100 val loss 7.3073
-2100 val perplexity 1491.0868
-2100 train 7.386800 (lr=6.4335e-05) (hash(x)=51167081)
-2300 val loss 7.3693
-2300 val perplexity 1586.4584
-2300 train 7.348310 (lr=4.5091e-05) (hash(x)=47377604)
-2200 val loss 7.3030
-2200 val perplexity 1484.7701
-2200 train 7.295847 (lr=6.3745e-05) (hash(x)=47994988)
-2400 val loss 7.3560
-2400 val perplexity 1565.5953
-2400 train 7.408455 (lr=4.4633e-05) (hash(x)=53554323)
-2300 val loss 7.2556
-2300 val perplexity 1416.0649
-2300 train 7.237532 (lr=6.3128e-05) (hash(x)=47377604)
-2500 val loss 7.3607
-2500 val perplexity 1573.0103
-2500 train 7.371253 (lr=4.4156e-05) (hash(x)=50780417)
-2400 val loss 7.2298
-2400 val perplexity 1379.9376
-2400 train 7.275304 (lr=6.2486e-05) (hash(x)=53554323)
-2600 val loss 7.3246
-2600 val perplexity 1517.1007
-2600 train 7.223897 (lr=4.3663e-05) (hash(x)=46453562)
-2500 val loss 7.2124
-2500 val perplexity 1356.2070
-2500 train 7.234367 (lr=6.1819e-05) (hash(x)=50780417)
-2700 val loss 7.3181
-2700 val perplexity 1507.3157
-2700 train 7.332546 (lr=4.3153e-05) (hash(x)=54404221)
-2600 val loss 7.1852
-2600 val perplexity 1319.7157
-2600 train 7.078228 (lr=6.1128e-05) (hash(x)=46453562)
-2800 val loss 7.3003
-2800 val perplexity 1480.7711
-2800 train 8.040918 (lr=4.2627e-05) (hash(x)=59318895)
-2700 val loss 7.1844
-2700 val perplexity 1318.7426
-2700 train 7.171799 (lr=6.0414e-05) (hash(x)=54404221)
-2900 val loss 7.3080
-2900 val perplexity 1492.1636
-2900 train 7.285590 (lr=4.2085e-05) (hash(x)=47845760)
-2800 val loss 7.1699
-2800 val perplexity 1299.7759
-2800 train 7.846885 (lr=5.9677e-05) (hash(x)=59318895)
-3000 val loss 7.2772
-3000 val perplexity 1446.9222
-3000 train 6.974861 (lr=4.1529e-05) (hash(x)=44336167)
-2900 val loss 7.1458
-2900 val perplexity 1268.7697
-2900 train 7.136559 (lr=5.8919e-05) (hash(x)=47845760)
-3100 val loss 7.2409
-3100 val perplexity 1395.3539
-3100 train 7.539340 (lr=4.0957e-05) (hash(x)=44479330)
-3000 val loss 7.1352
-3000 val perplexity 1255.3414
-3000 train 6.846057 (lr=5.8140e-05) (hash(x)=44336167)
-3200 val loss 7.2380
-3200 val perplexity 1391.3442
-3200 train 7.291301 (lr=4.0373e-05) (hash(x)=54593096)
-3100 val loss 7.1081
-3100 val perplexity 1221.8489
-3100 train 7.400929 (lr=5.7340e-05) (hash(x)=44479330)
-3300 val loss 7.2317
-3300 val perplexity 1382.6051
-3300 train 7.120160 (lr=3.9775e-05) (hash(x)=45347643)
-3200 val loss 7.1089
-3200 val perplexity 1222.8351
-3200 train 7.181127 (lr=5.6522e-05) (hash(x)=54593096)
-3400 val loss 7.1952
-3400 val perplexity 1332.9685
-3400 train 7.318199 (lr=3.9164e-05) (hash(x)=47797247)
-3300 val loss 7.0572
-3300 val perplexity 1161.2457
-3300 train 6.946620 (lr=5.5684e-05) (hash(x)=45347643)
-3500 val loss 7.1690
-3500 val perplexity 1298.5449
-3500 train 7.078238 (lr=3.8541e-05) (hash(x)=46115683)
-3400 val loss 7.0117
-3400 val perplexity 1109.5795
-3400 train 7.125251 (lr=5.4829e-05) (hash(x)=47797247)
-3600 val loss 7.1592
-3600 val perplexity 1285.8961
-3600 train 7.025664 (lr=3.7907e-05) (hash(x)=44502074)
-3500 val loss 6.9839
-3500 val perplexity 1079.1410
-3500 train 6.878851 (lr=5.3958e-05) (hash(x)=46115683)
-3700 val loss 7.1425
-3700 val perplexity 1264.6299
-3700 train 7.283512 (lr=3.7262e-05) (hash(x)=55388443)
-3600 val loss 6.9588
-3600 val perplexity 1052.3683
-3600 train 6.803612 (lr=5.3070e-05) (hash(x)=44502074)
-3800 val loss 7.1436
-3800 val perplexity 1266.0127
-3800 train 6.931653 (lr=3.6608e-05) (hash(x)=43790341)
-3900 val loss 7.1233
-3900 val perplexity 1240.5541
-3900 train 7.126774 (lr=3.5944e-05) (hash(x)=50013318)
-3700 val loss 6.9234
-3700 val perplexity 1015.7560
-3700 train 7.075602 (lr=5.2167e-05) (hash(x)=55388443)
-4000 val loss 7.1181
-4000 val perplexity 1234.0818
-4000 train 7.197229 (lr=3.5271e-05) (hash(x)=51704787)
-3800 val loss 6.9045
-3800 val perplexity 996.7182
-3800 train 6.700143 (lr=5.1251e-05) (hash(x)=43790341)
-4100 val loss 7.1075
-4100 val perplexity 1221.0736
-4100 train 7.178077 (lr=3.4590e-05) (hash(x)=50821964)
-3900 val loss 6.8876
-3900 val perplexity 980.0497
-3900 train 6.885145 (lr=5.0321e-05) (hash(x)=50013318)
-4200 val loss 7.1000
-4200 val perplexity 1211.9559
-4200 train 7.117894 (lr=3.3902e-05) (hash(x)=49675080)
-4000 val loss 6.8554
-4000 val perplexity 948.9904
-4000 train 6.929147 (lr=4.9379e-05) (hash(x)=51704787)
-4300 val loss 7.0877
-4300 val perplexity 1197.1761
-4300 train 6.751846 (lr=3.3207e-05) (hash(x)=43239281)
-4100 val loss 6.8422
-4100 val perplexity 936.5167
-4100 train 6.914855 (lr=4.8426e-05) (hash(x)=50821964)
-4400 val loss 7.0756
-4400 val perplexity 1182.7396
-4400 train 6.766150 (lr=3.2507e-05) (hash(x)=45076737)
-4200 val loss 6.8086
-4200 val perplexity 905.6094
-4200 train 6.836856 (lr=4.7463e-05) (hash(x)=49675080)
-4500 val loss 7.0691
-4500 val perplexity 1175.0457
-4500 train 7.197128 (lr=3.1801e-05) (hash(x)=57930262)
-4300 val loss 6.7909
-4300 val perplexity 889.7120
-4300 train 6.455951 (lr=4.6490e-05) (hash(x)=43239281)
-4600 val loss 7.0560
-4600 val perplexity 1159.7599
-4600 train 6.852347 (lr=3.1091e-05) (hash(x)=46721614)
-4400 val loss 6.7779
-4400 val perplexity 878.2114
-4400 train 6.462660 (lr=4.5509e-05) (hash(x)=45076737)
-4700 val loss 7.0485
-4700 val perplexity 1151.1281
-4700 train 6.876356 (lr=3.0377e-05) (hash(x)=49837920)
-4500 val loss 6.7576
-4500 val perplexity 860.5627
-4500 train 6.874951 (lr=4.4521e-05) (hash(x)=57930262)
-4800 val loss 7.0353
-4800 val perplexity 1135.9937
-4800 train 7.160434 (lr=2.9661e-05) (hash(x)=48380045)
-4600 val loss 6.7361
-4600 val perplexity 842.2592
-4600 train 6.513708 (lr=4.3527e-05) (hash(x)=46721614)
-4900 val loss 7.0306
-4900 val perplexity 1130.7338
-4900 train 6.909075 (lr=2.8942e-05) (hash(x)=44202577)
-4700 val loss 6.7285
-4700 val perplexity 835.9263
-4700 train 6.564556 (lr=4.2528e-05) (hash(x)=49837920)
-5000 val loss 7.0021
-5000 val perplexity 1098.8903
-5000 train 7.005834 (lr=2.8221e-05) (hash(x)=52038024)
-4800 val loss 6.7083
-4800 val perplexity 819.1469
-4800 train 6.879846 (lr=4.1525e-05) (hash(x)=48380045)
-5100 val loss 6.9806
-5100 val perplexity 1075.6146
-5100 train 7.158219 (lr=2.7500e-05) (hash(x)=53700038)
-4900 val loss 6.6987
-4900 val perplexity 811.3144
-4900 train 6.585225 (lr=4.0518e-05) (hash(x)=44202577)
-5200 val loss 6.9704
-5200 val perplexity 1064.6042
-5200 train 6.970703 (lr=2.6779e-05) (hash(x)=48137625)
-5000 val loss 6.6739
-5000 val perplexity 791.4561
-5000 train 6.682315 (lr=3.9510e-05) (hash(x)=52038024)
-5300 val loss 6.9584
-5300 val perplexity 1051.9539
-5300 train 6.836740 (lr=2.6058e-05) (hash(x)=43161573)
-5100 val loss 6.6505
-5100 val perplexity 773.1574
-5100 train 6.805350 (lr=3.8500e-05) (hash(x)=53700038)
-5400 val loss 6.9467
-5400 val perplexity 1039.6730
-5400 train 7.132883 (lr=2.5339e-05) (hash(x)=56673322)
-5200 val loss 6.6534
-5200 val perplexity 775.4347
-5200 train 6.657163 (lr=3.7490e-05) (hash(x)=48137625)
-5500 val loss 6.9299
-5500 val perplexity 1022.4167
-5500 train 7.111983 (lr=2.4623e-05) (hash(x)=53468295)
-5300 val loss 6.6340
-5300 val perplexity 760.5112
-5300 train 6.477094 (lr=3.6482e-05) (hash(x)=43161573)
-5600 val loss 6.9158
-5600 val perplexity 1008.0984
-5600 train 7.229111 (lr=2.3909e-05) (hash(x)=59287280)
-5400 val loss 6.6298
-5400 val perplexity 757.3374
-5400 train 6.820119 (lr=3.5475e-05) (hash(x)=56673322)
-5700 val loss 6.9053
-5700 val perplexity 997.5583
-5700 train 7.143720 (lr=2.3199e-05) (hash(x)=57575806)
-5500 val loss 6.6147
-5500 val perplexity 746.0064
-5500 train 6.824338 (lr=3.4472e-05) (hash(x)=53468295)
-5800 val loss 6.8949
-5800 val perplexity 987.2734
-5800 train 6.836562 (lr=2.2493e-05) (hash(x)=46897279)
-5600 val loss 6.6057
-5600 val perplexity 739.3046
-5600 train 6.898512 (lr=3.3473e-05) (hash(x)=59287280)
-5900 val loss 6.8898
-5900 val perplexity 982.2181
-5900 train 6.771390 (lr=2.1793e-05) (hash(x)=47565679)
-5700 val loss 6.5965
-5700 val perplexity 732.5139
-5700 train 6.830234 (lr=3.2479e-05) (hash(x)=57575806)
-6000 val loss 6.8769
-6000 val perplexity 969.5782
-6000 train 6.764159 (lr=2.1098e-05) (hash(x)=51590090)
-5800 val loss 6.5881
-5800 val perplexity 726.3762
-5800 train 6.606775 (lr=3.1491e-05) (hash(x)=46897279)
-6100 val loss 6.8705
-6100 val perplexity 963.4629
-6100 train 7.256005 (lr=2.0410e-05) (hash(x)=59732271)
-5900 val loss 6.5897
-5900 val perplexity 727.5562
-5900 train 6.480405 (lr=3.0510e-05) (hash(x)=47565679)
-6200 val loss 6.8648
-6200 val perplexity 957.9373
-6200 train 6.838703 (lr=1.9729e-05) (hash(x)=46394422)
-6000 val loss 6.5795
-6000 val perplexity 720.1879
-6000 train 6.441273 (lr=2.9537e-05) (hash(x)=51590090)
-6300 val loss 6.8538
-6300 val perplexity 947.4933
-6300 train 6.875601 (lr=1.9056e-05) (hash(x)=53748145)
-6100 val loss 6.5637
-6100 val perplexity 708.8743
-6100 train 7.011058 (lr=2.8574e-05) (hash(x)=59732271)
-6400 val loss 6.8485
-6400 val perplexity 942.4641
-6400 train 6.731420 (lr=1.8392e-05) (hash(x)=46054751)
-6200 val loss 6.5600
-6200 val perplexity 706.2467
-6200 train 6.588350 (lr=2.7621e-05) (hash(x)=46394422)
-6500 val loss 6.8386
-6500 val perplexity 933.1525
-6500 train 7.054920 (lr=1.7738e-05) (hash(x)=51816809)
-6300 val loss 6.5540
-6300 val perplexity 702.0677
-6300 train 6.569966 (lr=2.6679e-05) (hash(x)=53748145)
-6600 val loss 6.8378
-6600 val perplexity 932.3933
-6600 train 6.719009 (lr=1.7093e-05) (hash(x)=52453336)
-6400 val loss 6.5467
-6400 val perplexity 696.9398
-6400 train 6.418003 (lr=2.5749e-05) (hash(x)=46054751)
-6700 val loss 6.8274
-6700 val perplexity 922.7719
-6700 train 6.842637 (lr=1.6459e-05) (hash(x)=49108775)
-6800 val loss 6.8191
-6800 val perplexity 915.1290
-6800 train 6.744590 (lr=1.5836e-05) (hash(x)=46745396)
-6500 val loss 6.5368
-6500 val perplexity 690.0514
-6500 train 6.811157 (lr=2.4833e-05) (hash(x)=51816809)
-6900 val loss 6.8036
-6900 val perplexity 901.1028
-6900 train 6.938798 (lr=1.5225e-05) (hash(x)=46534986)
-6600 val loss 6.5317
-6600 val perplexity 686.5802
-6600 train 6.414201 (lr=2.3930e-05) (hash(x)=52453336)
-7000 val loss 6.7982
-7000 val perplexity 896.2391
-7000 train 7.138734 (lr=1.4627e-05) (hash(x)=49317888)
-6700 val loss 6.5316
-6700 val perplexity 686.5275
-6700 train 6.547672 (lr=2.3042e-05) (hash(x)=49108775)
-7100 val loss 6.7898
-7100 val perplexity 888.7249
-7100 train 6.800003 (lr=1.4043e-05) (hash(x)=50360484)
-6800 val loss 6.5200
-6800 val perplexity 678.6013
-6800 train 6.438734 (lr=2.2171e-05) (hash(x)=46745396)
-7200 val loss 6.7843
-7200 val perplexity 883.8223
-7200 train 6.675083 (lr=1.3471e-05) (hash(x)=49515094)
-6900 val loss 6.5101
-6900 val perplexity 671.8820
-6900 train 6.690259 (lr=2.1316e-05) (hash(x)=46534986)
-7300 val loss 6.7862
-7300 val perplexity 885.5561
-7300 train 6.775707 (lr=1.2915e-05) (hash(x)=51546861)
-7000 val loss 6.4946
-7000 val perplexity 661.5610
-7000 train 6.878696 (lr=2.0478e-05) (hash(x)=49317888)
-7400 val loss 6.7751
-7400 val perplexity 875.7767
-7400 train 6.777234 (lr=1.2373e-05) (hash(x)=48320948)
-7100 val loss 6.4876
-7100 val perplexity 656.9729
-7100 train 6.499301 (lr=1.9660e-05) (hash(x)=50360484)
-7500 val loss 6.7680
-7500 val perplexity 869.6076
-7500 train 6.685040 (lr=1.1847e-05) (hash(x)=40167457)
-7200 val loss 6.4838
-7200 val perplexity 654.4562
-7200 train 6.345729 (lr=1.8860e-05) (hash(x)=49515094)
-7600 val loss 6.7610
-7600 val perplexity 863.4881
-7600 train 6.716415 (lr=1.1337e-05) (hash(x)=49942165)
-7300 val loss 6.4852
-7300 val perplexity 655.3403
-7300 train 6.473186 (lr=1.8081e-05) (hash(x)=51546861)
-7700 val loss 6.7578
-7700 val perplexity 860.7277
-7700 train 6.508514 (lr=1.0844e-05) (hash(x)=48853311)
-7400 val loss 6.4816
-7400 val perplexity 653.0450
-7400 train 6.474329 (lr=1.7323e-05) (hash(x)=48320948)
-7800 val loss 6.7528
-7800 val perplexity 856.4212
-7800 train 6.658655 (lr=1.0367e-05) (hash(x)=48510117)
-7500 val loss 6.4817
-7500 val perplexity 653.0565
-7500 train 6.411730 (lr=1.6586e-05) (hash(x)=40167457)
-7900 val loss 6.7509
-7900 val perplexity 854.8618
-7900 train 6.762117 (lr=9.9088e-06) (hash(x)=48339781)
-7600 val loss 6.4654
-7600 val perplexity 642.5402
-7600 train 6.415967 (lr=1.5872e-05) (hash(x)=49942165)
-8000 val loss 6.7459
-8000 val perplexity 850.5717
-8000 train 6.878915 (lr=9.4682e-06) (hash(x)=54927320)
-7700 val loss 6.4599
-7700 val perplexity 638.9849
-7700 train 6.188712 (lr=1.5181e-05) (hash(x)=48853311)
-8100 val loss 6.7406
-8100 val perplexity 846.0574
-8100 train 6.488393 (lr=9.0461e-06) (hash(x)=46461786)
-7800 val loss 6.4578
-7800 val perplexity 637.6554
-7800 train 6.345714 (lr=1.4514e-05) (hash(x)=48510117)
-8200 val loss 6.7377
-8200 val perplexity 843.5936
-8200 train 6.667369 (lr=8.6430e-06) (hash(x)=51536260)
-7900 val loss 6.4579
-7900 val perplexity 637.7202
-7900 train 6.452045 (lr=1.3872e-05) (hash(x)=48339781)
-8300 val loss 6.7352
-8300 val perplexity 841.4913
-8300 train 6.494115 (lr=8.2593e-06) (hash(x)=44770722)
-8000 val loss 6.4506
-8000 val perplexity 633.1036
-8000 train 6.566749 (lr=1.3255e-05) (hash(x)=54927320)
-8400 val loss 6.7321
-8400 val perplexity 838.9099
-8400 train 6.740037 (lr=7.8953e-06) (hash(x)=50104957)
-8100 val loss 6.4473
-8100 val perplexity 631.0035
-8100 train 6.158883 (lr=1.2665e-05) (hash(x)=46461786)
-8500 val loss 6.7283
-8500 val perplexity 835.6963
-8500 train 6.834835 (lr=7.5515e-06) (hash(x)=50132971)
-8200 val loss 6.4479
-8200 val perplexity 631.3887
-8200 train 6.372143 (lr=1.2100e-05) (hash(x)=51536260)
-8600 val loss 6.7235
-8600 val perplexity 831.6856
-8600 train 6.676203 (lr=7.2282e-06) (hash(x)=52193699)
-8300 val loss 6.4479
-8300 val perplexity 631.3722
-8300 train 6.184940 (lr=1.1563e-05) (hash(x)=44770722)
-8700 val loss 6.7189
-8700 val perplexity 827.9468
-8700 train 6.706408 (lr=6.9257e-06) (hash(x)=47902319)
-8400 val loss 6.4421
-8400 val perplexity 627.6946
-8400 train 6.443494 (lr=1.1053e-05) (hash(x)=50104957)
-8800 val loss 6.7133
-8800 val perplexity 823.3087
-8800 train 6.985338 (lr=6.6444e-06) (hash(x)=54904230)
-8500 val loss 6.4391
-8500 val perplexity 625.8142
-8500 train 6.534442 (lr=1.0572e-05) (hash(x)=50132971)
-8900 val loss 6.7132
-8900 val perplexity 823.1639
-8900 train 6.608115 (lr=6.3845e-06) (hash(x)=46311615)
-8600 val loss 6.4317
-8600 val perplexity 621.2293
-8600 train 6.393053 (lr=1.0119e-05) (hash(x)=52193699)
-9000 val loss 6.7123
-9000 val perplexity 822.4835
-9000 train 6.588591 (lr=6.1462e-06) (hash(x)=48535188)
-8700 val loss 6.4298
-8700 val perplexity 620.0411
-8700 train 6.417854 (lr=9.6960e-06) (hash(x)=47902319)
-9100 val loss 6.7054
-9100 val perplexity 816.8428
-9100 train 6.734750 (lr=5.9300e-06) (hash(x)=51757372)
-8800 val loss 6.4236
-8800 val perplexity 616.1947
-8800 train 6.713367 (lr=9.3021e-06) (hash(x)=54904230)
-9200 val loss 6.7031
-9200 val perplexity 814.9101
-9200 train 6.588158 (lr=5.7359e-06) (hash(x)=51131708)
-8900 val loss 6.4212
-8900 val perplexity 614.7217
-8900 train 6.308746 (lr=8.9382e-06) (hash(x)=46311615)
-9300 val loss 6.7031
-9300 val perplexity 814.9078
-9300 train 6.644286 (lr=5.5641e-06) (hash(x)=44784276)
-9000 val loss 6.4235
-9000 val perplexity 616.1304
-9000 train 6.292938 (lr=8.6047e-06) (hash(x)=48535188)
-9400 val loss 6.7000
-9400 val perplexity 812.3867
-9400 train 6.797466 (lr=5.4149e-06) (hash(x)=51981169)
-9100 val loss 6.4145
-9100 val perplexity 610.6580
-9100 train 6.448308 (lr=8.3020e-06) (hash(x)=51757372)
-9500 val loss 6.6971
-9500 val perplexity 810.0333
-9500 train 6.648178 (lr=5.2884e-06) (hash(x)=47232936)
-9200 val loss 6.4128
-9200 val perplexity 609.6115
-9200 train 6.280666 (lr=8.0302e-06) (hash(x)=51131708)
-9600 val loss 6.6957
-9600 val perplexity 808.9078
-9600 train 6.730829 (lr=5.1847e-06) (hash(x)=53800450)
-9300 val loss 6.4138
-9300 val perplexity 610.1989
-9300 train 6.370407 (lr=7.7898e-06) (hash(x)=44784276)
-9700 val loss 6.6942
-9700 val perplexity 807.7153
-9700 train 6.812382 (lr=5.1040e-06) (hash(x)=55768123)
-9400 val loss 6.4115
-9400 val perplexity 608.8001
-9400 train 6.524776 (lr=7.5809e-06) (hash(x)=51981169)
-9800 val loss 6.6956
-9800 val perplexity 808.8314
-9800 train 6.635161 (lr=5.0462e-06) (hash(x)=47745177)
-9500 val loss 6.4100
-9500 val perplexity 607.8681
-9500 train 6.373538 (lr=7.4038e-06) (hash(x)=47232936)
-9900 val loss 6.6930
-9900 val perplexity 806.7683
-9900 train 6.937419 (lr=5.0116e-06) (hash(x)=56592246)
-9600 val loss 6.4066
-9600 val perplexity 605.8401
-9600 train 6.441170 (lr=7.2586e-06) (hash(x)=53800450)
-9999 val loss 6.6932
-9999 val perplexity 806.9299
-9700 val loss 6.4071
-9700 val perplexity 606.1292
-9700 train 6.529077 (lr=7.1456e-06) (hash(x)=55768123)
-9800 val loss 6.4078
-9800 val perplexity 606.5295
-9800 train 6.343826 (lr=7.0647e-06) (hash(x)=47745177)
-9900 val loss 6.4075
-9900 val perplexity 606.3977
-9900 train 6.676889 (lr=7.0162e-06) (hash(x)=56592246)
+3100 val loss 6.8108
+3100 val perplexity 907.6118
+3100 train 7.108453 (lr=8.1915e-05) (hash(x)=44479330)
+3200 val loss 6.8168
+3200 val perplexity 913.0430
+3200 train 6.861060 (lr=8.0745e-05) (hash(x)=54593096)
+3300 val loss 6.7666
+3300 val perplexity 868.3719
+3300 train 6.689936 (lr=7.9549e-05) (hash(x)=45347643)
+0 train 11.815376 (lr=7.5000e-07) (hash(x)=57791809)
+3400 val loss 6.7480
+3400 val perplexity 852.3517
+3400 train 6.868625 (lr=7.8328e-05) (hash(x)=47797247)
+100 val loss 9.4103
+100 val perplexity 12214.1318
+100 train 9.435302 (lr=7.5750e-05) (hash(x)=48211824)
+3500 val loss 6.7277
+3500 val perplexity 835.2067
+3500 train 6.586505 (lr=7.7082e-05) (hash(x)=46115683)
+200 val loss 7.8489
+200 val perplexity 2562.9167
+200 train 7.782820 (lr=1.5000e-04) (hash(x)=50375849)
+3600 val loss 6.7276
+3600 val perplexity 835.1287
+3600 train 6.583769 (lr=7.5814e-05) (hash(x)=44502074)
+300 val loss 7.7186
+300 val perplexity 2249.8511
+300 train 8.037295 (lr=1.4997e-04) (hash(x)=57250808)
+3700 val loss 6.6958
+3700 val perplexity 808.9891
+3700 train 6.847288 (lr=7.4525e-05) (hash(x)=55388443)
+3800 val loss 6.6831
+3800 val perplexity 798.8195
+3800 train 6.478742 (lr=7.3215e-05) (hash(x)=43790341)
+400 val loss 7.6896
+400 val perplexity 2185.4397
+400 train 8.281481 (lr=1.4986e-04) (hash(x)=62519858)
+3900 val loss 6.6852
+3900 val perplexity 800.4603
+3900 train 6.680488 (lr=7.1887e-05) (hash(x)=50013318)
+500 val loss 7.6498
+500 val perplexity 2100.2070
+500 train 7.555541 (lr=1.4969e-04) (hash(x)=47226806)
+600 val loss 7.6013
+600 val perplexity 2000.8110
+600 train 7.611244 (lr=1.4945e-04) (hash(x)=51149322)
+4000 val loss 6.6633
+4000 val perplexity 783.1157
+4000 train 6.724937 (lr=7.0541e-05) (hash(x)=51704787)
+700 val loss 7.5505
+700 val perplexity 1901.7231
+700 train 7.570621 (lr=1.4913e-04) (hash(x)=51564551)
+4100 val loss 6.6502
+4100 val perplexity 772.9760
+4100 train 6.690805 (lr=6.9180e-05) (hash(x)=50821964)
+800 val loss 7.4868
+800 val perplexity 1784.4075
+800 train 7.224887 (lr=1.4876e-04) (hash(x)=45093459)
+4200 val loss 6.6401
+4200 val perplexity 765.1970
+4200 train 6.676926 (lr=6.7804e-05) (hash(x)=49675080)
+900 val loss 7.4101
+900 val perplexity 1652.5939
+900 train 7.676486 (lr=1.4831e-04) (hash(x)=54988361)
+4300 val loss 6.6300
+4300 val perplexity 757.5169
+4300 train 6.297640 (lr=6.6414e-05) (hash(x)=43239281)
+1000 val loss 7.3711
+1000 val perplexity 1589.3545
+1000 train 7.263645 (lr=1.4779e-04) (hash(x)=47588648)
+4400 val loss 6.6187
+4400 val perplexity 748.9337
+4400 train 6.297700 (lr=6.5013e-05) (hash(x)=45076737)
+1100 val loss 7.3459
+1100 val perplexity 1549.8197
+1100 train 6.974550 (lr=1.4721e-04) (hash(x)=37984588)
+4500 val loss 6.6069
+4500 val perplexity 740.1642
+4500 train 6.723946 (lr=6.3602e-05) (hash(x)=57930262)
+1200 val loss 7.2713
+1200 val perplexity 1438.3854
+1200 train 7.423686 (lr=1.4656e-04) (hash(x)=56333817)
+4600 val loss 6.5956
+4600 val perplexity 731.8596
+4600 train 6.346147 (lr=6.2182e-05) (hash(x)=46721614)
+1300 val loss 7.2478
+1300 val perplexity 1405.0525
+1300 train 7.408738 (lr=1.4585e-04) (hash(x)=53454056)
+4700 val loss 6.5840
+4700 val perplexity 723.4270
+4700 train 6.413177 (lr=6.0754e-05) (hash(x)=49837920)
+1400 val loss 7.1724
+1400 val perplexity 1302.9978
+1400 train 7.307054 (lr=1.4507e-04) (hash(x)=55284163)
+4800 val loss 6.5830
+4800 val perplexity 722.7350
+4800 train 6.747669 (lr=5.9321e-05) (hash(x)=48380045)
+1500 val loss 7.1053
+1500 val perplexity 1218.3500
+1500 train 6.960086 (lr=1.4422e-04) (hash(x)=48162598)
+4900 val loss 6.5705
+4900 val perplexity 713.7289
+4900 train 6.455478 (lr=5.7883e-05) (hash(x)=44202577)
+1600 val loss 7.0936
+1600 val perplexity 1204.2488
+1600 train 7.176346 (lr=1.4332e-04) (hash(x)=54214535)
+5000 val loss 6.5548
+5000 val perplexity 702.6354
+5000 train 6.531763 (lr=5.6442e-05) (hash(x)=52038024)
+1700 val loss 7.0477
+1700 val perplexity 1150.2354
+1700 train 7.288129 (lr=1.4235e-04) (hash(x)=53525003)
+5100 val loss 6.5472
+5100 val perplexity 697.2672
+5100 train 6.682254 (lr=5.5000e-05) (hash(x)=53700038)
+1800 val loss 6.9886
+1800 val perplexity 1084.1967
+1800 train 7.055763 (lr=1.4131e-04) (hash(x)=51848994)
+5200 val loss 6.5449
+5200 val perplexity 695.6867
+5200 train 6.514779 (lr=5.3558e-05) (hash(x)=48137625)
+1900 val loss 6.9741
+1900 val perplexity 1068.6400
+1900 train 6.808705 (lr=1.4022e-04) (hash(x)=48405987)
+5300 val loss 6.5362
+5300 val perplexity 689.6800
+5300 train 6.367496 (lr=5.2117e-05) (hash(x)=43161573)
+2000 val loss 6.9172
+2000 val perplexity 1009.4669
+2000 train 7.280006 (lr=1.3907e-04) (hash(x)=58592291)
+2100 val loss 6.9460
+2100 val perplexity 1038.9731
+2100 train 7.032774 (lr=1.3786e-04) (hash(x)=51167081)
+5400 val loss 6.5369
+5400 val perplexity 690.1711
+5400 train 6.705731 (lr=5.0679e-05) (hash(x)=56673322)
+2200 val loss 6.8788
+2200 val perplexity 971.4705
+2200 train 6.855141 (lr=1.3660e-04) (hash(x)=47994988)
+5500 val loss 6.5188
+5500 val perplexity 677.7745
+5500 train 6.729006 (lr=4.9246e-05) (hash(x)=53468295)
+2300 val loss 6.9196
+2300 val perplexity 1011.8920
+2300 train 6.912843 (lr=1.3527e-04) (hash(x)=47377604)
+5600 val loss 6.5010
+5600 val perplexity 665.8105
+5600 train 6.796459 (lr=4.7818e-05) (hash(x)=59287280)
+2400 val loss 6.8704
+2400 val perplexity 963.3668
+2400 train 6.928603 (lr=1.3390e-04) (hash(x)=53554323)
+5700 val loss 6.5016
+5700 val perplexity 666.2183
+5700 train 6.710692 (lr=4.6398e-05) (hash(x)=57575806)
+2500 val loss 6.8641
+2500 val perplexity 957.2761
+2500 train 6.931783 (lr=1.3247e-04) (hash(x)=50780417)
+5800 val loss 6.4968
+5800 val perplexity 663.0387
+5800 train 6.534732 (lr=4.4987e-05) (hash(x)=46897279)
+2600 val loss 6.8701
+2600 val perplexity 963.0816
+2600 train 6.758279 (lr=1.3099e-04) (hash(x)=46453562)
+5900 val loss 6.4894
+5900 val perplexity 658.1490
+5900 train 6.375528 (lr=4.3586e-05) (hash(x)=47565679)
+2700 val loss 6.8107
+2700 val perplexity 907.5153
+2700 train 6.781026 (lr=1.2946e-04) (hash(x)=54404221)
+6000 val loss 6.4841
+6000 val perplexity 654.6342
+6000 train 6.333650 (lr=4.2196e-05) (hash(x)=51590090)
+2800 val loss 6.7924
+2800 val perplexity 891.0690
+2800 train 7.439435 (lr=1.2788e-04) (hash(x)=59318895)
+6100 val loss 6.4780
+6100 val perplexity 650.6495
+6100 train 6.915969 (lr=4.0820e-05) (hash(x)=59732271)
+2900 val loss 6.7875
+2900 val perplexity 886.6631
+2900 train 6.780449 (lr=1.2626e-04) (hash(x)=47845760)
+6200 val loss 6.4670
+6200 val perplexity 643.5385
+6200 train 6.530252 (lr=3.9459e-05) (hash(x)=46394422)
+3000 val loss 6.7641
+3000 val perplexity 866.1562
+3000 train 6.461198 (lr=1.2459e-04) (hash(x)=44336167)
+6300 val loss 6.4697
+6300 val perplexity 645.2667
+6300 train 6.481519 (lr=3.8113e-05) (hash(x)=53748145)
+3100 val loss 6.7478
+3100 val perplexity 852.1684
+3100 train 7.055303 (lr=1.2287e-04) (hash(x)=44479330)
+6400 val loss 6.4581
+6400 val perplexity 637.8579
+6400 train 6.312898 (lr=3.6785e-05) (hash(x)=46054751)
+3200 val loss 6.7569
+3200 val perplexity 859.9979
+3200 train 6.806598 (lr=1.2112e-04) (hash(x)=54593096)
+6500 val loss 6.4519
+6500 val perplexity 633.8904
+6500 train 6.736262 (lr=3.5475e-05) (hash(x)=51816809)
+3300 val loss 6.7278
+3300 val perplexity 835.3075
+3300 train 6.684790 (lr=1.1932e-04) (hash(x)=45347643)
+6600 val loss 6.4588
+6600 val perplexity 638.2906
+6600 train 6.274892 (lr=3.4186e-05) (hash(x)=52453336)
+3400 val loss 6.7000
+3400 val perplexity 812.3824
+3400 train 6.819838 (lr=1.1749e-04) (hash(x)=47797247)
+6700 val loss 6.4472
+6700 val perplexity 630.9039
+6700 train 6.439846 (lr=3.2918e-05) (hash(x)=49108775)
+3500 val loss 6.6888
+3500 val perplexity 803.3629
+3500 train 6.543953 (lr=1.1562e-04) (hash(x)=46115683)
+6800 val loss 6.4345
+6800 val perplexity 622.9869
+6800 train 6.340245 (lr=3.1672e-05) (hash(x)=46745396)
+3600 val loss 6.6714
+3600 val perplexity 789.5205
+3600 train 6.545329 (lr=1.1372e-04) (hash(x)=44502074)
+6900 val loss 6.4267
+6900 val perplexity 618.1205
+6900 train 6.602197 (lr=3.0451e-05) (hash(x)=46534986)
+3700 val loss 6.6690
+3700 val perplexity 787.6163
+3700 train 6.809144 (lr=1.1179e-04) (hash(x)=55388443)
+7000 val loss 6.4191
+7000 val perplexity 613.4761
+7000 train 6.785058 (lr=2.9255e-05) (hash(x)=49317888)
+3800 val loss 6.6656
+3800 val perplexity 784.8956
+3800 train 6.453290 (lr=1.0982e-04) (hash(x)=43790341)
+7100 val loss 6.4132
+7100 val perplexity 609.8173
+7100 train 6.441988 (lr=2.8085e-05) (hash(x)=50360484)
+3900 val loss 6.6528
+3900 val perplexity 774.9131
+3900 train 6.637487 (lr=1.0783e-04) (hash(x)=50013318)
+7200 val loss 6.4056
+7200 val perplexity 605.1969
+7200 train 6.250135 (lr=2.6943e-05) (hash(x)=49515094)
+4000 val loss 6.6511
+4000 val perplexity 773.6136
+4000 train 6.725606 (lr=1.0581e-04) (hash(x)=51704787)
+7300 val loss 6.4160
+7300 val perplexity 611.5779
+7300 train 6.397116 (lr=2.5830e-05) (hash(x)=51546861)
+4100 val loss 6.6422
+4100 val perplexity 766.7709
+4100 train 6.687906 (lr=1.0377e-04) (hash(x)=50821964)
+7400 val loss 6.4147
+7400 val perplexity 610.7546
+7400 train 6.393087 (lr=2.4746e-05) (hash(x)=48320948)
+4200 val loss 6.6313
+4200 val perplexity 758.4469
+4200 train 6.676696 (lr=1.0171e-04) (hash(x)=49675080)
+7500 val loss 6.4144
+7500 val perplexity 610.5697
+7500 train 6.344778 (lr=2.3694e-05) (hash(x)=40167457)
+4300 val loss 6.6157
+4300 val perplexity 746.7346
+4300 train 6.275900 (lr=9.9622e-05) (hash(x)=43239281)
+7600 val loss 6.3956
+7600 val perplexity 599.2078
+7600 train 6.340053 (lr=2.2674e-05) (hash(x)=49942165)
+4400 val loss 6.6233
+4400 val perplexity 752.4334
+4400 train 6.317001 (lr=9.7520e-05) (hash(x)=45076737)
+7700 val loss 6.3912
+7700 val perplexity 596.5556
+7700 train 6.109773 (lr=2.1687e-05) (hash(x)=48853311)
+4500 val loss 6.6173
+4500 val perplexity 747.8881
+4500 train 6.730391 (lr=9.5403e-05) (hash(x)=57930262)
+7800 val loss 6.3869
+7800 val perplexity 594.0394
+7800 train 6.283017 (lr=2.0735e-05) (hash(x)=48510117)
+4600 val loss 6.5940
+4600 val perplexity 730.6702
+4600 train 6.354017 (lr=9.3273e-05) (hash(x)=46721614)
+7900 val loss 6.3907
+7900 val perplexity 596.3019
+7900 train 6.387419 (lr=1.9818e-05) (hash(x)=48339781)
+4700 val loss 6.5861
+4700 val perplexity 724.9485
+4700 train 6.410762 (lr=9.1132e-05) (hash(x)=49837920)
+8000 val loss 6.3852
+8000 val perplexity 593.0266
+8000 train 6.477322 (lr=1.8936e-05) (hash(x)=54927320)
+4800 val loss 6.5926
+4800 val perplexity 729.6981
+4800 train 6.733562 (lr=8.8982e-05) (hash(x)=48380045)
+8100 val loss 6.3803
+8100 val perplexity 590.0946
+8100 train 6.103539 (lr=1.8092e-05) (hash(x)=46461786)
+4900 val loss 6.5911
+4900 val perplexity 728.6015
+4900 train 6.470277 (lr=8.6825e-05) (hash(x)=44202577)
+8200 val loss 6.3766
+8200 val perplexity 587.9359
+8200 train 6.291387 (lr=1.7286e-05) (hash(x)=51536260)
+5000 val loss 6.5647
+5000 val perplexity 709.6234
+5000 train 6.568296 (lr=8.4663e-05) (hash(x)=52038024)
+8300 val loss 6.3755
+8300 val perplexity 587.3049
+8300 train 6.118826 (lr=1.6519e-05) (hash(x)=44770722)
+5100 val loss 6.5647
+5100 val perplexity 709.6227
+5100 train 6.695176 (lr=8.2500e-05) (hash(x)=53700038)
+8400 val loss 6.3739
+8400 val perplexity 586.3613
+8400 train 6.378792 (lr=1.5791e-05) (hash(x)=50104957)
+5200 val loss 6.5524
+5200 val perplexity 700.9124
+5200 train 6.519544 (lr=8.0337e-05) (hash(x)=48137625)
+8500 val loss 6.3685
+8500 val perplexity 583.1942
+8500 train 6.459326 (lr=1.5103e-05) (hash(x)=50132971)
+5300 val loss 6.5438
+5300 val perplexity 694.8887
+5300 train 6.366642 (lr=7.8175e-05) (hash(x)=43161573)
+8600 val loss 6.3651
+8600 val perplexity 581.2270
+8600 train 6.323270 (lr=1.4456e-05) (hash(x)=52193699)
+5400 val loss 6.5453
+5400 val perplexity 695.9648
+5400 train 6.720807 (lr=7.6018e-05) (hash(x)=56673322)
+8700 val loss 6.3602
+8700 val perplexity 578.3399
+8700 train 6.344991 (lr=1.3851e-05) (hash(x)=47902319)
+5500 val loss 6.5354
+5500 val perplexity 689.1136
+5500 train 6.723743 (lr=7.3868e-05) (hash(x)=53468295)
+8800 val loss 6.3528
+8800 val perplexity 574.0699
+8800 train 6.642093 (lr=1.3289e-05) (hash(x)=54904230)
+5600 val loss 6.5390
+5600 val perplexity 691.6022
+5600 train 6.854773 (lr=7.1727e-05) (hash(x)=59287280)
+8900 val loss 6.3538
+8900 val perplexity 574.6966
+8900 train 6.230078 (lr=1.2769e-05) (hash(x)=46311615)
+5700 val loss 6.5331
+5700 val perplexity 687.5286
+5700 train 6.746007 (lr=6.9597e-05) (hash(x)=57575806)
+9000 val loss 6.3527
+9000 val perplexity 574.0275
+9000 train 6.200503 (lr=1.2292e-05) (hash(x)=48535188)
+5800 val loss 6.5352
+5800 val perplexity 688.9601
+5800 train 6.542769 (lr=6.7480e-05) (hash(x)=46897279)
+9100 val loss 6.3477
+9100 val perplexity 571.1558
+9100 train 6.375886 (lr=1.1860e-05) (hash(x)=51757372)
+5900 val loss 6.5230
+5900 val perplexity 680.6183
+5900 train 6.415868 (lr=6.5378e-05) (hash(x)=47565679)
+9200 val loss 6.3460
+9200 val perplexity 570.2044
+9200 train 6.207469 (lr=1.1472e-05) (hash(x)=51131708)
+6000 val loss 6.5250
+6000 val perplexity 682.0081
+6000 train 6.372841 (lr=6.3294e-05) (hash(x)=51590090)
+9300 val loss 6.3423
+9300 val perplexity 568.0868
+9300 train 6.299280 (lr=1.1128e-05) (hash(x)=44784276)
+6100 val loss 6.5130
+6100 val perplexity 673.8539
+6100 train 6.930476 (lr=6.1230e-05) (hash(x)=59732271)
+9400 val loss 6.3432
+9400 val perplexity 568.6069
+9400 train 6.450517 (lr=1.0830e-05) (hash(x)=51981169)
+6200 val loss 6.5090
+6200 val perplexity 671.1452
+9500 val loss 6.3407
+9500 val perplexity 567.1857
+6200 train 6.571561 (lr=5.9188e-05) (hash(x)=46394422)
+9500 train 6.312754 (lr=1.0577e-05) (hash(x)=47232936)
+9600 val loss 6.3410
+9600 val perplexity 567.3699
+9600 train 6.355495 (lr=1.0369e-05) (hash(x)=53800450)
+6300 val loss 6.5214
+6300 val perplexity 679.5251
+6300 train 6.526929 (lr=5.7169e-05) (hash(x)=53748145)
+9700 val loss 6.3406
+9700 val perplexity 567.1387
+9700 train 6.445551 (lr=1.0208e-05) (hash(x)=55768123)
+6400 val loss 6.5170
+6400 val perplexity 676.5420
+6400 train 6.375550 (lr=5.5177e-05) (hash(x)=46054751)
+9800 val loss 6.3420
+9800 val perplexity 567.9086
+9800 train 6.281539 (lr=1.0092e-05) (hash(x)=47745177)
+6500 val loss 6.5060
+6500 val perplexity 669.1499
+6500 train 6.733078 (lr=5.3213e-05) (hash(x)=51816809)
+9900 val loss 6.3431
+9900 val perplexity 568.5347
+9900 train 6.613006 (lr=1.0023e-05) (hash(x)=56592246)
+6600 val loss 6.4993
+6600 val perplexity 664.6857
+6600 train 6.381551 (lr=5.1279e-05) (hash(x)=52453336)
+9999 val loss 6.3466
+9999 val perplexity 570.5632
+6700 val loss 6.5058
+6700 val perplexity 669.0009
+6700 train 6.497029 (lr=4.9377e-05) (hash(x)=49108775)
+6800 val loss 6.4920
+6800 val perplexity 659.8412
+6800 train 6.398888 (lr=4.7509e-05) (hash(x)=46745396)
+6900 val loss 6.4899
+6900 val perplexity 658.4554
+6900 train 6.663884 (lr=4.5676e-05) (hash(x)=46534986)

attention_kindselective_n_heads2_seed1340/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c94c460d7c2e918bd19dd6c7d0a40fbf731522e2be5d922fe6d210895f4b6cd
+oid sha256:db5e71e2651a7e4c04adf1b1bdd2fd67898b9838ff7ce2857cd735f4766b5532
 size 38587970

attention_kindselective_n_heads2_seed1340/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:99967d2565438966d9e46b6de33f253f4b8e326c9f4cfe096170054f1ecb7401
+oid sha256:87a3a4e1a720bf19b1c7f850b83a50505c837892b3964a6f17b36ac335b8a606
 size 38587970

attention_kindselective_n_heads2_seed1340/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3fc8260bce2fd1ee0a25bb0aefdf1b927435a4e83a5f3001b7cd9a010becef3e
+oid sha256:43a6634ddb7efa2797ba98f1129965459e095430bafbbef119764211ecce5d4f
 size 38587970

attention_kindselective_n_heads2_seed1340/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ffa02ec9ceef9ae047bb6c9f2df20bb20a78f2a2d95be921f8573aad83aa4f34
+oid sha256:d3b59f7ce706fb6b80035e89fbc4a42332e13eb0a0bc36b121c007413b2e495b
 size 38587970

attention_kindselective_n_heads2_seed1340/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e00018265047cc47ecf529e2cb77c9da38ab5ffdb63eb4dd62a77f724a4e61f1
+oid sha256:ad182381a14e875a3c04a9d0fce780b0641e040b738f8454b13df6255a740e34
 size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1e45fc5874ff2b4f6e33851010911bf7688145d2c6254585473d064c73335ec
+oid sha256:89e51f2fe0afb3163bdb42b8d6c2d3968fb0c1d3a51b33ea7ae5061524b2b719
 size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3fa9a215741606396ff07a59b06d77363e8598933439c4e40b9445a216a12bff
+oid sha256:28aada8d81583aad9ff815ca63797851095f2d61265dbd7042c05b5071bebf1c
 size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b894c3c084b1f0ca1c18954ebede742cd1b3528fd4d2b7daeef15cd86b6a87b
+oid sha256:93e34e940783142b8f4cb2c5281bcbe179e89f9432b789eb0baeb86a876f3e07
 size 70895430