Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 7e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "7e-5_10240_2_1339", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "15e-5_10240_2_1339", "n_embd": 128}

attention_kindselective_n_heads2_seed1339/log2.txt

@@ -1,605 +1,534 @@
 max_steps: 10000
+900 val loss 7.5969
+900 val perplexity 1991.9968
+900 train 7.572263 (lr=9.8872e-05) (hash(x)=49436061)
+1000 val loss 7.5734
+1000 val perplexity 1945.8105
+1000 train 7.514424 (lr=9.8528e-05) (hash(x)=48818282)
 0 val loss 11.7444
 0 val perplexity 126046.6875
-0 val loss 11.7444
-0 val perplexity 126046.6875
-0 train 11.717561 (lr=2.5000e-07) (hash(x)=55241167)
-0 train 11.717561 (lr=3.5000e-07) (hash(x)=55241167)
-100 val loss 10.4190
-100 val perplexity 33488.9922
-100 train 10.355477 (lr=2.5250e-05) (hash(x)=46387919)
-100 val loss 10.2610
-100 val perplexity 28596.4766
-100 train 10.199209 (lr=3.5350e-05) (hash(x)=46387919)
-200 val loss 9.2623
-200 val perplexity 10532.9053
-200 train 9.287925 (lr=5.0000e-05) (hash(x)=51276369)
-200 val loss 8.4121
-200 val perplexity 4501.1421
-200 train 8.450884 (lr=7.0000e-05) (hash(x)=51276369)
-300 val loss 8.2291
-300 val perplexity 3748.2771
-300 train 8.188359 (lr=4.9988e-05) (hash(x)=48572728)
-300 val loss 7.8747
-300 val perplexity 2629.9775
-300 train 7.822401 (lr=6.9984e-05) (hash(x)=48572728)
-400 val loss 7.8660
-400 val perplexity 2606.9949
-400 train 7.633432 (lr=4.9954e-05) (hash(x)=44943400)
-400 val loss 7.6837
-400 val perplexity 2172.7102
-400 train 7.420489 (lr=6.9935e-05) (hash(x)=44943400)
-500 val loss 7.7288
-500 val perplexity 2272.7871
-500 train 7.606852 (lr=4.9896e-05) (hash(x)=48434590)
-500 val loss 7.6470
-500 val perplexity 2094.3916
-500 train 7.546611 (lr=6.9854e-05) (hash(x)=48434590)
-600 val loss 7.6689
-600 val perplexity 2140.6326
-600 train 7.521816 (lr=4.9815e-05) (hash(x)=47442266)
-600 val loss 7.6314
-600 val perplexity 2061.8621
-600 train 7.487195 (lr=6.9741e-05) (hash(x)=47442266)
-700 val loss 7.6457
-700 val perplexity 2091.6689
-700 train 7.700227 (lr=4.9712e-05) (hash(x)=56495712)
-700 val loss 7.6084
-700 val perplexity 2015.0454
-700 train 7.655407 (lr=6.9596e-05) (hash(x)=56495712)
-800 val loss 7.6335
-800 val perplexity 2066.2261
-800 train 7.637799 (lr=4.9585e-05) (hash(x)=50093738)
-800 val loss 7.5885
-800 val perplexity 1975.3878
-800 train 7.582485 (lr=6.9419e-05) (hash(x)=50093738)
-900 val loss 7.6268
-900 val perplexity 2052.5078
-900 train 7.591948 (lr=4.9436e-05) (hash(x)=49436061)
-900 val loss 7.5637
-900 val perplexity 1926.9707
-900 train 7.522345 (lr=6.9210e-05) (hash(x)=49436061)
-1000 val loss 7.6158
-1000 val perplexity 2030.0663
-1000 train 7.563162 (lr=4.9264e-05) (hash(x)=48818282)
-1000 val loss 7.5289
-1000 val perplexity 1860.9679
-1100 val loss 7.6007
-1100 val perplexity 1999.5815
-1000 train 7.465590 (lr=6.8970e-05) (hash(x)=48818282)
-1100 train 7.664679 (lr=4.9070e-05) (hash(x)=55387425)
-1200 val loss 7.5917
-1200 val perplexity 1981.7345
-1100 val loss 7.4848
-1100 val perplexity 1780.8323
-1200 train 7.537913 (lr=4.8854e-05) (hash(x)=48328378)
-1100 train 7.553554 (lr=6.8698e-05) (hash(x)=55387425)
-1300 val loss 7.5847
-1300 val perplexity 1967.7880
-1300 train 7.523361 (lr=4.8616e-05) (hash(x)=48956717)
-1200 val loss 7.4321
-1200 val perplexity 1689.3721
-1200 train 7.346271 (lr=6.8395e-05) (hash(x)=48328378)
-1400 val loss 7.5832
-1400 val perplexity 1964.9525
-1400 train 7.476568 (lr=4.8356e-05) (hash(x)=49782571)
-1300 val loss 7.3965
-1300 val perplexity 1630.3083
-1300 train 7.312257 (lr=6.8062e-05) (hash(x)=48956717)
-1500 val loss 7.5897
-1500 val perplexity 1977.6857
-1500 train 7.398695 (lr=4.8074e-05) (hash(x)=51081994)
-1400 val loss 7.3806
-1400 val perplexity 1604.5980
-1400 train 7.259795 (lr=6.7698e-05) (hash(x)=49782571)
-1600 val loss 7.5873
-1600 val perplexity 1972.9817
-1600 train 7.656009 (lr=4.7772e-05) (hash(x)=48791204)
-1500 val loss 7.3405
-1500 val perplexity 1541.5237
-1500 train 7.134542 (lr=6.7304e-05) (hash(x)=51081994)
-1700 val loss 7.5815
-1700 val perplexity 1961.5758
-1700 train 7.307205 (lr=4.7448e-05) (hash(x)=46425011)
-1600 val loss 7.3004
-1600 val perplexity 1480.8594
-1600 train 7.370697 (lr=6.6881e-05) (hash(x)=48791204)
-1800 val loss 7.5641
-1800 val perplexity 1927.8108
-1800 train 7.216667 (lr=4.7105e-05) (hash(x)=44919141)
-1700 val loss 7.2983
-1700 val perplexity 1477.7853
-1700 train 7.001472 (lr=6.6428e-05) (hash(x)=46425011)
-1900 val loss 7.5429
-1900 val perplexity 1887.2594
-1900 train 7.642665 (lr=4.6741e-05) (hash(x)=48299675)
-1800 val loss 7.2642
-1800 val perplexity 1428.2245
-1800 train 6.906070 (lr=6.5947e-05) (hash(x)=44919141)
-2000 val loss 7.5304
-2000 val perplexity 1863.8826
-2000 train 8.451202 (lr=4.6357e-05) (hash(x)=61804797)
-1900 val loss 7.2552
-1900 val perplexity 1415.4088
-1900 train 7.376617 (lr=6.5437e-05) (hash(x)=48299675)
-2100 val loss 7.5083
-2100 val perplexity 1823.1356
-2100 train 7.605269 (lr=4.5954e-05) (hash(x)=53638030)
-2000 val loss 7.1835
-2000 val perplexity 1317.5396
-2000 train 8.116750 (lr=6.4900e-05) (hash(x)=61804797)
-2200 val loss 7.4889
-2200 val perplexity 1788.0094
-2200 train 7.722961 (lr=4.5532e-05) (hash(x)=56364593)
-2100 val loss 7.1585
-2100 val perplexity 1284.9927
-2100 train 7.253870 (lr=6.4335e-05) (hash(x)=53638030)
-2300 val loss 7.4897
-2300 val perplexity 1789.4935
-2300 train 7.585523 (lr=4.5091e-05) (hash(x)=52945580)
-2200 val loss 7.1313
-2200 val perplexity 1250.5045
-2200 train 7.376586 (lr=6.3745e-05) (hash(x)=56364593)
-2400 val loss 7.4714
-2400 val perplexity 1757.0946
-2400 train 7.553903 (lr=4.4633e-05) (hash(x)=54595211)
-2300 val loss 7.1077
-2300 val perplexity 1221.3357
-2300 train 7.225344 (lr=6.3128e-05) (hash(x)=52945580)
-2500 val loss 7.4593
-2500 val perplexity 1735.9640
-2500 train 7.488019 (lr=4.4156e-05) (hash(x)=52323363)
-2400 val loss 7.0891
-2400 val perplexity 1198.8025
-2400 train 7.160409 (lr=6.2486e-05) (hash(x)=54595211)
-2600 val loss 7.4433
-2600 val perplexity 1708.3724
-2600 train 7.256810 (lr=4.3663e-05) (hash(x)=42652707)
-2500 val loss 7.0646
-2500 val perplexity 1169.7643
-2500 train 7.078955 (lr=6.1819e-05) (hash(x)=52323363)
-2700 val loss 7.4416
-2700 val perplexity 1705.4611
-2700 train 7.189201 (lr=4.3153e-05) (hash(x)=45687855)
-2600 val loss 7.0259
-2600 val perplexity 1125.3988
-2600 train 6.769950 (lr=6.1128e-05) (hash(x)=42652707)
-2800 val loss 7.4216
-2800 val perplexity 1671.7010
-2800 train 7.389744 (lr=4.2627e-05) (hash(x)=49962830)
-2700 val loss 7.0032
-2700 val perplexity 1100.1040
-2700 train 6.697522 (lr=6.0414e-05) (hash(x)=45687855)
-2900 val loss 7.4089
-2900 val perplexity 1650.5708
-2900 train 7.488739 (lr=4.2085e-05) (hash(x)=52958451)
-2800 val loss 6.9818
-2800 val perplexity 1076.8694
-2800 train 6.968762 (lr=5.9677e-05) (hash(x)=49962830)
-3000 val loss 7.3962
-3000 val perplexity 1629.7239
-3000 train 7.337688 (lr=4.1529e-05) (hash(x)=48896990)
-2900 val loss 6.9673
-2900 val perplexity 1061.3451
-2900 train 6.998752 (lr=5.8919e-05) (hash(x)=52958451)
-3100 val loss 7.3868
-3100 val perplexity 1614.4810
-3100 train 7.337799 (lr=4.0957e-05) (hash(x)=47064930)
-3000 val loss 6.9330
-3000 val perplexity 1025.5603
-3000 train 6.879025 (lr=5.8140e-05) (hash(x)=48896990)
-3200 val loss 7.3732
-3200 val perplexity 1592.6964
-3200 train 7.139594 (lr=4.0373e-05) (hash(x)=45145921)
-3100 val loss 6.9128
-3100 val perplexity 1005.0540
-3100 train 6.876074 (lr=5.7340e-05) (hash(x)=47064930)
-3300 val loss 7.3655
-3300 val perplexity 1580.4542
-3300 train 7.237511 (lr=3.9775e-05) (hash(x)=48085018)
-3200 val loss 6.8927
-3200 val perplexity 985.0220
-3200 train 6.601774 (lr=5.6522e-05) (hash(x)=45145921)
-3400 val loss 7.3462
-3400 val perplexity 1550.3525
-3400 train 7.371779 (lr=3.9164e-05) (hash(x)=49596053)
-3300 val loss 6.8761
-3300 val perplexity 968.8724
-3300 train 6.710589 (lr=5.5684e-05) (hash(x)=48085018)
-3500 val loss 7.3421
-3500 val perplexity 1543.9542
-3500 train 6.941387 (lr=3.8541e-05) (hash(x)=41135634)
-3400 val loss 6.8393
-3400 val perplexity 933.8585
-3400 train 6.835313 (lr=5.4829e-05) (hash(x)=49596053)
-3600 val loss 7.3236
-3600 val perplexity 1515.6783
-3600 train 7.334878 (lr=3.7907e-05) (hash(x)=54286330)
-3500 val loss 6.8254
-3500 val perplexity 920.9630
-3500 train 6.421352 (lr=5.3958e-05) (hash(x)=41135634)
-3700 val loss 7.3152
-3700 val perplexity 1502.9592
-3700 train 7.221657 (lr=3.7262e-05) (hash(x)=57061993)
-3600 val loss 6.8227
-3600 val perplexity 918.4611
-3600 train 6.826213 (lr=5.3070e-05) (hash(x)=54286330)
-3800 val loss 7.3070
-3800 val perplexity 1490.7235
-3800 train 6.914749 (lr=3.6608e-05) (hash(x)=46544884)
-3700 val loss 6.8169
-3700 val perplexity 913.1631
-3700 train 6.707649 (lr=5.2167e-05) (hash(x)=57061993)
-3900 val loss 7.3168
-3900 val perplexity 1505.3081
-3900 train 7.166756 (lr=3.5944e-05) (hash(x)=48937435)
-3800 val loss 6.8191
-3800 val perplexity 915.1304
-3800 train 6.409613 (lr=5.1251e-05) (hash(x)=46544884)
-4000 val loss 7.3382
-4000 val perplexity 1537.9268
-4000 train 7.348784 (lr=3.5271e-05) (hash(x)=54466186)
-3900 val loss 6.8155
-3900 val perplexity 911.8443
-3900 train 6.659271 (lr=5.0321e-05) (hash(x)=48937435)
-4100 val loss 7.2700
-4100 val perplexity 1436.5656
-4100 train 7.121221 (lr=3.4590e-05) (hash(x)=51079995)
-4000 val loss 6.7990
-4000 val perplexity 896.9698
-4000 train 6.853115 (lr=4.9379e-05) (hash(x)=54466186)
-4200 val loss 7.2588
-4200 val perplexity 1420.5469
-4200 train 7.411931 (lr=3.3902e-05) (hash(x)=56495581)
-4100 val loss 6.7819
-4100 val perplexity 881.7239
-4100 train 6.591974 (lr=4.8426e-05) (hash(x)=51079995)
-4300 val loss 7.2302
-4300 val perplexity 1380.4785
-4300 train 7.111370 (lr=3.3207e-05) (hash(x)=45095478)
-4200 val loss 6.7772
-4200 val perplexity 877.6279
-4200 train 6.916551 (lr=4.7463e-05) (hash(x)=56495581)
-4400 val loss 7.2184
-4400 val perplexity 1364.3369
-4400 train 7.173501 (lr=3.2507e-05) (hash(x)=49954028)
-4300 val loss 6.7472
-4300 val perplexity 851.6391
-4300 train 6.591402 (lr=4.6490e-05) (hash(x)=45095478)
-4500 val loss 7.2078
-4500 val perplexity 1349.9860
-4500 train 7.127753 (lr=3.1801e-05) (hash(x)=48975821)
-4400 val loss 6.7383
-4400 val perplexity 844.0901
-4400 train 6.701144 (lr=4.5509e-05) (hash(x)=49954028)
-4600 val loss 7.1827
-4600 val perplexity 1316.5165
-4600 train 7.240033 (lr=3.1091e-05) (hash(x)=49293028)
-4500 val loss 6.7848
-4500 val perplexity 884.3400
-4500 train 6.714043 (lr=4.4521e-05) (hash(x)=48975821)
-4700 val loss 7.1732
-4700 val perplexity 1304.0402
-4700 train 7.089901 (lr=3.0377e-05) (hash(x)=48077753)
-4600 val loss 6.6866
-4600 val perplexity 801.5901
-4600 train 6.766769 (lr=4.3527e-05) (hash(x)=49293028)
-4800 val loss 7.1638
-4800 val perplexity 1291.8126
-4800 train 6.971658 (lr=2.9661e-05) (hash(x)=45037879)
-4700 val loss 6.6807
-4700 val perplexity 796.8831
-4700 train 6.593471 (lr=4.2528e-05) (hash(x)=48077753)
-4900 val loss 7.1458
-4900 val perplexity 1268.8253
-4900 train 7.074956 (lr=2.8942e-05) (hash(x)=50320154)
-4800 val loss 6.6729
-4800 val perplexity 790.7224
-4800 train 6.540059 (lr=4.1525e-05) (hash(x)=45037879)
-5000 val loss 7.1412
-5000 val perplexity 1262.9335
-5000 train 7.337318 (lr=2.8221e-05) (hash(x)=55024523)
-4900 val loss 6.6512
-4900 val perplexity 773.7209
-4900 train 6.572750 (lr=4.0518e-05) (hash(x)=50320154)
-5100 val loss 7.1290
-5100 val perplexity 1247.6289
-5100 train 7.077831 (lr=2.7500e-05) (hash(x)=47570607)
-5000 val loss 6.6414
-5000 val perplexity 766.1722
-5000 train 6.879434 (lr=3.9510e-05) (hash(x)=55024523)
-5200 val loss 7.1245
-5200 val perplexity 1242.0054
-5200 train 7.220313 (lr=2.6779e-05) (hash(x)=54311349)
-5100 val loss 6.6317
-5100 val perplexity 758.7392
-5100 train 6.556490 (lr=3.8500e-05) (hash(x)=47570607)
-5300 val loss 7.1061
-5300 val perplexity 1219.3776
-5300 train 7.010005 (lr=2.6058e-05) (hash(x)=47178524)
-5200 val loss 6.6210
-5200 val perplexity 750.6971
-5200 train 6.755637 (lr=3.7490e-05) (hash(x)=54311349)
-5400 val loss 7.1027
-5400 val perplexity 1215.2343
-5400 train 7.118385 (lr=2.5339e-05) (hash(x)=49425088)
-5300 val loss 6.6153
-5300 val perplexity 746.3967
-5300 train 6.499031 (lr=3.6482e-05) (hash(x)=47178524)
-5500 val loss 7.0912
-5500 val perplexity 1201.3782
-5500 train 6.961625 (lr=2.4623e-05) (hash(x)=46383189)
-5400 val loss 6.6143
-5400 val perplexity 745.6650
-5400 train 6.656158 (lr=3.5475e-05) (hash(x)=49425088)
-5600 val loss 7.0903
-5600 val perplexity 1200.3011
-5600 train 7.287276 (lr=2.3909e-05) (hash(x)=54084990)
-5500 val loss 6.6047
-5500 val perplexity 738.5228
-5500 train 6.435539 (lr=3.4472e-05) (hash(x)=46383189)
-5700 val loss 7.0761
-5700 val perplexity 1183.2908
-5700 train 6.787469 (lr=2.3199e-05) (hash(x)=47384182)
-5600 val loss 6.5951
-5600 val perplexity 731.4849
-5800 val loss 7.0730
-5800 val perplexity 1179.7307
-5600 train 6.790871 (lr=3.3473e-05) (hash(x)=54084990)
-5800 train 6.992303 (lr=2.2493e-05) (hash(x)=51683744)
-5900 val loss 7.0678
-5900 val perplexity 1173.5835
-5700 val loss 6.5853
-5700 val perplexity 724.3849
-5900 train 7.068780 (lr=2.1793e-05) (hash(x)=52798836)
-5700 train 6.254418 (lr=3.2479e-05) (hash(x)=47384182)
-6000 val loss 7.0733
-6000 val perplexity 1179.9906
-6000 train 6.989652 (lr=2.1098e-05) (hash(x)=50945000)
-5800 val loss 6.5765
-5800 val perplexity 718.0475
-5800 train 6.463080 (lr=3.1491e-05) (hash(x)=51683744)
-6100 val loss 7.0610
-6100 val perplexity 1165.6578
-6100 train 6.829372 (lr=2.0410e-05) (hash(x)=48964427)
-5900 val loss 6.5714
-5900 val perplexity 714.3479
-5900 train 6.587032 (lr=3.0510e-05) (hash(x)=52798836)
-6200 val loss 7.0577
-6200 val perplexity 1161.8173
-6200 train 6.877470 (lr=1.9729e-05) (hash(x)=47148610)
-6000 val loss 6.5795
-6000 val perplexity 720.1752
-6000 train 6.458630 (lr=2.9537e-05) (hash(x)=50945000)
-6300 val loss 7.0582
-6300 val perplexity 1162.3004
-6300 train 6.992953 (lr=1.9056e-05) (hash(x)=50860553)
-6100 val loss 6.5734
-6100 val perplexity 715.7814
-6100 train 6.270368 (lr=2.8574e-05) (hash(x)=48964427)
-6400 val loss 7.0524
-6400 val perplexity 1155.6312
-6400 train 6.840018 (lr=1.8392e-05) (hash(x)=50553992)
-6200 val loss 6.5813
-6200 val perplexity 721.4669
-6200 train 6.344663 (lr=2.7621e-05) (hash(x)=47148610)
-6500 val loss 7.0418
-6500 val perplexity 1143.4537
-6500 train 6.901451 (lr=1.7738e-05) (hash(x)=52521332)
-6300 val loss 6.5646
-6300 val perplexity 709.5223
-6300 train 6.481687 (lr=2.6679e-05) (hash(x)=50860553)
-6600 val loss 7.0284
-6600 val perplexity 1128.1917
-6600 train 7.372125 (lr=1.7093e-05) (hash(x)=52609843)
-6400 val loss 6.5635
-6400 val perplexity 708.7479
-6400 train 6.332783 (lr=2.5749e-05) (hash(x)=50553992)
-6700 val loss 7.0170
-6700 val perplexity 1115.4137
-6700 train 7.073514 (lr=1.6459e-05) (hash(x)=53632957)
-6500 val loss 6.5573
-6500 val perplexity 704.3593
-6500 train 6.383666 (lr=2.4833e-05) (hash(x)=52521332)
-6800 val loss 7.0105
-6800 val perplexity 1108.2476
-6800 train 7.040189 (lr=1.5836e-05) (hash(x)=46927608)
-6600 val loss 6.5405
-6600 val perplexity 692.6306
-6600 train 6.884501 (lr=2.3930e-05) (hash(x)=52609843)
-6900 val loss 7.0013
-6900 val perplexity 1098.1124
-6900 train 7.049226 (lr=1.5225e-05) (hash(x)=54483057)
-6700 val loss 6.5188
-6700 val perplexity 677.7796
-6700 train 6.560914 (lr=2.3042e-05) (hash(x)=53632957)
-7000 val loss 6.9959
-7000 val perplexity 1092.1531
-7000 train 7.190527 (lr=1.4627e-05) (hash(x)=51493884)
-6800 val loss 6.5150
-6800 val perplexity 675.2225
-6800 train 6.525479 (lr=2.2171e-05) (hash(x)=46927608)
-7100 val loss 6.9874
-7100 val perplexity 1082.9354
-7100 train 7.136772 (lr=1.4043e-05) (hash(x)=53303341)
-6900 val loss 6.5077
-6900 val perplexity 670.2778
-6900 train 6.538913 (lr=2.1316e-05) (hash(x)=54483057)
-7200 val loss 6.9863
-7200 val perplexity 1081.6818
-7200 train 6.691040 (lr=1.3471e-05) (hash(x)=45272178)
-7000 val loss 6.4993
-7000 val perplexity 664.6807
-7000 train 6.666363 (lr=2.0478e-05) (hash(x)=51493884)
-7300 val loss 6.9778
-7300 val perplexity 1072.5254
-7300 train 6.909081 (lr=1.2915e-05) (hash(x)=50389872)
-7100 val loss 6.4939
-7100 val perplexity 661.1281
-7100 train 6.649939 (lr=1.9660e-05) (hash(x)=53303341)
-7400 val loss 6.9697
-7400 val perplexity 1063.8679
-7400 train 6.600302 (lr=1.2373e-05) (hash(x)=43796301)
-7200 val loss 6.4905
-7200 val perplexity 658.8703
-7200 train 6.180031 (lr=1.8860e-05) (hash(x)=45272178)
-7500 val loss 6.9651
-7500 val perplexity 1059.0161
-7500 train 6.886889 (lr=1.1847e-05) (hash(x)=47808686)
-7300 val loss 6.4860
-7300 val perplexity 655.9136
-7300 train 6.415273 (lr=1.8081e-05) (hash(x)=50389872)
-7600 val loss 6.9620
-7600 val perplexity 1055.6909
-7600 train 6.691191 (lr=1.1337e-05) (hash(x)=41936898)
-7400 val loss 6.4844
-7400 val perplexity 654.8358
-7400 train 6.127452 (lr=1.7323e-05) (hash(x)=43796301)
-7700 val loss 6.9674
-7700 val perplexity 1061.4114
-7700 train 7.146378 (lr=1.0844e-05) (hash(x)=57550318)
-7500 val loss 6.4803
-7500 val perplexity 652.1370
-7500 train 6.429677 (lr=1.6586e-05) (hash(x)=47808686)
-7800 val loss 6.9486
-7800 val perplexity 1041.6663
-7800 train 6.841320 (lr=1.0367e-05) (hash(x)=47485210)
-7600 val loss 6.4802
-7600 val perplexity 652.0792
-7600 train 6.200052 (lr=1.5872e-05) (hash(x)=41936898)
-7900 val loss 6.9450
-7900 val perplexity 1037.9823
-7900 train 7.281407 (lr=9.9088e-06) (hash(x)=53228688)
-7700 val loss 6.4803
-7700 val perplexity 652.1638
-7700 train 6.708580 (lr=1.5181e-05) (hash(x)=57550318)
-8000 val loss 6.9435
-8000 val perplexity 1036.3483
-8000 train 6.983154 (lr=9.4682e-06) (hash(x)=52018673)
-7800 val loss 6.4715
-7800 val perplexity 646.4274
-7800 train 6.346768 (lr=1.4514e-05) (hash(x)=47485210)
-8100 val loss 6.9386
-8100 val perplexity 1031.3220
-8100 train 6.757258 (lr=9.0461e-06) (hash(x)=47079349)
-7900 val loss 6.4728
-7900 val perplexity 647.3209
-7900 train 6.823822 (lr=1.3872e-05) (hash(x)=53228688)
-8200 val loss 6.9359
-8200 val perplexity 1028.5555
-8200 train 6.992998 (lr=8.6430e-06) (hash(x)=57921563)
-8000 val loss 6.4683
-8000 val perplexity 644.4054
-8000 train 6.525360 (lr=1.3255e-05) (hash(x)=52018673)
-8300 val loss 6.9300
-8300 val perplexity 1022.5231
-8300 train 6.509700 (lr=8.2593e-06) (hash(x)=45038933)
-8100 val loss 6.4681
-8100 val perplexity 644.2361
-8100 train 6.288201 (lr=1.2665e-05) (hash(x)=47079349)
-8400 val loss 6.9337
-8400 val perplexity 1026.2736
-8400 train 6.809709 (lr=7.8953e-06) (hash(x)=47763246)
-8200 val loss 6.4610
-8200 val perplexity 639.7166
-8200 train 6.510967 (lr=1.2100e-05) (hash(x)=57921563)
-8500 val loss 6.9268
-8500 val perplexity 1019.2207
-8500 train 6.919998 (lr=7.5515e-06) (hash(x)=56176595)
-8300 val loss 6.4590
-8300 val perplexity 638.4160
-8300 train 5.999636 (lr=1.1563e-05) (hash(x)=45038933)
-8600 val loss 6.9241
-8600 val perplexity 1016.4431
-8600 train 6.981564 (lr=7.2282e-06) (hash(x)=55184249)
-8400 val loss 6.4685
-8400 val perplexity 644.5172
-8400 train 6.311600 (lr=1.1053e-05) (hash(x)=47763246)
-8700 val loss 6.9220
-8700 val perplexity 1014.3809
-8700 train 6.897609 (lr=6.9257e-06) (hash(x)=46471646)
-8500 val loss 6.4614
-8500 val perplexity 639.9490
-8500 train 6.515769 (lr=1.0572e-05) (hash(x)=56176595)
-8800 val loss 6.9233
-8800 val perplexity 1015.7076
-8800 train 6.696762 (lr=6.6444e-06) (hash(x)=46233162)
-8600 val loss 6.4616
-8600 val perplexity 640.0672
-8600 train 6.535988 (lr=1.0119e-05) (hash(x)=55184249)
-8900 val loss 6.9317
-8900 val perplexity 1024.2495
-8900 train 6.854069 (lr=6.3845e-06) (hash(x)=47233684)
-8700 val loss 6.4591
-8700 val perplexity 638.4650
-8700 train 6.393760 (lr=9.6960e-06) (hash(x)=46471646)
-9000 val loss 6.9164
-9000 val perplexity 1008.7192
-9000 train 6.845828 (lr=6.1462e-06) (hash(x)=48374529)
-8800 val loss 6.4582
-8800 val perplexity 637.9197
-8800 train 6.175983 (lr=9.3021e-06) (hash(x)=46233162)
-9100 val loss 6.9079
-9100 val perplexity 1000.1794
-9100 train 6.852545 (lr=5.9300e-06) (hash(x)=48065371)
-8900 val loss 6.4608
-8900 val perplexity 639.6043
-8900 train 6.368966 (lr=8.9382e-06) (hash(x)=47233684)
-9200 val loss 6.9033
-9200 val perplexity 995.5312
-9200 train 6.823194 (lr=5.7359e-06) (hash(x)=47408078)
-9000 val loss 6.4567
-9000 val perplexity 636.9777
-9000 train 6.349148 (lr=8.6047e-06) (hash(x)=48374529)
-9300 val loss 6.9009
-9300 val perplexity 993.1728
-9300 train 6.908463 (lr=5.5641e-06) (hash(x)=50749781)
-9100 val loss 6.4471
-9100 val perplexity 630.8528
-9100 train 6.408804 (lr=8.3020e-06) (hash(x)=48065371)
-9400 val loss 6.8958
-9400 val perplexity 988.1155
-9400 train 7.069656 (lr=5.4149e-06) (hash(x)=48560169)
-9500 val loss 6.8921
-9500 val perplexity 984.4717
-9500 train 6.971089 (lr=5.2884e-06) (hash(x)=50936392)
-9200 val loss 6.4404
-9200 val perplexity 626.6569
-9200 train 6.438884 (lr=8.0302e-06) (hash(x)=47408078)
-9600 val loss 6.8880
-9600 val perplexity 980.4680
-9600 train 6.907552 (lr=5.1847e-06) (hash(x)=50651714)
-9300 val loss 6.4370
-9300 val perplexity 624.5002
-9300 train 6.419167 (lr=7.7898e-06) (hash(x)=50749781)
-9700 val loss 6.8876
-9700 val perplexity 980.0170
-9700 train 6.932286 (lr=5.1040e-06) (hash(x)=47311384)
-9400 val loss 6.4311
-9400 val perplexity 620.8464
-9400 train 6.620666 (lr=7.5809e-06) (hash(x)=48560169)
-9800 val loss 6.8858
-9800 val perplexity 978.2363
-9800 train 6.966930 (lr=5.0462e-06) (hash(x)=50921139)
-9500 val loss 6.4309
-9500 val perplexity 620.7466
-9500 train 6.564292 (lr=7.4038e-06) (hash(x)=50936392)
-9900 val loss 6.8855
-9900 val perplexity 977.9588
-9900 train 6.929035 (lr=5.0116e-06) (hash(x)=48142455)
-9600 val loss 6.4288
-9600 val perplexity 619.4521
-9600 train 6.451801 (lr=7.2586e-06) (hash(x)=50651714)
-9999 val loss 6.8814
-9999 val perplexity 973.9988
-9700 val loss 6.4277
-9700 val perplexity 618.7657
-9700 train 6.486131 (lr=7.1456e-06) (hash(x)=47311384)
-9800 val loss 6.4264
-9800 val perplexity 617.9319
-9800 train 6.528321 (lr=7.0647e-06) (hash(x)=50921139)
-9900 val loss 6.4334
-9900 val perplexity 622.3109
-9900 train 6.536893 (lr=7.0162e-06) (hash(x)=48142455)
-9999 val loss 6.4210
-9999 val perplexity 614.6432
+1100 val loss 7.5473
+1100 val perplexity 1895.5912
+1100 train 7.608188 (lr=9.8140e-05) (hash(x)=55387425)
+1200 val loss 7.5217
+1200 val perplexity 1847.7664
+1200 train 7.468529 (lr=9.7708e-05) (hash(x)=48328378)
+1300 val loss 7.4985
+1300 val perplexity 1805.4044
+1300 train 7.419605 (lr=9.7231e-05) (hash(x)=48956717)
+0 train 11.717561 (lr=7.5000e-07) (hash(x)=55241167)
+1400 val loss 7.5026
+1400 val perplexity 1812.7057
+1400 train 7.386037 (lr=9.6711e-05) (hash(x)=49782571)
+100 val loss 9.6965
+100 val perplexity 16261.2285
+100 train 9.628764 (lr=7.5750e-05) (hash(x)=46387919)
+1500 val loss 7.4773
+1500 val perplexity 1767.5096
+1500 train 7.258651 (lr=9.6149e-05) (hash(x)=51081994)
+200 val loss 8.0333
+200 val perplexity 3081.7817
+200 train 8.088078 (lr=1.5000e-04) (hash(x)=51276369)
+1600 val loss 7.4579
+1600 val perplexity 1733.5146
+1600 train 7.523229 (lr=9.5544e-05) (hash(x)=48791204)
+300 val loss 7.7162
+300 val perplexity 2244.4336
+300 train 7.639675 (lr=1.4997e-04) (hash(x)=48572728)
+1700 val loss 7.4387
+1700 val perplexity 1700.5919
+1700 train 7.161916 (lr=9.4897e-05) (hash(x)=46425011)
+400 val loss 7.6960
+400 val perplexity 2199.5452
+400 train 7.408287 (lr=1.4986e-04) (hash(x)=44943400)
+1800 val loss 7.4088
+1800 val perplexity 1650.4590
+1800 train 7.065498 (lr=9.4209e-05) (hash(x)=44919141)
+500 val loss 7.7190
+500 val perplexity 2250.7332
+500 train 7.623987 (lr=1.4969e-04) (hash(x)=48434590)
+1900 val loss 7.3505
+1900 val perplexity 1556.9557
+1900 train 7.455896 (lr=9.3481e-05) (hash(x)=48299675)
+600 val loss 7.6594
+600 val perplexity 2120.3806
+600 train 7.504135 (lr=1.4945e-04) (hash(x)=47442266)
+2000 val loss 7.3041
+2000 val perplexity 1486.4043
+2000 train 8.246761 (lr=9.2714e-05) (hash(x)=61804797)
+700 val loss 7.6184
+700 val perplexity 2035.3838
+700 train 7.673785 (lr=1.4913e-04) (hash(x)=56495712)
+2100 val loss 7.2584
+2100 val perplexity 1420.0369
+2100 train 7.342447 (lr=9.1908e-05) (hash(x)=53638030)
+800 val loss 7.5712
+800 val perplexity 1941.3823
+800 train 7.564387 (lr=1.4876e-04) (hash(x)=50093738)
+2200 val loss 7.2501
+2200 val perplexity 1408.2015
+2200 train 7.447799 (lr=9.1064e-05) (hash(x)=56364593)
+900 val loss 7.5101
+900 val perplexity 1826.3429
+900 train 7.466010 (lr=1.4831e-04) (hash(x)=49436061)
+2300 val loss 7.2280
+2300 val perplexity 1377.4703
+2300 train 7.307809 (lr=9.0182e-05) (hash(x)=52945580)
+1000 val loss 7.4440
+1000 val perplexity 1709.5248
+1000 train 7.373216 (lr=1.4779e-04) (hash(x)=48818282)
+2400 val loss 7.1955
+2400 val perplexity 1333.4473
+2400 train 7.284088 (lr=8.9265e-05) (hash(x)=54595211)
+1100 val loss 7.3930
+1100 val perplexity 1624.5348
+1100 train 7.437781 (lr=1.4721e-04) (hash(x)=55387425)
+2500 val loss 7.1905
+2500 val perplexity 1326.7396
+2500 train 7.198493 (lr=8.8313e-05) (hash(x)=52323363)
+1200 val loss 7.3864
+1200 val perplexity 1613.8221
+1200 train 7.297882 (lr=1.4656e-04) (hash(x)=48328378)
+2600 val loss 7.1661
+2600 val perplexity 1294.8401
+2600 train 6.969779 (lr=8.7326e-05) (hash(x)=42652707)
+1300 val loss 7.3559
+1300 val perplexity 1565.3617
+1300 train 7.271660 (lr=1.4585e-04) (hash(x)=48956717)
+2700 val loss 7.1652
+2700 val perplexity 1293.5731
+2700 train 6.893122 (lr=8.6306e-05) (hash(x)=45687855)
+1400 val loss 7.3621
+1400 val perplexity 1575.2109
+1400 train 7.228504 (lr=1.4507e-04) (hash(x)=49782571)
+2800 val loss 7.1404
+2800 val perplexity 1261.9781
+2800 train 7.139996 (lr=8.5254e-05) (hash(x)=49962830)
+1500 val loss 7.3566
+1500 val perplexity 1566.5236
+1500 train 7.131782 (lr=1.4422e-04) (hash(x)=51081994)
+2900 val loss 7.1124
+2900 val perplexity 1227.1224
+2900 train 7.157219 (lr=8.4170e-05) (hash(x)=52958451)
+1600 val loss 7.2937
+1600 val perplexity 1470.9889
+1600 train 7.357556 (lr=1.4332e-04) (hash(x)=48791204)
+3000 val loss 7.0789
+3000 val perplexity 1186.6748
+3000 train 7.020242 (lr=8.3057e-05) (hash(x)=48896990)
+1700 val loss 7.2649
+1700 val perplexity 1429.2798
+1700 train 6.956748 (lr=1.4235e-04) (hash(x)=46425011)
+3100 val loss 7.0680
+3100 val perplexity 1173.8483
+3100 train 7.037431 (lr=8.1915e-05) (hash(x)=47064930)
+1800 val loss 7.2177
+1800 val perplexity 1363.3796
+1800 train 6.878641 (lr=1.4131e-04) (hash(x)=44919141)
+3200 val loss 7.0466
+3200 val perplexity 1148.9044
+3200 train 6.800650 (lr=8.0745e-05) (hash(x)=45145921)
+1900 val loss 7.1876
+1900 val perplexity 1322.8931
+1900 train 7.273633 (lr=1.4022e-04) (hash(x)=48299675)
+3300 val loss 7.0223
+3300 val perplexity 1121.3967
+3300 train 6.873117 (lr=7.9549e-05) (hash(x)=48085018)
+2000 val loss 7.1324
+2000 val perplexity 1251.8201
+2000 train 7.983136 (lr=1.3907e-04) (hash(x)=61804797)
+3400 val loss 6.9982
+3400 val perplexity 1094.6818
+3400 train 7.005905 (lr=7.8328e-05) (hash(x)=49596053)
+2100 val loss 7.0736
+2100 val perplexity 1180.3406
+2100 train 7.182455 (lr=1.3786e-04) (hash(x)=53638030)
+3500 val loss 6.9891
+3500 val perplexity 1084.7898
+3500 train 6.591727 (lr=7.7082e-05) (hash(x)=41135634)
+2200 val loss 7.0376
+2200 val perplexity 1138.6738
+2200 train 7.247160 (lr=1.3660e-04) (hash(x)=56364593)
+3600 val loss 6.9701
+3600 val perplexity 1064.2880
+3600 train 6.984326 (lr=7.5814e-05) (hash(x)=54286330)
+2300 val loss 7.0180
+2300 val perplexity 1116.5631
+2300 train 7.111286 (lr=1.3527e-04) (hash(x)=52945580)
+3700 val loss 6.9778
+3700 val perplexity 1072.5734
+3700 train 6.879121 (lr=7.4525e-05) (hash(x)=57061993)
+2400 val loss 6.9654
+2400 val perplexity 1059.3838
+2400 train 7.038701 (lr=1.3390e-04) (hash(x)=54595211)
+3800 val loss 6.9521
+3800 val perplexity 1045.3574
+3800 train 6.536635 (lr=7.3215e-05) (hash(x)=46544884)
+2500 val loss 6.9307
+2500 val perplexity 1023.1781
+2500 train 6.927703 (lr=1.3247e-04) (hash(x)=52323363)
+3900 val loss 6.9418
+3900 val perplexity 1034.6000
+3900 train 6.773740 (lr=7.1887e-05) (hash(x)=48937435)
+2600 val loss 6.8754
+2600 val perplexity 968.1622
+2600 train 6.629884 (lr=1.3099e-04) (hash(x)=42652707)
+4000 val loss 6.9220
+4000 val perplexity 1014.3887
+4000 train 6.967721 (lr=7.0541e-05) (hash(x)=54466186)
+2700 val loss 6.8620
+2700 val perplexity 955.2903
+2700 train 6.563602 (lr=1.2946e-04) (hash(x)=45687855)
+4100 val loss 6.9034
+4100 val perplexity 995.6447
+4100 train 6.734161 (lr=6.9180e-05) (hash(x)=51079995)
+2800 val loss 6.8310
+2800 val perplexity 926.1013
+2800 train 6.844923 (lr=1.2788e-04) (hash(x)=49962830)
+4200 val loss 6.8986
+4200 val perplexity 990.8748
+4200 train 7.022826 (lr=6.7804e-05) (hash(x)=56495581)
+2900 val loss 6.8384
+2900 val perplexity 933.0102
+2900 train 6.857624 (lr=1.2626e-04) (hash(x)=52958451)
+4300 val loss 6.8568
+4300 val perplexity 950.3548
+4300 train 6.723518 (lr=6.6414e-05) (hash(x)=45095478)
+3000 val loss 6.7960
+3000 val perplexity 894.2563
+3000 train 6.743637 (lr=1.2459e-04) (hash(x)=48896990)
+4400 val loss 6.8469
+4400 val perplexity 940.9572
+4400 train 6.786045 (lr=6.5013e-05) (hash(x)=49954028)
+4500 val loss 6.8305
+4500 val perplexity 925.6757
+4500 train 6.764406 (lr=6.3602e-05) (hash(x)=48975821)
+3100 val loss 6.7878
+3100 val perplexity 887.0014
+3100 train 6.793625 (lr=1.2287e-04) (hash(x)=47064930)
+4600 val loss 6.8016
+4600 val perplexity 899.3201
+4600 train 6.895534 (lr=6.2182e-05) (hash(x)=49293028)
+3200 val loss 6.7811
+3200 val perplexity 881.0615
+3200 train 6.505876 (lr=1.2112e-04) (hash(x)=45145921)
+4700 val loss 6.7950
+4700 val perplexity 893.3932
+4700 train 6.722027 (lr=6.0754e-05) (hash(x)=48077753)
+3300 val loss 6.7835
+3300 val perplexity 883.1204
+3300 train 6.615720 (lr=1.1932e-04) (hash(x)=48085018)
+4800 val loss 6.7895
+4800 val perplexity 888.4474
+4800 train 6.651279 (lr=5.9321e-05) (hash(x)=45037879)
+3400 val loss 6.7838
+3400 val perplexity 883.4468
+3400 train 6.771408 (lr=1.1749e-04) (hash(x)=49596053)
+4900 val loss 6.7730
+4900 val perplexity 873.9567
+4900 train 6.689004 (lr=5.7883e-05) (hash(x)=50320154)
+3500 val loss 6.7643
+3500 val perplexity 866.3450
+3500 train 6.378889 (lr=1.1562e-04) (hash(x)=41135634)
+5000 val loss 6.7669
+5000 val perplexity 868.6503
+5000 train 6.983054 (lr=5.6442e-05) (hash(x)=55024523)
+3600 val loss 6.7291
+3600 val perplexity 836.3896
+3600 train 6.714438 (lr=1.1372e-04) (hash(x)=54286330)
+5100 val loss 6.7350
+5100 val perplexity 841.3091
+5100 train 6.674947 (lr=5.5000e-05) (hash(x)=47570607)
+3700 val loss 6.7328
+3700 val perplexity 839.4858
+3700 train 6.590438 (lr=1.1179e-04) (hash(x)=57061993)
+5200 val loss 6.7208
+5200 val perplexity 829.4543
+5200 train 6.841348 (lr=5.3558e-05) (hash(x)=54311349)
+3800 val loss 6.7147
+3800 val perplexity 824.4700
+3800 train 6.275104 (lr=1.0982e-04) (hash(x)=46544884)
+5300 val loss 6.7038
+5300 val perplexity 815.4897
+5300 train 6.609777 (lr=5.2117e-05) (hash(x)=47178524)
+3900 val loss 6.7199
+3900 val perplexity 828.7628
+3900 train 6.531300 (lr=1.0783e-04) (hash(x)=48937435)
+5400 val loss 6.6995
+5400 val perplexity 812.0095
+5400 train 6.718117 (lr=5.0679e-05) (hash(x)=49425088)
+4000 val loss 6.7199
+4000 val perplexity 828.7194
+4000 train 6.746758 (lr=1.0581e-04) (hash(x)=54466186)
+5500 val loss 6.6891
+5500 val perplexity 803.6185
+5500 train 6.531772 (lr=4.9246e-05) (hash(x)=46383189)
+4100 val loss 6.6825
+4100 val perplexity 798.3271
+4100 train 6.493564 (lr=1.0377e-04) (hash(x)=51079995)
+5600 val loss 6.6879
+5600 val perplexity 802.6320
+5600 train 6.864735 (lr=4.7818e-05) (hash(x)=54084990)
+4200 val loss 6.6782
+4200 val perplexity 794.8940
+4200 train 6.765399 (lr=1.0171e-04) (hash(x)=56495581)
+5700 val loss 6.6801
+5700 val perplexity 796.3665
+5700 train 6.365303 (lr=4.6398e-05) (hash(x)=47384182)
+4300 val loss 6.6532
+4300 val perplexity 775.2820
+4300 train 6.498065 (lr=9.9622e-05) (hash(x)=45095478)
+5800 val loss 6.6799
+5800 val perplexity 796.2286
+5800 train 6.573336 (lr=4.4987e-05) (hash(x)=51683744)
+4400 val loss 6.6397
+4400 val perplexity 764.8487
+4400 train 6.632647 (lr=9.7520e-05) (hash(x)=49954028)
+5900 val loss 6.6699
+5900 val perplexity 788.3209
+5900 train 6.679185 (lr=4.3586e-05) (hash(x)=52798836)
+4500 val loss 6.6403
+4500 val perplexity 765.3331
+4500 train 6.555310 (lr=9.5403e-05) (hash(x)=48975821)
+6000 val loss 6.6785
+6000 val perplexity 795.1404
+6000 train 6.579535 (lr=4.2196e-05) (hash(x)=50945000)
+4600 val loss 6.6170
+4600 val perplexity 747.6994
+4600 train 6.699085 (lr=9.3273e-05) (hash(x)=49293028)
+6100 val loss 6.6605
+6100 val perplexity 780.9373
+6100 train 6.376235 (lr=4.0820e-05) (hash(x)=48964427)
+4700 val loss 6.6228
+4700 val perplexity 752.0621
+6200 val loss 6.6637
+6200 val perplexity 783.4239
+4700 train 6.526177 (lr=9.1132e-05) (hash(x)=48077753)
+6200 train 6.446042 (lr=3.9459e-05) (hash(x)=47148610)
+6300 val loss 6.6562
+6300 val perplexity 777.5964
+6300 train 6.572690 (lr=3.8113e-05) (hash(x)=50860553)
+4800 val loss 6.6178
+4800 val perplexity 748.3293
+4800 train 6.518622 (lr=8.8982e-05) (hash(x)=45037879)
+6400 val loss 6.6432
+6400 val perplexity 767.5582
+6400 train 6.425728 (lr=3.6785e-05) (hash(x)=50553992)
+4900 val loss 6.5986
+4900 val perplexity 734.0506
+4900 train 6.506639 (lr=8.6825e-05) (hash(x)=50320154)
+6500 val loss 6.6449
+6500 val perplexity 768.8839
+6500 train 6.475725 (lr=3.5475e-05) (hash(x)=52521332)
+5000 val loss 6.5903
+5000 val perplexity 728.0233
+5000 train 6.842143 (lr=8.4663e-05) (hash(x)=55024523)
+6600 val loss 6.6217
+6600 val perplexity 751.2403
+6600 train 6.966245 (lr=3.4186e-05) (hash(x)=52609843)
+5100 val loss 6.5743
+5100 val perplexity 716.4435
+5100 train 6.487578 (lr=8.2500e-05) (hash(x)=47570607)
+6700 val loss 6.6003
+6700 val perplexity 735.3451
+6700 train 6.636974 (lr=3.2918e-05) (hash(x)=53632957)
+5200 val loss 6.5754
+5200 val perplexity 717.2426
+5200 train 6.693417 (lr=8.0337e-05) (hash(x)=54311349)
+6800 val loss 6.5942
+6800 val perplexity 730.8583
+6800 train 6.607226 (lr=3.1672e-05) (hash(x)=46927608)
+5300 val loss 6.5566
+5300 val perplexity 703.8453
+5300 train 6.460115 (lr=7.8175e-05) (hash(x)=47178524)
+6900 val loss 6.5882
+6900 val perplexity 726.4496
+6900 train 6.615112 (lr=3.0451e-05) (hash(x)=54483057)
+5400 val loss 6.5604
+5400 val perplexity 706.5711
+5400 train 6.593896 (lr=7.6018e-05) (hash(x)=49425088)
+7000 val loss 6.5774
+7000 val perplexity 718.6606
+7000 train 6.763172 (lr=2.9255e-05) (hash(x)=51493884)
+5500 val loss 6.5462
+5500 val perplexity 696.6125
+5500 train 6.380437 (lr=7.3868e-05) (hash(x)=46383189)
+7100 val loss 6.5651
+7100 val perplexity 709.8884
+7100 train 6.717602 (lr=2.8085e-05) (hash(x)=53303341)
+5600 val loss 6.5517
+5600 val perplexity 700.4420
+5600 train 6.718967 (lr=7.1727e-05) (hash(x)=54084990)
+7200 val loss 6.5605
+7200 val perplexity 706.6425
+7200 train 6.264062 (lr=2.6943e-05) (hash(x)=45272178)
+5700 val loss 6.5331
+5700 val perplexity 687.5358
+5700 train 6.188586 (lr=6.9597e-05) (hash(x)=47384182)
+7300 val loss 6.5546
+7300 val perplexity 702.4502
+7300 train 6.485512 (lr=2.5830e-05) (hash(x)=50389872)
+5800 val loss 6.5450
+5800 val perplexity 695.7803
+5800 train 6.420528 (lr=6.7480e-05) (hash(x)=51683744)
+7400 val loss 6.5482
+7400 val perplexity 698.0001
+7400 train 6.201846 (lr=2.4746e-05) (hash(x)=43796301)
+5900 val loss 6.5304
+5900 val perplexity 685.6703
+5900 train 6.529302 (lr=6.5378e-05) (hash(x)=52798836)
+7500 val loss 6.5417
+7500 val perplexity 693.4697
+7500 train 6.477395 (lr=2.3694e-05) (hash(x)=47808686)
+6000 val loss 6.5321
+6000 val perplexity 686.8110
+6000 train 6.415793 (lr=6.3294e-05) (hash(x)=50945000)
+7600 val loss 6.5397
+7600 val perplexity 692.0661
+7600 train 6.286121 (lr=2.2674e-05) (hash(x)=41936898)
+7700 val loss 6.5449
+7700 val perplexity 695.6599
+7700 train 6.755160 (lr=2.1687e-05) (hash(x)=57550318)
+6100 val loss 6.5229
+6100 val perplexity 680.5154
+6100 train 6.207138 (lr=6.1230e-05) (hash(x)=48964427)
+7800 val loss 6.5337
+7800 val perplexity 687.9454
+7800 train 6.407978 (lr=2.0735e-05) (hash(x)=47485210)
+6200 val loss 6.5283
+6200 val perplexity 684.2143
+6200 train 6.284555 (lr=5.9188e-05) (hash(x)=47148610)
+7900 val loss 6.5314
+7900 val perplexity 686.3399
+7900 train 6.898339 (lr=1.9818e-05) (hash(x)=53228688)
+6300 val loss 6.5233
+6300 val perplexity 680.8536
+6300 train 6.436521 (lr=5.7169e-05) (hash(x)=50860553)
+8000 val loss 6.5283
+8000 val perplexity 684.2267
+8000 train 6.578187 (lr=1.8936e-05) (hash(x)=52018673)
+6400 val loss 6.5221
+6400 val perplexity 679.9780
+6400 train 6.280699 (lr=5.5177e-05) (hash(x)=50553992)
+8100 val loss 6.5242
+8100 val perplexity 681.4324
+8100 train 6.353445 (lr=1.8092e-05) (hash(x)=47079349)
+6500 val loss 6.5162
+6500 val perplexity 675.9912
+6500 train 6.320277 (lr=5.3213e-05) (hash(x)=52521332)
+8200 val loss 6.5191
+8200 val perplexity 677.9484
+8200 train 6.559124 (lr=1.7286e-05) (hash(x)=57921563)
+6600 val loss 6.4862
+6600 val perplexity 656.0344
+6600 train 6.825990 (lr=5.1279e-05) (hash(x)=52609843)
+8300 val loss 6.5156
+8300 val perplexity 675.6302
+8300 train 6.059009 (lr=1.6519e-05) (hash(x)=45038933)
+6700 val loss 6.4759
+6700 val perplexity 649.3335
+6700 train 6.509449 (lr=4.9377e-05) (hash(x)=53632957)
+8400 val loss 6.5269
+8400 val perplexity 683.2643
+8400 train 6.374324 (lr=1.5791e-05) (hash(x)=47763246)
+6800 val loss 6.4683
+6800 val perplexity 644.3602
+6800 train 6.460664 (lr=4.7509e-05) (hash(x)=46927608)
+8500 val loss 6.5188
+8500 val perplexity 677.7935
+8500 train 6.563421 (lr=1.5103e-05) (hash(x)=56176595)
+6900 val loss 6.4662
+6900 val perplexity 643.0048
+6900 train 6.484306 (lr=4.5676e-05) (hash(x)=54483057)
+8600 val loss 6.5157
+8600 val perplexity 675.6860
+8600 train 6.585588 (lr=1.4456e-05) (hash(x)=55184249)
+7000 val loss 6.4562
+7000 val perplexity 636.6522
+7000 train 6.601372 (lr=4.3882e-05) (hash(x)=51493884)
+8700 val loss 6.5128
+8700 val perplexity 673.7424
+8700 train 6.452381 (lr=1.3851e-05) (hash(x)=46471646)
+7100 val loss 6.4422
+7100 val perplexity 627.7880
+7100 train 6.557719 (lr=4.2128e-05) (hash(x)=53303341)
+8800 val loss 6.5137
+8800 val perplexity 674.3239
+8800 train 6.222188 (lr=1.3289e-05) (hash(x)=46233162)
+7200 val loss 6.4518
+7200 val perplexity 633.8185
+7200 train 6.174304 (lr=4.0414e-05) (hash(x)=45272178)
+8900 val loss 6.5114
+8900 val perplexity 672.7723
+8900 train 6.433084 (lr=1.2769e-05) (hash(x)=47233684)
+7300 val loss 6.4358
+7300 val perplexity 623.7919
+7300 train 6.357533 (lr=3.8745e-05) (hash(x)=50389872)
+9000 val loss 6.5053
+9000 val perplexity 668.6711
+9000 train 6.403587 (lr=1.2292e-05) (hash(x)=48374529)
+7400 val loss 6.4336
+7400 val perplexity 622.3898
+7400 train 6.076984 (lr=3.7120e-05) (hash(x)=43796301)
+9100 val loss 6.4932
+9100 val perplexity 660.6213
+9100 train 6.443924 (lr=1.1860e-05) (hash(x)=48065371)
+7500 val loss 6.4282
+7500 val perplexity 619.0293
+9200 val loss 6.4862
+9200 val perplexity 656.0491
+9200 train 6.478733 (lr=1.1472e-05) (hash(x)=47408078)
+7500 train 6.375185 (lr=3.5541e-05) (hash(x)=47808686)
+9300 val loss 6.4858
+9300 val perplexity 655.7463
+9300 train 6.474623 (lr=1.1128e-05) (hash(x)=50749781)
+7600 val loss 6.4303
+7600 val perplexity 620.3874
+7600 train 6.170215 (lr=3.4011e-05) (hash(x)=41936898)
+9400 val loss 6.4775
+9400 val perplexity 650.3433
+9400 train 6.675423 (lr=1.0830e-05) (hash(x)=48560169)
+7700 val loss 6.4319
+7700 val perplexity 621.3718
+7700 train 6.636626 (lr=3.2531e-05) (hash(x)=57550318)
+9500 val loss 6.4770
+9500 val perplexity 650.0038
+9500 train 6.594353 (lr=1.0577e-05) (hash(x)=50936392)
+7800 val loss 6.4213
+7800 val perplexity 614.7798
+7800 train 6.287605 (lr=3.1102e-05) (hash(x)=47485210)
+9600 val loss 6.4729
+9600 val perplexity 647.3536
+9600 train 6.489072 (lr=1.0369e-05) (hash(x)=50651714)
+7900 val loss 6.4279
+7900 val perplexity 618.8944
+7900 train 6.767558 (lr=2.9726e-05) (hash(x)=53228688)
+9700 val loss 6.4730
+9700 val perplexity 647.3935
+9700 train 6.543466 (lr=1.0208e-05) (hash(x)=47311384)
+8000 val loss 6.4221
+8000 val perplexity 615.2637
+8000 train 6.469327 (lr=2.8405e-05) (hash(x)=52018673)
+9800 val loss 6.4721
+9800 val perplexity 646.8399
+9800 train 6.569210 (lr=1.0092e-05) (hash(x)=50921139)
+8100 val loss 6.4145
+8100 val perplexity 610.6134
+8100 train 6.244730 (lr=2.7138e-05) (hash(x)=47079349)
+9900 val loss 6.4766
+9900 val perplexity 649.7541
+9900 train 6.602388 (lr=1.0023e-05) (hash(x)=48142455)
+8200 val loss 6.4152
+8200 val perplexity 611.0695
+8200 train 6.430655 (lr=2.5929e-05) (hash(x)=57921563)
+9999 val loss 6.4663
+9999 val perplexity 643.0909
+8300 val loss 6.4102
+8300 val perplexity 607.9884
+8300 train 5.933331 (lr=2.4778e-05) (hash(x)=45038933)
+8400 val loss 6.4243
+8400 val perplexity 616.6609
+8400 train 6.261151 (lr=2.3686e-05) (hash(x)=47763246)
+8500 val loss 6.4191
+8500 val perplexity 613.4542
+8500 train 6.478737 (lr=2.2655e-05) (hash(x)=56176595)

attention_kindselective_n_heads2_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78ea3bcdb04430609e43ac1108a7b5dbf0450d0f99969c2ea13449b6a93310fb
+oid sha256:ae274553779f4440ebea69b957bc31051896ebee2d3258e12a570a1b866cbeeb
 size 38587970

attention_kindselective_n_heads2_seed1339/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:814089a077c8b23ef6a360929119aee50aedda8e9e4e98d0e7a7493c72ccf7f3
+oid sha256:26b8ade9b792c88be8ed413ee2fb84a1876a3caa9a0ff9735161613f62be9b07
 size 38587970

attention_kindselective_n_heads2_seed1339/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a220d7843b756c750dff05df665a2d3ac47d3a3d40239759a66d7d9ee0304d28
+oid sha256:108b45aea49deba3ee65399b9c6563aa2a3e5b3c9f4ff861fe0933f553ca579a
 size 38587970

attention_kindselective_n_heads2_seed1339/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e94c41ccd9890321419caef4669a9fe25ff3ccc0676eed6fdff9011850f032fc
+oid sha256:a57a691fc3cbc9e4af5a526d24a0a868b7545da36257b4a8a890e820e336a821
 size 38587970

attention_kindselective_n_heads2_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:461740930c03f23f42f01b5d8507f8a7b471d43e9d0f353d2581eb3d27820edb
+oid sha256:5ad72c0567bd0d1a62805941203c2f524d3cfdbbf3ba0188f4072436dcfbd8f9
 size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79cd2cab33452fea3f3ab1711cc885b686e94fd169470952d3f0ef5d133e5862
+oid sha256:226f99cb1aed7be3f245b40195adb0bb7cb364f7e3d93a2695839d21bf5a5a6c
 size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:135f20203befd7fa88831d72c8f43273b2a994f834af2da5d1f4459933fb3804
+oid sha256:a228f94ffeda43d25053c8d30e2a4cc11e7c0c35911468617171f4ab849dfcc4
 size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:621d654cf60841bfa9393523a818b267078c8a47c53b2db4e0c4625cdd9515ed
+oid sha256:dea87d02bb36351ad7cc4bbd01b389a9536dc03d472159789104875f6578d1e2
 size 70895430