Upload folder using huggingface_hub

attention_kindselective_n_heads8_seed1339/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads8_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 8, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2e-5_10240_8_1339", "n_embd": 512}

attention_kindselective_n_heads8_seed1339/dataloader_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
+size 964

attention_kindselective_n_heads8_seed1339/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f03ed2ebf741f15e13c79e6cc1e9a19b308450d81cc3b4d8d0338c63d77ca59
+size 964

attention_kindselective_n_heads8_seed1339/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82590037fb2eecbec961f7967a8dd1b8d85515d31a252f66b92b8139858a8b7c
+size 964

attention_kindselective_n_heads8_seed1339/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c544303717d12355a69b8ffb1eb109434e4fdccfd5a61279b6e8ba2e870d6700
+size 964

attention_kindselective_n_heads8_seed1339/log2.txt

@@ -0,0 +1,1209 @@
+max_steps: 10000
+0 val loss 11.0281
+0 val perplexity 61581.4062
+0 val loss 11.0281
+0 val perplexity 61581.6992
+0 val loss 11.0281
+0 val perplexity 61581.4062
+0 val loss 11.0281
+0 val perplexity 61582.3477
+0 train 11.019774 (lr=1.5000e-07) (hash(x)=55241167)
+0 train 11.019774 (lr=3.5000e-07) (hash(x)=55241167)
+0 train 11.019774 (lr=2.5000e-07) (hash(x)=55241167)
+0 train 11.019774 (lr=1.0000e-07) (hash(x)=55241167)
+100 val loss 9.7097
+100 val perplexity 16476.1328
+100 train 9.700400 (lr=1.5150e-05) (hash(x)=46387919)
+100 val loss 9.4717
+100 val perplexity 12986.8184
+100 val loss 9.5842
+100 val perplexity 14532.9561
+100 val loss 9.8417
+100 val perplexity 18802.3438
+100 train 9.463525 (lr=3.5350e-05) (hash(x)=46387919)
+100 train 9.577502 (lr=2.5250e-05) (hash(x)=46387919)
+100 train 9.837164 (lr=1.0100e-05) (hash(x)=46387919)
+200 val loss 9.0894
+200 val perplexity 8861.2422
+200 train 9.134879 (lr=3.0000e-05) (hash(x)=51276369)
+200 val loss 8.2561
+200 val perplexity 3851.0483
+200 train 8.325293 (lr=7.0000e-05) (hash(x)=51276369)
+200 val loss 8.1614
+200 val perplexity 3503.0369
+200 val loss 9.4203
+200 val perplexity 12335.9941
+200 train 8.220893 (lr=5.0000e-05) (hash(x)=51276369)
+200 train 9.459747 (lr=2.0000e-05) (hash(x)=51276369)
+300 val loss 8.1348
+300 val perplexity 3411.0425
+300 train 8.079185 (lr=2.9993e-05) (hash(x)=48572728)
+300 val loss 7.7588
+300 val perplexity 2342.0601
+300 train 7.683750 (lr=6.9984e-05) (hash(x)=48572728)
+300 val loss 8.6640
+300 val perplexity 5790.9355
+300 train 8.623919 (lr=1.9995e-05) (hash(x)=48572728)
+300 val loss 7.6377
+300 val perplexity 2075.0649
+300 train 7.547552 (lr=4.9988e-05) (hash(x)=48572728)
+400 val loss 7.7796
+400 val perplexity 2391.3811
+400 train 7.544574 (lr=2.9972e-05) (hash(x)=44943400)
+400 val loss 7.5697
+400 val perplexity 1938.5656
+400 train 7.291669 (lr=6.9935e-05) (hash(x)=44943400)
+400 val loss 8.0513
+400 val perplexity 3137.8838
+400 train 7.839612 (lr=1.9982e-05) (hash(x)=44943400)
+400 val loss 7.5305
+400 val perplexity 1864.0149
+400 train 7.253582 (lr=4.9954e-05) (hash(x)=44943400)
+500 val loss 7.6533
+500 val perplexity 2107.6501
+500 train 7.540147 (lr=2.9938e-05) (hash(x)=48434590)
+500 val loss 7.4707
+500 val perplexity 1755.8274
+500 train 7.338100 (lr=6.9854e-05) (hash(x)=48434590)
+500 val loss 7.7816
+500 val perplexity 2396.0164
+500 train 7.681769 (lr=1.9958e-05) (hash(x)=48434590)
+500 val loss 7.4639
+500 val perplexity 1743.9081
+500 train 7.340280 (lr=4.9896e-05) (hash(x)=48434590)
+600 val loss 7.5785
+600 val perplexity 1955.7003
+600 train 7.409949 (lr=2.9889e-05) (hash(x)=47442266)
+600 val loss 7.3962
+600 val perplexity 1629.8575
+600 val loss 7.6537
+600 val perplexity 2108.4563
+600 train 7.210290 (lr=6.9741e-05) (hash(x)=47442266)
+600 train 7.491193 (lr=1.9926e-05) (hash(x)=47442266)
+600 val loss 7.3922
+600 val perplexity 1623.2819
+600 train 7.205268 (lr=4.9815e-05) (hash(x)=47442266)
+700 val loss 7.5302
+700 val perplexity 1863.4373
+700 train 7.585056 (lr=2.9827e-05) (hash(x)=56495712)
+700 val loss 7.5849
+700 val perplexity 1968.2036
+700 val loss 7.3470
+700 val perplexity 1551.4893
+700 train 7.630278 (lr=1.9885e-05) (hash(x)=56495712)
+700 train 7.403419 (lr=6.9596e-05) (hash(x)=56495712)
+700 val loss 7.3310
+700 val perplexity 1526.8895
+700 train 7.399583 (lr=4.9712e-05) (hash(x)=56495712)
+800 val loss 7.4954
+800 val perplexity 1799.7959
+800 train 7.490317 (lr=2.9751e-05) (hash(x)=50093738)
+800 val loss 7.3051
+800 val perplexity 1487.8765
+800 val loss 7.5465
+800 val perplexity 1894.0831
+800 train 7.297331 (lr=6.9419e-05) (hash(x)=50093738)
+800 train 7.542716 (lr=1.9834e-05) (hash(x)=50093738)
+800 val loss 7.2942
+800 val perplexity 1471.7411
+800 train 7.273575 (lr=4.9585e-05) (hash(x)=50093738)
+900 val loss 7.4639
+900 val perplexity 1744.0095
+900 train 7.444237 (lr=2.9662e-05) (hash(x)=49436061)
+900 val loss 7.2642
+900 val perplexity 1428.2000
+900 val loss 7.5119
+900 val perplexity 1829.6874
+900 train 7.232059 (lr=6.9210e-05) (hash(x)=49436061)
+900 train 7.486966 (lr=1.9774e-05) (hash(x)=49436061)
+900 val loss 7.2523
+900 val perplexity 1411.3772
+900 train 7.216028 (lr=4.9436e-05) (hash(x)=49436061)
+1000 val loss 7.4476
+1000 val perplexity 1715.7721
+1000 train 7.375908 (lr=2.9558e-05) (hash(x)=48818282)
+1000 val loss 7.4869
+1000 val perplexity 1784.4542
+1000 val loss 7.2531
+1000 val perplexity 1412.5016
+1000 train 7.416676 (lr=1.9706e-05) (hash(x)=48818282)
+1000 train 7.184549 (lr=6.8970e-05) (hash(x)=48818282)
+1000 val loss 7.2196
+1000 val perplexity 1365.9194
+1000 train 7.146063 (lr=4.9264e-05) (hash(x)=48818282)
+1100 val loss 7.4264
+1100 val perplexity 1679.7290
+1100 train 7.496390 (lr=2.9442e-05) (hash(x)=55387425)
+1100 val loss 7.4593
+1100 val perplexity 1735.9408
+1100 val loss 7.2475
+1100 val perplexity 1404.6104
+1100 train 7.533914 (lr=1.9628e-05) (hash(x)=55387425)
+1100 train 7.310481 (lr=6.8698e-05) (hash(x)=55387425)
+1100 val loss 7.1829
+1100 val perplexity 1316.6659
+1100 train 7.246019 (lr=4.9070e-05) (hash(x)=55387425)
+1200 val loss 7.4108
+1200 val perplexity 1653.7014
+1200 train 7.301041 (lr=2.9312e-05) (hash(x)=48328378)
+1200 val loss 7.4398
+1200 val perplexity 1702.3777
+1200 train 7.347461 (lr=1.9542e-05) (hash(x)=48328378)
+1200 val loss 7.2117
+1200 val perplexity 1355.1605
+1200 train 7.083104 (lr=6.8395e-05) (hash(x)=48328378)
+1200 val loss 7.1377
+1200 val perplexity 1258.4778
+1200 train 6.990996 (lr=4.8854e-05) (hash(x)=48328378)
+1300 val loss 7.3916
+1300 val perplexity 1622.2303
+1300 train 7.309278 (lr=2.9169e-05) (hash(x)=48956717)
+1300 val loss 7.4174
+1300 val perplexity 1664.6868
+1300 train 7.348650 (lr=1.9446e-05) (hash(x)=48956717)
+1300 val loss 7.1916
+1300 val perplexity 1328.1700
+1300 train 7.095232 (lr=6.8062e-05) (hash(x)=48956717)
+1300 val loss 7.1050
+1300 val perplexity 1218.0630
+1300 train 6.989187 (lr=4.8616e-05) (hash(x)=48956717)
+1400 val loss 7.3837
+1400 val perplexity 1609.5293
+1400 train 7.285535 (lr=2.9013e-05) (hash(x)=49782571)
+1400 val loss 7.4088
+1400 val perplexity 1650.4393
+1400 train 7.314537 (lr=1.9342e-05) (hash(x)=49782571)
+1400 val loss 7.1905
+1400 val perplexity 1326.7339
+1400 train 7.081223 (lr=6.7698e-05) (hash(x)=49782571)
+1400 val loss 7.0826
+1400 val perplexity 1191.0359
+1400 train 6.962979 (lr=4.8356e-05) (hash(x)=49782571)
+1500 val loss 7.3719
+1500 val perplexity 1590.6541
+1500 train 7.180953 (lr=2.8845e-05) (hash(x)=51081994)
+1500 val loss 7.3955
+1500 val perplexity 1628.5710
+1500 train 7.199613 (lr=1.9230e-05) (hash(x)=51081994)
+1500 val loss 7.1761
+1500 val perplexity 1307.8132
+1500 train 6.960799 (lr=6.7304e-05) (hash(x)=51081994)
+1500 val loss 7.0469
+1500 val perplexity 1149.2792
+1500 train 6.789298 (lr=4.8074e-05) (hash(x)=51081994)
+1600 val loss 7.3619
+1600 val perplexity 1574.7723
+1600 train 7.437032 (lr=2.8663e-05) (hash(x)=48791204)
+1600 val loss 7.3823
+1600 val perplexity 1607.2921
+1600 train 7.458527 (lr=1.9109e-05) (hash(x)=48791204)
+1600 val loss 7.1631
+1600 val perplexity 1290.9561
+1600 train 7.228574 (lr=6.6881e-05) (hash(x)=48791204)
+1600 val loss 7.0259
+1600 val perplexity 1125.4299
+1600 train 7.080369 (lr=4.7772e-05) (hash(x)=48791204)
+1700 val loss 7.3512
+1700 val perplexity 1557.9872
+1700 train 7.064727 (lr=2.8469e-05) (hash(x)=46425011)
+1700 val loss 7.3648
+1700 val perplexity 1579.4250
+1700 train 7.079679 (lr=1.8979e-05) (hash(x)=46425011)
+1700 val loss 7.1553
+1700 val perplexity 1280.8689
+1700 train 6.839215 (lr=6.6428e-05) (hash(x)=46425011)
+1700 val loss 6.9937
+1700 val perplexity 1089.7976
+1700 train 6.657190 (lr=4.7448e-05) (hash(x)=46425011)
+1800 val loss 7.3486
+1800 val perplexity 1553.9711
+1800 train 7.003738 (lr=2.8263e-05) (hash(x)=44919141)
+1800 val loss 7.3539
+1800 val perplexity 1562.3319
+1800 train 7.010586 (lr=1.8842e-05) (hash(x)=44919141)
+1800 val loss 7.1675
+1800 val perplexity 1296.6541
+1800 train 6.812972 (lr=6.5947e-05) (hash(x)=44919141)
+1800 val loss 6.9646
+1800 val perplexity 1058.4592
+1800 train 6.574129 (lr=4.7105e-05) (hash(x)=44919141)
+1900 val loss 7.3377
+1900 val perplexity 1537.2053
+1900 train 7.431317 (lr=2.8044e-05) (hash(x)=48299675)
+1900 val loss 7.3345
+1900 val perplexity 1532.3342
+1900 train 7.430192 (lr=1.8696e-05) (hash(x)=48299675)
+1900 val loss 7.1367
+1900 val perplexity 1257.2877
+1900 val loss 6.9399
+1900 val perplexity 1032.6664
+1900 train 7.246999 (lr=6.5437e-05) (hash(x)=48299675)
+1900 train 7.059433 (lr=4.6741e-05) (hash(x)=48299675)
+2000 val loss 7.3206
+2000 val perplexity 1511.1768
+2000 train 8.112602 (lr=2.7814e-05) (hash(x)=61804797)
+2000 val loss 7.3156
+2000 val perplexity 1503.5886
+2000 train 8.138527 (lr=1.8543e-05) (hash(x)=61804797)
+2000 val loss 7.0982
+2000 val perplexity 1209.7937
+2000 val loss 6.9065
+2000 val perplexity 998.7697
+2000 train 7.933258 (lr=6.4900e-05) (hash(x)=61804797)
+2000 train 7.793611 (lr=4.6357e-05) (hash(x)=61804797)
+2100 val loss 7.3054
+2100 val perplexity 1488.3832
+2100 train 7.394943 (lr=2.7572e-05) (hash(x)=53638030)
+2100 val loss 7.2988
+2100 val perplexity 1478.5387
+2100 train 7.388790 (lr=1.8382e-05) (hash(x)=53638030)
+2100 val loss 7.0771
+2100 val perplexity 1184.5203
+2100 val loss 6.8708
+2100 val perplexity 963.7128
+2100 train 7.168970 (lr=6.4335e-05) (hash(x)=53638030)
+2100 train 6.969800 (lr=4.5954e-05) (hash(x)=53638030)
+2200 val loss 7.2858
+2200 val perplexity 1459.4324
+2200 train 7.543324 (lr=2.7319e-05) (hash(x)=56364593)
+2200 val loss 7.2805
+2200 val perplexity 1451.6970
+2200 train 7.541931 (lr=1.8213e-05) (hash(x)=56364593)
+2200 val loss 6.8395
+2200 val perplexity 934.0424
+2200 train 7.073711 (lr=4.5532e-05) (hash(x)=56364593)
+2200 val loss 7.0551
+2200 val perplexity 1158.7914
+2200 train 7.278820 (lr=6.3745e-05) (hash(x)=56364593)
+2300 val loss 7.2711
+2300 val perplexity 1438.1741
+2300 train 7.410802 (lr=2.7055e-05) (hash(x)=52945580)
+2300 val loss 7.2713
+2300 val perplexity 1438.3915
+2300 train 7.406998 (lr=1.8036e-05) (hash(x)=52945580)
+2300 val loss 6.8227
+2300 val perplexity 918.5023
+2300 train 6.977562 (lr=4.5091e-05) (hash(x)=52945580)
+2300 val loss 7.0360
+2300 val perplexity 1136.8781
+2300 train 7.169257 (lr=6.3128e-05) (hash(x)=52945580)
+2400 val loss 7.2607
+2400 val perplexity 1423.3226
+2400 train 7.313107 (lr=2.6780e-05) (hash(x)=54595211)
+2400 val loss 7.2538
+2400 val perplexity 1413.4213
+2400 train 7.296937 (lr=1.7853e-05) (hash(x)=54595211)
+2400 val loss 6.7938
+2400 val perplexity 892.2969
+2400 train 6.862321 (lr=4.4633e-05) (hash(x)=54595211)
+2400 val loss 7.0215
+2400 val perplexity 1120.5073
+2400 train 7.083766 (lr=6.2486e-05) (hash(x)=54595211)
+2500 val loss 7.2483
+2500 val perplexity 1405.7004
+2500 train 7.270072 (lr=2.6494e-05) (hash(x)=52323363)
+2500 val loss 7.2439
+2500 val perplexity 1399.5399
+2500 train 7.261842 (lr=1.7663e-05) (hash(x)=52323363)
+2500 val loss 6.7793
+2500 val perplexity 879.4120
+2500 val loss 7.0116
+2500 val perplexity 1109.4362
+2500 train 7.030132 (lr=6.1819e-05) (hash(x)=52323363)
+2500 train 6.792108 (lr=4.4156e-05) (hash(x)=52323363)
+2600 val loss 7.2389
+2600 val perplexity 1392.5264
+2600 train 6.972273 (lr=2.6198e-05) (hash(x)=42652707)
+2600 val loss 7.2260
+2600 val perplexity 1374.6908
+2600 train 6.964200 (lr=1.7465e-05) (hash(x)=42652707)
+2600 val loss 6.7583
+2600 val perplexity 861.1493
+2600 train 6.511856 (lr=4.3663e-05) (hash(x)=42652707)
+2600 val loss 7.0148
+2600 val perplexity 1113.0165
+2600 train 6.752173 (lr=6.1128e-05) (hash(x)=42652707)
+2700 val loss 7.2357
+2700 val perplexity 1388.0681
+2700 train 6.921242 (lr=2.5892e-05) (hash(x)=45687855)
+2700 val loss 7.2168
+2700 val perplexity 1362.0977
+2700 val loss 6.7482
+2700 val perplexity 852.5545
+2700 train 6.905418 (lr=1.7261e-05) (hash(x)=45687855)
+2700 train 6.414542 (lr=4.3153e-05) (hash(x)=45687855)
+2700 val loss 6.9994
+2700 val perplexity 1095.9656
+2700 train 6.681432 (lr=6.0414e-05) (hash(x)=45687855)
+2800 val loss 7.2221
+2800 val perplexity 1369.3673
+2800 train 7.194708 (lr=2.5576e-05) (hash(x)=49962830)
+2800 val loss 6.7228
+2800 val perplexity 831.1389
+2800 val loss 7.2040
+2800 val perplexity 1344.7562
+2800 train 6.732316 (lr=4.2627e-05) (hash(x)=49962830)
+2800 train 7.177933 (lr=1.7051e-05) (hash(x)=49962830)
+2800 val loss 7.0031
+2800 val perplexity 1099.9865
+2800 train 6.995296 (lr=5.9677e-05) (hash(x)=49962830)
+2900 val loss 7.2305
+2900 val perplexity 1380.9703
+2900 train 7.279706 (lr=2.5251e-05) (hash(x)=52958451)
+2900 val loss 6.7024
+2900 val perplexity 814.3538
+2900 val loss 7.1967
+2900 val perplexity 1335.0525
+2900 train 6.726287 (lr=4.2085e-05) (hash(x)=52958451)
+2900 train 7.253380 (lr=1.6834e-05) (hash(x)=52958451)
+2900 val loss 6.9826
+2900 val perplexity 1077.7458
+2900 train 7.006102 (lr=5.8919e-05) (hash(x)=52958451)
+3000 val loss 7.2079
+3000 val perplexity 1349.9924
+3000 train 7.143060 (lr=2.4917e-05) (hash(x)=48896990)
+3000 val loss 6.6892
+3000 val perplexity 803.7150
+3000 train 6.626854 (lr=4.1529e-05) (hash(x)=48896990)
+3000 val loss 7.1807
+3000 val perplexity 1313.8224
+3000 train 7.114826 (lr=1.6611e-05) (hash(x)=48896990)
+3000 val loss 6.9730
+3000 val perplexity 1067.4177
+3000 train 6.921494 (lr=5.8140e-05) (hash(x)=48896990)
+3100 val loss 7.2022
+3100 val perplexity 1342.3608
+3100 train 7.150681 (lr=2.4574e-05) (hash(x)=47064930)
+3100 val loss 6.6703
+3100 val perplexity 788.6352
+3100 train 6.664508 (lr=4.0957e-05) (hash(x)=47064930)
+3100 val loss 7.1689
+3100 val perplexity 1298.4149
+3100 train 7.118716 (lr=1.6383e-05) (hash(x)=47064930)
+3100 val loss 6.9743
+3100 val perplexity 1068.8025
+3100 train 6.925107 (lr=5.7340e-05) (hash(x)=47064930)
+3200 val loss 7.1856
+3200 val perplexity 1320.2545
+3200 train 6.912687 (lr=2.4224e-05) (hash(x)=45145921)
+3200 val loss 6.6590
+3200 val perplexity 779.7889
+3200 train 6.334053 (lr=4.0373e-05) (hash(x)=45145921)
+3200 val loss 7.1583
+3200 val perplexity 1284.7445
+3200 train 6.887141 (lr=1.6149e-05) (hash(x)=45145921)
+3200 val loss 6.9708
+3200 val perplexity 1065.0846
+3200 train 6.685108 (lr=5.6522e-05) (hash(x)=45145921)
+3300 val loss 7.1754
+3300 val perplexity 1306.8682
+3300 train 7.026076 (lr=2.3865e-05) (hash(x)=48085018)
+3300 val loss 6.6375
+3300 val perplexity 763.1765
+3300 train 6.467580 (lr=3.9775e-05) (hash(x)=48085018)
+3300 val loss 7.1463
+3300 val perplexity 1269.4426
+3300 train 6.998851 (lr=1.5910e-05) (hash(x)=48085018)
+3300 val loss 6.9690
+3300 val perplexity 1063.1316
+3300 train 6.808588 (lr=5.5684e-05) (hash(x)=48085018)
+3400 val loss 7.1656
+3400 val perplexity 1294.1870
+3400 train 7.203237 (lr=2.3498e-05) (hash(x)=49596053)
+3400 val loss 6.6357
+3400 val perplexity 761.8221
+3400 train 6.642577 (lr=3.9164e-05) (hash(x)=49596053)
+3400 val loss 7.1407
+3400 val perplexity 1262.3193
+3400 train 7.180227 (lr=1.5666e-05) (hash(x)=49596053)
+3400 val loss 6.9654
+3400 val perplexity 1059.3287
+3400 train 6.969168 (lr=5.4829e-05) (hash(x)=49596053)
+3500 val loss 7.1442
+3500 val perplexity 1266.8002
+3500 train 6.707533 (lr=2.3125e-05) (hash(x)=41135634)
+3500 val loss 6.6047
+3500 val perplexity 738.5925
+3500 train 6.213636 (lr=3.8541e-05) (hash(x)=41135634)
+3500 val loss 7.1232
+3500 val perplexity 1240.4121
+3500 train 6.685036 (lr=1.5416e-05) (hash(x)=41135634)
+3500 val loss 6.9724
+3500 val perplexity 1066.7339
+3500 train 6.579717 (lr=5.3958e-05) (hash(x)=41135634)
+3600 val loss 7.1358
+3600 val perplexity 1256.1060
+3600 train 7.137975 (lr=2.2744e-05) (hash(x)=54286330)
+3600 val loss 6.5831
+3600 val perplexity 722.7491
+3600 train 6.582256 (lr=3.7907e-05) (hash(x)=54286330)
+3600 val loss 7.1126
+3600 val perplexity 1227.3097
+3600 train 7.123320 (lr=1.5163e-05) (hash(x)=54286330)
+3600 val loss 6.9504
+3600 val perplexity 1043.5963
+3600 train 6.965904 (lr=5.3070e-05) (hash(x)=54286330)
+3700 val loss 7.1273
+3700 val perplexity 1245.5331
+3700 train 7.047629 (lr=2.2357e-05) (hash(x)=57061993)
+3700 val loss 6.5909
+3700 val perplexity 728.4212
+3700 train 6.439983 (lr=3.7262e-05) (hash(x)=57061993)
+3700 val loss 7.1064
+3700 val perplexity 1219.7939
+3700 train 7.037267 (lr=1.4905e-05) (hash(x)=57061993)
+3700 val loss 6.9561
+3700 val perplexity 1049.5469
+3700 train 6.876348 (lr=5.2167e-05) (hash(x)=57061993)
+3800 val loss 7.1073
+3800 val perplexity 1220.8978
+3800 train 6.682832 (lr=2.1965e-05) (hash(x)=46544884)
+3800 val loss 6.5705
+3800 val perplexity 713.7065
+3800 train 6.108221 (lr=3.6608e-05) (hash(x)=46544884)
+3800 val loss 7.0976
+3800 val perplexity 1209.0325
+3800 train 6.682347 (lr=1.4643e-05) (hash(x)=46544884)
+3800 val loss 6.9326
+3800 val perplexity 1025.1139
+3800 train 6.517674 (lr=5.1251e-05) (hash(x)=46544884)
+3900 val loss 7.0991
+3900 val perplexity 1210.8584
+3900 train 6.966492 (lr=2.1566e-05) (hash(x)=48937435)
+3900 val loss 6.5600
+3900 val perplexity 706.3019
+3900 train 6.408988 (lr=3.5944e-05) (hash(x)=48937435)
+3900 val loss 7.0855
+3900 val perplexity 1194.5229
+3900 train 6.953877 (lr=1.4377e-05) (hash(x)=48937435)
+3900 val loss 6.9300
+3900 val perplexity 1022.4562
+3900 train 6.793073 (lr=5.0321e-05) (hash(x)=48937435)
+4000 val loss 7.0941
+4000 val perplexity 1204.8864
+4000 train 7.138962 (lr=2.1162e-05) (hash(x)=54466186)
+4000 val loss 6.5605
+4000 val perplexity 706.6361
+4000 train 6.604020 (lr=3.5271e-05) (hash(x)=54466186)
+4000 val loss 7.0805
+4000 val perplexity 1188.5492
+4000 train 7.115824 (lr=1.4108e-05) (hash(x)=54466186)
+4000 val loss 6.9326
+4000 val perplexity 1025.1711
+4000 train 6.948184 (lr=4.9379e-05) (hash(x)=54466186)
+4100 val loss 7.0807
+4100 val perplexity 1188.8593
+4100 train 6.931328 (lr=2.0754e-05) (hash(x)=51079995)
+4100 val loss 6.5394
+4100 val perplexity 691.8721
+4100 train 6.342807 (lr=3.4590e-05) (hash(x)=51079995)
+4100 val loss 7.0660
+4100 val perplexity 1171.4478
+4100 train 6.912895 (lr=1.3836e-05) (hash(x)=51079995)
+4100 val loss 6.9133
+4100 val perplexity 1005.5554
+4100 train 6.757079 (lr=4.8426e-05) (hash(x)=51079995)
+4200 val loss 7.0770
+4200 val perplexity 1184.3700
+4200 train 7.238811 (lr=2.0341e-05) (hash(x)=56495581)
+4200 val loss 6.5314
+4200 val perplexity 686.3465
+4200 train 6.662952 (lr=3.3902e-05) (hash(x)=56495581)
+4200 val loss 7.0606
+4200 val perplexity 1165.1755
+4200 train 7.224622 (lr=1.3561e-05) (hash(x)=56495581)
+4200 val loss 6.9211
+4200 val perplexity 1013.4227
+4200 train 7.085524 (lr=4.7463e-05) (hash(x)=56495581)
+4300 val loss 7.0596
+4300 val perplexity 1163.9338
+4300 train 6.913306 (lr=1.9924e-05) (hash(x)=45095478)
+4300 val loss 6.5033
+4300 val perplexity 667.3305
+4300 train 6.326633 (lr=3.3207e-05) (hash(x)=45095478)
+4300 val loss 7.0338
+4300 val perplexity 1134.3417
+4300 train 6.887545 (lr=1.3283e-05) (hash(x)=45095478)
+4300 val loss 6.8885
+4300 val perplexity 980.9670
+4300 train 6.741586 (lr=4.6490e-05) (hash(x)=45095478)
+4400 val loss 7.0466
+4400 val perplexity 1148.9017
+4400 train 6.998871 (lr=1.9504e-05) (hash(x)=49954028)
+4400 val loss 6.4871
+4400 val perplexity 656.6105
+4400 train 6.504465 (lr=3.2507e-05) (hash(x)=49954028)
+4400 val loss 7.0249
+4400 val perplexity 1124.2563
+4400 train 6.952001 (lr=1.3003e-05) (hash(x)=49954028)
+4400 val loss 6.8805
+4400 val perplexity 973.1015
+4400 train 6.864935 (lr=4.5509e-05) (hash(x)=49954028)
+4500 val loss 7.0395
+4500 val perplexity 1140.8114
+4500 train 6.956896 (lr=1.9081e-05) (hash(x)=48975821)
+4500 val loss 6.4814
+4500 val perplexity 652.8572
+4500 train 6.403913 (lr=3.1801e-05) (hash(x)=48975821)
+4500 val loss 7.0197
+4500 val perplexity 1118.4393
+4500 train 6.932141 (lr=1.2720e-05) (hash(x)=48975821)
+4500 val loss 6.8784
+4500 val perplexity 971.0259
+4500 train 6.792545 (lr=4.4521e-05) (hash(x)=48975821)
+4600 val loss 7.0202
+4600 val perplexity 1119.0405
+4600 train 7.099740 (lr=1.8655e-05) (hash(x)=49293028)
+4600 val loss 6.4562
+4600 val perplexity 636.6337
+4600 train 6.527642 (lr=3.1091e-05) (hash(x)=49293028)
+4600 val loss 7.0010
+4600 val perplexity 1097.6952
+4600 train 7.073569 (lr=1.2436e-05) (hash(x)=49293028)
+4600 val loss 6.8440
+4600 val perplexity 938.2322
+4600 train 6.921262 (lr=4.3527e-05) (hash(x)=49293028)
+4700 val loss 7.0042
+4700 val perplexity 1101.2261
+4700 train 6.936911 (lr=1.8226e-05) (hash(x)=48077753)
+4700 val loss 6.4481
+4700 val perplexity 631.5083
+4700 train 6.353138 (lr=3.0377e-05) (hash(x)=48077753)
+4700 val loss 6.9942
+4700 val perplexity 1090.2617
+4700 train 6.922954 (lr=1.2151e-05) (hash(x)=48077753)
+4800 val loss 6.9923
+4800 val perplexity 1088.2677
+4800 train 6.816248 (lr=1.7796e-05) (hash(x)=45037879)
+4700 val loss 6.8457
+4700 val perplexity 939.7895
+4700 train 6.764469 (lr=4.2528e-05) (hash(x)=48077753)
+4800 val loss 6.4412
+4800 val perplexity 627.1833
+4800 train 6.349098 (lr=2.9661e-05) (hash(x)=45037879)
+4800 val loss 6.9846
+4800 val perplexity 1079.8929
+4800 train 6.818547 (lr=1.1864e-05) (hash(x)=45037879)
+4900 val loss 6.9870
+4900 val perplexity 1082.4578
+4900 train 6.892083 (lr=1.7365e-05) (hash(x)=50320154)
+4800 val loss 6.8367
+4800 val perplexity 931.3828
+4800 train 6.679591 (lr=4.1525e-05) (hash(x)=45037879)
+4900 val loss 6.4299
+4900 val perplexity 620.1333
+4900 train 6.340251 (lr=2.8942e-05) (hash(x)=50320154)
+4900 val loss 6.9748
+4900 val perplexity 1069.3352
+4900 train 6.880073 (lr=1.1577e-05) (hash(x)=50320154)
+5000 val loss 6.9746
+5000 val perplexity 1069.1670
+4900 val loss 6.8297
+4900 val perplexity 924.9001
+4900 train 6.744113 (lr=4.0518e-05) (hash(x)=50320154)
+5000 train 7.202687 (lr=1.6933e-05) (hash(x)=55024523)
+5000 val loss 6.4126
+5000 val perplexity 609.4899
+5000 train 6.674364 (lr=2.8221e-05) (hash(x)=55024523)
+5000 val loss 6.9676
+5000 val perplexity 1061.6569
+5000 train 7.191324 (lr=1.1288e-05) (hash(x)=55024523)
+5100 val loss 6.9586
+5100 val perplexity 1052.1821
+5100 train 6.892559 (lr=1.6500e-05) (hash(x)=47570607)
+5000 val loss 6.8270
+5000 val perplexity 922.4503
+5000 train 7.052451 (lr=3.9510e-05) (hash(x)=55024523)
+5100 val loss 6.3987
+5100 val perplexity 601.0562
+5100 train 6.313329 (lr=2.7500e-05) (hash(x)=47570607)
+5100 val loss 6.9547
+5100 val perplexity 1048.0336
+5100 train 6.890707 (lr=1.1000e-05) (hash(x)=47570607)
+5200 val loss 6.9542
+5200 val perplexity 1047.5540
+5200 train 7.065962 (lr=1.6067e-05) (hash(x)=54311349)
+5100 val loss 6.8143
+5100 val perplexity 910.7801
+5100 train 6.739725 (lr=3.8500e-05) (hash(x)=47570607)
+5200 val loss 6.3979
+5200 val perplexity 600.5731
+5200 train 6.529141 (lr=2.6779e-05) (hash(x)=54311349)
+5200 val loss 6.9452
+5200 val perplexity 1038.1858
+5200 train 7.059914 (lr=1.0712e-05) (hash(x)=54311349)
+5300 val loss 6.9438
+5300 val perplexity 1036.7140
+5300 train 6.836552 (lr=1.5635e-05) (hash(x)=47178524)
+5200 val loss 6.8222
+5200 val perplexity 917.9576
+5200 train 6.950159 (lr=3.7490e-05) (hash(x)=54311349)
+5300 val loss 6.3854
+5300 val perplexity 593.1091
+5300 train 6.271163 (lr=2.6058e-05) (hash(x)=47178524)
+5400 val loss 6.9342
+5400 val perplexity 1026.7831
+5300 val loss 6.9379
+5300 val perplexity 1030.6195
+5400 train 6.933376 (lr=1.5204e-05) (hash(x)=49425088)
+5300 train 6.834307 (lr=1.0423e-05) (hash(x)=47178524)
+5300 val loss 6.8130
+5300 val perplexity 909.6308
+5300 train 6.703873 (lr=3.6482e-05) (hash(x)=47178524)
+5400 val loss 6.3748
+5400 val perplexity 586.8486
+5400 train 6.402876 (lr=2.5339e-05) (hash(x)=49425088)
+5500 val loss 6.9271
+5500 val perplexity 1019.5517
+5500 train 6.762579 (lr=1.4774e-05) (hash(x)=46383189)
+5400 val loss 6.9295
+5400 val perplexity 1021.9454
+5400 train 6.937221 (lr=1.0136e-05) (hash(x)=49425088)
+5400 val loss 6.7971
+5400 val perplexity 895.2819
+5400 train 6.790603 (lr=3.5475e-05) (hash(x)=49425088)
+5500 val loss 6.3650
+5500 val perplexity 581.1469
+5500 train 6.185537 (lr=2.4623e-05) (hash(x)=46383189)
+5600 val loss 6.9271
+5600 val perplexity 1019.5128
+5600 train 7.123457 (lr=1.4345e-05) (hash(x)=54084990)
+5500 val loss 6.9233
+5500 val perplexity 1015.6611
+5500 train 6.753473 (lr=9.8491e-06) (hash(x)=46383189)
+5500 val loss 6.8003
+5500 val perplexity 898.1262
+5500 train 6.640832 (lr=3.4472e-05) (hash(x)=46383189)
+5600 val loss 6.3704
+5600 val perplexity 584.3188
+5600 train 6.532243 (lr=2.3909e-05) (hash(x)=54084990)
+5700 val loss 6.9151
+5700 val perplexity 1007.4103
+5700 train 6.593955 (lr=1.3919e-05) (hash(x)=47384182)
+5600 val loss 6.9220
+5600 val perplexity 1014.3364
+5600 train 7.129646 (lr=9.5636e-06) (hash(x)=54084990)
+5600 val loss 6.8021
+5600 val perplexity 899.7040
+5600 train 7.001111 (lr=3.3473e-05) (hash(x)=54084990)
+5700 val loss 6.3494
+5700 val perplexity 572.1433
+5700 train 6.033447 (lr=2.3199e-05) (hash(x)=47384182)
+5800 val loss 6.9078
+5800 val perplexity 1000.0802
+5800 train 6.791445 (lr=1.3496e-05) (hash(x)=51683744)
+5700 val loss 6.9151
+5700 val perplexity 1007.3843
+5700 train 6.592525 (lr=9.2796e-06) (hash(x)=47384182)
+5700 val loss 6.7996
+5700 val perplexity 897.4965
+5700 train 6.484940 (lr=3.2479e-05) (hash(x)=47384182)
+5800 val loss 6.3438
+5800 val perplexity 568.9437
+5800 train 6.210127 (lr=2.2493e-05) (hash(x)=51683744)
+5900 val loss 6.9021
+5900 val perplexity 994.3759
+5900 train 6.890203 (lr=1.3076e-05) (hash(x)=52798836)
+5800 val loss 6.9085
+5800 val perplexity 1000.7233
+5800 train 6.792042 (lr=8.9973e-06) (hash(x)=51683744)
+5800 val loss 6.7971
+5800 val perplexity 895.2123
+5800 train 6.675380 (lr=3.1491e-05) (hash(x)=51683744)
+5900 val loss 6.3346
+5900 val perplexity 563.7436
+5900 train 6.348949 (lr=2.1793e-05) (hash(x)=52798836)
+6000 val loss 6.8993
+6000 val perplexity 991.6287
+6000 train 6.782576 (lr=1.2659e-05) (hash(x)=50945000)
+5900 val loss 6.9018
+5900 val perplexity 994.1043
+5900 train 6.886106 (lr=8.7171e-06) (hash(x)=52798836)
+5900 val loss 6.7984
+5900 val perplexity 896.4216
+5900 train 6.783849 (lr=3.0510e-05) (hash(x)=52798836)
+6000 val loss 6.3380
+6000 val perplexity 565.6832
+6000 train 6.199138 (lr=2.1098e-05) (hash(x)=50945000)
+6100 val loss 6.8996
+6100 val perplexity 991.9120
+6100 train 6.640050 (lr=1.2246e-05) (hash(x)=48964427)
+6000 val loss 6.8999
+6000 val perplexity 992.2005
+6000 train 6.782024 (lr=8.4393e-06) (hash(x)=50945000)
+6000 val loss 6.7892
+6000 val perplexity 888.1768
+6000 train 6.678681 (lr=2.9537e-05) (hash(x)=50945000)
+6100 val loss 6.3275
+6100 val perplexity 559.7340
+6100 train 6.017313 (lr=2.0410e-05) (hash(x)=48964427)
+6200 val loss 6.8946
+6200 val perplexity 986.9628
+6200 train 6.686383 (lr=1.1838e-05) (hash(x)=47148610)
+6100 val loss 6.8952
+6100 val perplexity 987.5494
+6100 train 6.639012 (lr=8.1640e-06) (hash(x)=48964427)
+6100 val loss 6.7925
+6100 val perplexity 891.1420
+6100 train 6.528467 (lr=2.8574e-05) (hash(x)=48964427)
+6200 val loss 6.3256
+6200 val perplexity 558.7103
+6200 train 6.094476 (lr=1.9729e-05) (hash(x)=47148610)
+6300 val loss 6.8914
+6300 val perplexity 983.7978
+6300 train 6.825513 (lr=1.1434e-05) (hash(x)=50860553)
+6200 val loss 6.8907
+6200 val perplexity 983.0869
+6200 train 6.685598 (lr=7.8917e-06) (hash(x)=47148610)
+6200 val loss 6.7949
+6200 val perplexity 893.2441
+6200 train 6.577856 (lr=2.7621e-05) (hash(x)=47148610)
+6300 val loss 6.3159
+6300 val perplexity 553.2820
+6300 train 6.251698 (lr=1.9056e-05) (hash(x)=50860553)
+6400 val loss 6.8844
+6400 val perplexity 976.9618
+6400 train 6.688293 (lr=1.1035e-05) (hash(x)=50553992)
+6300 val loss 6.8884
+6300 val perplexity 980.7968
+6300 train 6.821770 (lr=7.6226e-06) (hash(x)=50860553)
+6300 val loss 6.7858
+6300 val perplexity 885.1630
+6300 train 6.732771 (lr=2.6679e-05) (hash(x)=50860553)
+6400 val loss 6.3137
+6400 val perplexity 552.0735
+6400 train 6.103390 (lr=1.8392e-05) (hash(x)=50553992)
+6500 val loss 6.8819
+6500 val perplexity 974.4667
+6500 train 6.751459 (lr=1.0643e-05) (hash(x)=52521332)
+6400 val loss 6.8830
+6400 val perplexity 975.5029
+6400 train 6.691499 (lr=7.3569e-06) (hash(x)=50553992)
+6400 val loss 6.7848
+6400 val perplexity 884.3045
+6400 train 6.593803 (lr=2.5749e-05) (hash(x)=50553992)
+6500 val loss 6.3159
+6500 val perplexity 553.3113
+6500 train 6.148011 (lr=1.7738e-05) (hash(x)=52521332)
+6600 val loss 6.8706
+6600 val perplexity 963.4868
+6600 train 7.219240 (lr=1.0256e-05) (hash(x)=52609843)
+6500 val loss 6.8780
+6500 val perplexity 970.6671
+6500 train 6.759769 (lr=7.0950e-06) (hash(x)=52521332)
+6500 val loss 6.7864
+6500 val perplexity 885.7056
+6500 train 6.640779 (lr=2.4833e-05) (hash(x)=52521332)
+6600 val loss 6.2923
+6600 val perplexity 540.3746
+6600 train 6.663771 (lr=1.7093e-05) (hash(x)=52609843)
+6700 val loss 6.8572
+6700 val perplexity 950.7396
+6700 train 6.936516 (lr=9.8753e-06) (hash(x)=53632957)
+6600 val loss 6.8663
+6600 val perplexity 959.3960
+6600 train 7.209294 (lr=6.8372e-06) (hash(x)=52609843)
+6600 val loss 6.7701
+6600 val perplexity 871.4236
+6600 train 7.132092 (lr=2.3930e-05) (hash(x)=52609843)
+6700 val loss 6.2836
+6700 val perplexity 535.7344
+6700 train 6.338120 (lr=1.6459e-05) (hash(x)=53632957)
+6800 val loss 6.8541
+6800 val perplexity 947.7671
+6800 train 6.899356 (lr=9.5017e-06) (hash(x)=46927608)
+6700 val loss 6.8581
+6700 val perplexity 951.5292
+6700 train 6.933838 (lr=6.5835e-06) (hash(x)=53632957)
+6700 val loss 6.7589
+6700 val perplexity 861.6755
+6700 train 6.818271 (lr=2.3042e-05) (hash(x)=53632957)
+6800 val loss 6.2808
+6800 val perplexity 534.2219
+6800 train 6.318456 (lr=1.5836e-05) (hash(x)=46927608)
+6900 val loss 6.8460
+6900 val perplexity 940.1440
+6900 train 6.898056 (lr=9.1353e-06) (hash(x)=54483057)
+6800 val loss 6.8557
+6800 val perplexity 949.2742
+6800 train 6.893238 (lr=6.3345e-06) (hash(x)=46927608)
+6800 val loss 6.7582
+6800 val perplexity 861.1172
+6800 train 6.799270 (lr=2.2171e-05) (hash(x)=46927608)
+6900 val loss 6.2722
+6900 val perplexity 529.6454
+6900 train 6.280756 (lr=1.5225e-05) (hash(x)=54483057)
+7000 val loss 6.8406
+7000 val perplexity 935.0188
+7000 train 7.017616 (lr=8.7764e-06) (hash(x)=51493884)
+6900 val loss 6.8503
+6900 val perplexity 944.1887
+6900 train 6.903790 (lr=6.0902e-06) (hash(x)=54483057)
+6900 val loss 6.7468
+6900 val perplexity 851.3662
+6900 train 6.786250 (lr=2.1316e-05) (hash(x)=54483057)
+7000 val loss 6.2622
+7000 val perplexity 524.3632
+7000 train 6.384635 (lr=1.4627e-05) (hash(x)=51493884)
+7100 val loss 6.8341
+7100 val perplexity 928.9899
+7100 train 6.986980 (lr=8.4255e-06) (hash(x)=53303341)
+7000 val loss 6.8478
+7000 val perplexity 941.8038
+7000 train 7.024706 (lr=5.8510e-06) (hash(x)=51493884)
+7000 val loss 6.7401
+7000 val perplexity 845.6735
+7000 train 6.893335 (lr=2.0478e-05) (hash(x)=51493884)
+7100 val loss 6.2533
+7100 val perplexity 519.7358
+7100 train 6.410233 (lr=1.4043e-05) (hash(x)=53303341)
+7200 val loss 6.8317
+7200 val perplexity 926.7962
+7200 train 6.506523 (lr=8.0829e-06) (hash(x)=45272178)
+7100 val loss 6.8382
+7100 val perplexity 932.8452
+7100 train 6.992830 (lr=5.6170e-06) (hash(x)=53303341)
+7100 val loss 6.7416
+7100 val perplexity 846.8945
+7100 train 6.887737 (lr=1.9660e-05) (hash(x)=53303341)
+7200 val loss 6.2521
+7200 val perplexity 519.1124
+7200 train 5.923408 (lr=1.3471e-05) (hash(x)=45272178)
+7300 val loss 6.8276
+7300 val perplexity 922.9598
+7300 train 6.765221 (lr=7.7489e-06) (hash(x)=50389872)
+7200 val loss 6.8349
+7200 val perplexity 929.7535
+7200 train 6.511398 (lr=5.3886e-06) (hash(x)=45272178)
+7200 val loss 6.7386
+7200 val perplexity 844.3676
+7200 train 6.421773 (lr=1.8860e-05) (hash(x)=45272178)
+7300 val loss 6.2463
+7300 val perplexity 516.0956
+7300 train 6.172292 (lr=1.2915e-05) (hash(x)=50389872)
+7400 val loss 6.8232
+7400 val perplexity 918.9180
+7400 train 6.427763 (lr=7.4239e-06) (hash(x)=43796301)
+7300 val loss 6.8288
+7300 val perplexity 924.1048
+7300 train 6.762436 (lr=5.1659e-06) (hash(x)=50389872)
+7300 val loss 6.7364
+7300 val perplexity 842.5316
+7300 train 6.675917 (lr=1.8081e-05) (hash(x)=50389872)
+7400 val loss 6.2405
+7400 val perplexity 513.1138
+7400 train 5.857545 (lr=1.2373e-05) (hash(x)=43796301)
+7500 val loss 6.8176
+7500 val perplexity 913.7742
+7500 train 6.729806 (lr=7.1083e-06) (hash(x)=47808686)
+7400 val loss 6.8260
+7400 val perplexity 921.4642
+7400 train 6.432824 (lr=4.9493e-06) (hash(x)=43796301)
+7400 val loss 6.7280
+7400 val perplexity 835.4955
+7400 train 6.339929 (lr=1.7323e-05) (hash(x)=43796301)
+7500 val loss 6.2347
+7500 val perplexity 510.1292
+7500 train 6.186483 (lr=1.1847e-05) (hash(x)=47808686)
+7600 val loss 6.8152
+7600 val perplexity 911.5635
+7600 train 6.492714 (lr=6.8023e-06) (hash(x)=41936898)
+7500 val loss 6.8228
+7500 val perplexity 918.5723
+7500 train 6.743688 (lr=4.7389e-06) (hash(x)=47808686)
+7500 val loss 6.7312
+7500 val perplexity 838.1662
+7500 train 6.651219 (lr=1.6586e-05) (hash(x)=47808686)
+7600 val loss 6.2323
+7600 val perplexity 508.9238
+7600 train 5.926254 (lr=1.1337e-05) (hash(x)=41936898)
+7700 val loss 6.8156
+7700 val perplexity 911.9543
+7700 train 7.035799 (lr=6.5062e-06) (hash(x)=57550318)
+7600 val loss 6.8198
+7600 val perplexity 915.7760
+7600 train 6.489659 (lr=4.5349e-06) (hash(x)=41936898)
+7600 val loss 6.7244
+7600 val perplexity 832.4652
+7600 train 6.405721 (lr=1.5872e-05) (hash(x)=41936898)
+7700 val loss 6.2320
+7700 val perplexity 508.7766
+7700 train 6.498176 (lr=1.0844e-05) (hash(x)=57550318)
+7800 val loss 6.8077
+7800 val perplexity 904.7552
+7800 train 6.667345 (lr=6.2205e-06) (hash(x)=47485210)
+7700 val loss 6.8177
+7700 val perplexity 913.8815
+7700 train 7.047784 (lr=4.3375e-06) (hash(x)=57550318)
+7700 val loss 6.7234
+7700 val perplexity 831.6507
+7700 train 6.950782 (lr=1.5181e-05) (hash(x)=57550318)
+7800 val loss 6.2312
+7800 val perplexity 508.3616
+7800 train 6.076318 (lr=1.0367e-05) (hash(x)=47485210)
+7900 val loss 6.8062
+7900 val perplexity 903.4692
+7900 train 7.151995 (lr=5.9453e-06) (hash(x)=53228688)
+7800 val loss 6.8134
+7800 val perplexity 909.9406
+7800 train 6.677879 (lr=4.1470e-06) (hash(x)=47485210)
+7800 val loss 6.7184
+7800 val perplexity 827.5126
+7800 train 6.569107 (lr=1.4514e-05) (hash(x)=47485210)
+7900 val loss 6.2215
+7900 val perplexity 503.4525
+7900 train 6.591218 (lr=9.9088e-06) (hash(x)=53228688)
+8000 val loss 6.8044
+8000 val perplexity 901.7992
+8000 train 6.852820 (lr=5.6809e-06) (hash(x)=52018673)
+7900 val loss 6.8108
+7900 val perplexity 907.6062
+7900 train 7.159123 (lr=3.9635e-06) (hash(x)=53228688)
+7900 val loss 6.7168
+7900 val perplexity 826.1689
+7900 train 7.056360 (lr=1.3872e-05) (hash(x)=53228688)
+8000 val loss 6.2179
+8000 val perplexity 501.6553
+8000 train 6.293939 (lr=9.4682e-06) (hash(x)=52018673)
+8100 val loss 6.7997
+8100 val perplexity 897.6198
+8100 train 6.593906 (lr=5.4277e-06) (hash(x)=47079349)
+8000 val loss 6.8103
+8000 val perplexity 907.1532
+8000 train 6.872995 (lr=3.7873e-06) (hash(x)=52018673)
+8000 val loss 6.7181
+8000 val perplexity 827.2534
+8000 train 6.770758 (lr=1.3255e-05) (hash(x)=52018673)
+8100 val loss 6.2129
+8100 val perplexity 499.1537
+8100 train 6.023364 (lr=9.0461e-06) (hash(x)=47079349)
+8200 val loss 6.7971
+8200 val perplexity 895.2606
+8200 train 6.860888 (lr=5.1858e-06) (hash(x)=57921563)
+8100 val loss 6.8062
+8100 val perplexity 903.4011
+8100 train 6.600097 (lr=3.6184e-06) (hash(x)=47079349)
+8100 val loss 6.7129
+8100 val perplexity 822.9833
+8100 train 6.511360 (lr=1.2665e-05) (hash(x)=47079349)
+8200 val loss 6.2123
+8200 val perplexity 498.8536
+8200 train 6.274234 (lr=8.6430e-06) (hash(x)=57921563)
+8300 val loss 6.7956
+8300 val perplexity 893.9357
+8300 train 6.333303 (lr=4.9556e-06) (hash(x)=45038933)
+8200 val loss 6.8044
+8200 val perplexity 901.7893
+8200 train 6.886234 (lr=3.4572e-06) (hash(x)=57921563)
+8200 val loss 6.7100
+8200 val perplexity 820.5785
+8200 train 6.749637 (lr=1.2100e-05) (hash(x)=57921563)
+8300 val loss 6.2076
+8300 val perplexity 496.5031
+8300 train 5.727176 (lr=8.2593e-06) (hash(x)=45038933)
+8400 val loss 6.8014
+8400 val perplexity 899.1366
+8400 train 6.681006 (lr=4.7372e-06) (hash(x)=47763246)
+8300 val loss 6.8026
+8300 val perplexity 900.1713
+8300 train 6.344654 (lr=3.3037e-06) (hash(x)=45038933)
+8400 val loss 6.2179
+8400 val perplexity 501.6299
+8400 train 6.083429 (lr=7.8953e-06) (hash(x)=47763246)
+8300 val loss 6.7081
+8300 val perplexity 818.9875
+8300 train 6.249794 (lr=1.1563e-05) (hash(x)=45038933)
+8500 val loss 6.7949
+8500 val perplexity 893.3076
+8500 train 6.825691 (lr=4.5309e-06) (hash(x)=56176595)
+8400 val loss 6.8055
+8400 val perplexity 902.7965
+8400 train 6.680042 (lr=3.1581e-06) (hash(x)=47763246)
+8500 val loss 6.2090
+8500 val perplexity 497.1946
+8500 train 6.305939 (lr=7.5515e-06) (hash(x)=56176595)
+8400 val loss 6.7128
+8400 val perplexity 822.8507
+8400 train 6.586521 (lr=1.1053e-05) (hash(x)=47763246)
+8600 val loss 6.7920
+8600 val perplexity 890.7125
+8600 train 6.854806 (lr=4.3369e-06) (hash(x)=55184249)
+8500 val loss 6.8023
+8500 val perplexity 899.9142
+8500 train 6.834250 (lr=3.0206e-06) (hash(x)=56176595)
+8600 val loss 6.2066
+8600 val perplexity 496.0045
+8600 train 6.306992 (lr=7.2282e-06) (hash(x)=55184249)
+8500 val loss 6.7070
+8500 val perplexity 818.1206
+8500 train 6.731531 (lr=1.0572e-05) (hash(x)=56176595)
+8700 val loss 6.7905
+8700 val perplexity 889.3621
+8700 train 6.757795 (lr=4.1554e-06) (hash(x)=46471646)
+8600 val loss 6.8010
+8600 val perplexity 898.7457
+8600 train 6.868478 (lr=2.8913e-06) (hash(x)=55184249)
+8700 val loss 6.2044
+8700 val perplexity 494.9356
+8700 train 6.142774 (lr=6.9257e-06) (hash(x)=46471646)
+8600 val loss 6.7051
+8600 val perplexity 816.5574
+8600 train 6.766881 (lr=1.0119e-05) (hash(x)=55184249)
+8800 val loss 6.7900
+8800 val perplexity 888.9067
+8800 train 6.528963 (lr=3.9866e-06) (hash(x)=46233162)
+8700 val loss 6.8006
+8700 val perplexity 898.3956
+8700 train 6.761759 (lr=2.7703e-06) (hash(x)=46471646)
+8800 val loss 6.2019
+8800 val perplexity 493.6650
+8800 train 5.902849 (lr=6.6444e-06) (hash(x)=46233162)
+8700 val loss 6.7049
+8700 val perplexity 816.4258
+8700 train 6.663841 (lr=9.6960e-06) (hash(x)=46471646)
+8900 val loss 6.7892
+8900 val perplexity 888.2064
+8900 train 6.687468 (lr=3.8307e-06) (hash(x)=47233684)
+8800 val loss 6.8005
+8800 val perplexity 898.3215
+8800 train 6.544139 (lr=2.6577e-06) (hash(x)=46233162)
+8900 val loss 6.2001
+8900 val perplexity 492.8117
+8900 train 6.111488 (lr=6.3845e-06) (hash(x)=47233684)
+8800 val loss 6.7042
+8800 val perplexity 815.7993
+8800 train 6.450424 (lr=9.3021e-06) (hash(x)=46233162)
+9000 val loss 6.7840
+9000 val perplexity 883.5842
+9000 train 6.694007 (lr=3.6877e-06) (hash(x)=48374529)
+8900 val loss 6.7976
+8900 val perplexity 895.6607
+8900 train 6.697267 (lr=2.5538e-06) (hash(x)=47233684)
+9000 val loss 6.1962
+9000 val perplexity 490.8607
+9000 train 6.080988 (lr=6.1462e-06) (hash(x)=48374529)
+9100 val loss 6.7769
+9100 val perplexity 877.3354
+9100 train 6.733949 (lr=3.5580e-06) (hash(x)=48065371)
+8900 val loss 6.7280
+8900 val perplexity 835.4911
+8900 train 6.629285 (lr=8.9382e-06) (hash(x)=47233684)
+9000 val loss 6.7929
+9000 val perplexity 891.4957
+9000 train 6.712595 (lr=2.4585e-06) (hash(x)=48374529)
+9100 val loss 6.1904
+9100 val perplexity 488.0458
+9100 train 6.196475 (lr=5.9300e-06) (hash(x)=48065371)
+9200 val loss 6.7719
+9200 val perplexity 872.9341
+9200 train 6.709245 (lr=3.4415e-06) (hash(x)=47408078)
+9000 val loss 6.7004
+9000 val perplexity 812.7133
+9000 train 6.625999 (lr=8.6047e-06) (hash(x)=48374529)
+9100 val loss 6.7882
+9100 val perplexity 887.2844
+9100 train 6.745033 (lr=2.3720e-06) (hash(x)=48065371)
+9200 val loss 6.1849
+9200 val perplexity 485.3546
+9200 train 6.187281 (lr=5.7359e-06) (hash(x)=47408078)
+9300 val loss 6.7691
+9300 val perplexity 870.5515
+9300 train 6.798889 (lr=3.3385e-06) (hash(x)=50749781)
+9100 val loss 6.6936
+9100 val perplexity 807.1893
+9100 train 6.639514 (lr=8.3020e-06) (hash(x)=48065371)
+9200 val loss 6.7833
+9200 val perplexity 882.9709
+9200 train 6.725305 (lr=2.2943e-06) (hash(x)=47408078)
+9300 val loss 6.1818
+9300 val perplexity 483.8589
+9300 train 6.174564 (lr=5.5641e-06) (hash(x)=50749781)
+9400 val loss 6.7662
+9400 val perplexity 868.0317
+9400 train 6.935981 (lr=3.2490e-06) (hash(x)=48560169)
+9200 val loss 6.6890
+9200 val perplexity 803.4981
+9200 train 6.640819 (lr=8.0302e-06) (hash(x)=47408078)
+9300 val loss 6.7810
+9300 val perplexity 880.9481
+9300 train 6.805525 (lr=2.2256e-06) (hash(x)=50749781)
+9400 val loss 6.1777
+9400 val perplexity 481.8840
+9400 train 6.386855 (lr=5.4149e-06) (hash(x)=48560169)
+9500 val loss 6.7656
+9500 val perplexity 867.4702
+9500 train 6.858753 (lr=3.1730e-06) (hash(x)=50936392)
+9300 val loss 6.6884
+9300 val perplexity 803.0623
+9300 train 6.706632 (lr=7.7898e-06) (hash(x)=50749781)
+9400 val loss 6.7790
+9400 val perplexity 879.1718
+9400 train 6.955186 (lr=2.1660e-06) (hash(x)=48560169)
+9500 val loss 6.1773
+9500 val perplexity 481.6834
+9500 train 6.323741 (lr=5.2884e-06) (hash(x)=50936392)
+9600 val loss 6.7621
+9600 val perplexity 864.4913
+9600 train 6.770996 (lr=3.1108e-06) (hash(x)=50651714)
+9400 val loss 6.6865
+9400 val perplexity 801.5171
+9400 train 6.862773 (lr=7.5809e-06) (hash(x)=48560169)
+9500 val loss 6.7787
+9500 val perplexity 878.9538
+9500 train 6.867536 (lr=2.1154e-06) (hash(x)=50936392)
+9600 val loss 6.1737
+9600 val perplexity 479.9369
+9600 train 6.183331 (lr=5.1847e-06) (hash(x)=50651714)
+9700 val loss 6.7618
+9700 val perplexity 864.1599
+9700 train 6.810016 (lr=3.0624e-06) (hash(x)=47311384)
+9500 val loss 6.6858
+9500 val perplexity 800.9467
+9500 train 6.774101 (lr=7.4038e-06) (hash(x)=50936392)
+9600 val loss 6.7750
+9600 val perplexity 875.6749
+9600 train 6.782158 (lr=2.0739e-06) (hash(x)=50651714)
+9700 val loss 6.1739
+9700 val perplexity 480.0685
+9700 train 6.242620 (lr=5.1040e-06) (hash(x)=47311384)
+9800 val loss 6.7606
+9800 val perplexity 863.1872
+9800 train 6.851887 (lr=3.0277e-06) (hash(x)=50921139)
+9600 val loss 6.6821
+9600 val perplexity 798.0090
+9600 train 6.700386 (lr=7.2586e-06) (hash(x)=50651714)
+9700 val loss 6.7748
+9700 val perplexity 875.5066
+9700 train 6.824248 (lr=2.0416e-06) (hash(x)=47311384)
+9700 val loss 6.6819
+9700 val perplexity 797.8264
+9700 train 6.707674 (lr=7.1456e-06) (hash(x)=47311384)
+9800 val loss 6.1714
+9800 val perplexity 478.8529
+9900 val loss 6.7642
+9900 val perplexity 866.2962
+9800 train 6.277335 (lr=5.0462e-06) (hash(x)=50921139)
+9900 train 6.832372 (lr=3.0069e-06) (hash(x)=48142455)
+9800 val loss 6.7740
+9800 val perplexity 874.7659
+9800 train 6.852812 (lr=2.0185e-06) (hash(x)=50921139)
+9800 val loss 6.6834
+9800 val perplexity 799.0583
+9800 train 6.780736 (lr=7.0647e-06) (hash(x)=50921139)
+9900 val loss 6.7763
+9900 val perplexity 876.8536
+9900 train 6.845608 (lr=2.0046e-06) (hash(x)=48142455)
+9900 val loss 6.6816
+9900 val perplexity 797.6224
+9900 train 6.752887 (lr=7.0162e-06) (hash(x)=48142455)
+9999 val loss 6.7563
+9999 val perplexity 859.4526
+9900 val loss 6.1866
+9900 val perplexity 486.1883
+9900 train 6.339453 (lr=5.0116e-06) (hash(x)=48142455)
+9999 val loss 6.6808
+9999 val perplexity 796.9465
+9999 val loss 6.7708
+9999 val perplexity 872.0459
+9999 val loss 6.1683
+9999 val perplexity 477.3951

attention_kindselective_n_heads8_seed1339/model_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cc97daafc5f6ded218c83e1b779eb3d5e97ba3598ab35305df344c8869610e8
+size 257976706

attention_kindselective_n_heads8_seed1339/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cb8ebeeccc09ab5d42dc52154f3bc963855ad035936bf90089675a51ce9c382
+size 257976706

attention_kindselective_n_heads8_seed1339/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80a4b48de41d0bff6b808707d66b00458018754a102bccc0170d3dabd0a8017a
+size 257976706

attention_kindselective_n_heads8_seed1339/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1382134aa91d192f7966000d70b476917de03bd95cdf60fc4eaeb69c0ae25798
+size 257976706

attention_kindselective_n_heads8_seed1339/optimizer_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eecd2916e9fcb62023114ef3d168d16a02311d6db718b9ed79b44949ec4fdbf0
+size 509672838

attention_kindselective_n_heads8_seed1339/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3688091768d19206e9078a5e8894f0b8c17e39cea9c6e9bbf6c10abb705e06a0
+size 509672838

attention_kindselective_n_heads8_seed1339/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a603a4dc1aa4548b8438e55cbd52bf233b711699976c0fba008d5aaee5ba4c8e
+size 509672838

attention_kindselective_n_heads8_seed1339/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:206e13efb97150555c1ca2b999f82739aba7206cd7894c146f5a95fbec5bed4f
+size 509672838