Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.0001, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "10e-5_10240_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_7/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 50000, "warmup_steps": 200, "group": "wider_is_better_7", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 7e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "7e-5_10240_4_1340", "n_embd": 256}

attention_kindselective_n_heads4_seed1340/dataloader_10000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3858f6c832feea78a674d8c5c384061cc7d4f22cddbd0a2be6de33bc91e2c72
+size 964

attention_kindselective_n_heads4_seed1340/dataloader_12500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab3779d33c2e0a7873fcd8c39402e44260740665950323ad1445480ec339965a
+size 964

attention_kindselective_n_heads4_seed1340/dataloader_42500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf6d24c78d89100d146bce9f26be940db3d71092473d9b55db97d6b35531eac2
+size 964

attention_kindselective_n_heads4_seed1340/dataloader_45000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22bb90b43d81f3da5454f91a70e1ed29aeb2f470a727ce38390ff8a5c4924889
+size 964

attention_kindselective_n_heads4_seed1340/dataloader_47500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55507725e6988f190e4963078652fafa6b68e8d4f79221387612612babf3e1c1
+size 964

attention_kindselective_n_heads4_seed1340/dataloader_49999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47776cddb8021172f048a950b83f25b692cb340214b800ce3837c15ceb58907c
+size 964

attention_kindselective_n_heads4_seed1340/log2.txt

@@ -1,592 +1,1113 @@
-max_steps: 10000
+max_steps: 50000
+1100 val loss 7.3709
+1100 val perplexity 1589.1416
+1100 train 6.997883 (lr=9.9927e-05) (hash(x)=37984588)
+40100 val loss 5.8536
+40100 val perplexity 348.4711
+40100 train 5.793257 (lr=9.2472e-06) (hash(x)=48035170)
+1200 val loss 7.3289
+1200 val perplexity 1523.7263
+1200 train 7.504585 (lr=9.9910e-05) (hash(x)=56333817)
 0 val loss 11.2703
 0 val perplexity 78458.0234
-0 train 11.272942 (lr=7.5000e-07) (hash(x)=57791809)
-100 val loss 9.1994
-100 val perplexity 9891.5127
-100 train 9.216334 (lr=7.5750e-05) (hash(x)=48211824)
-200 val loss 7.7349
-200 val perplexity 2286.7070
-200 train 7.673902 (lr=1.5000e-04) (hash(x)=50375849)
-0 train 11.272942 (lr=5.0000e-07) (hash(x)=57791809)
-300 val loss 7.6076
-300 val perplexity 2013.5125
-300 train 7.925827 (lr=1.4997e-04) (hash(x)=57250808)
-100 val loss 9.5575
-100 val perplexity 14150.3975
-100 train 9.571983 (lr=5.0500e-05) (hash(x)=48211824)
-400 val loss 7.5727
-400 val perplexity 1944.4767
-400 train 8.180161 (lr=1.4986e-04) (hash(x)=62519858)
-200 val loss 8.0717
-200 val perplexity 3202.6396
-200 train 8.018360 (lr=1.0000e-04) (hash(x)=50375849)
-500 val loss 7.5053
-500 val perplexity 1817.6418
-500 train 7.404214 (lr=1.4969e-04) (hash(x)=47226806)
-300 val loss 7.6542
-300 val perplexity 2109.4519
-300 train 7.955753 (lr=9.9977e-05) (hash(x)=57250808)
-600 val loss 7.4856
-600 val perplexity 1782.2722
-600 train 7.516066 (lr=1.4945e-04) (hash(x)=51149322)
-400 val loss 7.6014
-400 val perplexity 2001.0934
-400 train 8.229440 (lr=9.9908e-05) (hash(x)=62519858)
-700 val loss 7.4777
-700 val perplexity 1768.1790
-700 train 7.494768 (lr=1.4913e-04) (hash(x)=51564551)
-500 val loss 7.5660
-500 val perplexity 1931.4121
-500 train 7.453595 (lr=9.9792e-05) (hash(x)=47226806)
-800 val loss 7.4734
-800 val perplexity 1760.6666
-800 train 7.244524 (lr=1.4876e-04) (hash(x)=45093459)
-600 val loss 7.5616
-600 val perplexity 1922.8881
-600 train 7.599857 (lr=9.9631e-05) (hash(x)=51149322)
-900 val loss 7.4478
-900 val perplexity 1716.0348
-900 train 7.690024 (lr=1.4831e-04) (hash(x)=54988361)
-700 val loss 7.5504
-700 val perplexity 1901.4637
-700 train 7.545293 (lr=9.9423e-05) (hash(x)=51564551)
-1000 val loss 7.4670
-1000 val perplexity 1749.4158
-1000 train 7.367063 (lr=1.4779e-04) (hash(x)=47588648)
-800 val loss 7.5201
-800 val perplexity 1844.7793
-800 train 7.278911 (lr=9.9170e-05) (hash(x)=45093459)
-1100 val loss 7.4328
-1100 val perplexity 1690.5156
-1100 train 7.082927 (lr=1.4721e-04) (hash(x)=37984588)
-900 val loss 7.5207
-900 val perplexity 1845.7920
-900 train 7.809628 (lr=9.8872e-05) (hash(x)=54988361)
-1200 val loss 7.3818
-1200 val perplexity 1606.4569
-1200 train 7.558693 (lr=1.4656e-04) (hash(x)=56333817)
-1000 val loss 7.5099
-1000 val perplexity 1825.9989
-1000 train 7.391204 (lr=9.8528e-05) (hash(x)=47588648)
-1300 val loss 7.3642
-1300 val perplexity 1578.5209
-1300 train 7.491827 (lr=1.4585e-04) (hash(x)=53454056)
-1100 val loss 7.4699
-1100 val perplexity 1754.4908
-1100 train 7.058946 (lr=9.8140e-05) (hash(x)=37984588)
-1400 val loss 7.3266
-1400 val perplexity 1520.1290
-1400 train 7.428581 (lr=1.4507e-04) (hash(x)=55284163)
-1200 val loss 7.4695
-1200 val perplexity 1753.7841
-1200 train 7.647787 (lr=9.7708e-05) (hash(x)=56333817)
-1500 val loss 7.3234
-1500 val perplexity 1515.3085
-1500 train 7.260311 (lr=1.4422e-04) (hash(x)=48162598)
-1300 val loss 7.4999
-1300 val perplexity 1807.8933
-1300 train 7.617690 (lr=9.7231e-05) (hash(x)=53454056)
-1600 val loss 7.2709
-1600 val perplexity 1437.7860
-1600 train 7.330367 (lr=1.4332e-04) (hash(x)=54214535)
-1400 val loss 7.4034
-1400 val perplexity 1641.5452
-1400 train 7.515925 (lr=9.6711e-05) (hash(x)=55284163)
-1700 val loss 7.3486
-1700 val perplexity 1554.0712
-1700 train 7.561377 (lr=1.4235e-04) (hash(x)=53525003)
-1500 val loss 7.3976
-1500 val perplexity 1631.9938
-1500 train 7.301088 (lr=9.6149e-05) (hash(x)=48162598)
-1800 val loss 7.2820
-1800 val perplexity 1453.8403
-1800 train 7.356642 (lr=1.4131e-04) (hash(x)=51848994)
-1600 val loss 7.4057
-1600 val perplexity 1645.3561
-1600 train 7.469926 (lr=9.5544e-05) (hash(x)=54214535)
-1900 val loss 7.3030
-1900 val perplexity 1484.7439
-1900 train 7.137829 (lr=1.4022e-04) (hash(x)=48405987)
-1700 val loss 7.3512
-1700 val perplexity 1558.1410
-1700 train 7.569810 (lr=9.4897e-05) (hash(x)=53525003)
-2000 val loss 7.2506
-2000 val perplexity 1409.0001
-2000 train 7.602566 (lr=1.3907e-04) (hash(x)=58592291)
-1800 val loss 7.3285
-1800 val perplexity 1523.0544
-1800 train 7.415604 (lr=9.4209e-05) (hash(x)=51848994)
-2100 val loss 7.2532
-2100 val perplexity 1412.6349
-2100 train 7.341368 (lr=1.3786e-04) (hash(x)=51167081)
-1900 val loss 7.3569
-1900 val perplexity 1566.9419
-1900 train 7.183960 (lr=9.3481e-05) (hash(x)=48405987)
-2200 val loss 7.2590
-2200 val perplexity 1420.7731
-2200 train 7.260748 (lr=1.3660e-04) (hash(x)=47994988)
-2000 val loss 7.3472
-2000 val perplexity 1551.8170
-2000 train 7.699927 (lr=9.2714e-05) (hash(x)=58592291)
-2300 val loss 7.2521
-2300 val perplexity 1411.0697
-2300 train 7.250330 (lr=1.3527e-04) (hash(x)=47377604)
-2100 val loss 7.3025
-2100 val perplexity 1483.9901
-2100 train 7.402580 (lr=9.1908e-05) (hash(x)=51167081)
-2400 val loss 7.2556
-2400 val perplexity 1416.0265
-2400 train 7.304438 (lr=1.3390e-04) (hash(x)=53554323)
-2200 val loss 7.3241
-2200 val perplexity 1516.3789
-2200 train 7.340111 (lr=9.1064e-05) (hash(x)=47994988)
-2500 val loss 7.3073
-2500 val perplexity 1491.1608
-2500 train 7.313736 (lr=1.3247e-04) (hash(x)=50780417)
-2300 val loss 7.2707
-2300 val perplexity 1437.5735
-2300 train 7.255850 (lr=9.0182e-05) (hash(x)=47377604)
-2600 val loss 7.2764
-2600 val perplexity 1445.7822
-2600 train 7.182763 (lr=1.3099e-04) (hash(x)=46453562)
-2400 val loss 7.2498
-2400 val perplexity 1407.8685
-2400 train 7.306433 (lr=8.9265e-05) (hash(x)=53554323)
-2700 val loss 7.2792
-2700 val perplexity 1449.8409
-2700 train 7.287231 (lr=1.2946e-04) (hash(x)=54404221)
-2500 val loss 7.2853
-2500 val perplexity 1458.7178
-2500 train 7.310247 (lr=8.8313e-05) (hash(x)=50780417)
-2800 val loss 7.2837
-2800 val perplexity 1456.3575
-2800 train 7.949651 (lr=1.2788e-04) (hash(x)=59318895)
-2600 val loss 7.2457
-2600 val perplexity 1402.0936
-2600 train 7.150898 (lr=8.7326e-05) (hash(x)=46453562)
-2900 val loss 7.2333
-2900 val perplexity 1384.8267
-2900 train 7.221962 (lr=1.2626e-04) (hash(x)=47845760)
-2700 val loss 7.2470
-2700 val perplexity 1403.8638
-2700 train 7.253963 (lr=8.6306e-05) (hash(x)=54404221)
-3000 val loss 7.2471
-3000 val perplexity 1403.9862
-3000 train 6.979670 (lr=1.2459e-04) (hash(x)=44336167)
-2800 val loss 7.2707
-2800 val perplexity 1437.6050
-2800 train 7.978097 (lr=8.5254e-05) (hash(x)=59318895)
-3100 val loss 7.2154
-3100 val perplexity 1360.2776
-3100 train 7.447736 (lr=1.2287e-04) (hash(x)=44479330)
-2900 val loss 7.2657
-2900 val perplexity 1430.3938
-2900 train 7.266902 (lr=8.4170e-05) (hash(x)=47845760)
-3200 val loss 7.2325
-3200 val perplexity 1383.7072
-3200 train 7.285394 (lr=1.2112e-04) (hash(x)=54593096)
-3000 val loss 7.2402
-3000 val perplexity 1394.3390
-3000 train 6.979912 (lr=8.3057e-05) (hash(x)=44336167)
-3300 val loss 7.2123
-3300 val perplexity 1356.0752
-3300 train 7.084377 (lr=1.1932e-04) (hash(x)=45347643)
-3100 val loss 7.2144
-3100 val perplexity 1358.8521
-3100 train 7.504636 (lr=8.1915e-05) (hash(x)=44479330)
-3400 val loss 7.2006
-3400 val perplexity 1340.2240
-3400 train 7.304337 (lr=1.1749e-04) (hash(x)=47797247)
-3200 val loss 7.2444
-3200 val perplexity 1400.2949
-3200 train 7.282361 (lr=8.0745e-05) (hash(x)=54593096)
-3500 val loss 7.1473
-3500 val perplexity 1270.7017
-3500 train 7.050789 (lr=1.1562e-04) (hash(x)=46115683)
-3300 val loss 7.2259
-3300 val perplexity 1374.5964
-3300 train 7.102276 (lr=7.9549e-05) (hash(x)=45347643)
-3600 val loss 7.1462
-3600 val perplexity 1269.2581
-3600 train 6.955878 (lr=1.1372e-04) (hash(x)=44502074)
-3400 val loss 7.2068
-3400 val perplexity 1348.5294
-3400 train 7.334606 (lr=7.8328e-05) (hash(x)=47797247)
-3700 val loss 7.1355
-3700 val perplexity 1255.8024
-3700 train 7.249683 (lr=1.1179e-04) (hash(x)=55388443)
-3500 val loss 7.1718
-3500 val perplexity 1302.1456
-3500 train 7.073598 (lr=7.7082e-05) (hash(x)=46115683)
-3800 val loss 7.1516
-3800 val perplexity 1276.1625
-3800 train 6.977693 (lr=1.0982e-04) (hash(x)=43790341)
-3600 val loss 7.1680
-3600 val perplexity 1297.1829
-3600 train 7.016492 (lr=7.5814e-05) (hash(x)=44502074)
-3900 val loss 7.1724
-3900 val perplexity 1302.9469
-3900 train 7.196667 (lr=1.0783e-04) (hash(x)=50013318)
-3700 val loss 7.1518
-3700 val perplexity 1276.4393
-3700 train 7.289806 (lr=7.4525e-05) (hash(x)=55388443)
-4000 val loss 7.1715
-4000 val perplexity 1301.7539
-4000 train 7.247557 (lr=1.0581e-04) (hash(x)=51704787)
-3800 val loss 7.1593
-3800 val perplexity 1286.0034
-3800 train 6.950498 (lr=7.3215e-05) (hash(x)=43790341)
-4100 val loss 7.1315
-4100 val perplexity 1250.7228
-4100 train 7.198262 (lr=1.0377e-04) (hash(x)=50821964)
-3900 val loss 7.1414
-3900 val perplexity 1263.2340
-3900 train 7.136201 (lr=7.1887e-05) (hash(x)=50013318)
-4200 val loss 7.1552
-4200 val perplexity 1280.7772
-4200 train 7.200123 (lr=1.0171e-04) (hash(x)=49675080)
-4000 val loss 7.1388
-4000 val perplexity 1259.8586
-4000 train 7.210222 (lr=7.0541e-05) (hash(x)=51704787)
-4300 val loss 7.1326
-4300 val perplexity 1252.0898
-4300 train 6.829738 (lr=9.9622e-05) (hash(x)=43239281)
-4100 val loss 7.1296
-4100 val perplexity 1248.3210
-4100 train 7.190423 (lr=6.9180e-05) (hash(x)=50821964)
-4400 val loss 7.1795
-4400 val perplexity 1312.2640
-4400 train 6.915021 (lr=9.7520e-05) (hash(x)=45076737)
-4200 val loss 7.1301
-4200 val perplexity 1249.0374
-4200 train 7.166576 (lr=6.7804e-05) (hash(x)=49675080)
-4500 val loss 7.1361
-4500 val perplexity 1256.5530
-4500 train 7.226453 (lr=9.5403e-05) (hash(x)=57930262)
-4300 val loss 7.1375
-4300 val perplexity 1258.2251
-4300 train 6.828071 (lr=6.6414e-05) (hash(x)=43239281)
-4600 val loss 7.1293
-4600 val perplexity 1247.9628
-4600 train 6.932743 (lr=9.3273e-05) (hash(x)=46721614)
-4400 val loss 7.1356
-4400 val perplexity 1255.8755
-4400 train 6.854441 (lr=6.5013e-05) (hash(x)=45076737)
-4700 val loss 7.1447
-4700 val perplexity 1267.3850
-4700 train 7.003017 (lr=9.1132e-05) (hash(x)=49837920)
-4500 val loss 7.1212
-4500 val perplexity 1237.9576
-4500 train 7.217383 (lr=6.3602e-05) (hash(x)=57930262)
-4800 val loss 7.1396
-4800 val perplexity 1260.9735
-4800 train 7.256048 (lr=8.8982e-05) (hash(x)=48380045)
-4600 val loss 7.1148
-4600 val perplexity 1230.0862
-4600 train 6.938160 (lr=6.2182e-05) (hash(x)=46721614)
-4900 val loss 7.1166
-4900 val perplexity 1232.3048
-4900 train 7.003822 (lr=8.6825e-05) (hash(x)=44202577)
-4700 val loss 7.1160
-4700 val perplexity 1231.4736
-4700 train 6.945326 (lr=6.0754e-05) (hash(x)=49837920)
-5000 val loss 7.1251
-5000 val perplexity 1242.7483
-5000 train 7.123570 (lr=8.4663e-05) (hash(x)=52038024)
-4800 val loss 7.1030
-4800 val perplexity 1215.5854
-4800 train 7.221812 (lr=5.9321e-05) (hash(x)=48380045)
-5100 val loss 7.0881
-5100 val perplexity 1197.5981
-5100 train 7.271852 (lr=8.2500e-05) (hash(x)=53700038)
-4900 val loss 7.0901
-4900 val perplexity 1200.0150
-4900 train 6.977787 (lr=5.7883e-05) (hash(x)=44202577)
-5200 val loss 7.0702
-5200 val perplexity 1176.3390
-5200 train 7.041414 (lr=8.0337e-05) (hash(x)=48137625)
-5000 val loss 7.0778
-5000 val perplexity 1185.3610
-5000 train 7.090936 (lr=5.6442e-05) (hash(x)=52038024)
-5300 val loss 7.0610
-5300 val perplexity 1165.6478
-5300 train 6.929040 (lr=7.8175e-05) (hash(x)=43161573)
-5100 val loss 7.0503
-5100 val perplexity 1153.2191
-5100 train 7.230675 (lr=5.5000e-05) (hash(x)=53700038)
-5400 val loss 7.0882
-5400 val perplexity 1197.7939
-5400 train 7.294000 (lr=7.6018e-05) (hash(x)=56673322)
-5200 val loss 7.0469
-5200 val perplexity 1149.2523
-5200 train 7.025224 (lr=5.3558e-05) (hash(x)=48137625)
-5500 val loss 7.0674
-5500 val perplexity 1173.1265
-5500 train 7.230914 (lr=7.3868e-05) (hash(x)=53468295)
-5300 val loss 7.0382
-5300 val perplexity 1139.3370
-5300 train 6.922906 (lr=5.2117e-05) (hash(x)=43161573)
-5600 val loss 7.0512
-5600 val perplexity 1154.2776
-5600 train 7.362746 (lr=7.1727e-05) (hash(x)=59287280)
-5400 val loss 7.0431
-5400 val perplexity 1144.9668
-5400 train 7.235172 (lr=5.0679e-05) (hash(x)=56673322)
-5700 val loss 7.0557
-5700 val perplexity 1159.4276
-5700 train 7.293931 (lr=6.9597e-05) (hash(x)=57575806)
-5500 val loss 7.0248
-5500 val perplexity 1124.2102
-5500 train 7.204517 (lr=4.9246e-05) (hash(x)=53468295)
-5800 val loss 7.0477
-5800 val perplexity 1150.1986
-5800 train 6.968884 (lr=6.7480e-05) (hash(x)=46897279)
-5600 val loss 7.0278
-5600 val perplexity 1127.5576
-5600 train 7.337178 (lr=4.7818e-05) (hash(x)=59287280)
-5900 val loss 7.0581
-5900 val perplexity 1162.2921
-5900 train 6.934814 (lr=6.5378e-05) (hash(x)=47565679)
-5700 val loss 7.0104
-5700 val perplexity 1108.0684
-5700 train 7.266252 (lr=4.6398e-05) (hash(x)=57575806)
-5800 val loss 6.9975
-5800 val perplexity 1093.8611
-5800 train 6.937305 (lr=4.4987e-05) (hash(x)=46897279)
-6000 val loss 7.0583
-6000 val perplexity 1162.4668
-6000 train 6.945346 (lr=6.3294e-05) (hash(x)=51590090)
-5900 val loss 7.0068
-5900 val perplexity 1104.0801
-5900 train 6.872268 (lr=4.3586e-05) (hash(x)=47565679)
-6100 val loss 7.0384
-6100 val perplexity 1139.5461
-6100 train 7.429985 (lr=6.1230e-05) (hash(x)=59732271)
-6000 val loss 6.9759
-6000 val perplexity 1070.5658
-6000 train 6.855279 (lr=4.2196e-05) (hash(x)=51590090)
-6200 val loss 7.0536
-6200 val perplexity 1157.0466
-6200 train 7.013299 (lr=5.9188e-05) (hash(x)=46394422)
-6100 val loss 6.9696
-6100 val perplexity 1063.7852
-6100 train 7.359732 (lr=4.0820e-05) (hash(x)=59732271)
-6200 val loss 6.9557
-6200 val perplexity 1049.0975
-6200 train 6.948706 (lr=3.9459e-05) (hash(x)=46394422)
-6300 val loss 7.0336
-6300 val perplexity 1134.1621
-6300 train 7.057237 (lr=5.7169e-05) (hash(x)=53748145)
-6300 val loss 6.9495
-6300 val perplexity 1042.5876
-6300 train 6.970834 (lr=3.8113e-05) (hash(x)=53748145)
-6400 val loss 7.0264
-6400 val perplexity 1125.9877
-6400 train 6.902154 (lr=5.5177e-05) (hash(x)=46054751)
-6400 val loss 6.9335
-6400 val perplexity 1026.0587
-6400 train 6.816382 (lr=3.6785e-05) (hash(x)=46054751)
-6500 val loss 6.9250
-6500 val perplexity 1017.3809
-6500 train 7.130113 (lr=3.5475e-05) (hash(x)=51816809)
-6500 val loss 7.0105
-6500 val perplexity 1108.1730
-6500 train 7.192453 (lr=5.3213e-05) (hash(x)=51816809)
-6600 val loss 6.9167
-6600 val perplexity 1009.0242
-6600 train 6.753559 (lr=3.4186e-05) (hash(x)=52453336)
-6600 val loss 7.0129
-6600 val perplexity 1110.8755
-6600 train 6.860138 (lr=5.1279e-05) (hash(x)=52453336)
-6700 val loss 6.9028
-6700 val perplexity 995.0874
-6700 train 6.934293 (lr=3.2918e-05) (hash(x)=49108775)
-6800 val loss 6.8958
-6800 val perplexity 988.1391
-6800 train 6.812268 (lr=3.1672e-05) (hash(x)=46745396)
-6700 val loss 6.9999
-6700 val perplexity 1096.5558
-6700 train 7.014852 (lr=4.9377e-05) (hash(x)=49108775)
-6900 val loss 6.8921
-6900 val perplexity 984.4801
-6900 train 7.014542 (lr=3.0451e-05) (hash(x)=46534986)
-6800 val loss 7.0024
-6800 val perplexity 1099.2440
-6800 train 6.900960 (lr=4.7509e-05) (hash(x)=46745396)
-7000 val loss 6.8785
-7000 val perplexity 971.1417
-7000 train 7.170157 (lr=2.9255e-05) (hash(x)=49317888)
-7100 val loss 6.8711
-7100 val perplexity 964.0015
-7100 train 6.891702 (lr=2.8085e-05) (hash(x)=50360484)
-6900 val loss 6.9860
-6900 val perplexity 1081.3929
-6900 train 7.101790 (lr=4.5676e-05) (hash(x)=46534986)
-7200 val loss 6.8633
-7200 val perplexity 956.5050
-7200 train 6.759152 (lr=2.6943e-05) (hash(x)=49515094)
-7000 val loss 6.9851
-7000 val perplexity 1080.4318
-7000 train 7.236263 (lr=4.3882e-05) (hash(x)=49317888)
-7300 val loss 6.8535
-7300 val perplexity 947.2141
-7300 train 6.831221 (lr=2.5830e-05) (hash(x)=51546861)
-7400 val loss 6.8538
-7400 val perplexity 947.4413
-7400 train 6.841997 (lr=2.4746e-05) (hash(x)=48320948)
-7100 val loss 6.9737
-7100 val perplexity 1068.2135
-7100 train 6.995534 (lr=4.2128e-05) (hash(x)=50360484)
-7500 val loss 6.8490
-7500 val perplexity 942.9482
-7500 train 6.744160 (lr=2.3694e-05) (hash(x)=40167457)
-7200 val loss 6.9736
-7200 val perplexity 1068.0883
-7200 train 6.880153 (lr=4.0414e-05) (hash(x)=49515094)
-7600 val loss 6.8423
-7600 val perplexity 936.6636
-7600 train 6.787523 (lr=2.2674e-05) (hash(x)=49942165)
-7300 val loss 6.9733
-7300 val perplexity 1067.7426
-7300 train 6.945283 (lr=3.8745e-05) (hash(x)=51546861)
-7700 val loss 6.8377
-7700 val perplexity 932.3729
-7700 train 6.593207 (lr=2.1687e-05) (hash(x)=48853311)
-7800 val loss 6.8349
-7800 val perplexity 929.7131
-7800 train 6.757367 (lr=2.0735e-05) (hash(x)=48510117)
-7400 val loss 6.9641
-7400 val perplexity 1058.0121
-7400 train 6.963182 (lr=3.7120e-05) (hash(x)=48320948)
-7900 val loss 6.8322
-7900 val perplexity 927.1918
-7900 train 6.843838 (lr=1.9818e-05) (hash(x)=48339781)
-7500 val loss 6.9680
-7500 val perplexity 1062.0619
-7500 train 6.844945 (lr=3.5541e-05) (hash(x)=40167457)
-8000 val loss 6.8184
-8000 val perplexity 914.4979
-8000 train 6.935864 (lr=1.8936e-05) (hash(x)=54927320)
-8100 val loss 6.8138
-8100 val perplexity 910.3555
-8100 train 6.570134 (lr=1.8092e-05) (hash(x)=46461786)
-7600 val loss 6.9606
-7600 val perplexity 1054.3075
-7600 train 6.885594 (lr=3.4011e-05) (hash(x)=49942165)
-8200 val loss 6.8039
-8200 val perplexity 901.3185
-8200 train 6.741070 (lr=1.7286e-05) (hash(x)=51536260)
-7700 val loss 6.9648
-7700 val perplexity 1058.6656
-7700 train 6.718952 (lr=3.2531e-05) (hash(x)=48853311)
-8300 val loss 6.7965
-8300 val perplexity 894.6973
-8300 train 6.555057 (lr=1.6519e-05) (hash(x)=44770722)
-8400 val loss 6.7932
-8400 val perplexity 891.7589
-8400 train 6.784442 (lr=1.5791e-05) (hash(x)=50104957)
-7800 val loss 6.9621
-7800 val perplexity 1055.7982
-7800 train 6.868794 (lr=3.1102e-05) (hash(x)=48510117)
-8500 val loss 6.7882
-8500 val perplexity 887.2971
-8500 train 6.884346 (lr=1.5103e-05) (hash(x)=50132971)
-7900 val loss 6.9607
-7900 val perplexity 1054.3367
-7900 train 6.978685 (lr=2.9726e-05) (hash(x)=48339781)
-8600 val loss 6.7788
-8600 val perplexity 879.0137
-8600 train 6.735719 (lr=1.4456e-05) (hash(x)=52193699)
-8700 val loss 6.7764
-8700 val perplexity 876.9025
-8700 train 6.774114 (lr=1.3851e-05) (hash(x)=47902319)
-8000 val loss 6.9558
-8000 val perplexity 1049.2601
-8000 train 7.068865 (lr=2.8405e-05) (hash(x)=54927320)
-8800 val loss 6.7649
-8800 val perplexity 866.8619
-8800 train 7.029649 (lr=1.3289e-05) (hash(x)=54904230)
-8100 val loss 6.9537
-8100 val perplexity 1047.0311
-8100 train 6.719138 (lr=2.7138e-05) (hash(x)=46461786)
-8900 val loss 6.7611
-8900 val perplexity 863.6302
-8900 train 6.677911 (lr=1.2769e-05) (hash(x)=46311615)
-9000 val loss 6.7601
-9000 val perplexity 862.7271
-9000 train 6.630702 (lr=1.2292e-05) (hash(x)=48535188)
-8200 val loss 6.9487
-8200 val perplexity 1041.8079
-8200 train 6.892978 (lr=2.5929e-05) (hash(x)=51536260)
-9100 val loss 6.7496
-9100 val perplexity 853.6826
-9100 train 6.778772 (lr=1.1860e-05) (hash(x)=51757372)
-8300 val loss 6.9447
-8300 val perplexity 1037.6498
-8300 train 6.711561 (lr=2.4778e-05) (hash(x)=44770722)
-9200 val loss 6.7470
-9200 val perplexity 851.5067
-9200 train 6.636959 (lr=1.1472e-05) (hash(x)=51131708)
-8400 val loss 6.9416
-8400 val perplexity 1034.3774
-8400 train 6.938999 (lr=2.3686e-05) (hash(x)=50104957)
-9300 val loss 6.7415
-9300 val perplexity 846.8267
-9300 train 6.686966 (lr=1.1128e-05) (hash(x)=44784276)
-8500 val loss 6.9362
-8500 val perplexity 1028.8685
-8500 train 7.034618 (lr=2.2655e-05) (hash(x)=50132971)
-9400 val loss 6.7376
-9400 val perplexity 843.5647
-9400 train 6.831891 (lr=1.0830e-05) (hash(x)=51981169)
-8600 val loss 6.9281
-8600 val perplexity 1020.5430
-8600 train 6.879009 (lr=2.1685e-05) (hash(x)=52193699)
-9500 val loss 6.7318
-9500 val perplexity 838.6479
-9500 train 6.684929 (lr=1.0577e-05) (hash(x)=47232936)
-8700 val loss 6.9265
-8700 val perplexity 1018.9719
-8700 train 6.928772 (lr=2.0777e-05) (hash(x)=47902319)
-9600 val loss 6.7301
-9600 val perplexity 837.1972
-9600 train 6.762636 (lr=1.0369e-05) (hash(x)=53800450)
-8800 val loss 6.9224
-8800 val perplexity 1014.7075
-8800 train 7.177705 (lr=1.9933e-05) (hash(x)=54904230)
-9700 val loss 6.7257
-9700 val perplexity 833.5361
-9700 train 6.842899 (lr=1.0208e-05) (hash(x)=55768123)
-8900 val loss 6.9157
-8900 val perplexity 1008.0081
-8900 train 6.824413 (lr=1.9153e-05) (hash(x)=46311615)
-9800 val loss 6.7242
-9800 val perplexity 832.2679
-9800 train 6.656127 (lr=1.0092e-05) (hash(x)=47745177)
-9900 val loss 6.7249
-9900 val perplexity 832.9302
-9900 train 6.977684 (lr=1.0023e-05) (hash(x)=56592246)
-9000 val loss 6.9129
-9000 val perplexity 1005.1110
-9000 train 6.765456 (lr=1.8439e-05) (hash(x)=48535188)
-9999 val loss 6.7242
-9999 val perplexity 832.3088
-9100 val loss 6.9100
-9100 val perplexity 1002.2089
-9100 train 6.928776 (lr=1.7790e-05) (hash(x)=51757372)
-9200 val loss 6.9081
-9200 val perplexity 1000.3215
-9200 train 6.806732 (lr=1.7208e-05) (hash(x)=51131708)
-9300 val loss 6.9059
-9300 val perplexity 998.1198
-9300 train 6.841065 (lr=1.6692e-05) (hash(x)=44784276)
-9400 val loss 6.9072
-9400 val perplexity 999.4905
-9400 train 6.993570 (lr=1.6245e-05) (hash(x)=51981169)
-9500 val loss 6.9063
-9500 val perplexity 998.5016
-9500 train 6.858905 (lr=1.5865e-05) (hash(x)=47232936)
-9600 val loss 6.9042
-9600 val perplexity 996.4511
-9600 train 6.943208 (lr=1.5554e-05) (hash(x)=53800450)
+1300 val loss 7.3412
+1300 val perplexity 1542.5471
+1300 train 7.483638 (lr=9.9892e-05) (hash(x)=53454056)
+40200 val loss 5.8481
+40200 val perplexity 346.5657
+40200 train 5.675584 (lr=9.1646e-06) (hash(x)=46651322)
+1400 val loss 7.2560
+1400 val perplexity 1416.6073
+1400 train 7.362457 (lr=9.9871e-05) (hash(x)=55284163)
+40300 val loss 5.8508
+40300 val perplexity 347.5022
+40300 train 5.707502 (lr=9.0827e-06) (hash(x)=46378099)
+1500 val loss 7.3036
+1500 val perplexity 1485.6129
+1500 train 7.158344 (lr=9.9849e-05) (hash(x)=48162598)
+0 train 11.272942 (lr=3.5000e-07) (hash(x)=57791809)
+1600 val loss 7.2907
+1600 val perplexity 1466.6249
+1600 train 7.350899 (lr=9.9825e-05) (hash(x)=54214535)
+40400 val loss 5.8443
+40400 val perplexity 345.2604
+40400 train 5.696771 (lr=9.0015e-06) (hash(x)=46495438)
+100 val loss 9.7259
+100 val perplexity 16745.7324
+100 train 9.735422 (lr=3.5350e-05) (hash(x)=48211824)
+1700 val loss 7.2683
+1700 val perplexity 1434.0831
+1700 train 7.474521 (lr=9.9799e-05) (hash(x)=53525003)
+200 val loss 8.1014
+200 val perplexity 3298.9370
+200 train 8.058278 (lr=7.0000e-05) (hash(x)=50375849)
+40500 val loss 5.8388
+40500 val perplexity 343.3772
+40500 train 5.996032 (lr=8.9211e-06) (hash(x)=52059147)
+1800 val loss 7.2337
+1800 val perplexity 1385.3571
+1800 train 7.303545 (lr=9.9771e-05) (hash(x)=51848994)
+300 val loss 7.6442
+300 val perplexity 2088.4370
+300 train 7.951274 (lr=6.9999e-05) (hash(x)=57250808)
+400 val loss 7.5686
+400 val perplexity 1936.4075
+400 train 8.221130 (lr=6.9997e-05) (hash(x)=62519858)
+40600 val loss 5.8407
+40600 val perplexity 344.0050
+40600 train 5.996709 (lr=8.8414e-06) (hash(x)=54885045)
+1900 val loss 7.2157
+1900 val perplexity 1360.6837
+1900 train 7.053372 (lr=9.9741e-05) (hash(x)=48405987)
+500 val loss 7.5147
+500 val perplexity 1834.7661
+500 train 7.395103 (lr=6.9994e-05) (hash(x)=47226806)
+40700 val loss 5.8398
+40700 val perplexity 343.7109
+40700 train 6.230294 (lr=8.7624e-06) (hash(x)=53213971)
+600 val loss 7.4733
+600 val perplexity 1760.4728
+600 train 7.511261 (lr=6.9990e-05) (hash(x)=51149322)
+2000 val loss 7.1997
+2000 val perplexity 1338.9938
+2000 train 7.547995 (lr=9.9710e-05) (hash(x)=58592291)
+700 val loss 7.4403
+700 val perplexity 1703.2408
+700 train 7.431039 (lr=6.9984e-05) (hash(x)=51564551)
+40800 val loss 5.8359
+40800 val perplexity 342.3692
+40800 train 5.498599 (lr=8.6842e-06) (hash(x)=45133794)
+2100 val loss 7.1929
+2100 val perplexity 1329.9009
+2100 train 7.272621 (lr=9.9677e-05) (hash(x)=51167081)
+800 val loss 7.4083
+800 val perplexity 1649.6486
+800 train 7.160203 (lr=6.9977e-05) (hash(x)=45093459)
+40900 val loss 5.8379
+40900 val perplexity 343.0574
+40900 train 5.988767 (lr=8.6068e-06) (hash(x)=56546547)
+900 val loss 7.3864
+900 val perplexity 1613.9653
+900 train 7.685310 (lr=6.9969e-05) (hash(x)=54988361)
+2200 val loss 7.1644
+2200 val perplexity 1292.5675
+2200 train 7.169672 (lr=9.9642e-05) (hash(x)=47994988)
+1000 val loss 7.3699
+1000 val perplexity 1587.5004
+1000 train 7.238439 (lr=6.9960e-05) (hash(x)=47588648)
+41000 val loss 5.8354
+41000 val perplexity 342.1882
+41000 train 6.221378 (lr=8.5301e-06) (hash(x)=49552270)
+2300 val loss 7.1801
+2300 val perplexity 1313.0338
+2300 train 7.161005 (lr=9.9606e-05) (hash(x)=47377604)
+1100 val loss 7.3479
+1100 val perplexity 1552.9474
+1100 train 6.955181 (lr=6.9949e-05) (hash(x)=37984588)
+1200 val loss 7.3201
+1200 val perplexity 1510.3381
+1200 train 7.489902 (lr=6.9937e-05) (hash(x)=56333817)
+41100 val loss 5.8354
+41100 val perplexity 342.2155
+41100 train 6.067726 (lr=8.4541e-06) (hash(x)=51222370)
+2400 val loss 7.1777
+2400 val perplexity 1309.8896
+2400 train 7.223081 (lr=9.9567e-05) (hash(x)=53554323)
+1300 val loss 7.3031
+1300 val perplexity 1484.8395
+1300 train 7.442085 (lr=6.9924e-05) (hash(x)=53454056)
+41200 val loss 5.8369
+41200 val perplexity 342.7026
+41200 train 5.903095 (lr=8.3789e-06) (hash(x)=50883608)
+2500 val loss 7.1743
+2500 val perplexity 1305.3995
+2500 train 7.199438 (lr=9.9527e-05) (hash(x)=50780417)
+1400 val loss 7.2946
+1400 val perplexity 1472.2584
+1400 train 7.407144 (lr=6.9910e-05) (hash(x)=55284163)
+1500 val loss 7.3019
+1500 val perplexity 1483.1653
+1500 train 7.143060 (lr=6.9894e-05) (hash(x)=48162598)
+41300 val loss 5.8382
+41300 val perplexity 343.1662
+41300 train 6.058548 (lr=8.3045e-06) (hash(x)=52996828)
+2600 val loss 7.1702
+2600 val perplexity 1300.0759
+2600 train 7.071150 (lr=9.9485e-05) (hash(x)=46453562)
+1600 val loss 7.2579
+1600 val perplexity 1419.3342
+1600 train 7.321304 (lr=6.9877e-05) (hash(x)=54214535)
+41400 val loss 5.8361
+41400 val perplexity 342.4566
+41400 train 5.811955 (lr=8.2308e-06) (hash(x)=48822716)
+2700 val loss 7.2418
+2700 val perplexity 1396.6613
+2700 train 7.236431 (lr=9.9442e-05) (hash(x)=54404221)
+1700 val loss 7.2608
+1700 val perplexity 1423.4137
+1700 train 7.450740 (lr=6.9859e-05) (hash(x)=53525003)
+1800 val loss 7.2223
+1800 val perplexity 1369.6931
+1800 train 7.294693 (lr=6.9840e-05) (hash(x)=51848994)
+41500 val loss 5.8375
+41500 val perplexity 342.9267
+41500 train 5.816633 (lr=8.1579e-06) (hash(x)=50936577)
+2800 val loss 7.1606
+2800 val perplexity 1287.7007
+2800 train 7.852295 (lr=9.9396e-05) (hash(x)=59318895)
+1900 val loss 7.2239
+1900 val perplexity 1371.8162
+1900 train 7.046358 (lr=6.9819e-05) (hash(x)=48405987)
+41600 val loss 5.8377
+41600 val perplexity 343.0029
+41600 train 5.881047 (lr=8.0858e-06) (hash(x)=44375046)
+2900 val loss 7.1619
+2900 val perplexity 1289.3486
+2900 train 7.149612 (lr=9.9349e-05) (hash(x)=47845760)
+2000 val loss 7.2136
+2000 val perplexity 1357.7904
+2000 train 7.559543 (lr=6.9797e-05) (hash(x)=58592291)
+2100 val loss 7.2009
+2100 val perplexity 1340.6804
+2100 train 7.274922 (lr=6.9774e-05) (hash(x)=51167081)
+41700 val loss 5.8362
+41700 val perplexity 342.4806
+41700 train 5.599354 (lr=8.0144e-06) (hash(x)=44060021)
+3000 val loss 7.1622
+3000 val perplexity 1289.7507
+3000 train 6.873025 (lr=9.9300e-05) (hash(x)=44336167)
+2200 val loss 7.1883
+2200 val perplexity 1323.8024
+2200 train 7.174533 (lr=6.9750e-05) (hash(x)=47994988)
+41800 val loss 5.8358
+41800 val perplexity 342.3502
+41800 train 6.128627 (lr=7.9438e-06) (hash(x)=57765221)
+2300 val loss 7.2004
+2300 val perplexity 1340.0284
+3100 val loss 7.1277
+3100 val perplexity 1246.0415
+2300 train 7.185493 (lr=6.9724e-05) (hash(x)=47377604)
+3100 train 7.390228 (lr=9.9249e-05) (hash(x)=44479330)
+2400 val loss 7.1732
+2400 val perplexity 1303.9562
+2400 train 7.220889 (lr=6.9697e-05) (hash(x)=53554323)
+3200 val loss 7.1646
+3200 val perplexity 1292.8153
+3200 train 7.241447 (lr=9.9197e-05) (hash(x)=54593096)
+41900 val loss 5.8348
+41900 val perplexity 341.9940
+41900 train 5.597972 (lr=7.8740e-06) (hash(x)=46051470)
+2500 val loss 7.1729
+2500 val perplexity 1303.6392
+2500 train 7.214406 (lr=6.9669e-05) (hash(x)=50780417)
+3300 val loss 7.1778
+3300 val perplexity 1309.9883
+3300 train 7.088508 (lr=9.9142e-05) (hash(x)=45347643)
+2600 val loss 7.1263
+2600 val perplexity 1244.2609
+2600 train 7.028562 (lr=6.9640e-05) (hash(x)=46453562)
+42000 val loss 5.8415
+42000 val perplexity 344.2894
+42000 train 6.066402 (lr=7.8050e-06) (hash(x)=52077616)
+2700 val loss 7.1115
+2700 val perplexity 1225.9312
+2700 train 7.080199 (lr=6.9609e-05) (hash(x)=54404221)
+3400 val loss 7.1109
+3400 val perplexity 1225.2228
+3400 train 7.224405 (lr=9.9086e-05) (hash(x)=47797247)
+42100 val loss 5.8336
+42100 val perplexity 341.5976
+42100 train 5.685880 (lr=7.7368e-06) (hash(x)=47845199)
+2800 val loss 7.1089
+2800 val perplexity 1222.8175
+2800 train 7.750087 (lr=6.9577e-05) (hash(x)=59318895)
+2900 val loss 7.1042
+2900 val perplexity 1217.0992
+2900 train 7.102781 (lr=6.9544e-05) (hash(x)=47845760)
+3500 val loss 7.1129
+3500 val perplexity 1227.7335
+3500 train 7.017200 (lr=9.9028e-05) (hash(x)=46115683)
+42200 val loss 5.8312
+42200 val perplexity 340.7710
+42200 train 6.088301 (lr=7.6693e-06) (hash(x)=51549823)
+3000 val loss 7.0754
+3000 val perplexity 1182.5112
+3000 train 6.773137 (lr=6.9510e-05) (hash(x)=44336167)
+3600 val loss 7.1055
+3600 val perplexity 1218.6689
+3600 train 6.937795 (lr=9.8969e-05) (hash(x)=44502074)
+42300 val loss 5.8243
+42300 val perplexity 338.4359
+42300 train 5.965167 (lr=7.6027e-06) (hash(x)=56922131)
+3100 val loss 7.0511
+3100 val perplexity 1154.1840
+3100 train 7.331172 (lr=6.9474e-05) (hash(x)=44479330)
+3200 val loss 7.0365
+3200 val perplexity 1137.3547
+3200 train 7.123116 (lr=6.9438e-05) (hash(x)=54593096)
+3700 val loss 7.0926
+3700 val perplexity 1202.9988
+3700 train 7.232045 (lr=9.8908e-05) (hash(x)=55388443)
+42400 val loss 5.8262
+42400 val perplexity 339.0629
+42400 train 5.787869 (lr=7.5368e-06) (hash(x)=49004372)
+3300 val loss 7.0396
+3300 val perplexity 1140.9235
+3300 train 6.961339 (lr=6.9400e-05) (hash(x)=45347643)
+3800 val loss 7.0881
+3800 val perplexity 1197.5861
+3800 train 6.871030 (lr=9.8845e-05) (hash(x)=43790341)
+42500 val loss 5.8224
+42500 val perplexity 337.7739
+42500 train 5.810077 (lr=7.4717e-06) (hash(x)=50651839)
+3400 val loss 7.0278
+3400 val perplexity 1127.5500
+3400 train 7.154087 (lr=6.9360e-05) (hash(x)=47797247)
+3500 val loss 7.0127
+3500 val perplexity 1110.6456
+3500 train 6.888947 (lr=6.9320e-05) (hash(x)=46115683)
+3900 val loss 7.0927
+3900 val perplexity 1203.1331
+3900 train 7.089813 (lr=9.8780e-05) (hash(x)=50013318)
+42600 val loss 5.8226
+42600 val perplexity 337.8404
+42600 train 6.146226 (lr=7.4074e-06) (hash(x)=50767721)
+3600 val loss 6.9927
+3600 val perplexity 1088.6067
+3600 train 6.841239 (lr=6.9278e-05) (hash(x)=44502074)
+4000 val loss 7.0888
+4000 val perplexity 1198.4447
+4000 train 7.173865 (lr=9.8713e-05) (hash(x)=51704787)
+42700 val loss 5.8274
+42700 val perplexity 339.4667
+42700 train 5.559310 (lr=7.3440e-06) (hash(x)=49099183)
+3700 val loss 6.9963
+3700 val perplexity 1092.6323
+3700 train 7.130051 (lr=6.9235e-05) (hash(x)=55388443)
+3800 val loss 6.9835
+3800 val perplexity 1078.6748
+3800 train 6.767923 (lr=6.9191e-05) (hash(x)=43790341)
+4100 val loss 7.0791
+4100 val perplexity 1186.8921
+4100 train 7.136350 (lr=9.8645e-05) (hash(x)=50821964)
+42800 val loss 5.8205
+42800 val perplexity 337.1293
+42800 train 6.236982 (lr=7.2813e-06) (hash(x)=42272413)
+3900 val loss 6.9793
+3900 val perplexity 1074.1350
+3900 train 6.971794 (lr=6.9146e-05) (hash(x)=50013318)
+4200 val loss 7.1331
+4200 val perplexity 1252.7737
+4200 train 7.145490 (lr=9.8575e-05) (hash(x)=49675080)
+42900 val loss 5.8239
+42900 val perplexity 338.2816
+42900 train 5.616502 (lr=7.2194e-06) (hash(x)=48582863)
+4000 val loss 6.9700
+4000 val perplexity 1064.2661
+4000 train 7.038657 (lr=6.9099e-05) (hash(x)=51704787)
+4100 val loss 6.9672
+4100 val perplexity 1061.2119
+4100 train 7.042075 (lr=6.9051e-05) (hash(x)=50821964)
+4300 val loss 7.1120
+4300 val perplexity 1226.5977
+4300 train 6.766655 (lr=9.8503e-05) (hash(x)=43239281)
+43000 val loss 5.8217
+43000 val perplexity 337.5590
+43000 train 5.796019 (lr=7.1583e-06) (hash(x)=48703446)
+4200 val loss 6.9870
+4200 val perplexity 1082.4434
+4200 train 6.998270 (lr=6.9002e-05) (hash(x)=49675080)
+4400 val loss 7.1208
+4400 val perplexity 1237.4618
+4400 train 6.820117 (lr=9.8430e-05) (hash(x)=45076737)
+4300 val loss 6.9632
+4300 val perplexity 1056.9971
+4300 train 6.617874 (lr=6.8952e-05) (hash(x)=43239281)
+43100 val loss 5.8203
+43100 val perplexity 337.0722
+43100 train 5.743421 (lr=7.0981e-06) (hash(x)=48730321)
+4400 val loss 6.9505
+4400 val perplexity 1043.6844
+4400 train 6.636121 (lr=6.8901e-05) (hash(x)=45076737)
+4500 val loss 7.1026
+4500 val perplexity 1215.1410
+4500 train 7.188966 (lr=9.8355e-05) (hash(x)=57930262)
+43200 val loss 5.8184
+43200 val perplexity 336.4272
+43200 train 6.053661 (lr=7.0386e-06) (hash(x)=56536090)
+4500 val loss 6.9386
+4500 val perplexity 1031.2871
+4500 train 7.046677 (lr=6.8848e-05) (hash(x)=57930262)
+4600 val loss 7.0943
+4600 val perplexity 1205.0604
+4600 train 6.878264 (lr=9.8278e-05) (hash(x)=46721614)
+4600 val loss 6.9329
+4600 val perplexity 1025.4572
+4600 train 6.712116 (lr=6.8794e-05) (hash(x)=46721614)
+43300 val loss 5.8176
+43300 val perplexity 336.1527
+43300 train 5.962167 (lr=6.9800e-06) (hash(x)=54154116)
+4700 val loss 6.9091
+4700 val perplexity 1001.3547
+4700 train 6.706449 (lr=6.8739e-05) (hash(x)=49837920)
+4700 val loss 7.0600
+4700 val perplexity 1164.4540
+4700 train 6.883050 (lr=9.8199e-05) (hash(x)=49837920)
+4800 val loss 6.8954
+4800 val perplexity 987.7471
+4800 train 7.025270 (lr=6.8683e-05) (hash(x)=48380045)
+43400 val loss 5.8258
+43400 val perplexity 338.9396
+43400 train 5.723001 (lr=6.9222e-06) (hash(x)=50058055)
+4800 val loss 7.0530
+4800 val perplexity 1156.3136
+4900 val loss 6.8825
+4900 val perplexity 975.0323
+4800 train 7.163837 (lr=9.8119e-05) (hash(x)=48380045)
+4900 train 6.764085 (lr=6.8626e-05) (hash(x)=44202577)
+43500 val loss 5.8194
+43500 val perplexity 336.7863
+43500 train 5.749609 (lr=6.8652e-06) (hash(x)=48743802)
+5000 val loss 6.8525
+5000 val perplexity 946.2589
+5000 train 6.858996 (lr=6.8567e-05) (hash(x)=52038024)
+4900 val loss 7.0500
+4900 val perplexity 1152.8193
+4900 train 6.937270 (lr=9.8036e-05) (hash(x)=44202577)
+5100 val loss 6.8317
+5100 val perplexity 926.8068
+5100 train 7.012638 (lr=6.8507e-05) (hash(x)=53700038)
+43600 val loss 5.8188
+43600 val perplexity 336.5771
+43600 train 5.458486 (lr=6.8090e-06) (hash(x)=42792886)
+5200 val loss 6.8274
+5200 val perplexity 922.8022
+5200 train 6.814481 (lr=6.8446e-05) (hash(x)=48137625)
+5000 val loss 7.0187
+5000 val perplexity 1117.3700
+5000 train 7.044736 (lr=9.7953e-05) (hash(x)=52038024)
+43700 val loss 5.8163
+43700 val perplexity 335.7397
+43700 train 6.229331 (lr=6.7537e-06) (hash(x)=56446070)
+5300 val loss 6.8135
+5300 val perplexity 910.0942
+5300 train 6.670180 (lr=6.8384e-05) (hash(x)=43161573)
+5100 val loss 7.0031
+5100 val perplexity 1100.0636
+5100 train 7.189824 (lr=9.7867e-05) (hash(x)=53700038)
+5400 val loss 6.8088
+5400 val perplexity 905.7951
+5400 train 7.007874 (lr=6.8320e-05) (hash(x)=56673322)
+43800 val loss 5.8178
+43800 val perplexity 336.2392
+43800 train 5.653908 (lr=6.6992e-06) (hash(x)=45584354)
+5500 val loss 6.8175
+5500 val perplexity 913.7433
+5500 train 6.971771 (lr=6.8256e-05) (hash(x)=53468295)
+5200 val loss 6.9942
+5200 val perplexity 1090.2700
+5200 train 6.960814 (lr=9.7780e-05) (hash(x)=48137625)
+43900 val loss 5.8161
+43900 val perplexity 335.6672
+43900 train 5.664525 (lr=6.6455e-06) (hash(x)=49339253)
+5600 val loss 6.7675
+5600 val perplexity 869.1533
+5600 train 7.060785 (lr=6.8190e-05) (hash(x)=59287280)
+5300 val loss 6.9828
+5300 val perplexity 1077.8917
+5300 train 6.854216 (lr=9.7691e-05) (hash(x)=43161573)
+5700 val loss 6.7728
+5700 val perplexity 873.7137
+5700 train 7.033890 (lr=6.8123e-05) (hash(x)=57575806)
+44000 val loss 5.8158
+44000 val perplexity 335.5459
+44000 train 5.635589 (lr=6.5926e-06) (hash(x)=46183203)
+5800 val loss 6.7736
+5800 val perplexity 874.4873
+5800 train 6.723006 (lr=6.8055e-05) (hash(x)=46897279)
+5400 val loss 6.9617
+5400 val perplexity 1055.4479
+5400 train 7.139781 (lr=9.7600e-05) (hash(x)=56673322)
+44100 val loss 5.8135
+44100 val perplexity 334.7792
+44100 train 5.802389 (lr=6.5406e-06) (hash(x)=47849630)
+5900 val loss 6.7703
+5900 val perplexity 871.5712
+5900 train 6.636249 (lr=6.7985e-05) (hash(x)=47565679)
+5500 val loss 6.9375
+5500 val perplexity 1030.2126
+5500 train 7.112161 (lr=9.7508e-05) (hash(x)=53468295)
+6000 val loss 6.7804
+6000 val perplexity 880.4198
+6000 train 6.648599 (lr=6.7915e-05) (hash(x)=51590090)
+44200 val loss 5.8127
+44200 val perplexity 334.5341
+44200 train 6.146719 (lr=6.4894e-06) (hash(x)=49834275)
+6100 val loss 6.7618
+6100 val perplexity 864.2370
+6100 train 7.137344 (lr=6.7843e-05) (hash(x)=59732271)
+5600 val loss 6.9085
+5600 val perplexity 1000.7457
+5600 train 7.224924 (lr=9.7414e-05) (hash(x)=59287280)
+44300 val loss 5.8095
+44300 val perplexity 333.4532
+44300 train 6.254009 (lr=6.4390e-06) (hash(x)=62535257)
+6200 val loss 6.7642
+6200 val perplexity 866.2368
+6200 train 6.741118 (lr=6.7770e-05) (hash(x)=46394422)
+5700 val loss 6.9043
+5700 val perplexity 996.5828
+5700 train 7.165204 (lr=9.7318e-05) (hash(x)=57575806)
+6300 val loss 6.7599
+6300 val perplexity 862.5737
+6300 train 6.768298 (lr=6.7696e-05) (hash(x)=53748145)
+44400 val loss 5.8078
+44400 val perplexity 332.8983
+44400 train 5.866293 (lr=6.3895e-06) (hash(x)=49253957)
+6400 val loss 6.7398
+6400 val perplexity 845.4134
+6400 train 6.611226 (lr=6.7621e-05) (hash(x)=46054751)
+5800 val loss 6.8937
+5800 val perplexity 986.0107
+5800 train 6.843620 (lr=9.7221e-05) (hash(x)=46897279)
+6500 val loss 6.7447
+6500 val perplexity 849.5231
+6500 train 7.008954 (lr=6.7545e-05) (hash(x)=51816809)
+44500 val loss 5.8086
+44500 val perplexity 333.1573
+44500 train 5.787062 (lr=6.3408e-06) (hash(x)=55368339)
+5900 val loss 6.8839
+5900 val perplexity 976.4327
+5900 train 6.751657 (lr=9.7122e-05) (hash(x)=47565679)
+6600 val loss 6.7294
+6600 val perplexity 836.6632
+6600 train 6.619945 (lr=6.7467e-05) (hash(x)=52453336)
+44600 val loss 5.8085
+44600 val perplexity 333.1111
+44600 train 5.920068 (lr=6.2929e-06) (hash(x)=47098476)
+6700 val loss 6.7275
+6700 val perplexity 835.0275
+6700 train 6.752491 (lr=6.7389e-05) (hash(x)=49108775)
+6000 val loss 6.8816
+6000 val perplexity 974.1851
+6000 train 6.755291 (lr=9.7021e-05) (hash(x)=51590090)
+6800 val loss 6.7240
+6800 val perplexity 832.1128
+6800 train 6.615003 (lr=6.7309e-05) (hash(x)=46745396)
+44700 val loss 5.8119
+44700 val perplexity 334.2415
+44700 train 5.744551 (lr=6.2459e-06) (hash(x)=48280562)
+6100 val loss 6.8433
+6100 val perplexity 937.6213
+6100 train 7.231963 (lr=9.6919e-05) (hash(x)=59732271)
+6900 val loss 6.7129
+6900 val perplexity 822.9876
+6900 train 6.874073 (lr=6.7228e-05) (hash(x)=46534986)
+44800 val loss 5.8111
+44800 val perplexity 333.9803
+44800 train 6.081460 (lr=6.1998e-06) (hash(x)=55591638)
+7000 val loss 6.7105
+7000 val perplexity 821.0043
+7000 train 7.102075 (lr=6.7146e-05) (hash(x)=49317888)
+6200 val loss 6.8340
+6200 val perplexity 928.8552
+6200 train 6.807478 (lr=9.6815e-05) (hash(x)=46394422)
+7100 val loss 6.7149
+7100 val perplexity 824.5883
+7100 train 6.716461 (lr=6.7063e-05) (hash(x)=50360484)
+44900 val loss 5.8106
+44900 val perplexity 333.8185
+44900 train 5.963002 (lr=6.1545e-06) (hash(x)=53757748)
+6300 val loss 6.8275
+6300 val perplexity 922.9268
+6300 train 6.854854 (lr=9.6709e-05) (hash(x)=53748145)
+7200 val loss 6.7289
+7200 val perplexity 836.1858
+7200 train 6.611360 (lr=6.6978e-05) (hash(x)=49515094)
+45000 val loss 5.8083
+45000 val perplexity 333.0545
+45000 train 5.965748 (lr=6.1100e-06) (hash(x)=51685087)
+7300 val loss 6.7329
+7300 val perplexity 839.6050
+7300 train 6.724388 (lr=6.6893e-05) (hash(x)=51546861)
+6400 val loss 6.8335
+6400 val perplexity 928.4501
+6400 train 6.675262 (lr=9.6602e-05) (hash(x)=46054751)
+7400 val loss 6.7554
+7400 val perplexity 858.6415
+7400 train 6.747338 (lr=6.6806e-05) (hash(x)=48320948)
+45100 val loss 5.8066
+45100 val perplexity 332.4754
+45100 train 5.804672 (lr=6.0664e-06) (hash(x)=50093774)
+6500 val loss 6.8217
+6500 val perplexity 917.5349
+6500 train 7.011214 (lr=9.6493e-05) (hash(x)=51816809)
+7500 val loss 6.7581
+7500 val perplexity 861.0138
+7500 train 6.601406 (lr=6.6718e-05) (hash(x)=40167457)
+45200 val loss 5.8079
+45200 val perplexity 332.9253
+45200 train 5.531459 (lr=6.0237e-06) (hash(x)=43460450)
+7600 val loss 6.7487
+7600 val perplexity 852.9794
+7600 train 6.694093 (lr=6.6630e-05) (hash(x)=49942165)
+6600 val loss 6.8168
+6600 val perplexity 913.0578
+6600 train 6.709125 (lr=9.6382e-05) (hash(x)=52453336)
+7700 val loss 6.7365
+7700 val perplexity 842.6336
+7700 train 6.467313 (lr=6.6540e-05) (hash(x)=48853311)
+45300 val loss 5.8075
+45300 val perplexity 332.7967
+45300 train 6.013202 (lr=5.9818e-06) (hash(x)=49935488)
+6700 val loss 6.7775
+6700 val perplexity 877.8773
+6700 train 6.801557 (lr=9.6270e-05) (hash(x)=49108775)
+7800 val loss 6.6977
+7800 val perplexity 810.5665
+7800 train 6.587886 (lr=6.6448e-05) (hash(x)=48510117)
+7900 val loss 6.6830
+7900 val perplexity 798.6885
+7900 train 6.674704 (lr=6.6356e-05) (hash(x)=48339781)
+45400 val loss 5.8084
+45400 val perplexity 333.0822
+45400 train 5.796069 (lr=5.9407e-06) (hash(x)=49447929)
+6800 val loss 6.7620
+6800 val perplexity 864.3433
+6800 train 6.641977 (lr=9.6156e-05) (hash(x)=46745396)
+8000 val loss 6.6692
+8000 val perplexity 787.8011
+8000 train 6.791812 (lr=6.6263e-05) (hash(x)=54927320)
+45500 val loss 5.8073
+45500 val perplexity 332.7321
+45500 train 5.758606 (lr=5.9005e-06) (hash(x)=50713904)
+8100 val loss 6.6685
+8100 val perplexity 787.1943
+8100 train 6.383808 (lr=6.6169e-05) (hash(x)=46461786)
+6900 val loss 6.7623
+6900 val perplexity 864.6273
+6900 train 6.878799 (lr=9.6040e-05) (hash(x)=46534986)
+8200 val loss 6.6542
+8200 val perplexity 776.0621
+8200 train 6.567985 (lr=6.6073e-05) (hash(x)=51536260)
+45600 val loss 5.8281
+45600 val perplexity 339.7057
+45600 train 5.737741 (lr=5.8612e-06) (hash(x)=47674606)
+7000 val loss 6.7450
+7000 val perplexity 849.7970
+7000 train 7.090025 (lr=9.5923e-05) (hash(x)=49317888)
+8300 val loss 6.6445
+8300 val perplexity 768.5745
+8300 train 6.385070 (lr=6.5976e-05) (hash(x)=44770722)
+45700 val loss 5.8069
+45700 val perplexity 332.6016
+45700 train 5.718114 (lr=5.8227e-06) (hash(x)=51539617)
+8400 val loss 6.6291
+8400 val perplexity 756.7707
+8400 train 6.630779 (lr=6.5879e-05) (hash(x)=50104957)
+7100 val loss 6.7150
+7100 val perplexity 824.7158
+7100 train 6.732314 (lr=9.5804e-05) (hash(x)=50360484)
+8500 val loss 6.6186
+8500 val perplexity 748.8944
+8500 train 6.730096 (lr=6.5780e-05) (hash(x)=50132971)
+45800 val loss 5.8064
+45800 val perplexity 332.4121
+45800 train 5.777891 (lr=5.7851e-06) (hash(x)=44448785)
+7200 val loss 6.7163
+7200 val perplexity 825.7302
+7200 train 6.610737 (lr=9.5683e-05) (hash(x)=49515094)
+8600 val loss 6.6052
+8600 val perplexity 738.9441
+8600 train 6.559481 (lr=6.5680e-05) (hash(x)=52193699)
+45900 val loss 5.8067
+45900 val perplexity 332.5286
+45900 train 5.633348 (lr=5.7484e-06) (hash(x)=51499105)
+8700 val loss 6.5946
+8700 val perplexity 731.1027
+8700 train 6.601634 (lr=6.5579e-05) (hash(x)=47902319)
+7300 val loss 6.7274
+7300 val perplexity 834.9750
+7300 train 6.727911 (lr=9.5561e-05) (hash(x)=51546861)
+8800 val loss 6.5905
+8800 val perplexity 728.1722
+8800 train 6.874655 (lr=6.5477e-05) (hash(x)=54904230)
+46000 val loss 5.8019
+46000 val perplexity 330.9422
+46000 train 5.717524 (lr=5.7125e-06) (hash(x)=48359464)
+8900 val loss 6.5658
+8900 val perplexity 710.3594
+7400 val loss 6.7125
+7400 val perplexity 822.6318
+8900 train 6.456809 (lr=6.5374e-05) (hash(x)=46311615)
+7400 train 6.713262 (lr=9.5437e-05) (hash(x)=48320948)
+46100 val loss 5.8034
+46100 val perplexity 331.4209
+46100 train 5.856997 (lr=5.6775e-06) (hash(x)=51885986)
+9000 val loss 6.5762
+9000 val perplexity 717.8407
+9000 train 6.467047 (lr=6.5270e-05) (hash(x)=48535188)
+7500 val loss 6.7149
+7500 val perplexity 824.6104
+7500 train 6.552715 (lr=9.5312e-05) (hash(x)=40167457)
+9100 val loss 6.5482
+9100 val perplexity 697.9618
+9100 train 6.596004 (lr=6.5164e-05) (hash(x)=51757372)
+46200 val loss 5.8014
+46200 val perplexity 330.7549
+46200 train 6.350838 (lr=5.6434e-06) (hash(x)=65186615)
+9200 val loss 6.5645
+9200 val perplexity 709.4841
+9200 train 6.451322 (lr=6.5058e-05) (hash(x)=51131708)
+7600 val loss 6.6876
+7600 val perplexity 802.4150
+7600 train 6.612953 (lr=9.5185e-05) (hash(x)=49942165)
+9300 val loss 6.5492
+9300 val perplexity 698.7104
+9300 train 6.486413 (lr=6.4951e-05) (hash(x)=44784276)
+46300 val loss 5.8012
+46300 val perplexity 330.6895
+46300 train 5.776049 (lr=5.6101e-06) (hash(x)=49626999)
+7700 val loss 6.6817
+7700 val perplexity 797.6483
+7700 train 6.419989 (lr=9.5057e-05) (hash(x)=48853311)
+9400 val loss 6.5449
+9400 val perplexity 695.6748
+9400 train 6.656165 (lr=6.4842e-05) (hash(x)=51981169)
+46400 val loss 5.7988
+46400 val perplexity 329.9163
+46400 train 5.685073 (lr=5.5777e-06) (hash(x)=43325701)
+9500 val loss 6.5243
+9500 val perplexity 681.5016
+9500 train 6.455208 (lr=6.4733e-05) (hash(x)=47232936)
+7800 val loss 6.7101
+7800 val perplexity 820.6340
+7800 train 6.605823 (lr=9.4926e-05) (hash(x)=48510117)
+9600 val loss 6.5283
+9600 val perplexity 684.2013
+9600 train 6.555980 (lr=6.4622e-05) (hash(x)=53800450)
+46500 val loss 5.8014
+46500 val perplexity 330.7682
+46500 train 6.095033 (lr=5.5462e-06) (hash(x)=54028595)
+7900 val loss 6.6982
+7900 val perplexity 810.9125
+7900 train 6.697021 (lr=9.4795e-05) (hash(x)=48339781)
+9700 val loss 6.5110
+9700 val perplexity 672.4990
+9700 train 6.611804 (lr=6.4511e-05) (hash(x)=55768123)
+46600 val loss 5.8004
+46600 val perplexity 330.4477
+46600 train 5.646887 (lr=5.5156e-06) (hash(x)=44519175)
+9800 val loss 6.5201
+9800 val perplexity 678.6589
+9800 train 6.437235 (lr=6.4398e-05) (hash(x)=47745177)
+8000 val loss 6.7015
+8000 val perplexity 813.6622
+8000 train 6.827817 (lr=9.4661e-05) (hash(x)=54927320)
+46700 val loss 5.8020
+46700 val perplexity 330.9630
+46700 train 5.859873 (lr=5.4858e-06) (hash(x)=48357998)
+9900 val loss 6.5059
+9900 val perplexity 669.1097
+9900 train 6.771001 (lr=6.4284e-05) (hash(x)=56592246)
+8100 val loss 6.7125
+8100 val perplexity 822.6165
+8100 train 6.440390 (lr=9.4526e-05) (hash(x)=46461786)
+10000 val loss 6.5050
+10000 val perplexity 668.4611
+10000 train 6.518945 (lr=6.4170e-05) (hash(x)=51655963)
+46800 val loss 5.8007
+46800 val perplexity 330.5271
+46800 train 5.972873 (lr=5.4569e-06) (hash(x)=55911353)
+10100 val loss 6.5228
+10100 val perplexity 680.4901
+10100 train 6.419588 (lr=6.4054e-05) (hash(x)=49809511)
+8200 val loss 6.6683
+8200 val perplexity 787.0919
+8200 train 6.587510 (lr=9.4390e-05) (hash(x)=51536260)
+46900 val loss 5.8022
+46900 val perplexity 331.0330
+46900 train 5.741434 (lr=5.4289e-06) (hash(x)=47897187)
+10200 val loss 6.5020
+10200 val perplexity 666.4471
+10200 train 6.091907 (lr=6.3937e-05) (hash(x)=42297812)
+8300 val loss 6.7038
+8300 val perplexity 815.4675
+8300 train 6.438375 (lr=9.4252e-05) (hash(x)=44770722)
+10300 val loss 6.4973
+10300 val perplexity 663.3268
+10300 train 6.167974 (lr=6.3820e-05) (hash(x)=55529820)
+47000 val loss 5.7987
+47000 val perplexity 329.8616
+47000 train 5.359689 (lr=5.4017e-06) (hash(x)=43196571)
+10400 val loss 6.5084
+10400 val perplexity 670.7539
+10400 train 6.523131 (lr=6.3701e-05) (hash(x)=53255684)
+8400 val loss 6.6674
+8400 val perplexity 786.3558
+8400 train 6.660437 (lr=9.4112e-05) (hash(x)=50104957)
+10500 val loss 6.4665
+10500 val perplexity 643.2431
+10500 train 6.590051 (lr=6.3581e-05) (hash(x)=54306191)
+47100 val loss 5.7992
+47100 val perplexity 330.0356
+47100 train 5.657062 (lr=5.3755e-06) (hash(x)=51224987)
+8500 val loss 6.6906
+8500 val perplexity 804.8203
+8500 train 6.818645 (lr=9.3971e-05) (hash(x)=50132971)
+10600 val loss 6.4734
+10600 val perplexity 647.6887
+10600 train 6.714907 (lr=6.3460e-05) (hash(x)=60130567)
+47200 val loss 5.8016
+47200 val perplexity 330.8251
+47200 train 5.508888 (lr=5.3501e-06) (hash(x)=47943697)
+10700 val loss 6.4934
+10700 val perplexity 660.7467
+10700 train 6.420184 (lr=6.3339e-05) (hash(x)=50074737)
+8600 val loss 6.6548
+8600 val perplexity 776.4918
+8600 train 6.595765 (lr=9.3828e-05) (hash(x)=52193699)
+10800 val loss 6.4459
+10800 val perplexity 630.1255
+10800 train 6.556093 (lr=6.3216e-05) (hash(x)=51547220)
+47300 val loss 5.8012
+47300 val perplexity 330.6942
+47300 train 5.713035 (lr=5.3256e-06) (hash(x)=47351003)
+10900 val loss 6.4832
+10900 val perplexity 654.0450
+10900 train 6.565746 (lr=6.3092e-05) (hash(x)=55943981)
+8700 val loss 6.6440
+8700 val perplexity 768.1542
+8700 train 6.659471 (lr=9.3684e-05) (hash(x)=47902319)
+47400 val loss 5.8019
+47400 val perplexity 330.9364
+47400 train 5.875019 (lr=5.3020e-06) (hash(x)=55562243)
+11000 val loss 6.4434
+11000 val perplexity 628.5189
+11000 train 6.483435 (lr=6.2968e-05) (hash(x)=46444570)
+8800 val loss 6.6390
+8800 val perplexity 764.3397
+8800 train 6.929801 (lr=9.3538e-05) (hash(x)=54904230)
+11100 val loss 6.4500
+11100 val perplexity 632.7070
+11100 train 6.459686 (lr=6.2842e-05) (hash(x)=49589063)
+47500 val loss 5.8001
+47500 val perplexity 330.3456
+47500 train 5.675170 (lr=5.2792e-06) (hash(x)=53544850)
+11200 val loss 6.4506
+11200 val perplexity 633.0538
+11200 train 6.457446 (lr=6.2715e-05) (hash(x)=51392283)
+8900 val loss 6.6381
+8900 val perplexity 763.6228
+8900 train 6.531604 (lr=9.3391e-05) (hash(x)=46311615)
+47600 val loss 5.8024
+47600 val perplexity 331.0871
+47600 train 5.985342 (lr=5.2574e-06) (hash(x)=43634907)
+11300 val loss 6.4364
+11300 val perplexity 624.1540
+11300 train 6.397132 (lr=6.2588e-05) (hash(x)=45081133)
+9000 val loss 6.6386
+9000 val perplexity 764.0009
+9000 train 6.519309 (lr=9.3242e-05) (hash(x)=48535188)
+11400 val loss 6.4289
+11400 val perplexity 619.4864
+11400 train 6.524812 (lr=6.2459e-05) (hash(x)=53700397)
+47700 val loss 5.7999
+47700 val perplexity 330.2570
+47700 train 5.776142 (lr=5.2364e-06) (hash(x)=47909383)
+11500 val loss 6.4207
+11500 val perplexity 614.4413
+11500 train 6.142918 (lr=6.2330e-05) (hash(x)=43839088)
+9100 val loss 6.6102
+9100 val perplexity 742.6683
+9100 train 6.649930 (lr=9.3092e-05) (hash(x)=51757372)
+47800 val loss 5.7953
+47800 val perplexity 328.7346
+47800 train 5.674577 (lr=5.2163e-06) (hash(x)=45871079)
+11600 val loss 6.4324
+11600 val perplexity 621.6376
+11600 train 6.465709 (lr=6.2199e-05) (hash(x)=48088111)
+9200 val loss 6.6127
+9200 val perplexity 744.4720
+9200 train 6.500169 (lr=9.2940e-05) (hash(x)=51131708)
+11700 val loss 6.4299
+11700 val perplexity 620.0857
+11700 train 6.745090 (lr=6.2068e-05) (hash(x)=55108226)
+47900 val loss 5.7951
+47900 val perplexity 328.6861
+47900 train 5.784526 (lr=5.1972e-06) (hash(x)=47333324)
+11800 val loss 6.4176
+11800 val perplexity 612.5077
+11800 train 6.684676 (lr=6.1936e-05) (hash(x)=58524839)
+9300 val loss 6.5997
+9300 val perplexity 734.8544
+9300 train 6.554953 (lr=9.2786e-05) (hash(x)=44784276)
+48000 val loss 5.7948
+48000 val perplexity 328.5829
+48000 train 5.876556 (lr=5.1788e-06) (hash(x)=52758020)
+11900 val loss 6.4150
+11900 val perplexity 610.9667
+11900 train 6.081831 (lr=6.1802e-05) (hash(x)=43864078)
+9400 val loss 6.5917
+9400 val perplexity 729.0008
+9400 train 6.715466 (lr=9.2632e-05) (hash(x)=51981169)
+12000 val loss 6.4093
+12000 val perplexity 607.4685
+12000 train 6.095185 (lr=6.1668e-05) (hash(x)=43448544)
+48100 val loss 5.7937
+48100 val perplexity 328.2226
+48100 train 5.802242 (lr=5.1614e-06) (hash(x)=49806349)
+12100 val loss 6.3978
+12100 val perplexity 600.5402
+12100 train 6.396197 (lr=6.1533e-05) (hash(x)=55200399)
+9500 val loss 6.6014
+9500 val perplexity 736.1489
+9500 train 6.523407 (lr=9.2475e-05) (hash(x)=47232936)
+12200 val loss 6.4067
+12200 val perplexity 605.8679
+48200 val loss 5.7914
+48200 val perplexity 327.4593
+12200 train 6.546592 (lr=6.1397e-05) (hash(x)=57627314)
+48200 train 5.868467 (lr=5.1449e-06) (hash(x)=53220839)
+9600 val loss 6.5844
+9600 val perplexity 723.6813
+9600 train 6.593096 (lr=9.2317e-05) (hash(x)=53800450)
+12300 val loss 6.4145
+12300 val perplexity 610.6370
+12300 train 6.693074 (lr=6.1260e-05) (hash(x)=53617087)
+48300 val loss 5.7940
+48300 val perplexity 328.3305
+48300 train 5.822805 (lr=5.1293e-06) (hash(x)=56052541)
+12400 val loss 6.4177
+12400 val perplexity 612.5649
+12400 train 6.262563 (lr=6.1122e-05) (hash(x)=51135678)
+9700 val loss 6.5799
+9700 val perplexity 720.4314
+9700 train 6.675194 (lr=9.2158e-05) (hash(x)=55768123)
+48400 val loss 5.7905
+48400 val perplexity 327.1818
+48400 train 5.670732 (lr=5.1145e-06) (hash(x)=44482356)
+12500 val loss 6.4237
+12500 val perplexity 616.2940
+12500 train 6.428000 (lr=6.0984e-05) (hash(x)=48025130)
+12600 val loss 6.4127
+12600 val perplexity 609.5201
+9800 val loss 6.5909
+9800 val perplexity 728.4636
+12600 train 6.470107 (lr=6.0844e-05) (hash(x)=52135695)
+9800 train 6.517842 (lr=9.1997e-05) (hash(x)=47745177)
+48500 val loss 5.7897
+48500 val perplexity 326.9112
+48500 train 5.520173 (lr=5.1007e-06) (hash(x)=45714818)
+12700 val loss 6.4382
+12700 val perplexity 625.2830
+12700 train 6.303355 (lr=6.0703e-05) (hash(x)=51888613)
+9900 val loss 6.5974
+9900 val perplexity 733.2065
+9900 train 6.867185 (lr=9.1835e-05) (hash(x)=56592246)
+48600 val loss 5.7913
+48600 val perplexity 327.4302
+48600 train 5.555061 (lr=5.0877e-06) (hash(x)=49476556)
+12800 val loss 6.4182
+12800 val perplexity 612.8937
+12800 train 6.351943 (lr=6.0562e-05) (hash(x)=50418818)
+12900 val loss 6.4053
+12900 val perplexity 605.0304
+12900 train 7.257672 (lr=6.0420e-05) (hash(x)=58649585)
+10000 val loss 6.6087
+10000 val perplexity 741.5094
+10000 train 6.601171 (lr=9.1671e-05) (hash(x)=51655963)
+48700 val loss 5.7916
+48700 val perplexity 327.5210
+48700 train 5.483535 (lr=5.0756e-06) (hash(x)=42508579)
+13000 val loss 6.3998
+13000 val perplexity 601.7260
+13000 train 6.663928 (lr=6.0277e-05) (hash(x)=54567307)
+10100 val loss 6.5917
+10100 val perplexity 728.9938
+10100 train 6.492918 (lr=9.1506e-05) (hash(x)=49809511)
+13100 val loss 6.3887
+13100 val perplexity 595.0941
+13100 train 6.565659 (lr=6.0133e-05) (hash(x)=52071473)
+48800 val loss 5.7920
+48800 val perplexity 327.6678
+48800 train 6.004168 (lr=5.0644e-06) (hash(x)=52737449)
+13200 val loss 6.3823
+13200 val perplexity 591.2615
+13200 train 6.148626 (lr=5.9988e-05) (hash(x)=46293092)
+10200 val loss 6.5790
+10200 val perplexity 719.8549
+10200 train 6.172993 (lr=9.1339e-05) (hash(x)=42297812)
+48900 val loss 5.7912
+48900 val perplexity 327.4165
+48900 train 5.636753 (lr=5.0542e-06) (hash(x)=47057569)
+13300 val loss 6.3882
+13300 val perplexity 594.8053
+13300 train 6.513732 (lr=5.9842e-05) (hash(x)=56511467)
+10300 val loss 6.5718
+10300 val perplexity 714.6194
+10300 train 6.288669 (lr=9.1171e-05) (hash(x)=55529820)
+13400 val loss 6.3714
+13400 val perplexity 584.8615
+13400 train 6.486885 (lr=5.9695e-05) (hash(x)=54753763)
+49000 val loss 5.7940
+49000 val perplexity 328.3136
+49000 train 5.770644 (lr=5.0448e-06) (hash(x)=49908975)
+13500 val loss 6.3630
+13500 val perplexity 579.9861
+13500 train 6.510321 (lr=5.9548e-05) (hash(x)=53610247)
+10400 val loss 6.5765
+10400 val perplexity 717.9875
+10400 train 6.559584 (lr=9.1001e-05) (hash(x)=53255684)
+49100 val loss 5.7920
+49100 val perplexity 327.6564
+49100 train 5.614700 (lr=5.0363e-06) (hash(x)=48427414)
+13600 val loss 6.3478
+13600 val perplexity 571.2138
+13600 train 6.225275 (lr=5.9400e-05) (hash(x)=47526249)
+10500 val loss 6.5377
+10500 val perplexity 690.7025
+10500 train 6.650892 (lr=9.0830e-05) (hash(x)=54306191)
+13700 val loss 6.3612
+13700 val perplexity 578.9458
+13700 train 6.217256 (lr=5.9251e-05) (hash(x)=51185517)
+49200 val loss 5.7897
+49200 val perplexity 326.9223
+49200 train 5.618961 (lr=5.0286e-06) (hash(x)=50246074)
+13800 val loss 6.3604
+13800 val perplexity 578.4911
+13800 train 6.032686 (lr=5.9101e-05) (hash(x)=45953529)
+10600 val loss 6.5441
+10600 val perplexity 695.1240
+10600 train 6.779467 (lr=9.0658e-05) (hash(x)=60130567)
+49300 val loss 5.7902
+49300 val perplexity 327.0729
+49300 train 5.844873 (lr=5.0219e-06) (hash(x)=47715359)
+13900 val loss 6.3799
+13900 val perplexity 589.8560
+13900 train 6.307417 (lr=5.8950e-05) (hash(x)=47238157)
+14000 val loss 6.3578
+14000 val perplexity 577.0012
+14000 train 6.338602 (lr=5.8799e-05) (hash(x)=54250750)
+10700 val loss 6.5191
+10700 val perplexity 677.9451
+10700 train 6.443255 (lr=9.0484e-05) (hash(x)=50074737)
+49400 val loss 5.7951
+49400 val perplexity 328.7013
+49400 train 5.789733 (lr=5.0161e-06) (hash(x)=50175867)
+14100 val loss 6.3644
+14100 val perplexity 580.7674
+14100 train 6.351201 (lr=5.8646e-05) (hash(x)=48198552)
+10800 val loss 6.5320
+10800 val perplexity 686.7475
+10800 train 6.645676 (lr=9.0308e-05) (hash(x)=51547220)
+14200 val loss 6.3626
+14200 val perplexity 579.7433
+49500 val loss 5.7927
+49500 val perplexity 327.8968
+14200 train 6.497223 (lr=5.8493e-05) (hash(x)=52020690)
+49500 train 5.631345 (lr=5.0112e-06) (hash(x)=49336040)
+14300 val loss 6.3541
+14300 val perplexity 574.8251
+14300 train 6.156728 (lr=5.8339e-05) (hash(x)=45165483)
+10900 val loss 6.5470
+10900 val perplexity 697.1835
+10900 train 6.629424 (lr=9.0132e-05) (hash(x)=55943981)
+49600 val loss 5.7945
+49600 val perplexity 328.5037
+49600 train 5.834286 (lr=5.0072e-06) (hash(x)=52039357)
+14400 val loss 6.3595
+14400 val perplexity 577.9749
+14400 train 6.475995 (lr=5.8184e-05) (hash(x)=52184072)
+11000 val loss 6.5333
+11000 val perplexity 687.6558
+11000 train 6.565971 (lr=8.9954e-05) (hash(x)=46444570)
+14500 val loss 6.3595
+14500 val perplexity 577.9520
+14500 train 6.706707 (lr=5.8029e-05) (hash(x)=56046436)
+49700 val loss 5.7948
+49700 val perplexity 328.5858
+49700 train 5.590436 (lr=5.0040e-06) (hash(x)=47568707)
+14600 val loss 6.3534
+14600 val perplexity 574.4607
+14600 train 6.230230 (lr=5.7872e-05) (hash(x)=52029694)
+11100 val loss 6.5385
+11100 val perplexity 691.2264
+11100 train 6.547019 (lr=8.9774e-05) (hash(x)=49589063)
+49800 val loss 5.7962
+49800 val perplexity 329.0321
+49800 train 5.719766 (lr=5.0018e-06) (hash(x)=48451274)
+14700 val loss 6.3699
+14700 val perplexity 584.0244
+14700 train 6.211578 (lr=5.7715e-05) (hash(x)=50258224)
+11200 val loss 6.5242
+11200 val perplexity 681.4029
+11200 train 6.523566 (lr=8.9593e-05) (hash(x)=51392283)
+14800 val loss 6.3676
+14800 val perplexity 582.6866
+14800 train 5.692614 (lr=5.7558e-05) (hash(x)=42112262)
+49900 val loss 5.7984
+49900 val perplexity 329.7762
+49900 train 5.486496 (lr=5.0004e-06) (hash(x)=44523603)
+14900 val loss 6.3608
+14900 val perplexity 578.7267
+14900 train 6.101720 (lr=5.7399e-05) (hash(x)=47219933)
+11300 val loss 6.5394
+11300 val perplexity 691.8585
+11300 train 6.495648 (lr=8.9411e-05) (hash(x)=45081133)
+49999 val loss 5.7909
+49999 val perplexity 327.3206
+15000 val loss 6.4009
+15000 val perplexity 602.3866
+15000 train 6.002666 (lr=5.7240e-05) (hash(x)=58309309)
+11400 val loss 6.5171
+11400 val perplexity 676.6274
+11400 train 6.611968 (lr=8.9227e-05) (hash(x)=53700397)
+15100 val loss 6.3498
+15100 val perplexity 572.3857
+15100 train 6.254695 (lr=5.7079e-05) (hash(x)=48756049)
+15200 val loss 6.3577
+15200 val perplexity 576.9346
+15200 train 6.047930 (lr=5.6919e-05) (hash(x)=49791737)
+11500 val loss 6.5199
+11500 val perplexity 678.5172
+11500 train 6.260948 (lr=8.9043e-05) (hash(x)=43839088)
+15300 val loss 6.3336
+15300 val perplexity 563.2054
+15300 train 6.302131 (lr=5.6757e-05) (hash(x)=53084126)
+11600 val loss 6.5099
+11600 val perplexity 671.7664
+11600 train 6.543172 (lr=8.8856e-05) (hash(x)=48088111)
+15400 val loss 6.3310
+15400 val perplexity 561.6938
+15400 train 6.422942 (lr=5.6595e-05) (hash(x)=55041679)
+15500 val loss 6.3303
+15500 val perplexity 561.3511
+15500 train 6.124517 (lr=5.6432e-05) (hash(x)=43249867)
+11700 val loss 6.5147
+11700 val perplexity 674.9705
+11700 train 6.829165 (lr=8.8668e-05) (hash(x)=55108226)
+15600 val loss 6.3211
+15600 val perplexity 556.1926
+15600 train 6.312618 (lr=5.6268e-05) (hash(x)=49006517)
+15700 val loss 6.3168
+15700 val perplexity 553.8163
+15700 train 6.645555 (lr=5.6104e-05) (hash(x)=60986839)
+11800 val loss 6.5320
+11800 val perplexity 686.7426
+11800 train 6.764431 (lr=8.8479e-05) (hash(x)=58524839)
+15800 val loss 6.3371
+15800 val perplexity 565.1663
+15800 train 6.246183 (lr=5.5938e-05) (hash(x)=48354906)
+11900 val loss 6.5044
+11900 val perplexity 668.0739
+11900 train 6.170547 (lr=8.8289e-05) (hash(x)=43864078)
+15900 val loss 6.3422
+15900 val perplexity 568.0605
+15900 train 6.207072 (lr=5.5773e-05) (hash(x)=52679780)
+16000 val loss 6.3104
+16000 val perplexity 550.2414
+16000 train 6.475496 (lr=5.5606e-05) (hash(x)=58049587)
+12000 val loss 6.5055
+12000 val perplexity 668.7924
+12000 train 6.222349 (lr=8.8097e-05) (hash(x)=43448544)

attention_kindselective_n_heads4_seed1340/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e5d1d69356d9c198afaea93cf7ccf66f2d5809310214a3e417a834f53abe451a
+oid sha256:e856700b2ba8c1c78fec371f511980a9165c3a304e65b1e2ecf19f9a9ba2c4f5
 size 92843394

attention_kindselective_n_heads4_seed1340/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3969de9631fa1091a1c449d2a388db43ecd8c763683bd08da56a12468a2b209a
+oid sha256:6ce0f91188a61cf2a20045bdc8d3dc3070278a30ac99975978bf3d232dd03f70
 size 92843394

attention_kindselective_n_heads4_seed1340/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf4eafbd63635551ef8204a5b1e4fa01c7111a78c8eb15db22b64873e52bad4b
+oid sha256:21edaf2540d7212d65d0bd210419f7d6a0f9281cde1d06f864c3ec1d63fff124
 size 92843394

attention_kindselective_n_heads4_seed1340/model_10000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e28d809d0626f302c7278753b8c13072c86cc80bea7caeb91264050622dd5e7
+size 92843394

attention_kindselective_n_heads4_seed1340/model_12500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69a6cd3908266914a289ba86b8a7d2af8ce51bf4034f6b670c6730502d6f65d4
+size 92843394

attention_kindselective_n_heads4_seed1340/model_42500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e937e5bde413b5f1d5cd2e7a03428c77cca6ac8ef7547cc1d8aa0565c33ea0d
+size 92843394

attention_kindselective_n_heads4_seed1340/model_45000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03ef3fa0f39e905fced4c8e9f6d4c4c36d3dfdec8a8589b65a5f4c6363469015
+size 92843394

attention_kindselective_n_heads4_seed1340/model_47500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58579aca525c6640a50ed6de3f5bc997ccf568b1c74ac74080a8413181c72725
+size 92843394

attention_kindselective_n_heads4_seed1340/model_49999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddd2671bc53d3d47c2a3597b04a352fe6e56e1aa82a19bed3ee6f4e205890842
+size 92843394

attention_kindselective_n_heads4_seed1340/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:06179ff7fbca3cf6f6d3d1546cd7e6f27c331a63a9a71de710896c329a40b635
+oid sha256:71d5d80bc8621c01c7517a9c8966efcf316b98161adb2e069c5201828111ea9b
 size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:534c88f2b95620226732b552d992f6d228437ef793d6a67da093fc7cc42ce5e8
+oid sha256:4cf0986c29b0869c7e9016f8a369414e67e321e3e73344a91dc6a3451b88ff99
 size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f9ae9b15de67f44a4e1d5dcb110a3995dd6867e2ff4a70347ffd32b561b4229
+oid sha256:f45a3b4647bd40c9e04b365a1e5c1703f4207fa3698dc385fc2cfd5bc0ea4bd5
 size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_10000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06bc9c8845c72cf18f14e45b43f7c4c19b17aa50a69cff163f981ea53f88744f
+size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_12500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61803c8687b0d4b6de778e759324ef063faecd26bcfb6e193e45505fe759b8d9
+size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_42500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dffbcca299af84d6d052050b7c9c7230f9052fbb489bbc7104607f111ccd656e
+size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_45000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b4dba3c28b0a3140da1cda29187583bfb0ffbc5b49bd20693dae0ef1a68772c
+size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_47500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee4537d10c929400a183bc037015dc153913766e3bc65eeecde61d2e9ef87311
+size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_49999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5280d94beeaac57235e9ca63bb2521ca98af393b93180fe9a674caafe9bb168
+size 179406214