Upload folder using huggingface_hub

attention_kindselective_n_heads8_seed1341/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads8_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 8, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_10240_8_1341", "n_embd": 512}

attention_kindselective_n_heads8_seed1341/dataloader_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
+size 964

attention_kindselective_n_heads8_seed1341/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f03ed2ebf741f15e13c79e6cc1e9a19b308450d81cc3b4d8d0338c63d77ca59
+size 964

attention_kindselective_n_heads8_seed1341/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82590037fb2eecbec961f7967a8dd1b8d85515d31a252f66b92b8139858a8b7c
+size 964

attention_kindselective_n_heads8_seed1341/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c544303717d12355a69b8ffb1eb109434e4fdccfd5a61279b6e8ba2e870d6700
+size 964

attention_kindselective_n_heads8_seed1341/log2.txt

@@ -0,0 +1,1209 @@
+max_steps: 10000
+0 val loss 11.0680
+0 val perplexity 64089.5781
+0 val loss 11.0680
+0 val perplexity 64089.5781
+0 val loss 11.0680
+0 val perplexity 64089.5781
+0 val loss 11.0680
+0 val perplexity 64089.5781
+0 train 11.048663 (lr=1.0000e-07) (hash(x)=47078120)
+0 train 11.048663 (lr=2.5000e-07) (hash(x)=47078120)
+0 train 11.048663 (lr=3.5000e-07) (hash(x)=47078120)
+0 train 11.048676 (lr=1.5000e-07) (hash(x)=47078120)
+100 val loss 9.8889
+100 val perplexity 19711.1328
+100 train 9.874242 (lr=1.0100e-05) (hash(x)=43429388)
+100 val loss 9.4787
+100 val perplexity 13077.7705
+100 val loss 9.7282
+100 val perplexity 16783.9766
+100 train 9.448478 (lr=3.5350e-05) (hash(x)=43429388)
+100 val loss 9.5889
+100 val perplexity 14601.5840
+100 train 9.713765 (lr=1.5150e-05) (hash(x)=43429388)
+100 train 9.563825 (lr=2.5250e-05) (hash(x)=43429388)
+200 val loss 9.4174
+200 val perplexity 12300.9268
+200 train 9.477206 (lr=2.0000e-05) (hash(x)=52929681)
+200 val loss 8.3919
+200 val perplexity 4411.0986
+200 val loss 8.0984
+200 val perplexity 3289.3396
+200 train 8.515907 (lr=5.0000e-05) (hash(x)=52929681)
+200 val loss 9.1730
+200 val perplexity 9633.6963
+200 train 8.248425 (lr=7.0000e-05) (hash(x)=52929681)
+200 train 9.244697 (lr=3.0000e-05) (hash(x)=52929681)
+300 val loss 8.8455
+300 val perplexity 6943.1738
+300 train 8.874369 (lr=1.9995e-05) (hash(x)=49930367)
+300 val loss 7.7464
+300 val perplexity 2313.2307
+300 train 7.757118 (lr=4.9988e-05) (hash(x)=49930367)
+300 val loss 7.6647
+300 val perplexity 2131.7766
+300 val loss 8.2538
+300 val perplexity 3842.1523
+300 train 7.684315 (lr=6.9984e-05) (hash(x)=49930367)
+300 train 8.276509 (lr=2.9993e-05) (hash(x)=49930367)
+400 val loss 8.2382
+400 val perplexity 3782.5420
+400 train 8.121539 (lr=1.9982e-05) (hash(x)=48542946)
+400 val loss 7.5870
+400 val perplexity 1972.4014
+400 train 7.386682 (lr=4.9954e-05) (hash(x)=48542946)
+400 val loss 7.5243
+400 val perplexity 1852.4517
+400 val loss 7.8717
+400 val perplexity 2621.8926
+400 train 7.326735 (lr=6.9935e-05) (hash(x)=48542946)
+400 train 7.709966 (lr=2.9972e-05) (hash(x)=48542946)
+500 val loss 7.9134
+500 val perplexity 2733.5874
+500 train 8.054297 (lr=1.9958e-05) (hash(x)=55286048)
+500 val loss 7.6640
+500 val perplexity 2130.2502
+500 val loss 7.5074
+500 val perplexity 1821.5359
+500 val loss 7.4518
+500 val perplexity 1722.9878
+500 train 7.837950 (lr=2.9938e-05) (hash(x)=55286048)
+500 train 7.714735 (lr=4.9896e-05) (hash(x)=55286048)
+500 train 7.667075 (lr=6.9854e-05) (hash(x)=55286048)
+600 val loss 7.7274
+600 val perplexity 2269.6853
+600 train 7.917661 (lr=1.9926e-05) (hash(x)=51577760)
+600 val loss 7.5601
+600 val perplexity 1920.0514
+600 train 7.793221 (lr=2.9889e-05) (hash(x)=51577760)
+600 val loss 7.4491
+600 val perplexity 1718.2578
+600 val loss 7.3973
+600 val perplexity 1631.6017
+600 train 7.691466 (lr=4.9815e-05) (hash(x)=51577760)
+600 train 7.632826 (lr=6.9741e-05) (hash(x)=51577760)
+700 val loss 7.6300
+700 val perplexity 2059.1208
+700 train 7.758367 (lr=1.9885e-05) (hash(x)=57433471)
+700 val loss 7.4870
+700 val perplexity 1784.6235
+700 train 7.647078 (lr=2.9827e-05) (hash(x)=57433471)
+700 val loss 7.3347
+700 val perplexity 1532.5104
+700 train 7.507759 (lr=6.9596e-05) (hash(x)=57433471)
+700 val loss 7.3973
+700 val perplexity 1631.5386
+700 train 7.570853 (lr=4.9712e-05) (hash(x)=57433471)
+800 val loss 7.5663
+800 val perplexity 1931.9979
+800 train 7.500276 (lr=1.9834e-05) (hash(x)=49799291)
+800 val loss 7.4398
+800 val perplexity 1702.4897
+800 train 7.366629 (lr=2.9751e-05) (hash(x)=49799291)
+800 val loss 7.3000
+800 val perplexity 1480.3511
+800 train 7.210431 (lr=6.9419e-05) (hash(x)=49799291)
+800 val loss 7.3632
+800 val perplexity 1576.9290
+800 train 7.281087 (lr=4.9585e-05) (hash(x)=49799291)
+900 val loss 7.5209
+900 val perplexity 1846.1362
+900 train 7.526892 (lr=1.9774e-05) (hash(x)=49502839)
+900 val loss 7.4191
+900 val perplexity 1667.5374
+900 train 7.422843 (lr=2.9662e-05) (hash(x)=49502839)
+900 val loss 7.2387
+900 val perplexity 1392.2841
+900 train 7.215050 (lr=6.9210e-05) (hash(x)=49502839)
+900 val loss 7.3366
+900 val perplexity 1535.4969
+900 train 7.333570 (lr=4.9436e-05) (hash(x)=49502839)
+1000 val loss 7.4822
+1000 val perplexity 1776.1934
+1000 train 7.707015 (lr=1.9706e-05) (hash(x)=51142904)
+1000 val loss 7.3874
+1000 val perplexity 1615.5121
+1000 train 7.649211 (lr=2.9558e-05) (hash(x)=51142904)
+1000 val loss 7.1808
+1000 val perplexity 1313.9176
+1000 train 7.483747 (lr=6.8970e-05) (hash(x)=51142904)
+1000 val loss 7.3008
+1000 val perplexity 1481.4540
+1000 train 7.578658 (lr=4.9264e-05) (hash(x)=51142904)
+1100 val loss 7.4532
+1100 val perplexity 1725.2938
+1100 train 7.515146 (lr=1.9628e-05) (hash(x)=52751086)
+1100 val loss 7.3649
+1100 val perplexity 1579.5704
+1100 train 7.431870 (lr=2.9442e-05) (hash(x)=52751086)
+1100 val loss 7.1476
+1100 val perplexity 1271.1102
+1100 train 7.207986 (lr=6.8698e-05) (hash(x)=52751086)
+1100 val loss 7.2667
+1100 val perplexity 1431.8098
+1100 train 7.324548 (lr=4.9070e-05) (hash(x)=52751086)
+1200 val loss 7.4319
+1200 val perplexity 1688.9976
+1200 train 7.388729 (lr=1.9542e-05) (hash(x)=51538621)
+1200 val loss 7.3563
+1200 val perplexity 1566.0082
+1200 train 7.301937 (lr=2.9312e-05) (hash(x)=51538621)
+1200 val loss 7.1481
+1200 val perplexity 1271.6545
+1200 train 7.103180 (lr=6.8395e-05) (hash(x)=51538621)
+1200 val loss 7.2521
+1200 val perplexity 1411.0454
+1200 train 7.200856 (lr=4.8854e-05) (hash(x)=51538621)
+1300 val loss 7.4095
+1300 val perplexity 1651.6470
+1300 train 7.466151 (lr=1.9446e-05) (hash(x)=52034040)
+1300 val loss 7.3408
+1300 val perplexity 1541.8721
+1300 train 7.412338 (lr=2.9169e-05) (hash(x)=52034040)
+1300 val loss 7.0932
+1300 val perplexity 1203.7023
+1300 train 7.166739 (lr=6.8062e-05) (hash(x)=52034040)
+1300 val loss 7.2236
+1300 val perplexity 1371.4453
+1300 train 7.305515 (lr=4.8616e-05) (hash(x)=52034040)
+1400 val loss 7.3843
+1400 val perplexity 1610.4229
+1400 train 7.408330 (lr=1.9342e-05) (hash(x)=50640105)
+1400 val loss 7.3167
+1400 val perplexity 1505.1840
+1400 train 7.352315 (lr=2.9013e-05) (hash(x)=50640105)
+1400 val loss 7.0789
+1400 val perplexity 1186.6193
+1400 train 7.134275 (lr=6.7698e-05) (hash(x)=50640105)
+1400 val loss 7.2080
+1400 val perplexity 1350.1521
+1400 train 7.266603 (lr=4.8356e-05) (hash(x)=50640105)
+1500 val loss 7.3619
+1500 val perplexity 1574.9045
+1500 train 7.287918 (lr=1.9230e-05) (hash(x)=49016270)
+1500 val loss 7.2903
+1500 val perplexity 1465.9382
+1500 train 7.219070 (lr=2.8845e-05) (hash(x)=49016270)
+1500 val loss 7.0480
+1500 val perplexity 1150.5200
+1500 train 6.976660 (lr=6.7304e-05) (hash(x)=49016270)
+1500 val loss 7.1885
+1500 val perplexity 1324.1281
+1500 train 7.115997 (lr=4.8074e-05) (hash(x)=49016270)
+1600 val loss 7.3388
+1600 val perplexity 1538.8591
+1600 train 7.111701 (lr=1.9109e-05) (hash(x)=46100488)
+1600 val loss 7.2736
+1600 val perplexity 1441.7178
+1600 train 7.037468 (lr=2.8663e-05) (hash(x)=46100488)
+1600 val loss 7.0191
+1600 val perplexity 1117.7505
+1600 train 6.790306 (lr=6.6881e-05) (hash(x)=46100488)
+1600 val loss 7.1701
+1600 val perplexity 1299.9185
+1600 train 6.944131 (lr=4.7772e-05) (hash(x)=46100488)
+1700 val loss 7.3212
+1700 val perplexity 1512.0359
+1700 train 7.369852 (lr=1.8979e-05) (hash(x)=49185350)
+1700 val loss 7.2640
+1700 val perplexity 1427.8956
+1700 train 7.317546 (lr=2.8469e-05) (hash(x)=49185350)
+1700 val loss 7.0044
+1700 val perplexity 1101.4866
+1700 train 7.065816 (lr=6.6428e-05) (hash(x)=49185350)
+1700 val loss 7.1566
+1700 val perplexity 1282.5233
+1700 train 7.217639 (lr=4.7448e-05) (hash(x)=49185350)
+1800 val loss 7.3080
+1800 val perplexity 1492.1678
+1800 train 7.221244 (lr=1.8842e-05) (hash(x)=48024574)
+1800 val loss 7.2471
+1800 val perplexity 1403.9862
+1800 train 7.151842 (lr=2.8263e-05) (hash(x)=48024574)
+1800 val loss 6.9785
+1800 val perplexity 1073.2799
+1800 train 6.877046 (lr=6.5947e-05) (hash(x)=48024574)
+1800 val loss 7.1461
+1800 val perplexity 1269.1606
+1900 val loss 7.2891
+1900 val perplexity 1464.3209
+1800 train 7.040640 (lr=4.7105e-05) (hash(x)=48024574)
+1900 train 7.115648 (lr=1.8696e-05) (hash(x)=45823189)
+1900 val loss 7.2267
+1900 val perplexity 1375.6698
+1900 train 7.034698 (lr=2.8044e-05) (hash(x)=45823189)
+1900 val loss 6.9555
+1900 val perplexity 1048.9285
+1900 train 6.739227 (lr=6.5437e-05) (hash(x)=45823189)
+2000 val loss 7.2767
+2000 val perplexity 1446.2552
+2000 train 7.108987 (lr=1.8543e-05) (hash(x)=45703932)
+1900 val loss 7.1339
+1900 val perplexity 1253.7621
+1900 train 6.941078 (lr=4.6741e-05) (hash(x)=45823189)
+2000 val loss 7.2035
+2000 val perplexity 1344.1517
+2000 train 7.027834 (lr=2.7814e-05) (hash(x)=45703932)
+2000 val loss 6.9407
+2000 val perplexity 1033.5043
+2000 train 6.743909 (lr=6.4900e-05) (hash(x)=45703932)
+2100 val loss 7.2564
+2100 val perplexity 1417.1031
+2100 train 7.887122 (lr=1.8382e-05) (hash(x)=58570170)
+2000 val loss 7.1274
+2000 val perplexity 1245.6257
+2000 train 6.943665 (lr=4.6357e-05) (hash(x)=45703932)
+2100 val loss 7.1834
+2100 val perplexity 1317.3862
+2100 train 7.836891 (lr=2.7572e-05) (hash(x)=58570170)
+2100 val loss 6.9112
+2100 val perplexity 1003.4402
+2100 train 7.606760 (lr=6.4335e-05) (hash(x)=58570170)
+2200 val loss 7.2513
+2200 val perplexity 1409.9761
+2200 train 7.275453 (lr=1.8213e-05) (hash(x)=55262880)
+2100 val loss 7.1219
+2100 val perplexity 1238.8298
+2100 train 7.786014 (lr=4.5954e-05) (hash(x)=58570170)
+2200 val loss 7.1710
+2200 val perplexity 1301.1643
+2200 train 7.189610 (lr=2.7319e-05) (hash(x)=55262880)
+2200 val loss 6.9619
+2200 val perplexity 1055.6140
+2200 train 6.981012 (lr=6.3745e-05) (hash(x)=55262880)
+2300 val loss 7.2239
+2300 val perplexity 1371.8573
+2300 train 6.981198 (lr=1.8036e-05) (hash(x)=46415497)
+2200 val loss 7.1250
+2200 val perplexity 1242.6696
+2200 train 7.142995 (lr=4.5532e-05) (hash(x)=55262880)
+2300 val loss 7.1466
+2300 val perplexity 1269.8204
+2300 train 6.890901 (lr=2.7055e-05) (hash(x)=46415497)
+2400 val loss 7.2045
+2400 val perplexity 1345.4766
+2400 train 7.125453 (lr=1.7853e-05) (hash(x)=49272278)
+2300 val loss 6.8938
+2300 val perplexity 986.1673
+2300 train 6.622870 (lr=6.3128e-05) (hash(x)=46415497)
+2300 val loss 7.1014
+2300 val perplexity 1213.7175
+2300 train 6.864444 (lr=4.5091e-05) (hash(x)=46415497)
+2400 val loss 7.1125
+2400 val perplexity 1227.2383
+2400 train 7.022389 (lr=2.6780e-05) (hash(x)=49272278)
+2500 val loss 7.1825
+2500 val perplexity 1316.1329
+2500 train 7.040866 (lr=1.7663e-05) (hash(x)=48390803)
+2400 val loss 6.8477
+2400 val perplexity 941.7234
+2400 train 6.740791 (lr=6.2486e-05) (hash(x)=49272278)
+2400 val loss 7.0815
+2400 val perplexity 1189.7360
+2400 train 6.993971 (lr=4.4633e-05) (hash(x)=49272278)
+2500 val loss 7.0937
+2500 val perplexity 1204.3481
+2500 train 6.950256 (lr=2.6494e-05) (hash(x)=48390803)
+2500 val loss 6.8303
+2500 val perplexity 925.4961
+2600 val loss 7.1665
+2600 val perplexity 1295.2786
+2600 train 7.031013 (lr=1.7465e-05) (hash(x)=47450116)
+2500 train 6.699334 (lr=6.1819e-05) (hash(x)=48390803)
+2500 val loss 7.0623
+2500 val perplexity 1167.1173
+2500 train 6.923334 (lr=4.4156e-05) (hash(x)=48390803)
+2600 val loss 7.0709
+2600 val perplexity 1177.1604
+2600 train 6.918849 (lr=2.6198e-05) (hash(x)=47450116)
+2700 val loss 7.1567
+2700 val perplexity 1282.6627
+2700 train 7.217981 (lr=1.7261e-05) (hash(x)=52681152)
+2600 val loss 6.8076
+2600 val perplexity 904.6650
+2600 train 6.673070 (lr=6.1128e-05) (hash(x)=47450116)
+2600 val loss 7.0478
+2600 val perplexity 1150.3018
+2600 train 6.902493 (lr=4.3663e-05) (hash(x)=47450116)
+2700 val loss 7.0496
+2700 val perplexity 1152.3412
+2700 train 7.117827 (lr=2.5892e-05) (hash(x)=52681152)
+2800 val loss 7.1363
+2800 val perplexity 1256.7675
+2800 train 7.209763 (lr=1.7051e-05) (hash(x)=50664094)
+2700 val loss 6.7852
+2700 val perplexity 884.6849
+2700 train 6.864418 (lr=6.0414e-05) (hash(x)=52681152)
+2700 val loss 7.0395
+2700 val perplexity 1140.7711
+2700 train 7.108398 (lr=4.3153e-05) (hash(x)=52681152)
+2800 val loss 7.0391
+2800 val perplexity 1140.3605
+2800 train 7.111782 (lr=2.5576e-05) (hash(x)=50664094)
+2900 val loss 7.1164
+2900 val perplexity 1231.9611
+2900 train 6.889511 (lr=1.6834e-05) (hash(x)=47067144)
+2800 val loss 6.7762
+2800 val perplexity 876.7094
+2800 train 6.850021 (lr=5.9677e-05) (hash(x)=50664094)
+2800 val loss 7.0292
+2800 val perplexity 1129.1125
+2800 train 7.094847 (lr=4.2627e-05) (hash(x)=50664094)
+2900 val loss 7.0108
+2900 val perplexity 1108.5176
+2900 train 6.776527 (lr=2.5251e-05) (hash(x)=47067144)
+3000 val loss 7.0995
+3000 val perplexity 1211.4059
+3000 train 6.896778 (lr=1.6611e-05) (hash(x)=45015009)
+2900 val loss 6.7621
+2900 val perplexity 864.4884
+2900 train 6.564690 (lr=5.8919e-05) (hash(x)=47067144)
+2900 val loss 7.0175
+2900 val perplexity 1115.9957
+2900 train 6.791795 (lr=4.2085e-05) (hash(x)=47067144)
+3000 val loss 6.9916
+3000 val perplexity 1087.4424
+3000 train 6.785407 (lr=2.4917e-05) (hash(x)=45015009)
+3100 val loss 7.0851
+3100 val perplexity 1194.0815
+3100 train 6.896208 (lr=1.6383e-05) (hash(x)=45245896)
+3000 val loss 6.7417
+3000 val perplexity 847.0032
+3000 train 6.542316 (lr=5.8140e-05) (hash(x)=45015009)
+3000 val loss 7.0100
+3000 val perplexity 1107.6675
+3000 train 6.804945 (lr=4.1529e-05) (hash(x)=45015009)
+3100 val loss 6.9760
+3100 val perplexity 1070.6133
+3100 train 6.779830 (lr=2.4574e-05) (hash(x)=45245896)
+3200 val loss 7.0673
+3200 val perplexity 1172.9346
+3200 train 7.026747 (lr=1.6149e-05) (hash(x)=49995942)
+3100 val loss 6.7318
+3100 val perplexity 838.6891
+3100 train 6.545393 (lr=5.7340e-05) (hash(x)=45245896)
+3100 val loss 6.9952
+3100 val perplexity 1091.3998
+3100 train 6.805078 (lr=4.0957e-05) (hash(x)=45245896)
+3200 val loss 6.9604
+3200 val perplexity 1054.0768
+3200 train 6.913863 (lr=2.4224e-05) (hash(x)=49995942)
+3300 val loss 7.0569
+3300 val perplexity 1160.8708
+3300 train 6.930760 (lr=1.5910e-05) (hash(x)=52311504)
+3200 val loss 6.7166
+3200 val perplexity 825.9999
+3200 train 6.681448 (lr=5.6522e-05) (hash(x)=49995942)
+3200 val loss 6.9841
+3200 val perplexity 1079.2999
+3200 train 6.941720 (lr=4.0373e-05) (hash(x)=49995942)
+3300 val loss 6.9465
+3300 val perplexity 1039.4871
+3300 train 6.819232 (lr=2.3865e-05) (hash(x)=52311504)
+3400 val loss 7.0490
+3400 val perplexity 1151.7167
+3400 train 7.042104 (lr=1.5666e-05) (hash(x)=44332917)
+3300 val loss 6.7106
+3300 val perplexity 821.0791
+3300 train 6.580021 (lr=5.5684e-05) (hash(x)=52311504)
+3300 val loss 6.9826
+3300 val perplexity 1077.7489
+3300 train 6.858939 (lr=3.9775e-05) (hash(x)=52311504)
+3400 val loss 6.9332
+3400 val perplexity 1025.7731
+3400 train 6.936850 (lr=2.3498e-05) (hash(x)=44332917)
+3500 val loss 7.0323
+3500 val perplexity 1132.6359
+3500 train 7.139286 (lr=1.5416e-05) (hash(x)=56517159)
+3400 val loss 6.7010
+3400 val perplexity 813.2463
+3400 train 6.689954 (lr=5.4829e-05) (hash(x)=44332917)
+3400 val loss 6.9820
+3400 val perplexity 1077.1057
+3400 train 6.967134 (lr=3.9164e-05) (hash(x)=44332917)
+3500 val loss 6.9179
+3500 val perplexity 1010.1714
+3500 train 7.036409 (lr=2.3125e-05) (hash(x)=56517159)
+3600 val loss 7.0287
+3600 val perplexity 1128.5656
+3600 train 6.907561 (lr=1.5163e-05) (hash(x)=50720920)
+3500 val loss 6.6896
+3500 val perplexity 803.9983
+3500 train 6.777810 (lr=5.3958e-05) (hash(x)=56517159)
+3500 val loss 6.9596
+3500 val perplexity 1053.2172
+3500 train 7.056179 (lr=3.8541e-05) (hash(x)=56517159)
+3600 val loss 6.9100
+3600 val perplexity 1002.2887
+3600 train 6.787700 (lr=2.2744e-05) (hash(x)=50720920)
+3700 val loss 7.0122
+3700 val perplexity 1110.0436
+3700 train 7.504549 (lr=1.4905e-05) (hash(x)=62727701)
+3600 val loss 6.6984
+3600 val perplexity 811.0815
+3600 train 6.556833 (lr=5.3070e-05) (hash(x)=50720920)
+3600 val loss 6.9576
+3600 val perplexity 1051.0614
+3600 train 6.823028 (lr=3.7907e-05) (hash(x)=50720920)
+3700 val loss 6.8997
+3700 val perplexity 991.9734
+3700 train 7.375038 (lr=2.2357e-05) (hash(x)=62727701)
+3800 val loss 7.0017
+3800 val perplexity 1098.4895
+3800 train 6.844926 (lr=1.4643e-05) (hash(x)=54772539)
+3700 val loss 6.6738
+3700 val perplexity 791.3810
+3700 train 7.154129 (lr=5.2167e-05) (hash(x)=62727701)
+3700 val loss 6.9509
+3700 val perplexity 1044.1140
+3700 train 7.432554 (lr=3.7262e-05) (hash(x)=62727701)
+3800 val loss 6.8858
+3800 val perplexity 978.2759
+3800 train 6.742421 (lr=2.1965e-05) (hash(x)=54772539)
+3900 val loss 6.9911
+3900 val perplexity 1086.9178
+3900 train 6.996634 (lr=1.4377e-05) (hash(x)=52274485)
+3800 val loss 6.6617
+3800 val perplexity 781.8557
+3800 train 6.517062 (lr=5.1251e-05) (hash(x)=54772539)
+3800 val loss 6.9500
+3800 val perplexity 1043.1793
+3800 train 6.794149 (lr=3.6608e-05) (hash(x)=54772539)
+3900 val loss 6.8733
+3900 val perplexity 966.1602
+3900 train 6.879457 (lr=2.1566e-05) (hash(x)=52274485)
+4000 val loss 6.9853
+4000 val perplexity 1080.6085
+4000 train 6.692829 (lr=1.4108e-05) (hash(x)=50118307)
+3900 val loss 6.6532
+3900 val perplexity 775.2562
+3900 train 6.649825 (lr=5.0321e-05) (hash(x)=52274485)
+4000 val loss 6.8697
+4000 val perplexity 962.6872
+3900 val loss 6.9389
+3900 val perplexity 1031.6731
+4000 train 6.566363 (lr=2.1162e-05) (hash(x)=50118307)
+3900 train 6.947895 (lr=3.5944e-05) (hash(x)=52274485)
+4100 val loss 6.9691
+4100 val perplexity 1063.2795
+4100 train 6.554935 (lr=1.3836e-05) (hash(x)=42771647)
+4000 val loss 6.6577
+4000 val perplexity 778.7485
+4000 train 6.355518 (lr=4.9379e-05) (hash(x)=50118307)
+4100 val loss 6.8494
+4100 val perplexity 943.3229
+4100 train 6.423832 (lr=2.0754e-05) (hash(x)=42771647)
+4000 val loss 6.9340
+4000 val perplexity 1026.6221
+4000 train 6.642421 (lr=3.5271e-05) (hash(x)=50118307)
+4200 val loss 6.9589
+4200 val perplexity 1052.4305
+4200 train 7.068559 (lr=1.3561e-05) (hash(x)=51748836)
+4100 val loss 6.6376
+4100 val perplexity 763.2496
+4100 train 6.206556 (lr=4.8426e-05) (hash(x)=42771647)
+4200 val loss 6.8396
+4200 val perplexity 934.1204
+4200 train 6.950802 (lr=2.0341e-05) (hash(x)=51748836)
+4100 val loss 6.9343
+4100 val perplexity 1026.8899
+4100 train 6.515369 (lr=3.4590e-05) (hash(x)=42771647)
+4300 val loss 6.9447
+4300 val perplexity 1037.6448
+4300 train 6.956426 (lr=1.3283e-05) (hash(x)=49021280)
+4200 val loss 6.6271
+4200 val perplexity 755.2573
+4200 train 6.723958 (lr=4.7463e-05) (hash(x)=51748836)
+4300 val loss 6.8180
+4300 val perplexity 914.1560
+4300 train 6.831802 (lr=1.9924e-05) (hash(x)=49021280)
+4400 val loss 6.9332
+4400 val perplexity 1025.7461
+4400 train 7.020908 (lr=1.3003e-05) (hash(x)=55200309)
+4200 val loss 6.9201
+4200 val perplexity 1012.4133
+4200 train 7.022127 (lr=3.3902e-05) (hash(x)=51748836)
+4300 val loss 6.6103
+4300 val perplexity 742.6902
+4300 train 6.629007 (lr=4.6490e-05) (hash(x)=49021280)
+4400 val loss 6.7991
+4400 val perplexity 897.0301
+4400 train 6.883133 (lr=1.9504e-05) (hash(x)=55200309)
+4500 val loss 6.9237
+4500 val perplexity 1016.1121
+4500 train 7.022612 (lr=1.2720e-05) (hash(x)=52085049)
+4300 val loss 6.9037
+4300 val perplexity 995.9319
+4300 train 6.916924 (lr=3.3207e-05) (hash(x)=49021280)
+4400 val loss 6.6076
+4400 val perplexity 740.7267
+4400 train 6.694966 (lr=4.5509e-05) (hash(x)=55200309)
+4500 val loss 6.7862
+4500 val perplexity 885.5211
+4500 train 6.876517 (lr=1.9081e-05) (hash(x)=52085049)
+4600 val loss 6.9170
+4600 val perplexity 1009.3023
+4600 train 6.909331 (lr=1.2436e-05) (hash(x)=48935595)
+4400 val loss 6.8934
+4400 val perplexity 985.7658
+4400 train 6.987580 (lr=3.2507e-05) (hash(x)=55200309)
+4500 val loss 6.5932
+4500 val perplexity 730.0834
+4500 train 6.665695 (lr=4.4521e-05) (hash(x)=52085049)
+4600 val loss 6.7758
+4600 val perplexity 876.3883
+4600 train 6.771263 (lr=1.8655e-05) (hash(x)=48935595)
+4700 val loss 6.9094
+4700 val perplexity 1001.5992
+4700 train 7.105185 (lr=1.2151e-05) (hash(x)=49182380)
+4500 val loss 6.8863
+4500 val perplexity 978.7505
+4500 train 6.974458 (lr=3.1801e-05) (hash(x)=52085049)
+4600 val loss 6.5769
+4600 val perplexity 718.3427
+4600 train 6.586899 (lr=4.3527e-05) (hash(x)=48935595)
+4800 val loss 6.8964
+4800 val perplexity 988.7254
+4700 val loss 6.7589
+4700 val perplexity 861.7277
+4800 train 6.694540 (lr=1.1864e-05) (hash(x)=43941929)
+4700 train 6.979102 (lr=1.8226e-05) (hash(x)=49182380)
+4600 val loss 6.8921
+4600 val perplexity 984.4585
+4600 train 6.891096 (lr=3.1091e-05) (hash(x)=48935595)
+4700 val loss 6.5731
+4700 val perplexity 715.5541
+4700 train 6.820342 (lr=4.2528e-05) (hash(x)=49182380)
+4800 val loss 6.7492
+4800 val perplexity 853.3676
+4900 val loss 6.8910
+4900 val perplexity 983.3855
+4900 train 7.058427 (lr=1.1577e-05) (hash(x)=51852773)
+4800 train 6.550427 (lr=1.7796e-05) (hash(x)=43941929)
+4700 val loss 6.8718
+4700 val perplexity 964.7123
+4700 train 7.057541 (lr=3.0377e-05) (hash(x)=49182380)
+4800 val loss 6.5603
+4800 val perplexity 706.4680
+4800 train 6.383238 (lr=4.1525e-05) (hash(x)=43941929)
+5000 val loss 6.8868
+5000 val perplexity 979.2640
+4900 val loss 6.7417
+4900 val perplexity 846.9903
+4900 train 6.903447 (lr=1.7365e-05) (hash(x)=51852773)
+5000 train 6.477450 (lr=1.1288e-05) (hash(x)=40509616)
+4800 val loss 6.8656
+4800 val perplexity 958.6776
+4800 train 6.658672 (lr=2.9661e-05) (hash(x)=43941929)
+4900 val loss 6.5601
+4900 val perplexity 706.3693
+4900 train 6.714787 (lr=4.0518e-05) (hash(x)=51852773)
+5000 val loss 6.7326
+5000 val perplexity 839.3625
+5100 val loss 6.8764
+5100 val perplexity 969.1464
+5100 train 7.171636 (lr=1.1000e-05) (hash(x)=57585369)
+5000 train 6.303533 (lr=1.6933e-05) (hash(x)=40509616)
+4900 val loss 6.8608
+4900 val perplexity 954.1130
+4900 train 7.019123 (lr=2.8942e-05) (hash(x)=51852773)
+5000 val loss 6.5599
+5000 val perplexity 706.1871
+5000 train 6.147030 (lr=3.9510e-05) (hash(x)=40509616)
+5200 val loss 6.8709
+5200 val perplexity 963.7726
+5200 train 6.894734 (lr=1.0712e-05) (hash(x)=51042313)
+5100 val loss 6.7199
+5100 val perplexity 828.7194
+5100 train 6.972271 (lr=1.6500e-05) (hash(x)=57585369)
+5000 val loss 6.8607
+5000 val perplexity 954.0266
+5000 train 6.446211 (lr=2.8221e-05) (hash(x)=40509616)
+5100 val loss 6.5459
+5100 val perplexity 696.3658
+5100 train 6.742117 (lr=3.8500e-05) (hash(x)=57585369)
+5300 val loss 6.8614
+5300 val perplexity 954.7220
+5300 train 7.021107 (lr=1.0423e-05) (hash(x)=52001684)
+5200 val loss 6.7183
+5200 val perplexity 827.4274
+5200 train 6.726495 (lr=1.6067e-05) (hash(x)=51042313)
+5100 val loss 6.8445
+5100 val perplexity 938.6766
+5100 train 7.109215 (lr=2.7500e-05) (hash(x)=57585369)
+5200 val loss 6.5457
+5200 val perplexity 696.2642
+5400 val loss 6.8580
+5400 val perplexity 951.4784
+5200 train 6.539655 (lr=3.7490e-05) (hash(x)=51042313)
+5400 train 6.782125 (lr=1.0136e-05) (hash(x)=48831647)
+5300 val loss 6.7052
+5300 val perplexity 816.6333
+5300 train 6.861244 (lr=1.5635e-05) (hash(x)=52001684)
+5200 val loss 6.8397
+5200 val perplexity 934.1863
+5200 train 6.856593 (lr=2.6779e-05) (hash(x)=51042313)
+5500 val loss 6.8556
+5500 val perplexity 949.1456
+5500 train 7.206544 (lr=9.8491e-06) (hash(x)=50192069)
+5300 val loss 6.5281
+5300 val perplexity 684.1060
+5300 train 6.689152 (lr=3.6482e-05) (hash(x)=52001684)
+5400 val loss 6.7006
+5400 val perplexity 812.8881
+5400 train 6.625731 (lr=1.5204e-05) (hash(x)=48831647)
+5300 val loss 6.8333
+5300 val perplexity 928.2460
+5300 train 6.993705 (lr=2.6058e-05) (hash(x)=52001684)
+5600 val loss 6.8466
+5600 val perplexity 940.6655
+5600 train 6.654557 (lr=9.5636e-06) (hash(x)=47208852)
+5400 val loss 6.5233
+5400 val perplexity 680.8095
+5400 train 6.443679 (lr=3.5475e-05) (hash(x)=48831647)
+5500 val loss 6.6987
+5500 val perplexity 811.3495
+5500 train 7.087494 (lr=1.4774e-05) (hash(x)=50192069)
+5400 val loss 6.8253
+5400 val perplexity 920.8331
+5400 train 6.749354 (lr=2.5339e-05) (hash(x)=48831647)
+5700 val loss 6.8428
+5700 val perplexity 937.1461
+5700 train 6.436399 (lr=9.2796e-06) (hash(x)=44061694)
+5500 val loss 6.5335
+5500 val perplexity 687.8297
+5500 train 6.943909 (lr=3.4472e-05) (hash(x)=50192069)
+5600 val loss 6.6884
+5600 val perplexity 803.0125
+5600 train 6.500677 (lr=1.4345e-05) (hash(x)=47208852)
+5500 val loss 6.8219
+5500 val perplexity 917.7541
+5500 train 7.200629 (lr=2.4623e-05) (hash(x)=50192069)
+5800 val loss 6.8371
+5800 val perplexity 931.7457
+5800 train 7.215187 (lr=8.9973e-06) (hash(x)=56513279)
+5600 val loss 6.5178
+5600 val perplexity 677.0574
+5700 val loss 6.6834
+5700 val perplexity 799.0583
+5600 train 6.333476 (lr=3.3473e-05) (hash(x)=47208852)
+5700 train 6.274059 (lr=1.3919e-05) (hash(x)=44061694)
+5600 val loss 6.8152
+5600 val perplexity 911.5791
+5600 train 6.629575 (lr=2.3909e-05) (hash(x)=47208852)
+5900 val loss 6.8312
+5900 val perplexity 926.2846
+5900 train 6.992841 (lr=8.7171e-06) (hash(x)=50412818)
+5800 val loss 6.6761
+5800 val perplexity 793.2364
+5700 val loss 6.5278
+5700 val perplexity 683.8582
+5800 train 7.079706 (lr=1.3496e-05) (hash(x)=56513279)
+5700 train 6.099132 (lr=3.2479e-05) (hash(x)=44061694)
+5700 val loss 6.8144
+5700 val perplexity 910.8314
+5700 train 6.405194 (lr=2.3199e-05) (hash(x)=44061694)
+6000 val loss 6.8262
+6000 val perplexity 921.7257
+6000 train 6.613876 (lr=8.4393e-06) (hash(x)=47159634)
+5900 val loss 6.6691
+5900 val perplexity 787.6990
+5900 train 6.819257 (lr=1.3076e-05) (hash(x)=50412818)
+5800 val loss 6.5141
+5800 val perplexity 674.6172
+5800 train 6.898224 (lr=3.1491e-05) (hash(x)=56513279)
+5800 val loss 6.8042
+5800 val perplexity 901.6212
+5800 train 7.188510 (lr=2.2493e-05) (hash(x)=56513279)
+6100 val loss 6.8191
+6100 val perplexity 915.1422
+6100 train 6.918497 (lr=8.1640e-06) (hash(x)=54312795)
+6000 val loss 6.6627
+6000 val perplexity 782.6782
+6000 train 6.458968 (lr=1.2659e-05) (hash(x)=47159634)
+5900 val loss 6.5100
+5900 val perplexity 671.8426
+5900 train 6.649952 (lr=3.0510e-05) (hash(x)=50412818)
+5900 val loss 6.8038
+5900 val perplexity 901.3005
+5900 train 6.949226 (lr=2.1793e-05) (hash(x)=50412818)
+6200 val loss 6.8130
+6200 val perplexity 909.6113
+6200 train 6.990555 (lr=7.8917e-06) (hash(x)=54187587)
+6100 val loss 6.6554
+6100 val perplexity 776.9844
+6100 train 6.756692 (lr=1.2246e-05) (hash(x)=54312795)
+6000 val loss 6.5064
+6000 val perplexity 669.4238
+6000 train 6.328054 (lr=2.9537e-05) (hash(x)=47159634)
+6000 val loss 6.7957
+6000 val perplexity 894.0056
+6000 train 6.589294 (lr=2.1098e-05) (hash(x)=47159634)
+6300 val loss 6.8062
+6300 val perplexity 903.3907
+6300 train 6.916306 (lr=7.6226e-06) (hash(x)=53620387)
+6200 val loss 6.6462
+6200 val perplexity 769.8491
+6200 train 6.825405 (lr=1.1838e-05) (hash(x)=54187587)
+6100 val loss 6.4902
+6100 val perplexity 658.6664
+6100 train 6.581672 (lr=2.8574e-05) (hash(x)=54312795)
+6100 val loss 6.7919
+6100 val perplexity 890.6199
+6100 train 6.885747 (lr=2.0410e-05) (hash(x)=54312795)
+6400 val loss 6.7978
+6400 val perplexity 895.8444
+6400 train 6.737978 (lr=7.3569e-06) (hash(x)=48761774)
+6300 val loss 6.6364
+6300 val perplexity 762.3516
+6300 train 6.762627 (lr=1.1434e-05) (hash(x)=53620387)
+6200 val loss 6.4772
+6200 val perplexity 650.1536
+6200 train 6.615796 (lr=2.7621e-05) (hash(x)=54187587)
+6200 val loss 6.7828
+6200 val perplexity 882.5361
+6200 train 6.953031 (lr=1.9729e-05) (hash(x)=54187587)
+6500 val loss 6.7933
+6500 val perplexity 891.8179
+6500 train 7.034630 (lr=7.0950e-06) (hash(x)=56690281)
+6400 val loss 6.6275
+6400 val perplexity 755.6107
+6400 train 6.551711 (lr=1.1035e-05) (hash(x)=48761774)
+6300 val loss 6.4718
+6300 val perplexity 646.6768
+6300 train 6.592690 (lr=2.6679e-05) (hash(x)=53620387)
+6300 val loss 6.7704
+6300 val perplexity 871.6821
+6300 train 6.878792 (lr=1.9056e-05) (hash(x)=53620387)
+6600 val loss 6.7888
+6600 val perplexity 887.8833
+6600 train 6.580529 (lr=6.8372e-06) (hash(x)=42985269)
+6500 val loss 6.6192
+6500 val perplexity 749.3109
+6500 train 6.858943 (lr=1.0643e-05) (hash(x)=56690281)
+6400 val loss 6.4743
+6400 val perplexity 648.2544
+6400 train 6.393586 (lr=2.5749e-05) (hash(x)=48761774)
+6400 val loss 6.7612
+6400 val perplexity 863.6722
+6400 train 6.697858 (lr=1.8392e-05) (hash(x)=48761774)
+6700 val loss 6.7863
+6700 val perplexity 885.5966
+6700 train 6.804174 (lr=6.5835e-06) (hash(x)=53315447)
+6600 val loss 6.6162
+6600 val perplexity 747.0964
+6600 train 6.430020 (lr=1.0256e-05) (hash(x)=42985269)
+6500 val loss 6.4627
+6500 val perplexity 640.7616
+6500 train 6.703842 (lr=2.4833e-05) (hash(x)=56690281)
+6500 val loss 6.7577
+6500 val perplexity 860.6866
+6500 train 7.011350 (lr=1.7738e-05) (hash(x)=56690281)
+6800 val loss 6.7796
+6800 val perplexity 879.7517
+6800 train 7.271560 (lr=6.3345e-06) (hash(x)=61577166)
+6700 val loss 6.6127
+6700 val perplexity 744.4901
+6700 train 6.654109 (lr=9.8753e-06) (hash(x)=53315447)
+6600 val loss 6.4587
+6600 val perplexity 638.2580
+6600 train 6.275012 (lr=2.3930e-05) (hash(x)=42985269)
+6900 val loss 6.7779
+6900 val perplexity 878.2637
+6900 train 6.883226 (lr=6.0902e-06) (hash(x)=54641005)
+6600 val loss 6.7546
+6600 val perplexity 857.9875
+6600 train 6.544014 (lr=1.7093e-05) (hash(x)=42985269)
+6800 val loss 6.6055
+6800 val perplexity 739.1291
+6800 train 7.100903 (lr=9.5017e-06) (hash(x)=61577166)
+6700 val loss 6.4604
+6700 val perplexity 639.3189
+6700 train 6.494969 (lr=2.3042e-05) (hash(x)=53315447)
+7000 val loss 6.7713
+7000 val perplexity 872.4722
+7000 train 7.330594 (lr=5.8510e-06) (hash(x)=60579512)
+6700 val loss 6.7507
+6700 val perplexity 854.6246
+6700 train 6.784648 (lr=1.6459e-05) (hash(x)=53315447)
+6900 val loss 6.5959
+6900 val perplexity 732.0701
+6900 train 6.713835 (lr=9.1353e-06) (hash(x)=54641005)
+6800 val loss 6.4498
+6800 val perplexity 632.5809
+6800 train 6.904761 (lr=2.2171e-05) (hash(x)=61577166)
+7100 val loss 6.7694
+7100 val perplexity 870.7997
+7100 train 6.665070 (lr=5.6170e-06) (hash(x)=53151549)
+6800 val loss 6.7475
+6800 val perplexity 851.9592
+6800 train 7.248035 (lr=1.5836e-05) (hash(x)=61577166)
+7000 val loss 6.5925
+7000 val perplexity 729.6163
+7000 train 7.168230 (lr=8.7764e-06) (hash(x)=60579512)
+6900 val loss 6.4431
+6900 val perplexity 628.3522
+6900 train 6.581858 (lr=2.1316e-05) (hash(x)=54641005)
+7200 val loss 6.7639
+7200 val perplexity 865.9951
+7200 train 7.761684 (lr=5.3886e-06) (hash(x)=71842455)
+6900 val loss 6.7337
+6900 val perplexity 840.2455
+6900 train 6.854381 (lr=1.5225e-05) (hash(x)=54641005)
+7100 val loss 6.5893
+7100 val perplexity 727.2419
+7100 train 6.488562 (lr=8.4255e-06) (hash(x)=53151549)
+7000 val loss 6.4382
+7000 val perplexity 625.2898
+7000 train 6.964822 (lr=2.0478e-05) (hash(x)=60579512)
+7300 val loss 6.7625
+7300 val perplexity 864.8347
+7300 train 6.528047 (lr=5.1659e-06) (hash(x)=44516452)
+7000 val loss 6.7305
+7000 val perplexity 837.5577
+7000 train 7.277111 (lr=1.4627e-05) (hash(x)=60579512)
+7200 val loss 6.5813
+7200 val perplexity 721.4806
+7200 train 7.597327 (lr=8.0829e-06) (hash(x)=71842455)
+7100 val loss 6.4414
+7100 val perplexity 627.2758
+7100 train 6.345554 (lr=1.9660e-05) (hash(x)=53151549)
+7400 val loss 6.7592
+7400 val perplexity 861.9483
+7400 train 6.410480 (lr=4.9493e-06) (hash(x)=42667710)
+7100 val loss 6.7267
+7100 val perplexity 834.4161
+7100 train 6.616548 (lr=1.4043e-05) (hash(x)=53151549)
+7300 val loss 6.5815
+7300 val perplexity 721.5986
+7300 train 6.349890 (lr=7.7489e-06) (hash(x)=44516452)
+7200 val loss 6.4361
+7200 val perplexity 623.9594
+7200 train 7.351032 (lr=1.8860e-05) (hash(x)=71842455)
+7500 val loss 6.7555
+7500 val perplexity 858.8078
+7500 train 6.505149 (lr=4.7389e-06) (hash(x)=47050797)
+7200 val loss 6.7204
+7200 val perplexity 829.1766
+7200 train 7.733423 (lr=1.3471e-05) (hash(x)=71842455)
+7400 val loss 6.5739
+7400 val perplexity 716.1360
+7400 train 6.221677 (lr=7.4239e-06) (hash(x)=42667710)
+7300 val loss 6.4313
+7300 val perplexity 620.9935
+7300 train 6.198883 (lr=1.8081e-05) (hash(x)=44516452)
+7600 val loss 6.7532
+7600 val perplexity 856.8362
+7600 train 6.666946 (lr=4.5349e-06) (hash(x)=49785056)
+7300 val loss 6.7178
+7300 val perplexity 827.0081
+7300 train 6.492526 (lr=1.2915e-05) (hash(x)=44516452)
+7500 val loss 6.5720
+7500 val perplexity 714.8130
+7500 train 6.316886 (lr=7.1083e-06) (hash(x)=47050797)
+7400 val loss 6.4278
+7400 val perplexity 618.8268
+7400 train 6.063058 (lr=1.7323e-05) (hash(x)=42667710)
+7700 val loss 6.7499
+7700 val perplexity 853.9936
+7700 train 6.595833 (lr=4.3375e-06) (hash(x)=53232030)
+7400 val loss 6.7131
+7400 val perplexity 823.0838
+7400 train 6.368190 (lr=1.2373e-05) (hash(x)=42667710)
+7600 val loss 6.5660
+7600 val perplexity 710.5325
+7600 train 6.483896 (lr=6.8023e-06) (hash(x)=49785056)
+7500 val loss 6.4225
+7500 val perplexity 615.5386
+7500 train 6.168694 (lr=1.6586e-05) (hash(x)=47050797)
+7800 val loss 6.7466
+7800 val perplexity 851.1527
+7800 train 6.579307 (lr=4.1470e-06) (hash(x)=48049749)
+7500 val loss 6.7074
+7500 val perplexity 818.4573
+7500 train 6.456196 (lr=1.1847e-05) (hash(x)=47050797)
+7700 val loss 6.5620
+7700 val perplexity 707.7010
+7700 train 6.402081 (lr=6.5062e-06) (hash(x)=53232030)
+7600 val loss 6.4167
+7600 val perplexity 611.9720
+7600 train 6.321131 (lr=1.5872e-05) (hash(x)=49785056)
+7900 val loss 6.7407
+7900 val perplexity 846.1240
+7900 train 6.525646 (lr=3.9635e-06) (hash(x)=44768513)
+7600 val loss 6.7034
+7600 val perplexity 815.2051
+7600 train 6.616943 (lr=1.1337e-05) (hash(x)=49785056)
+7800 val loss 6.5586
+7800 val perplexity 705.2940
+7800 train 6.375649 (lr=6.2205e-06) (hash(x)=48049749)
+7700 val loss 6.4153
+7700 val perplexity 611.1429
+7700 train 6.237316 (lr=1.5181e-05) (hash(x)=53232030)
+8000 val loss 6.7368
+8000 val perplexity 842.8249
+8000 train 6.609796 (lr=3.7873e-06) (hash(x)=46228039)
+7700 val loss 6.7032
+7700 val perplexity 814.9871
+7700 train 6.551203 (lr=1.0844e-05) (hash(x)=53232030)
+7900 val loss 6.5509
+7900 val perplexity 699.8884
+7900 train 6.348945 (lr=5.9453e-06) (hash(x)=44768513)
+7800 val loss 6.4185
+7800 val perplexity 613.0842
+7800 train 6.233474 (lr=1.4514e-05) (hash(x)=48049749)
+8100 val loss 6.7341
+8100 val perplexity 840.5965
+8100 train 7.155693 (lr=3.6184e-06) (hash(x)=60017091)
+7800 val loss 6.6980
+7800 val perplexity 810.7551
+7800 train 6.527575 (lr=1.0367e-05) (hash(x)=48049749)
+8000 val loss 6.5460
+8000 val perplexity 696.4471
+8000 train 6.408615 (lr=5.6809e-06) (hash(x)=46228039)
+7900 val loss 6.4048
+7900 val perplexity 604.7163
+7900 train 6.222796 (lr=1.3872e-05) (hash(x)=44768513)
+8200 val loss 6.7300
+8200 val perplexity 837.1589
+8200 train 6.721200 (lr=3.4572e-06) (hash(x)=49910198)
+7900 val loss 6.6898
+7900 val perplexity 804.1443
+7900 train 6.489498 (lr=9.9088e-06) (hash(x)=44768513)
+8100 val loss 6.5439
+8100 val perplexity 695.0140
+8100 train 6.955059 (lr=5.4277e-06) (hash(x)=60017091)
+8300 val loss 6.7268
+8300 val perplexity 834.4623
+8300 train 7.089801 (lr=3.3037e-06) (hash(x)=57919055)
+8000 val loss 6.4018
+8000 val perplexity 602.9116
+8000 train 6.253847 (lr=1.3255e-05) (hash(x)=46228039)
+8000 val loss 6.6854
+8000 val perplexity 800.5988
+8000 train 6.562236 (lr=9.4682e-06) (hash(x)=46228039)
+8200 val loss 6.5399
+8200 val perplexity 692.2427
+8200 train 6.549590 (lr=5.1858e-06) (hash(x)=49910198)
+8400 val loss 6.7268
+8400 val perplexity 834.4695
+8400 train 6.833617 (lr=3.1581e-06) (hash(x)=49694964)
+8100 val loss 6.3971
+8100 val perplexity 600.1071
+8100 train 6.769569 (lr=1.2665e-05) (hash(x)=60017091)
+8100 val loss 6.6832
+8100 val perplexity 798.8370
+8100 train 7.114765 (lr=9.0461e-06) (hash(x)=60017091)
+8300 val loss 6.5345
+8300 val perplexity 688.5184
+8300 train 6.907079 (lr=4.9556e-06) (hash(x)=57919055)
+8500 val loss 6.7234
+8500 val perplexity 831.6531
+8500 train 6.736595 (lr=3.0206e-06) (hash(x)=53762585)
+8200 val loss 6.3928
+8200 val perplexity 597.5019
+8200 train 6.411548 (lr=1.2100e-05) (hash(x)=49910198)
+8200 val loss 6.6775
+8200 val perplexity 794.3458
+8200 train 6.668144 (lr=8.6430e-06) (hash(x)=49910198)
+8400 val loss 6.5352
+8400 val perplexity 688.9457
+8400 train 6.652549 (lr=4.7372e-06) (hash(x)=49694964)
+8600 val loss 6.7207
+8600 val perplexity 829.4001
+8600 train 6.771939 (lr=2.8913e-06) (hash(x)=51166973)
+8300 val loss 6.3882
+8300 val perplexity 594.7954
+8300 train 6.720931 (lr=1.1563e-05) (hash(x)=57919055)
+8300 val loss 6.6720
+8300 val perplexity 789.9393
+8300 train 7.051010 (lr=8.2593e-06) (hash(x)=57919055)
+8500 val loss 6.5307
+8500 val perplexity 685.8688
+8500 train 6.550485 (lr=4.5309e-06) (hash(x)=53762585)
+8700 val loss 6.7181
+8700 val perplexity 827.2309
+8700 train 6.789652 (lr=2.7703e-06) (hash(x)=53968049)
+8400 val loss 6.3887
+8400 val perplexity 595.0958
+8400 train 6.482622 (lr=1.1053e-05) (hash(x)=49694964)
+8400 val loss 6.6700
+8400 val perplexity 788.4261
+8400 train 6.787225 (lr=7.8953e-06) (hash(x)=49694964)
+8600 val loss 6.5288
+8600 val perplexity 684.6053
+8600 train 6.580200 (lr=4.3369e-06) (hash(x)=51166973)
+8800 val loss 6.7160
+8800 val perplexity 825.5129
+8800 train 6.815679 (lr=2.6577e-06) (hash(x)=59231056)
+8500 val loss 6.3836
+8500 val perplexity 592.0306
+8500 train 6.408396 (lr=1.0572e-05) (hash(x)=53762585)
+8500 val loss 6.6665
+8500 val perplexity 785.6351
+8500 train 6.681557 (lr=7.5515e-06) (hash(x)=53762585)
+8700 val loss 6.5256
+8700 val perplexity 682.3744
+8900 val loss 6.7144
+8900 val perplexity 824.1870
+8700 train 6.605652 (lr=4.1554e-06) (hash(x)=53968049)
+8900 train 6.623099 (lr=2.5538e-06) (hash(x)=50488048)
+8600 val loss 6.3830
+8600 val perplexity 591.6888
+8600 train 6.420757 (lr=1.0119e-05) (hash(x)=51166973)
+8600 val loss 6.6634
+8600 val perplexity 783.2065
+8600 train 6.713762 (lr=7.2282e-06) (hash(x)=51166973)
+9000 val loss 6.7123
+9000 val perplexity 822.4427
+9000 train 6.414406 (lr=2.4585e-06) (hash(x)=44492956)
+8800 val loss 6.5224
+8800 val perplexity 680.2186
+8800 train 6.619614 (lr=3.9866e-06) (hash(x)=59231056)
+8700 val loss 6.3795
+8700 val perplexity 589.6207
+8700 train 6.453899 (lr=9.6960e-06) (hash(x)=53968049)
+8700 val loss 6.6606
+8700 val perplexity 781.0009
+8700 train 6.732889 (lr=6.9257e-06) (hash(x)=53968049)
+9100 val loss 6.7135
+9100 val perplexity 823.4442
+9100 train 6.769123 (lr=2.3720e-06) (hash(x)=51134989)
+8900 val loss 6.5213
+8900 val perplexity 679.4908
+8900 train 6.445029 (lr=3.8307e-06) (hash(x)=50488048)
+8800 val loss 6.3741
+8800 val perplexity 586.4824
+8800 train 6.473434 (lr=9.3021e-06) (hash(x)=59231056)
+8800 val loss 6.6562
+8800 val perplexity 777.5667
+8800 train 6.754422 (lr=6.6444e-06) (hash(x)=59231056)
+9200 val loss 6.7107
+9200 val perplexity 821.1844
+9200 train 6.514919 (lr=2.2943e-06) (hash(x)=48636056)
+9000 val loss 6.5202
+9000 val perplexity 678.7302
+9000 train 6.222732 (lr=3.6877e-06) (hash(x)=44492956)
+8900 val loss 6.3754
+8900 val perplexity 587.2310
+8900 train 6.260254 (lr=8.9382e-06) (hash(x)=50488048)
+9300 val loss 6.7099
+9300 val perplexity 820.5175
+9300 train 6.680036 (lr=2.2256e-06) (hash(x)=50200551)
+8900 val loss 6.6543
+8900 val perplexity 776.0824
+8900 train 6.573340 (lr=6.3845e-06) (hash(x)=50488048)
+9100 val loss 6.5226
+9100 val perplexity 680.3639
+9100 train 6.545186 (lr=3.5580e-06) (hash(x)=51134989)
+9000 val loss 6.3725
+9000 val perplexity 585.4980
+9000 train 6.081764 (lr=8.6047e-06) (hash(x)=44492956)
+9400 val loss 6.7092
+9400 val perplexity 819.8851
+9400 train 6.530241 (lr=2.1660e-06) (hash(x)=48057228)
+9000 val loss 6.6524
+9000 val perplexity 774.6767
+9000 train 6.361135 (lr=6.1462e-06) (hash(x)=44492956)
+9200 val loss 6.5177
+9200 val perplexity 676.9999
+9200 train 6.318551 (lr=3.4415e-06) (hash(x)=48636056)
+9100 val loss 6.3738
+9100 val perplexity 586.2674
+9100 train 6.352215 (lr=8.3020e-06) (hash(x)=51134989)
+9500 val loss 6.7056
+9500 val perplexity 816.9913
+9500 train 6.484961 (lr=2.1154e-06) (hash(x)=48125171)
+9100 val loss 6.6548
+9100 val perplexity 776.4929
+9100 train 6.679288 (lr=5.9300e-06) (hash(x)=51134989)
+9300 val loss 6.5164
+9300 val perplexity 676.1420
+9300 train 6.469476 (lr=3.3385e-06) (hash(x)=50200551)
+9200 val loss 6.3685
+9200 val perplexity 583.1851
+9200 train 6.174442 (lr=8.0302e-06) (hash(x)=48636056)
+9600 val loss 6.7056
+9600 val perplexity 816.9484
+9600 train 6.674135 (lr=2.0739e-06) (hash(x)=53375853)
+9200 val loss 6.6493
+9200 val perplexity 772.2683
+9200 train 6.453731 (lr=5.7359e-06) (hash(x)=48636056)
+9400 val loss 6.5164
+9400 val perplexity 676.1276
+9400 train 6.336080 (lr=3.2490e-06) (hash(x)=48057228)
+9300 val loss 6.3682
+9300 val perplexity 583.0174
+9300 train 6.318146 (lr=7.7898e-06) (hash(x)=50200551)
+9700 val loss 6.7019
+9700 val perplexity 813.9532
+9700 train 7.475645 (lr=2.0416e-06) (hash(x)=53924631)
+9300 val loss 6.6490
+9300 val perplexity 772.0294
+9300 train 6.610506 (lr=5.5641e-06) (hash(x)=50200551)
+9500 val loss 6.5129
+9500 val perplexity 673.7589
+9500 train 6.297124 (lr=3.1730e-06) (hash(x)=48125171)
+9400 val loss 6.3696
+9400 val perplexity 583.8489
+9400 train 6.185252 (lr=7.5809e-06) (hash(x)=48057228)
+9800 val loss 6.6990
+9800 val perplexity 811.5999
+9800 train 6.732530 (lr=2.0185e-06) (hash(x)=48895047)
+9400 val loss 6.6472
+9400 val perplexity 770.5902
+9400 train 6.478499 (lr=5.4149e-06) (hash(x)=48057228)
+9600 val loss 6.5126
+9600 val perplexity 673.5715
+9600 train 6.459769 (lr=3.1108e-06) (hash(x)=53375853)
+9500 val loss 6.3674
+9500 val perplexity 582.5411
+9500 train 6.146292 (lr=7.4038e-06) (hash(x)=48125171)
+9900 val loss 6.6963
+9900 val perplexity 809.4128
+9900 train 6.532602 (lr=2.0046e-06) (hash(x)=44269923)
+9500 val loss 6.6440
+9500 val perplexity 768.1942
+9500 train 6.421329 (lr=5.2884e-06) (hash(x)=48125171)
+9700 val loss 6.5079
+9700 val perplexity 670.4332
+9700 train 7.313867 (lr=3.0624e-06) (hash(x)=53924631)
+9600 val loss 6.3645
+9600 val perplexity 580.8505
+9600 train 6.295037 (lr=7.2586e-06) (hash(x)=53375853)
+9999 val loss 6.6978
+9999 val perplexity 810.6125
+9600 val loss 6.6432
+9600 val perplexity 767.5720
+9600 train 6.602069 (lr=5.1847e-06) (hash(x)=53375853)
+9800 val loss 6.5053
+9800 val perplexity 668.6689
+9800 train 6.551248 (lr=3.0277e-06) (hash(x)=48895047)
+9700 val loss 6.3601
+9700 val perplexity 578.3121
+9700 train 7.180431 (lr=7.1456e-06) (hash(x)=53924631)
+9700 val loss 6.6379
+9700 val perplexity 763.5204
+9700 train 7.429144 (lr=5.1040e-06) (hash(x)=53924631)
+9900 val loss 6.5029
+9900 val perplexity 667.0677
+9900 train 6.344713 (lr=3.0069e-06) (hash(x)=44269923)
+9800 val loss 6.3598
+9800 val perplexity 578.1113
+9800 train 6.405945 (lr=7.0647e-06) (hash(x)=48895047)
+9800 val loss 6.6355
+9800 val perplexity 761.6445
+9800 train 6.674919 (lr=5.0462e-06) (hash(x)=48895047)
+9999 val loss 6.5035
+9999 val perplexity 667.4902
+9900 val loss 6.6316
+9900 val perplexity 758.6907
+9900 train 6.478441 (lr=5.0116e-06) (hash(x)=44269923)
+9900 val loss 6.3549
+9900 val perplexity 575.2765
+9900 train 6.186326 (lr=7.0162e-06) (hash(x)=44269923)
+9999 val loss 6.6315
+9999 val perplexity 758.6360
+9999 val loss 6.3557
+9999 val perplexity 575.7891

attention_kindselective_n_heads8_seed1341/model_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e860e923a556180e7e2dade3cce2e338a0c6e4ced0183df1bef4b801ac4ed341
+size 257976706

attention_kindselective_n_heads8_seed1341/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f185c3159438e039d4fc515e37f75f445947587e36943760fb5202a8375b6af4
+size 257976706

attention_kindselective_n_heads8_seed1341/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee0c71f6868299f0a2ff54ed1ec8d3aeaf7c4572b36fbacc656c3c31f9cbf2a1
+size 257976706

attention_kindselective_n_heads8_seed1341/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef70a6883ed7ace4af08d40c7efad1fdddd7b9a3923d2826bd87c81631ed7525
+size 257976706

attention_kindselective_n_heads8_seed1341/optimizer_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bd67e3d6826de3f55ad5d5f1c04b26f72da5c15b4892f1263072c7dfd955a14
+size 509672838

attention_kindselective_n_heads8_seed1341/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba33e8595171564264bbf96f4a16fc0a8b17e06aca96550034da8efea6848d93
+size 509672838

attention_kindselective_n_heads8_seed1341/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a08c66319f568af505a5def305e7297e4f583c6ba4f94fce35a6492eaaf6622d
+size 509672838

attention_kindselective_n_heads8_seed1341/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0de2886281b4cfd84dfa959b6a06d6011db4a724bda0944d7119a668fadebcb4
+size 509672838