Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_2_1339", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 7e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "7e-5_10240_2_1339", "n_embd": 128}

attention_kindselective_n_heads2_seed1339/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
+oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
 size 964

attention_kindselective_n_heads2_seed1339/dataloader_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+oid sha256:7f03ed2ebf741f15e13c79e6cc1e9a19b308450d81cc3b4d8d0338c63d77ca59
 size 964

attention_kindselective_n_heads2_seed1339/dataloader_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+oid sha256:82590037fb2eecbec961f7967a8dd1b8d85515d31a252f66b92b8139858a8b7c
 size 964

attention_kindselective_n_heads2_seed1339/dataloader_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+oid sha256:c544303717d12355a69b8ffb1eb109434e4fdccfd5a61279b6e8ba2e870d6700
 size 964

attention_kindselective_n_heads2_seed1339/log2.txt

@@ -1,303 +1,605 @@
 max_steps: 10000
-0 val loss 11.7467
-0 val perplexity 126331.5391
-0 train 11.740535 (lr=1.5000e-07) (hash(x)=150724848)
-100 val loss 10.5531
-100 val perplexity 38296.7969
-100 train 10.582304 (lr=1.5150e-05) (hash(x)=149217061)
-200 val loss 9.7262
-200 val perplexity 16750.0762
-200 train 9.707881 (lr=3.0000e-05) (hash(x)=149910534)
-300 val loss 8.5429
-300 val perplexity 5130.0444
-300 train 8.481003 (lr=2.9993e-05) (hash(x)=142185643)
-400 val loss 7.9918
-400 val perplexity 2956.5107
-400 train 8.018432 (lr=2.9972e-05) (hash(x)=148123706)
-500 val loss 7.7440
-500 val perplexity 2307.6658
-500 train 7.625622 (lr=2.9938e-05) (hash(x)=149952383)
-600 val loss 7.6583
-600 val perplexity 2118.1472
-600 train 7.492473 (lr=2.9889e-05) (hash(x)=146678221)
-700 val loss 7.6015
-700 val perplexity 2001.2423
-700 train 7.618642 (lr=2.9827e-05) (hash(x)=156180736)
-800 val loss 7.5754
-800 val perplexity 1949.5923
-800 train 7.563412 (lr=2.9751e-05) (hash(x)=151700982)
-900 val loss 7.5546
-900 val perplexity 1909.5804
-900 train 7.378692 (lr=2.9662e-05) (hash(x)=147288467)
-1000 val loss 7.5409
-1000 val perplexity 1883.4944
-1000 train 7.593865 (lr=2.9558e-05) (hash(x)=156182087)
-1100 val loss 7.5328
-1100 val perplexity 1868.2631
-1100 train 7.420054 (lr=2.9442e-05) (hash(x)=147861550)
-1200 val loss 7.5242
-1200 val perplexity 1852.2574
-1200 train 7.404797 (lr=2.9312e-05) (hash(x)=149318660)
-1300 val loss 7.5252
-1300 val perplexity 1854.2634
-1300 train 7.397731 (lr=2.9169e-05) (hash(x)=151631103)
-1400 val loss 7.5227
-1400 val perplexity 1849.5629
-1400 train 7.571448 (lr=2.9013e-05) (hash(x)=150482428)
-1500 val loss 7.5023
-1500 val perplexity 1812.2666
-1500 train 7.358995 (lr=2.8845e-05) (hash(x)=144458617)
-1600 val loss 7.4812
-1600 val perplexity 1774.3268
-1600 train 7.361270 (lr=2.8663e-05) (hash(x)=143268605)
-1700 val loss 7.4645
-1700 val perplexity 1744.9453
-1700 train 7.396727 (lr=2.8469e-05) (hash(x)=150817011)
-1800 val loss 7.4566
-1800 val perplexity 1731.2513
-1800 train 7.454478 (lr=2.8263e-05) (hash(x)=152322423)
-1900 val loss 7.4449
-1900 val perplexity 1711.1436
-1900 train 7.286794 (lr=2.8044e-05) (hash(x)=143574126)
-2000 val loss 7.4327
-2000 val perplexity 1690.3496
-2000 train 7.217881 (lr=2.7814e-05) (hash(x)=147904298)
-2100 val loss 7.4200
-2100 val perplexity 1669.0591
-2100 train 7.191989 (lr=2.7572e-05) (hash(x)=144719400)
-2200 val loss 7.4000
-2200 val perplexity 1635.9027
-2200 train 7.653648 (lr=2.7319e-05) (hash(x)=154343147)
-2300 val loss 7.3701
-2300 val perplexity 1587.8517
-2300 train 7.430760 (lr=2.7055e-05) (hash(x)=154372635)
-2400 val loss 7.3509
-2400 val perplexity 1557.6462
-2400 train 7.204432 (lr=2.6780e-05) (hash(x)=141843115)
-2500 val loss 7.3293
-2500 val perplexity 1524.2460
-2500 train 7.223310 (lr=2.6494e-05) (hash(x)=149417679)
-2600 val loss 7.3065
-2600 val perplexity 1489.9070
-2600 train 7.167060 (lr=2.6198e-05) (hash(x)=145279030)
-2700 val loss 7.2811
-2700 val perplexity 1452.5431
-2700 train 7.161134 (lr=2.5892e-05) (hash(x)=146363911)
-2800 val loss 7.2621
-2800 val perplexity 1425.2209
-2800 train 7.244130 (lr=2.5576e-05) (hash(x)=152507639)
-2900 val loss 7.2392
-2900 val perplexity 1392.9362
-2900 train 7.028590 (lr=2.5251e-05) (hash(x)=139296191)
-3000 val loss 7.2215
-3000 val perplexity 1368.5748
-3000 train 7.244356 (lr=2.4917e-05) (hash(x)=148473774)
-3100 val loss 7.1779
-3100 val perplexity 1310.1744
-3100 train 7.289028 (lr=2.4574e-05) (hash(x)=154130527)
-3200 val loss 7.1512
-3200 val perplexity 1275.6593
-3200 train 7.077964 (lr=2.4224e-05) (hash(x)=151117002)
-3300 val loss 7.1260
-3300 val perplexity 1243.9489
-3300 train 7.227322 (lr=2.3865e-05) (hash(x)=151035383)
-3400 val loss 7.0967
-3400 val perplexity 1207.9542
-3400 train 7.183464 (lr=2.3498e-05) (hash(x)=150614749)
-3500 val loss 7.0740
-3500 val perplexity 1180.8242
-3500 train 7.129180 (lr=2.3125e-05) (hash(x)=151431391)
-3600 val loss 7.0466
-3600 val perplexity 1148.9330
-3600 train 6.987071 (lr=2.2744e-05) (hash(x)=151140797)
-3700 val loss 7.0165
-3700 val perplexity 1114.8676
-3700 train 7.249604 (lr=2.2357e-05) (hash(x)=154608340)
-3800 val loss 6.9886
-3800 val perplexity 1084.2018
-3800 train 7.016414 (lr=2.1965e-05) (hash(x)=169641309)
-3900 val loss 6.9617
-3900 val perplexity 1055.3744
-3900 train 6.910209 (lr=2.1566e-05) (hash(x)=150592479)
-4000 val loss 6.9326
-4000 val perplexity 1025.2000
-4000 train 6.902418 (lr=2.1162e-05) (hash(x)=155495317)
-4100 val loss 6.9115
-4100 val perplexity 1003.7407
-4100 train 7.114289 (lr=2.0754e-05) (hash(x)=170323970)
-4200 val loss 6.8869
-4200 val perplexity 979.3690
-4200 train 6.689180 (lr=2.0341e-05) (hash(x)=140854360)
-4300 val loss 6.8560
-4300 val perplexity 949.5716
-4300 train 6.847187 (lr=1.9924e-05) (hash(x)=148926644)
-4400 val loss 6.8237
-4400 val perplexity 919.4199
-4400 train 6.867269 (lr=1.9504e-05) (hash(x)=149997871)
-4500 val loss 6.7989
-4500 val perplexity 896.8582
-4500 train 6.745916 (lr=1.9081e-05) (hash(x)=149219289)
-4600 val loss 6.7731
-4600 val perplexity 874.0312
-4600 train 6.579295 (lr=1.8655e-05) (hash(x)=139836197)
-4700 val loss 6.7550
-4700 val perplexity 858.2997
-4700 train 6.710022 (lr=1.8226e-05) (hash(x)=150722883)
-4800 val loss 6.7303
-4800 val perplexity 837.4395
-4800 train 6.927665 (lr=1.7796e-05) (hash(x)=166864648)
-4900 val loss 6.7064
-4900 val perplexity 817.6269
-4900 train 6.820303 (lr=1.7365e-05) (hash(x)=157746743)
-5000 val loss 6.6780
-5000 val perplexity 794.7618
-5000 train 6.646445 (lr=1.6933e-05) (hash(x)=153032257)
-5100 val loss 6.6568
-5100 val perplexity 778.0281
-5100 train 6.741875 (lr=1.6500e-05) (hash(x)=151862371)
-5200 val loss 6.6357
-5200 val perplexity 761.8054
-5200 train 6.965239 (lr=1.6067e-05) (hash(x)=168862995)
-5300 val loss 6.6194
-5300 val perplexity 749.4685
-5300 train 6.594924 (lr=1.5635e-05) (hash(x)=148243567)
-5400 val loss 6.6047
-5400 val perplexity 738.5341
-5400 train 6.526024 (lr=1.5204e-05) (hash(x)=145381742)
-5500 val loss 6.5829
-5500 val perplexity 722.6185
-5500 train 6.733404 (lr=1.4774e-05) (hash(x)=155602174)
-5600 val loss 6.5620
-5600 val perplexity 707.6592
-5600 train 6.655446 (lr=1.4345e-05) (hash(x)=160637672)
-5700 val loss 6.5480
-5700 val perplexity 697.8254
-5700 train 6.487575 (lr=1.3919e-05) (hash(x)=152101131)
-5800 val loss 6.5325
-5800 val perplexity 687.1360
-5800 train 6.491645 (lr=1.3496e-05) (hash(x)=152327089)
-5900 val loss 6.5261
-5900 val perplexity 682.7640
-5900 train 6.454029 (lr=1.3076e-05) (hash(x)=151094543)
-6000 val loss 6.5148
-6000 val perplexity 675.0368
-6000 train 6.621383 (lr=1.2659e-05) (hash(x)=160233315)
-6100 val loss 6.4954
-6100 val perplexity 662.1202
-6100 train 6.595299 (lr=1.2246e-05) (hash(x)=158295337)
-6200 val loss 6.4802
-6200 val perplexity 652.1130
-6200 train 6.442608 (lr=1.1838e-05) (hash(x)=151269817)
-6300 val loss 6.4713
-6300 val perplexity 646.3232
-6300 train 6.575415 (lr=1.1434e-05) (hash(x)=156912756)
-6400 val loss 6.4603
-6400 val perplexity 639.2235
-6400 train 6.395647 (lr=1.1035e-05) (hash(x)=125779637)
-6500 val loss 6.4595
-6500 val perplexity 638.7330
-6500 train 6.357163 (lr=1.0643e-05) (hash(x)=148157661)
-6600 val loss 6.4493
-6600 val perplexity 632.2552
-6600 train 6.514656 (lr=1.0256e-05) (hash(x)=152820049)
-6700 val loss 6.4382
-6700 val perplexity 625.2979
-6700 train 6.391938 (lr=9.8753e-06) (hash(x)=155818717)
-6800 val loss 6.4283
-6800 val perplexity 619.1412
-6800 train 6.463234 (lr=9.5017e-06) (hash(x)=158833881)
-6900 val loss 6.4202
-6900 val perplexity 614.1420
-6900 train 6.413707 (lr=9.1353e-06) (hash(x)=164984909)
-7000 val loss 6.4139
-7000 val perplexity 610.2620
-7000 train 6.291857 (lr=8.7764e-06) (hash(x)=151172493)
-7100 val loss 6.4110
-7100 val perplexity 608.5116
-7100 train 6.417180 (lr=8.4255e-06) (hash(x)=154023633)
-7200 val loss 6.4018
-7200 val perplexity 602.9565
-7200 train 6.265404 (lr=8.0829e-06) (hash(x)=139351250)
-7300 val loss 6.3948
-7300 val perplexity 598.6977
-7300 train 6.346671 (lr=7.7489e-06) (hash(x)=158719458)
-7400 val loss 6.3858
-7400 val perplexity 593.3685
-7400 train 6.486239 (lr=7.4239e-06) (hash(x)=162250810)
-7500 val loss 6.3777
-7500 val perplexity 588.5457
-7500 train 6.237173 (lr=7.1083e-06) (hash(x)=142827557)
-7600 val loss 6.3751
-7600 val perplexity 587.0210
-7600 train 6.279722 (lr=6.8023e-06) (hash(x)=147830801)
-7700 val loss 6.3701
-7700 val perplexity 584.0976
-7700 train 6.498811 (lr=6.5062e-06) (hash(x)=165233314)
-7800 val loss 6.3656
-7800 val perplexity 581.5134
-7800 train 6.361984 (lr=6.2205e-06) (hash(x)=160654717)
-7900 val loss 6.3624
-7900 val perplexity 579.6442
-7900 train 6.218457 (lr=5.9453e-06) (hash(x)=140712586)
-8000 val loss 6.3538
-8000 val perplexity 574.6667
-8000 train 6.490506 (lr=5.6809e-06) (hash(x)=150309061)
-8100 val loss 6.3507
-8100 val perplexity 572.8998
-8100 train 6.403825 (lr=5.4277e-06) (hash(x)=158805466)
-8200 val loss 6.3458
-8200 val perplexity 570.1182
-8200 train 6.204818 (lr=5.1858e-06) (hash(x)=142814078)
-8300 val loss 6.3424
-8300 val perplexity 568.1640
-8300 train 6.298479 (lr=4.9556e-06) (hash(x)=148340417)
-8400 val loss 6.3400
-8400 val perplexity 566.7823
-8400 train 6.272801 (lr=4.7372e-06) (hash(x)=157132639)
-8500 val loss 6.3395
-8500 val perplexity 566.5011
-8500 train 6.289837 (lr=4.5309e-06) (hash(x)=152994240)
-8600 val loss 6.3312
-8600 val perplexity 561.8377
-8600 train 6.268798 (lr=4.3369e-06) (hash(x)=152210154)
-8700 val loss 6.3291
-8700 val perplexity 560.6460
-8700 train 6.383264 (lr=4.1554e-06) (hash(x)=149631247)
-8800 val loss 6.3247
-8800 val perplexity 558.1961
-8800 train 6.177348 (lr=3.9866e-06) (hash(x)=145617784)
-8900 val loss 6.3224
-8900 val perplexity 556.9269
-8900 train 6.424631 (lr=3.8307e-06) (hash(x)=155984970)
-9000 val loss 6.3215
-9000 val perplexity 556.4344
-9000 train 6.062396 (lr=3.6877e-06) (hash(x)=143433013)
-9100 val loss 6.3194
-9100 val perplexity 555.2563
-9100 train 6.223702 (lr=3.5580e-06) (hash(x)=152507533)
-9200 val loss 6.3143
-9200 val perplexity 552.4195
-9200 train 6.450141 (lr=3.4415e-06) (hash(x)=162400650)
-9300 val loss 6.3106
-9300 val perplexity 550.4012
-9300 train 6.372156 (lr=3.3385e-06) (hash(x)=148419068)
-9400 val loss 6.3096
-9400 val perplexity 549.8129
-9400 train 6.264310 (lr=3.2490e-06) (hash(x)=145025169)
-9500 val loss 6.3079
-9500 val perplexity 548.9128
-9500 train 6.279797 (lr=3.1730e-06) (hash(x)=145104235)
-9600 val loss 6.3075
-9600 val perplexity 548.6482
-9600 train 6.231697 (lr=3.1108e-06) (hash(x)=150499797)
-9700 val loss 6.3053
-9700 val perplexity 547.4417
-9700 train 6.115508 (lr=3.0624e-06) (hash(x)=139690022)
-9800 val loss 6.3002
-9800 val perplexity 544.6938
-9800 train 6.280835 (lr=3.0277e-06) (hash(x)=145645279)
-9900 val loss 6.2976
-9900 val perplexity 543.2441
-9900 train 6.197763 (lr=3.0069e-06) (hash(x)=150103432)
-9999 val loss 6.2956
-9999 val perplexity 542.2076
+0 val loss 11.7444
+0 val perplexity 126046.6875
+0 val loss 11.7444
+0 val perplexity 126046.6875
+0 train 11.717561 (lr=2.5000e-07) (hash(x)=55241167)
+0 train 11.717561 (lr=3.5000e-07) (hash(x)=55241167)
+100 val loss 10.4190
+100 val perplexity 33488.9922
+100 train 10.355477 (lr=2.5250e-05) (hash(x)=46387919)
+100 val loss 10.2610
+100 val perplexity 28596.4766
+100 train 10.199209 (lr=3.5350e-05) (hash(x)=46387919)
+200 val loss 9.2623
+200 val perplexity 10532.9053
+200 train 9.287925 (lr=5.0000e-05) (hash(x)=51276369)
+200 val loss 8.4121
+200 val perplexity 4501.1421
+200 train 8.450884 (lr=7.0000e-05) (hash(x)=51276369)
+300 val loss 8.2291
+300 val perplexity 3748.2771
+300 train 8.188359 (lr=4.9988e-05) (hash(x)=48572728)
+300 val loss 7.8747
+300 val perplexity 2629.9775
+300 train 7.822401 (lr=6.9984e-05) (hash(x)=48572728)
+400 val loss 7.8660
+400 val perplexity 2606.9949
+400 train 7.633432 (lr=4.9954e-05) (hash(x)=44943400)
+400 val loss 7.6837
+400 val perplexity 2172.7102
+400 train 7.420489 (lr=6.9935e-05) (hash(x)=44943400)
+500 val loss 7.7288
+500 val perplexity 2272.7871
+500 train 7.606852 (lr=4.9896e-05) (hash(x)=48434590)
+500 val loss 7.6470
+500 val perplexity 2094.3916
+500 train 7.546611 (lr=6.9854e-05) (hash(x)=48434590)
+600 val loss 7.6689
+600 val perplexity 2140.6326
+600 train 7.521816 (lr=4.9815e-05) (hash(x)=47442266)
+600 val loss 7.6314
+600 val perplexity 2061.8621
+600 train 7.487195 (lr=6.9741e-05) (hash(x)=47442266)
+700 val loss 7.6457
+700 val perplexity 2091.6689
+700 train 7.700227 (lr=4.9712e-05) (hash(x)=56495712)
+700 val loss 7.6084
+700 val perplexity 2015.0454
+700 train 7.655407 (lr=6.9596e-05) (hash(x)=56495712)
+800 val loss 7.6335
+800 val perplexity 2066.2261
+800 train 7.637799 (lr=4.9585e-05) (hash(x)=50093738)
+800 val loss 7.5885
+800 val perplexity 1975.3878
+800 train 7.582485 (lr=6.9419e-05) (hash(x)=50093738)
+900 val loss 7.6268
+900 val perplexity 2052.5078
+900 train 7.591948 (lr=4.9436e-05) (hash(x)=49436061)
+900 val loss 7.5637
+900 val perplexity 1926.9707
+900 train 7.522345 (lr=6.9210e-05) (hash(x)=49436061)
+1000 val loss 7.6158
+1000 val perplexity 2030.0663
+1000 train 7.563162 (lr=4.9264e-05) (hash(x)=48818282)
+1000 val loss 7.5289
+1000 val perplexity 1860.9679
+1100 val loss 7.6007
+1100 val perplexity 1999.5815
+1000 train 7.465590 (lr=6.8970e-05) (hash(x)=48818282)
+1100 train 7.664679 (lr=4.9070e-05) (hash(x)=55387425)
+1200 val loss 7.5917
+1200 val perplexity 1981.7345
+1100 val loss 7.4848
+1100 val perplexity 1780.8323
+1200 train 7.537913 (lr=4.8854e-05) (hash(x)=48328378)
+1100 train 7.553554 (lr=6.8698e-05) (hash(x)=55387425)
+1300 val loss 7.5847
+1300 val perplexity 1967.7880
+1300 train 7.523361 (lr=4.8616e-05) (hash(x)=48956717)
+1200 val loss 7.4321
+1200 val perplexity 1689.3721
+1200 train 7.346271 (lr=6.8395e-05) (hash(x)=48328378)
+1400 val loss 7.5832
+1400 val perplexity 1964.9525
+1400 train 7.476568 (lr=4.8356e-05) (hash(x)=49782571)
+1300 val loss 7.3965
+1300 val perplexity 1630.3083
+1300 train 7.312257 (lr=6.8062e-05) (hash(x)=48956717)
+1500 val loss 7.5897
+1500 val perplexity 1977.6857
+1500 train 7.398695 (lr=4.8074e-05) (hash(x)=51081994)
+1400 val loss 7.3806
+1400 val perplexity 1604.5980
+1400 train 7.259795 (lr=6.7698e-05) (hash(x)=49782571)
+1600 val loss 7.5873
+1600 val perplexity 1972.9817
+1600 train 7.656009 (lr=4.7772e-05) (hash(x)=48791204)
+1500 val loss 7.3405
+1500 val perplexity 1541.5237
+1500 train 7.134542 (lr=6.7304e-05) (hash(x)=51081994)
+1700 val loss 7.5815
+1700 val perplexity 1961.5758
+1700 train 7.307205 (lr=4.7448e-05) (hash(x)=46425011)
+1600 val loss 7.3004
+1600 val perplexity 1480.8594
+1600 train 7.370697 (lr=6.6881e-05) (hash(x)=48791204)
+1800 val loss 7.5641
+1800 val perplexity 1927.8108
+1800 train 7.216667 (lr=4.7105e-05) (hash(x)=44919141)
+1700 val loss 7.2983
+1700 val perplexity 1477.7853
+1700 train 7.001472 (lr=6.6428e-05) (hash(x)=46425011)
+1900 val loss 7.5429
+1900 val perplexity 1887.2594
+1900 train 7.642665 (lr=4.6741e-05) (hash(x)=48299675)
+1800 val loss 7.2642
+1800 val perplexity 1428.2245
+1800 train 6.906070 (lr=6.5947e-05) (hash(x)=44919141)
+2000 val loss 7.5304
+2000 val perplexity 1863.8826
+2000 train 8.451202 (lr=4.6357e-05) (hash(x)=61804797)
+1900 val loss 7.2552
+1900 val perplexity 1415.4088
+1900 train 7.376617 (lr=6.5437e-05) (hash(x)=48299675)
+2100 val loss 7.5083
+2100 val perplexity 1823.1356
+2100 train 7.605269 (lr=4.5954e-05) (hash(x)=53638030)
+2000 val loss 7.1835
+2000 val perplexity 1317.5396
+2000 train 8.116750 (lr=6.4900e-05) (hash(x)=61804797)
+2200 val loss 7.4889
+2200 val perplexity 1788.0094
+2200 train 7.722961 (lr=4.5532e-05) (hash(x)=56364593)
+2100 val loss 7.1585
+2100 val perplexity 1284.9927
+2100 train 7.253870 (lr=6.4335e-05) (hash(x)=53638030)
+2300 val loss 7.4897
+2300 val perplexity 1789.4935
+2300 train 7.585523 (lr=4.5091e-05) (hash(x)=52945580)
+2200 val loss 7.1313
+2200 val perplexity 1250.5045
+2200 train 7.376586 (lr=6.3745e-05) (hash(x)=56364593)
+2400 val loss 7.4714
+2400 val perplexity 1757.0946
+2400 train 7.553903 (lr=4.4633e-05) (hash(x)=54595211)
+2300 val loss 7.1077
+2300 val perplexity 1221.3357
+2300 train 7.225344 (lr=6.3128e-05) (hash(x)=52945580)
+2500 val loss 7.4593
+2500 val perplexity 1735.9640
+2500 train 7.488019 (lr=4.4156e-05) (hash(x)=52323363)
+2400 val loss 7.0891
+2400 val perplexity 1198.8025
+2400 train 7.160409 (lr=6.2486e-05) (hash(x)=54595211)
+2600 val loss 7.4433
+2600 val perplexity 1708.3724
+2600 train 7.256810 (lr=4.3663e-05) (hash(x)=42652707)
+2500 val loss 7.0646
+2500 val perplexity 1169.7643
+2500 train 7.078955 (lr=6.1819e-05) (hash(x)=52323363)
+2700 val loss 7.4416
+2700 val perplexity 1705.4611
+2700 train 7.189201 (lr=4.3153e-05) (hash(x)=45687855)
+2600 val loss 7.0259
+2600 val perplexity 1125.3988
+2600 train 6.769950 (lr=6.1128e-05) (hash(x)=42652707)
+2800 val loss 7.4216
+2800 val perplexity 1671.7010
+2800 train 7.389744 (lr=4.2627e-05) (hash(x)=49962830)
+2700 val loss 7.0032
+2700 val perplexity 1100.1040
+2700 train 6.697522 (lr=6.0414e-05) (hash(x)=45687855)
+2900 val loss 7.4089
+2900 val perplexity 1650.5708
+2900 train 7.488739 (lr=4.2085e-05) (hash(x)=52958451)
+2800 val loss 6.9818
+2800 val perplexity 1076.8694
+2800 train 6.968762 (lr=5.9677e-05) (hash(x)=49962830)
+3000 val loss 7.3962
+3000 val perplexity 1629.7239
+3000 train 7.337688 (lr=4.1529e-05) (hash(x)=48896990)
+2900 val loss 6.9673
+2900 val perplexity 1061.3451
+2900 train 6.998752 (lr=5.8919e-05) (hash(x)=52958451)
+3100 val loss 7.3868
+3100 val perplexity 1614.4810
+3100 train 7.337799 (lr=4.0957e-05) (hash(x)=47064930)
+3000 val loss 6.9330
+3000 val perplexity 1025.5603
+3000 train 6.879025 (lr=5.8140e-05) (hash(x)=48896990)
+3200 val loss 7.3732
+3200 val perplexity 1592.6964
+3200 train 7.139594 (lr=4.0373e-05) (hash(x)=45145921)
+3100 val loss 6.9128
+3100 val perplexity 1005.0540
+3100 train 6.876074 (lr=5.7340e-05) (hash(x)=47064930)
+3300 val loss 7.3655
+3300 val perplexity 1580.4542
+3300 train 7.237511 (lr=3.9775e-05) (hash(x)=48085018)
+3200 val loss 6.8927
+3200 val perplexity 985.0220
+3200 train 6.601774 (lr=5.6522e-05) (hash(x)=45145921)
+3400 val loss 7.3462
+3400 val perplexity 1550.3525
+3400 train 7.371779 (lr=3.9164e-05) (hash(x)=49596053)
+3300 val loss 6.8761
+3300 val perplexity 968.8724
+3300 train 6.710589 (lr=5.5684e-05) (hash(x)=48085018)
+3500 val loss 7.3421
+3500 val perplexity 1543.9542
+3500 train 6.941387 (lr=3.8541e-05) (hash(x)=41135634)
+3400 val loss 6.8393
+3400 val perplexity 933.8585
+3400 train 6.835313 (lr=5.4829e-05) (hash(x)=49596053)
+3600 val loss 7.3236
+3600 val perplexity 1515.6783
+3600 train 7.334878 (lr=3.7907e-05) (hash(x)=54286330)
+3500 val loss 6.8254
+3500 val perplexity 920.9630
+3500 train 6.421352 (lr=5.3958e-05) (hash(x)=41135634)
+3700 val loss 7.3152
+3700 val perplexity 1502.9592
+3700 train 7.221657 (lr=3.7262e-05) (hash(x)=57061993)
+3600 val loss 6.8227
+3600 val perplexity 918.4611
+3600 train 6.826213 (lr=5.3070e-05) (hash(x)=54286330)
+3800 val loss 7.3070
+3800 val perplexity 1490.7235
+3800 train 6.914749 (lr=3.6608e-05) (hash(x)=46544884)
+3700 val loss 6.8169
+3700 val perplexity 913.1631
+3700 train 6.707649 (lr=5.2167e-05) (hash(x)=57061993)
+3900 val loss 7.3168
+3900 val perplexity 1505.3081
+3900 train 7.166756 (lr=3.5944e-05) (hash(x)=48937435)
+3800 val loss 6.8191
+3800 val perplexity 915.1304
+3800 train 6.409613 (lr=5.1251e-05) (hash(x)=46544884)
+4000 val loss 7.3382
+4000 val perplexity 1537.9268
+4000 train 7.348784 (lr=3.5271e-05) (hash(x)=54466186)
+3900 val loss 6.8155
+3900 val perplexity 911.8443
+3900 train 6.659271 (lr=5.0321e-05) (hash(x)=48937435)
+4100 val loss 7.2700
+4100 val perplexity 1436.5656
+4100 train 7.121221 (lr=3.4590e-05) (hash(x)=51079995)
+4000 val loss 6.7990
+4000 val perplexity 896.9698
+4000 train 6.853115 (lr=4.9379e-05) (hash(x)=54466186)
+4200 val loss 7.2588
+4200 val perplexity 1420.5469
+4200 train 7.411931 (lr=3.3902e-05) (hash(x)=56495581)
+4100 val loss 6.7819
+4100 val perplexity 881.7239
+4100 train 6.591974 (lr=4.8426e-05) (hash(x)=51079995)
+4300 val loss 7.2302
+4300 val perplexity 1380.4785
+4300 train 7.111370 (lr=3.3207e-05) (hash(x)=45095478)
+4200 val loss 6.7772
+4200 val perplexity 877.6279
+4200 train 6.916551 (lr=4.7463e-05) (hash(x)=56495581)
+4400 val loss 7.2184
+4400 val perplexity 1364.3369
+4400 train 7.173501 (lr=3.2507e-05) (hash(x)=49954028)
+4300 val loss 6.7472
+4300 val perplexity 851.6391
+4300 train 6.591402 (lr=4.6490e-05) (hash(x)=45095478)
+4500 val loss 7.2078
+4500 val perplexity 1349.9860
+4500 train 7.127753 (lr=3.1801e-05) (hash(x)=48975821)
+4400 val loss 6.7383
+4400 val perplexity 844.0901
+4400 train 6.701144 (lr=4.5509e-05) (hash(x)=49954028)
+4600 val loss 7.1827
+4600 val perplexity 1316.5165
+4600 train 7.240033 (lr=3.1091e-05) (hash(x)=49293028)
+4500 val loss 6.7848
+4500 val perplexity 884.3400
+4500 train 6.714043 (lr=4.4521e-05) (hash(x)=48975821)
+4700 val loss 7.1732
+4700 val perplexity 1304.0402
+4700 train 7.089901 (lr=3.0377e-05) (hash(x)=48077753)
+4600 val loss 6.6866
+4600 val perplexity 801.5901
+4600 train 6.766769 (lr=4.3527e-05) (hash(x)=49293028)
+4800 val loss 7.1638
+4800 val perplexity 1291.8126
+4800 train 6.971658 (lr=2.9661e-05) (hash(x)=45037879)
+4700 val loss 6.6807
+4700 val perplexity 796.8831
+4700 train 6.593471 (lr=4.2528e-05) (hash(x)=48077753)
+4900 val loss 7.1458
+4900 val perplexity 1268.8253
+4900 train 7.074956 (lr=2.8942e-05) (hash(x)=50320154)
+4800 val loss 6.6729
+4800 val perplexity 790.7224
+4800 train 6.540059 (lr=4.1525e-05) (hash(x)=45037879)
+5000 val loss 7.1412
+5000 val perplexity 1262.9335
+5000 train 7.337318 (lr=2.8221e-05) (hash(x)=55024523)
+4900 val loss 6.6512
+4900 val perplexity 773.7209
+4900 train 6.572750 (lr=4.0518e-05) (hash(x)=50320154)
+5100 val loss 7.1290
+5100 val perplexity 1247.6289
+5100 train 7.077831 (lr=2.7500e-05) (hash(x)=47570607)
+5000 val loss 6.6414
+5000 val perplexity 766.1722
+5000 train 6.879434 (lr=3.9510e-05) (hash(x)=55024523)
+5200 val loss 7.1245
+5200 val perplexity 1242.0054
+5200 train 7.220313 (lr=2.6779e-05) (hash(x)=54311349)
+5100 val loss 6.6317
+5100 val perplexity 758.7392
+5100 train 6.556490 (lr=3.8500e-05) (hash(x)=47570607)
+5300 val loss 7.1061
+5300 val perplexity 1219.3776
+5300 train 7.010005 (lr=2.6058e-05) (hash(x)=47178524)
+5200 val loss 6.6210
+5200 val perplexity 750.6971
+5200 train 6.755637 (lr=3.7490e-05) (hash(x)=54311349)
+5400 val loss 7.1027
+5400 val perplexity 1215.2343
+5400 train 7.118385 (lr=2.5339e-05) (hash(x)=49425088)
+5300 val loss 6.6153
+5300 val perplexity 746.3967
+5300 train 6.499031 (lr=3.6482e-05) (hash(x)=47178524)
+5500 val loss 7.0912
+5500 val perplexity 1201.3782
+5500 train 6.961625 (lr=2.4623e-05) (hash(x)=46383189)
+5400 val loss 6.6143
+5400 val perplexity 745.6650
+5400 train 6.656158 (lr=3.5475e-05) (hash(x)=49425088)
+5600 val loss 7.0903
+5600 val perplexity 1200.3011
+5600 train 7.287276 (lr=2.3909e-05) (hash(x)=54084990)
+5500 val loss 6.6047
+5500 val perplexity 738.5228
+5500 train 6.435539 (lr=3.4472e-05) (hash(x)=46383189)
+5700 val loss 7.0761
+5700 val perplexity 1183.2908
+5700 train 6.787469 (lr=2.3199e-05) (hash(x)=47384182)
+5600 val loss 6.5951
+5600 val perplexity 731.4849
+5800 val loss 7.0730
+5800 val perplexity 1179.7307
+5600 train 6.790871 (lr=3.3473e-05) (hash(x)=54084990)
+5800 train 6.992303 (lr=2.2493e-05) (hash(x)=51683744)
+5900 val loss 7.0678
+5900 val perplexity 1173.5835
+5700 val loss 6.5853
+5700 val perplexity 724.3849
+5900 train 7.068780 (lr=2.1793e-05) (hash(x)=52798836)
+5700 train 6.254418 (lr=3.2479e-05) (hash(x)=47384182)
+6000 val loss 7.0733
+6000 val perplexity 1179.9906
+6000 train 6.989652 (lr=2.1098e-05) (hash(x)=50945000)
+5800 val loss 6.5765
+5800 val perplexity 718.0475
+5800 train 6.463080 (lr=3.1491e-05) (hash(x)=51683744)
+6100 val loss 7.0610
+6100 val perplexity 1165.6578
+6100 train 6.829372 (lr=2.0410e-05) (hash(x)=48964427)
+5900 val loss 6.5714
+5900 val perplexity 714.3479
+5900 train 6.587032 (lr=3.0510e-05) (hash(x)=52798836)
+6200 val loss 7.0577
+6200 val perplexity 1161.8173
+6200 train 6.877470 (lr=1.9729e-05) (hash(x)=47148610)
+6000 val loss 6.5795
+6000 val perplexity 720.1752
+6000 train 6.458630 (lr=2.9537e-05) (hash(x)=50945000)
+6300 val loss 7.0582
+6300 val perplexity 1162.3004
+6300 train 6.992953 (lr=1.9056e-05) (hash(x)=50860553)
+6100 val loss 6.5734
+6100 val perplexity 715.7814
+6100 train 6.270368 (lr=2.8574e-05) (hash(x)=48964427)
+6400 val loss 7.0524
+6400 val perplexity 1155.6312
+6400 train 6.840018 (lr=1.8392e-05) (hash(x)=50553992)
+6200 val loss 6.5813
+6200 val perplexity 721.4669
+6200 train 6.344663 (lr=2.7621e-05) (hash(x)=47148610)
+6500 val loss 7.0418
+6500 val perplexity 1143.4537
+6500 train 6.901451 (lr=1.7738e-05) (hash(x)=52521332)
+6300 val loss 6.5646
+6300 val perplexity 709.5223
+6300 train 6.481687 (lr=2.6679e-05) (hash(x)=50860553)
+6600 val loss 7.0284
+6600 val perplexity 1128.1917
+6600 train 7.372125 (lr=1.7093e-05) (hash(x)=52609843)
+6400 val loss 6.5635
+6400 val perplexity 708.7479
+6400 train 6.332783 (lr=2.5749e-05) (hash(x)=50553992)
+6700 val loss 7.0170
+6700 val perplexity 1115.4137
+6700 train 7.073514 (lr=1.6459e-05) (hash(x)=53632957)
+6500 val loss 6.5573
+6500 val perplexity 704.3593
+6500 train 6.383666 (lr=2.4833e-05) (hash(x)=52521332)
+6800 val loss 7.0105
+6800 val perplexity 1108.2476
+6800 train 7.040189 (lr=1.5836e-05) (hash(x)=46927608)
+6600 val loss 6.5405
+6600 val perplexity 692.6306
+6600 train 6.884501 (lr=2.3930e-05) (hash(x)=52609843)
+6900 val loss 7.0013
+6900 val perplexity 1098.1124
+6900 train 7.049226 (lr=1.5225e-05) (hash(x)=54483057)
+6700 val loss 6.5188
+6700 val perplexity 677.7796
+6700 train 6.560914 (lr=2.3042e-05) (hash(x)=53632957)
+7000 val loss 6.9959
+7000 val perplexity 1092.1531
+7000 train 7.190527 (lr=1.4627e-05) (hash(x)=51493884)
+6800 val loss 6.5150
+6800 val perplexity 675.2225
+6800 train 6.525479 (lr=2.2171e-05) (hash(x)=46927608)
+7100 val loss 6.9874
+7100 val perplexity 1082.9354
+7100 train 7.136772 (lr=1.4043e-05) (hash(x)=53303341)
+6900 val loss 6.5077
+6900 val perplexity 670.2778
+6900 train 6.538913 (lr=2.1316e-05) (hash(x)=54483057)
+7200 val loss 6.9863
+7200 val perplexity 1081.6818
+7200 train 6.691040 (lr=1.3471e-05) (hash(x)=45272178)
+7000 val loss 6.4993
+7000 val perplexity 664.6807
+7000 train 6.666363 (lr=2.0478e-05) (hash(x)=51493884)
+7300 val loss 6.9778
+7300 val perplexity 1072.5254
+7300 train 6.909081 (lr=1.2915e-05) (hash(x)=50389872)
+7100 val loss 6.4939
+7100 val perplexity 661.1281
+7100 train 6.649939 (lr=1.9660e-05) (hash(x)=53303341)
+7400 val loss 6.9697
+7400 val perplexity 1063.8679
+7400 train 6.600302 (lr=1.2373e-05) (hash(x)=43796301)
+7200 val loss 6.4905
+7200 val perplexity 658.8703
+7200 train 6.180031 (lr=1.8860e-05) (hash(x)=45272178)
+7500 val loss 6.9651
+7500 val perplexity 1059.0161
+7500 train 6.886889 (lr=1.1847e-05) (hash(x)=47808686)
+7300 val loss 6.4860
+7300 val perplexity 655.9136
+7300 train 6.415273 (lr=1.8081e-05) (hash(x)=50389872)
+7600 val loss 6.9620
+7600 val perplexity 1055.6909
+7600 train 6.691191 (lr=1.1337e-05) (hash(x)=41936898)
+7400 val loss 6.4844
+7400 val perplexity 654.8358
+7400 train 6.127452 (lr=1.7323e-05) (hash(x)=43796301)
+7700 val loss 6.9674
+7700 val perplexity 1061.4114
+7700 train 7.146378 (lr=1.0844e-05) (hash(x)=57550318)
+7500 val loss 6.4803
+7500 val perplexity 652.1370
+7500 train 6.429677 (lr=1.6586e-05) (hash(x)=47808686)
+7800 val loss 6.9486
+7800 val perplexity 1041.6663
+7800 train 6.841320 (lr=1.0367e-05) (hash(x)=47485210)
+7600 val loss 6.4802
+7600 val perplexity 652.0792
+7600 train 6.200052 (lr=1.5872e-05) (hash(x)=41936898)
+7900 val loss 6.9450
+7900 val perplexity 1037.9823
+7900 train 7.281407 (lr=9.9088e-06) (hash(x)=53228688)
+7700 val loss 6.4803
+7700 val perplexity 652.1638
+7700 train 6.708580 (lr=1.5181e-05) (hash(x)=57550318)
+8000 val loss 6.9435
+8000 val perplexity 1036.3483
+8000 train 6.983154 (lr=9.4682e-06) (hash(x)=52018673)
+7800 val loss 6.4715
+7800 val perplexity 646.4274
+7800 train 6.346768 (lr=1.4514e-05) (hash(x)=47485210)
+8100 val loss 6.9386
+8100 val perplexity 1031.3220
+8100 train 6.757258 (lr=9.0461e-06) (hash(x)=47079349)
+7900 val loss 6.4728
+7900 val perplexity 647.3209
+7900 train 6.823822 (lr=1.3872e-05) (hash(x)=53228688)
+8200 val loss 6.9359
+8200 val perplexity 1028.5555
+8200 train 6.992998 (lr=8.6430e-06) (hash(x)=57921563)
+8000 val loss 6.4683
+8000 val perplexity 644.4054
+8000 train 6.525360 (lr=1.3255e-05) (hash(x)=52018673)
+8300 val loss 6.9300
+8300 val perplexity 1022.5231
+8300 train 6.509700 (lr=8.2593e-06) (hash(x)=45038933)
+8100 val loss 6.4681
+8100 val perplexity 644.2361
+8100 train 6.288201 (lr=1.2665e-05) (hash(x)=47079349)
+8400 val loss 6.9337
+8400 val perplexity 1026.2736
+8400 train 6.809709 (lr=7.8953e-06) (hash(x)=47763246)
+8200 val loss 6.4610
+8200 val perplexity 639.7166
+8200 train 6.510967 (lr=1.2100e-05) (hash(x)=57921563)
+8500 val loss 6.9268
+8500 val perplexity 1019.2207
+8500 train 6.919998 (lr=7.5515e-06) (hash(x)=56176595)
+8300 val loss 6.4590
+8300 val perplexity 638.4160
+8300 train 5.999636 (lr=1.1563e-05) (hash(x)=45038933)
+8600 val loss 6.9241
+8600 val perplexity 1016.4431
+8600 train 6.981564 (lr=7.2282e-06) (hash(x)=55184249)
+8400 val loss 6.4685
+8400 val perplexity 644.5172
+8400 train 6.311600 (lr=1.1053e-05) (hash(x)=47763246)
+8700 val loss 6.9220
+8700 val perplexity 1014.3809
+8700 train 6.897609 (lr=6.9257e-06) (hash(x)=46471646)
+8500 val loss 6.4614
+8500 val perplexity 639.9490
+8500 train 6.515769 (lr=1.0572e-05) (hash(x)=56176595)
+8800 val loss 6.9233
+8800 val perplexity 1015.7076
+8800 train 6.696762 (lr=6.6444e-06) (hash(x)=46233162)
+8600 val loss 6.4616
+8600 val perplexity 640.0672
+8600 train 6.535988 (lr=1.0119e-05) (hash(x)=55184249)
+8900 val loss 6.9317
+8900 val perplexity 1024.2495
+8900 train 6.854069 (lr=6.3845e-06) (hash(x)=47233684)
+8700 val loss 6.4591
+8700 val perplexity 638.4650
+8700 train 6.393760 (lr=9.6960e-06) (hash(x)=46471646)
+9000 val loss 6.9164
+9000 val perplexity 1008.7192
+9000 train 6.845828 (lr=6.1462e-06) (hash(x)=48374529)
+8800 val loss 6.4582
+8800 val perplexity 637.9197
+8800 train 6.175983 (lr=9.3021e-06) (hash(x)=46233162)
+9100 val loss 6.9079
+9100 val perplexity 1000.1794
+9100 train 6.852545 (lr=5.9300e-06) (hash(x)=48065371)
+8900 val loss 6.4608
+8900 val perplexity 639.6043
+8900 train 6.368966 (lr=8.9382e-06) (hash(x)=47233684)
+9200 val loss 6.9033
+9200 val perplexity 995.5312
+9200 train 6.823194 (lr=5.7359e-06) (hash(x)=47408078)
+9000 val loss 6.4567
+9000 val perplexity 636.9777
+9000 train 6.349148 (lr=8.6047e-06) (hash(x)=48374529)
+9300 val loss 6.9009
+9300 val perplexity 993.1728
+9300 train 6.908463 (lr=5.5641e-06) (hash(x)=50749781)
+9100 val loss 6.4471
+9100 val perplexity 630.8528
+9100 train 6.408804 (lr=8.3020e-06) (hash(x)=48065371)
+9400 val loss 6.8958
+9400 val perplexity 988.1155
+9400 train 7.069656 (lr=5.4149e-06) (hash(x)=48560169)
+9500 val loss 6.8921
+9500 val perplexity 984.4717
+9500 train 6.971089 (lr=5.2884e-06) (hash(x)=50936392)
+9200 val loss 6.4404
+9200 val perplexity 626.6569
+9200 train 6.438884 (lr=8.0302e-06) (hash(x)=47408078)
+9600 val loss 6.8880
+9600 val perplexity 980.4680
+9600 train 6.907552 (lr=5.1847e-06) (hash(x)=50651714)
+9300 val loss 6.4370
+9300 val perplexity 624.5002
+9300 train 6.419167 (lr=7.7898e-06) (hash(x)=50749781)
+9700 val loss 6.8876
+9700 val perplexity 980.0170
+9700 train 6.932286 (lr=5.1040e-06) (hash(x)=47311384)
+9400 val loss 6.4311
+9400 val perplexity 620.8464
+9400 train 6.620666 (lr=7.5809e-06) (hash(x)=48560169)
+9800 val loss 6.8858
+9800 val perplexity 978.2363
+9800 train 6.966930 (lr=5.0462e-06) (hash(x)=50921139)
+9500 val loss 6.4309
+9500 val perplexity 620.7466
+9500 train 6.564292 (lr=7.4038e-06) (hash(x)=50936392)
+9900 val loss 6.8855
+9900 val perplexity 977.9588
+9900 train 6.929035 (lr=5.0116e-06) (hash(x)=48142455)
+9600 val loss 6.4288
+9600 val perplexity 619.4521
+9600 train 6.451801 (lr=7.2586e-06) (hash(x)=50651714)
+9999 val loss 6.8814
+9999 val perplexity 973.9988
+9700 val loss 6.4277
+9700 val perplexity 618.7657
+9700 train 6.486131 (lr=7.1456e-06) (hash(x)=47311384)
+9800 val loss 6.4264
+9800 val perplexity 617.9319
+9800 train 6.528321 (lr=7.0647e-06) (hash(x)=50921139)
+9900 val loss 6.4334
+9900 val perplexity 622.3109
+9900 train 6.536893 (lr=7.0162e-06) (hash(x)=48142455)
+9999 val loss 6.4210
+9999 val perplexity 614.6432

attention_kindselective_n_heads2_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:992fce12960917e67fe333978c01ab8a1168c2d58c5ba978e00dcd7988513450
+oid sha256:78ea3bcdb04430609e43ac1108a7b5dbf0450d0f99969c2ea13449b6a93310fb
 size 38587970

attention_kindselective_n_heads2_seed1339/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d43af45ee6b4af132415e0df0d43a6b60b0f73a569bd2830225f0bc786cc385
+oid sha256:814089a077c8b23ef6a360929119aee50aedda8e9e4e98d0e7a7493c72ccf7f3
 size 38587970

attention_kindselective_n_heads2_seed1339/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc7a6bea127db219249fd3c5b87fd5cef8b65412dd927d2d06137bf15ecd6810
+oid sha256:a220d7843b756c750dff05df665a2d3ac47d3a3d40239759a66d7d9ee0304d28
 size 38587970

attention_kindselective_n_heads2_seed1339/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae43b53e4572a8ffdc6daa9c7dfa2cf2422a024f985558fc1223d73da608af4c
+oid sha256:e94c41ccd9890321419caef4669a9fe25ff3ccc0676eed6fdff9011850f032fc
 size 38587970

attention_kindselective_n_heads2_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4638b256edfc13bb0531ad0b4c5a3e9dc58e85390cc0e5f22b814d704239c6bf
+oid sha256:461740930c03f23f42f01b5d8507f8a7b471d43e9d0f353d2581eb3d27820edb
 size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:581649abe991b6fa8953ce33ce20d06ca6d4b8019177d33625611a2777a77bc5
+oid sha256:79cd2cab33452fea3f3ab1711cc885b686e94fd169470952d3f0ef5d133e5862
 size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:125f33f6a9ef4d45f8d4d711ef5cf84fa22e33cd6388b65ac4876c47b07070f9
+oid sha256:135f20203befd7fa88831d72c8f43273b2a994f834af2da5d1f4459933fb3804
 size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf6bff0001314e87e1c262e977deff42b8efa0a930960c8b56a22c26f720e4bb
+oid sha256:621d654cf60841bfa9393523a818b267078c8a47c53b2db4e0c4625cdd9515ed
 size 70895430