Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_2_1341", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads2_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_10240_2_1341", "n_embd": 128}

attention_kindselective_n_heads2_seed1341/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
+oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
 size 964

attention_kindselective_n_heads2_seed1341/dataloader_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+oid sha256:7f03ed2ebf741f15e13c79e6cc1e9a19b308450d81cc3b4d8d0338c63d77ca59
 size 964

attention_kindselective_n_heads2_seed1341/dataloader_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+oid sha256:82590037fb2eecbec961f7967a8dd1b8d85515d31a252f66b92b8139858a8b7c
 size 964

attention_kindselective_n_heads2_seed1341/dataloader_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+oid sha256:c544303717d12355a69b8ffb1eb109434e4fdccfd5a61279b6e8ba2e870d6700
 size 964

attention_kindselective_n_heads2_seed1341/log2.txt

@@ -1,303 +1,591 @@
 max_steps: 10000
-0 val loss 11.7722
-0 val perplexity 129595.2031
-0 train 11.774795 (lr=1.5000e-07) (hash(x)=145079536)
-100 val loss 10.3638
-100 val perplexity 31690.6855
-100 train 10.310390 (lr=1.5150e-05) (hash(x)=154745873)
-200 val loss 9.4344
-200 val perplexity 12511.1768
-200 train 9.417608 (lr=3.0000e-05) (hash(x)=155800595)
-300 val loss 8.1776
-300 val perplexity 3560.4011
-300 train 8.142024 (lr=2.9993e-05) (hash(x)=148595389)
-400 val loss 7.7316
-400 val perplexity 2279.2727
-400 train 7.647736 (lr=2.9972e-05) (hash(x)=145606733)
-500 val loss 7.5773
-500 val perplexity 1953.3274
-500 train 7.406966 (lr=2.9938e-05) (hash(x)=138221231)
-600 val loss 7.5203
-600 val perplexity 1845.0748
-600 train 7.525246 (lr=2.9889e-05) (hash(x)=150367139)
-700 val loss 7.4772
-700 val perplexity 1767.2173
-700 train 7.618254 (lr=2.9827e-05) (hash(x)=155579314)
-800 val loss 7.4219
-800 val perplexity 1672.1436
-800 train 7.494927 (lr=2.9751e-05) (hash(x)=155747374)
-900 val loss 7.3563
-900 val perplexity 1566.0425
-900 train 7.568688 (lr=2.9662e-05) (hash(x)=159334575)
-1000 val loss 7.2908
-1000 val perplexity 1466.7249
-1000 train 7.166207 (lr=2.9558e-05) (hash(x)=140604760)
-1100 val loss 7.2271
-1100 val perplexity 1376.2406
-1100 train 7.053119 (lr=2.9442e-05) (hash(x)=146208052)
-1200 val loss 7.1591
-1200 val perplexity 1285.7166
-1200 train 7.065541 (lr=2.9312e-05) (hash(x)=148404734)
-1300 val loss 7.0993
-1300 val perplexity 1211.0946
-1300 train 7.030000 (lr=2.9169e-05) (hash(x)=155681970)
-1400 val loss 7.0369
-1400 val perplexity 1137.8608
-1400 train 6.990728 (lr=2.9013e-05) (hash(x)=148115934)
-1500 val loss 6.9659
-1500 val perplexity 1059.9108
-1500 train 7.128124 (lr=2.8845e-05) (hash(x)=157074034)
-1600 val loss 6.9095
-1600 val perplexity 1001.7750
-1600 train 6.662536 (lr=2.8663e-05) (hash(x)=137464699)
-1700 val loss 6.8588
-1700 val perplexity 952.2664
-1700 train 7.000303 (lr=2.8469e-05) (hash(x)=166955614)
-1800 val loss 6.8125
-1800 val perplexity 909.1738
-1800 train 6.636445 (lr=2.8263e-05) (hash(x)=143886042)
-1900 val loss 6.7785
-1900 val perplexity 878.7321
-1900 train 7.344856 (lr=2.8044e-05) (hash(x)=193610391)
-2000 val loss 6.7289
-2000 val perplexity 836.2564
-2000 train 6.923176 (lr=2.7814e-05) (hash(x)=163799796)
-2100 val loss 6.6765
-2100 val perplexity 793.5648
-2100 train 6.666200 (lr=2.7572e-05) (hash(x)=154107339)
-2200 val loss 6.6449
-2200 val perplexity 768.8541
-2200 train 6.642457 (lr=2.7319e-05) (hash(x)=144592844)
-2300 val loss 6.6128
-2300 val perplexity 744.5838
-2300 train 6.680394 (lr=2.7055e-05) (hash(x)=154535861)
-2400 val loss 6.5805
-2400 val perplexity 720.9152
-2400 train 7.345472 (lr=2.6780e-05) (hash(x)=204706354)
-2500 val loss 6.5576
-2500 val perplexity 704.6126
-2500 train 6.345363 (lr=2.6494e-05) (hash(x)=143783202)
-2600 val loss 6.5345
-2600 val perplexity 688.5145
-2600 train 6.449786 (lr=2.6198e-05) (hash(x)=150862210)
-2700 val loss 6.5102
-2700 val perplexity 671.9762
-2700 train 6.551790 (lr=2.5892e-05) (hash(x)=150753426)
-2800 val loss 6.4929
-2800 val perplexity 660.4471
-2800 train 6.511770 (lr=2.5576e-05) (hash(x)=147766811)
-2900 val loss 6.4571
-2900 val perplexity 637.2186
-2900 train 6.394937 (lr=2.5251e-05) (hash(x)=151177814)
-3000 val loss 6.4394
-3000 val perplexity 626.0252
-3000 train 6.307624 (lr=2.4917e-05) (hash(x)=135925327)
-3100 val loss 6.4255
-3100 val perplexity 617.3714
-3100 train 6.066052 (lr=2.4574e-05) (hash(x)=124842353)
-3200 val loss 6.4126
-3200 val perplexity 609.4990
-3200 train 6.509407 (lr=2.4224e-05) (hash(x)=160440642)
-3300 val loss 6.3849
-3300 val perplexity 592.8026
-3300 train 6.300091 (lr=2.3865e-05) (hash(x)=143017131)
-3400 val loss 6.3705
-3400 val perplexity 584.3425
-3400 train 6.720880 (lr=2.3498e-05) (hash(x)=148143474)
-3500 val loss 6.3546
-3500 val perplexity 575.1470
-3500 train 6.168210 (lr=2.3125e-05) (hash(x)=148732639)
-3600 val loss 6.3427
-3600 val perplexity 568.3472
-3600 train 6.334900 (lr=2.2744e-05) (hash(x)=146333803)
-3700 val loss 6.3290
-3700 val perplexity 560.5775
-3700 train 6.129919 (lr=2.2357e-05) (hash(x)=143611445)
-3800 val loss 6.3270
-3800 val perplexity 559.4955
-3800 train 6.108684 (lr=2.1965e-05) (hash(x)=141272115)
-3900 val loss 6.3116
-3900 val perplexity 550.9077
-3900 train 6.246768 (lr=2.1566e-05) (hash(x)=149161848)
-4000 val loss 6.2831
-4000 val perplexity 535.4652
-4000 train 6.355687 (lr=2.1162e-05) (hash(x)=152036955)
-4100 val loss 6.2690
-4100 val perplexity 527.9527
-4100 train 6.317615 (lr=2.0754e-05) (hash(x)=153260968)
-4200 val loss 6.2611
-4200 val perplexity 523.7897
-4200 train 6.157646 (lr=2.0341e-05) (hash(x)=147629478)
-4300 val loss 6.2524
-4300 val perplexity 519.2458
-4300 train 6.222133 (lr=1.9924e-05) (hash(x)=161701080)
-4400 val loss 6.2445
-4400 val perplexity 515.1727
-4400 train 6.297623 (lr=1.9504e-05) (hash(x)=155290524)
-4500 val loss 6.2363
-4500 val perplexity 510.9671
-4500 train 6.020789 (lr=1.9081e-05) (hash(x)=144726593)
-4600 val loss 6.2316
-4600 val perplexity 508.5655
-4600 train 6.176429 (lr=1.8655e-05) (hash(x)=149517202)
-4700 val loss 6.2135
-4700 val perplexity 499.4560
-4700 train 6.223347 (lr=1.8226e-05) (hash(x)=139403008)
-4800 val loss 6.1999
-4800 val perplexity 492.6792
-4800 train 6.289749 (lr=1.7796e-05) (hash(x)=153387238)
-4900 val loss 6.1991
-4900 val perplexity 492.2868
-4900 train 5.946225 (lr=1.7365e-05) (hash(x)=139881569)
-5000 val loss 6.1858
-5000 val perplexity 485.8128
-5000 train 6.200039 (lr=1.6933e-05) (hash(x)=152602580)
-5100 val loss 6.1775
-5100 val perplexity 481.7774
-5100 train 6.222368 (lr=1.6500e-05) (hash(x)=150598460)
-5200 val loss 6.1737
-5200 val perplexity 479.9440
-5200 train 6.049888 (lr=1.6067e-05) (hash(x)=147660496)
-5300 val loss 6.1657
-5300 val perplexity 476.1298
-5300 train 5.907603 (lr=1.5635e-05) (hash(x)=147693848)
-5400 val loss 6.1717
-5400 val perplexity 478.9965
-5400 train 5.929591 (lr=1.5204e-05) (hash(x)=136048145)
-5500 val loss 6.1501
-5500 val perplexity 468.7782
-5500 train 6.171415 (lr=1.4774e-05) (hash(x)=140313123)
-5600 val loss 6.1436
-5600 val perplexity 465.7047
-5600 train 6.169950 (lr=1.4345e-05) (hash(x)=156997993)
-5700 val loss 6.1348
-5700 val perplexity 461.6267
-5700 train 6.146784 (lr=1.3919e-05) (hash(x)=143513880)
-5800 val loss 6.1298
-5800 val perplexity 459.3479
-5800 train 6.255326 (lr=1.3496e-05) (hash(x)=153736628)
-5900 val loss 6.1278
-5900 val perplexity 458.4455
-5900 train 6.270812 (lr=1.3076e-05) (hash(x)=166385417)
-6000 val loss 6.1195
-6000 val perplexity 454.6567
-6000 train 6.350556 (lr=1.2659e-05) (hash(x)=153216517)
-6100 val loss 6.1183
-6100 val perplexity 454.0780
-6100 train 6.004011 (lr=1.2246e-05) (hash(x)=145387545)
-6200 val loss 6.1155
-6200 val perplexity 452.8155
-6200 train 6.235756 (lr=1.1838e-05) (hash(x)=156306460)
-6300 val loss 6.1022
-6300 val perplexity 446.8536
-6300 train 6.034924 (lr=1.1434e-05) (hash(x)=148610447)
-6400 val loss 6.0974
-6400 val perplexity 444.7107
-6400 train 6.314558 (lr=1.1035e-05) (hash(x)=158343535)
-6500 val loss 6.0914
-6500 val perplexity 442.0321
-6500 train 6.262345 (lr=1.0643e-05) (hash(x)=157742557)
-6600 val loss 6.0901
-6600 val perplexity 441.4777
-6600 train 5.989834 (lr=1.0256e-05) (hash(x)=150457308)
-6700 val loss 6.0879
-6700 val perplexity 440.4988
-6700 train 5.953698 (lr=9.8753e-06) (hash(x)=142426205)
-6800 val loss 6.0852
-6800 val perplexity 439.3283
-6800 train 5.884865 (lr=9.5017e-06) (hash(x)=151522525)
-6900 val loss 6.0779
-6900 val perplexity 436.1267
-6900 train 6.016484 (lr=9.1353e-06) (hash(x)=152029630)
-7000 val loss 6.0745
-7000 val perplexity 434.6117
-7000 train 5.922952 (lr=8.7764e-06) (hash(x)=142978954)
-7100 val loss 6.0687
-7100 val perplexity 432.1179
-7100 train 5.947993 (lr=8.4255e-06) (hash(x)=148064359)
-7200 val loss 6.0660
-7200 val perplexity 430.9639
-7200 train 5.966048 (lr=8.0829e-06) (hash(x)=155771845)
-7300 val loss 6.0634
-7300 val perplexity 429.8407
-7300 train 5.861534 (lr=7.7489e-06) (hash(x)=141639844)
-7400 val loss 6.0639
-7400 val perplexity 430.0664
-7400 train 6.113844 (lr=7.4239e-06) (hash(x)=153466021)
-7500 val loss 6.0577
-7500 val perplexity 427.3977
-7500 train 6.058067 (lr=7.1083e-06) (hash(x)=148843587)
-7600 val loss 6.0520
-7600 val perplexity 424.9706
-7600 train 6.189840 (lr=6.8023e-06) (hash(x)=152053933)
-7700 val loss 6.0507
-7700 val perplexity 424.4133
-7700 train 5.898217 (lr=6.5062e-06) (hash(x)=141564003)
-7800 val loss 6.0465
-7800 val perplexity 422.6144
-7800 train 5.920815 (lr=6.2205e-06) (hash(x)=148617843)
-7900 val loss 6.0456
-7900 val perplexity 422.2512
-7900 train 6.015367 (lr=5.9453e-06) (hash(x)=146417668)
-8000 val loss 6.0450
-8000 val perplexity 421.9865
-8000 train 5.850304 (lr=5.6809e-06) (hash(x)=150165488)
-8100 val loss 6.0408
-8100 val perplexity 420.2381
-8100 train 6.013474 (lr=5.4277e-06) (hash(x)=144818855)
-8200 val loss 6.0372
-8200 val perplexity 418.7222
-8200 train 6.119427 (lr=5.1858e-06) (hash(x)=143074954)
-8300 val loss 6.0340
-8300 val perplexity 417.3623
-8300 train 5.923309 (lr=4.9556e-06) (hash(x)=145172494)
-8400 val loss 6.0337
-8400 val perplexity 417.2652
-8400 train 6.065529 (lr=4.7372e-06) (hash(x)=152284393)
-8500 val loss 6.0314
-8500 val perplexity 416.2838
-8500 train 5.882653 (lr=4.5309e-06) (hash(x)=147044705)
-8600 val loss 6.0320
-8600 val perplexity 416.5469
-8600 train 5.929805 (lr=4.3369e-06) (hash(x)=147762608)
-8700 val loss 6.0279
-8700 val perplexity 414.8306
-8700 train 6.042439 (lr=4.1554e-06) (hash(x)=141994567)
-8800 val loss 6.0255
-8800 val perplexity 413.8477
-8800 train 6.112911 (lr=3.9866e-06) (hash(x)=152552896)
-8900 val loss 6.0231
-8900 val perplexity 412.8680
-8900 train 5.859866 (lr=3.8307e-06) (hash(x)=139714454)
-9000 val loss 6.0218
-9000 val perplexity 412.3357
-9000 train 6.052674 (lr=3.6877e-06) (hash(x)=152274419)
-9100 val loss 6.0214
-9100 val perplexity 412.1412
-9100 train 6.103989 (lr=3.5580e-06) (hash(x)=160536885)
-9200 val loss 6.0210
-9200 val perplexity 411.9943
-9200 train 5.841023 (lr=3.4415e-06) (hash(x)=138386899)
-9300 val loss 6.0192
-9300 val perplexity 411.2347
-9300 train 6.074156 (lr=3.3385e-06) (hash(x)=149957457)
-9400 val loss 6.0161
-9400 val perplexity 409.9883
-9400 train 6.065225 (lr=3.2490e-06) (hash(x)=161671548)
-9500 val loss 6.0157
-9500 val perplexity 409.8131
-9500 train 6.159393 (lr=3.1730e-06) (hash(x)=171857400)
-9600 val loss 6.0129
-9600 val perplexity 408.6819
-9600 train 5.782595 (lr=3.1108e-06) (hash(x)=135378471)
-9700 val loss 6.0124
-9700 val perplexity 408.4582
-9700 train 5.940501 (lr=3.0624e-06) (hash(x)=149514591)
-9800 val loss 6.0114
-9800 val perplexity 408.0481
-9800 train 5.993114 (lr=3.0277e-06) (hash(x)=152679072)
-9900 val loss 6.0094
-9900 val perplexity 407.2200
-9900 train 5.938741 (lr=3.0069e-06) (hash(x)=150415193)
-9999 val loss 6.0089
-9999 val perplexity 407.0517
+0 val loss 11.7712
+0 val perplexity 129466.1094
+0 val loss 11.7712
+0 val perplexity 129466.1094
+0 train 11.770625 (lr=3.5000e-07) (hash(x)=47078120)
+0 train 11.770625 (lr=2.5000e-07) (hash(x)=47078120)
+100 val loss 10.0727
+100 val perplexity 23688.1914
+100 train 10.071457 (lr=3.5350e-05) (hash(x)=43429388)
+100 val loss 10.2257
+100 val perplexity 27604.7012
+100 train 10.230239 (lr=2.5250e-05) (hash(x)=43429388)
+200 val loss 8.5407
+200 val perplexity 5118.7700
+200 train 8.701613 (lr=7.0000e-05) (hash(x)=52929681)
+200 val loss 9.0275
+200 val perplexity 8328.7500
+200 train 9.171297 (lr=5.0000e-05) (hash(x)=52929681)
+300 val loss 7.7996
+300 val perplexity 2439.5366
+300 train 7.821887 (lr=6.9984e-05) (hash(x)=49930367)
+300 val loss 7.9612
+300 val perplexity 2867.4238
+300 train 7.983008 (lr=4.9988e-05) (hash(x)=49930367)
+400 val loss 7.6933
+400 val perplexity 2193.5842
+400 train 7.520776 (lr=6.9935e-05) (hash(x)=48542946)
+400 val loss 7.7024
+400 val perplexity 2213.6982
+400 train 7.530426 (lr=4.9954e-05) (hash(x)=48542946)
+500 val loss 7.6582
+500 val perplexity 2117.9321
+500 train 7.850955 (lr=4.9896e-05) (hash(x)=55286048)
+500 val loss 7.6649
+500 val perplexity 2132.2778
+500 train 7.856366 (lr=6.9854e-05) (hash(x)=55286048)
+600 val loss 7.6322
+600 val perplexity 2063.6858
+600 train 7.885755 (lr=4.9815e-05) (hash(x)=51577760)
+600 val loss 7.6414
+600 val perplexity 2082.6948
+600 train 7.883400 (lr=6.9741e-05) (hash(x)=51577760)
+700 val loss 7.5973
+700 val perplexity 1992.7426
+700 train 7.745926 (lr=4.9712e-05) (hash(x)=57433471)
+700 val loss 7.6175
+700 val perplexity 2033.5338
+700 train 7.751812 (lr=6.9596e-05) (hash(x)=57433471)
+800 val loss 7.5873
+800 val perplexity 1972.9855
+800 train 7.505336 (lr=4.9585e-05) (hash(x)=49799291)
+800 val loss 7.5945
+800 val perplexity 1987.2598
+800 train 7.511701 (lr=6.9419e-05) (hash(x)=49799291)
+900 val loss 7.5716
+900 val perplexity 1942.1638
+900 train 7.573707 (lr=4.9436e-05) (hash(x)=49502839)
+900 val loss 7.5740
+900 val perplexity 1946.9084
+900 train 7.579421 (lr=6.9210e-05) (hash(x)=49502839)
+1000 val loss 7.5416
+1000 val perplexity 1884.8977
+1000 train 7.802229 (lr=4.9264e-05) (hash(x)=51142904)
+1000 val loss 7.5218
+1000 val perplexity 1847.9504
+1000 train 7.800779 (lr=6.8970e-05) (hash(x)=51142904)
+1100 val loss 7.5367
+1100 val perplexity 1875.6093
+1100 train 7.609015 (lr=4.9070e-05) (hash(x)=52751086)
+1100 val loss 7.5060
+1100 val perplexity 1818.8705
+1100 train 7.595724 (lr=6.8698e-05) (hash(x)=52751086)
+1200 val loss 7.5167
+1200 val perplexity 1838.4198
+1200 train 7.513431 (lr=4.8854e-05) (hash(x)=51538621)
+1200 val loss 7.4812
+1200 val perplexity 1774.2981
+1200 train 7.480718 (lr=6.8395e-05) (hash(x)=51538621)
+1300 val loss 7.5203
+1300 val perplexity 1845.1267
+1300 train 7.599769 (lr=4.8616e-05) (hash(x)=52034040)
+1300 val loss 7.4708
+1300 val perplexity 1755.9396
+1300 train 7.550740 (lr=6.8062e-05) (hash(x)=52034040)
+1400 val loss 7.4884
+1400 val perplexity 1787.2448
+1400 train 7.515759 (lr=4.8356e-05) (hash(x)=50640105)
+1400 val loss 7.4255
+1400 val perplexity 1678.2040
+1400 train 7.458371 (lr=6.7698e-05) (hash(x)=50640105)
+1500 val loss 7.4658
+1500 val perplexity 1747.2191
+1500 train 7.407789 (lr=4.8074e-05) (hash(x)=49016270)
+1500 val loss 7.3886
+1500 val perplexity 1617.4746
+1500 train 7.317589 (lr=6.7304e-05) (hash(x)=49016270)
+1600 val loss 7.4415
+1600 val perplexity 1705.3562
+1600 train 7.206252 (lr=4.7772e-05) (hash(x)=46100488)
+1700 val loss 7.4194
+1700 val perplexity 1668.0033
+1700 train 7.459988 (lr=4.7448e-05) (hash(x)=49185350)
+1600 val loss 7.3380
+1600 val perplexity 1537.6078
+1600 train 7.104100 (lr=6.6881e-05) (hash(x)=46100488)
+1800 val loss 7.3992
+1800 val perplexity 1634.6200
+1800 train 7.278754 (lr=4.7105e-05) (hash(x)=48024574)
+1700 val loss 7.2826
+1700 val perplexity 1454.7106
+1700 train 7.309137 (lr=6.6428e-05) (hash(x)=49185350)
+1900 val loss 7.3710
+1900 val perplexity 1589.1908
+1900 train 7.176445 (lr=4.6741e-05) (hash(x)=45823189)
+1800 val loss 7.2495
+1800 val perplexity 1407.4564
+1800 train 7.130040 (lr=6.5947e-05) (hash(x)=48024574)
+2000 val loss 7.3510
+2000 val perplexity 1557.7837
+2000 train 7.191787 (lr=4.6357e-05) (hash(x)=45703932)
+1900 val loss 7.2065
+1900 val perplexity 1348.1270
+1900 train 7.000680 (lr=6.5437e-05) (hash(x)=45823189)
+2100 val loss 7.3243
+2100 val perplexity 1516.6754
+2100 train 7.937493 (lr=4.5954e-05) (hash(x)=58570170)
+2000 val loss 7.1571
+2000 val perplexity 1283.1888
+2000 train 6.968543 (lr=6.4900e-05) (hash(x)=45703932)
+2200 val loss 7.3308
+2200 val perplexity 1526.6041
+2200 train 7.352339 (lr=4.5532e-05) (hash(x)=55262880)
+2100 val loss 7.1143
+2100 val perplexity 1229.3721
+2100 train 7.765666 (lr=6.4335e-05) (hash(x)=58570170)
+2300 val loss 7.2808
+2300 val perplexity 1452.1927
+2300 train 7.059850 (lr=4.5091e-05) (hash(x)=46415497)
+2200 val loss 7.0929
+2200 val perplexity 1203.3556
+2200 train 7.112502 (lr=6.3745e-05) (hash(x)=55262880)
+2400 val loss 7.2422
+2400 val perplexity 1397.1962
+2400 train 7.170742 (lr=4.4633e-05) (hash(x)=49272278)
+2300 val loss 7.0295
+2300 val perplexity 1129.4420
+2300 train 6.759797 (lr=6.3128e-05) (hash(x)=46415497)
+2500 val loss 7.2227
+2500 val perplexity 1370.1537
+2500 train 7.073672 (lr=4.4156e-05) (hash(x)=48390803)
+2400 val loss 6.9712
+2400 val perplexity 1065.5042
+2400 train 6.889137 (lr=6.2486e-05) (hash(x)=49272278)
+2600 val loss 7.1874
+2600 val perplexity 1322.6218
+2600 train 7.069160 (lr=4.3663e-05) (hash(x)=47450116)
+2500 val loss 6.9361
+2500 val perplexity 1028.7850
+2500 train 6.783605 (lr=6.1819e-05) (hash(x)=48390803)
+2700 val loss 7.1763
+2700 val perplexity 1308.1176
+2700 train 7.254964 (lr=4.3153e-05) (hash(x)=52681152)
+2800 val loss 7.1497
+2800 val perplexity 1273.7507
+2800 train 7.206081 (lr=4.2627e-05) (hash(x)=50664094)
+2600 val loss 6.8966
+2600 val perplexity 988.8942
+2600 train 6.776588 (lr=6.1128e-05) (hash(x)=47450116)
+2900 val loss 7.1360
+2900 val perplexity 1256.4001
+2900 train 6.937007 (lr=4.2085e-05) (hash(x)=47067144)
+2700 val loss 6.8688
+2700 val perplexity 961.7508
+2700 train 6.956961 (lr=6.0414e-05) (hash(x)=52681152)
+3000 val loss 7.1024
+3000 val perplexity 1214.9301
+3000 train 6.913496 (lr=4.1529e-05) (hash(x)=45015009)
+2800 val loss 6.8370
+2800 val perplexity 931.7004
+2800 train 6.900253 (lr=5.9677e-05) (hash(x)=50664094)
+3100 val loss 7.0774
+3100 val perplexity 1184.8473
+3100 train 6.905035 (lr=4.0957e-05) (hash(x)=45245896)
+2900 val loss 6.8098
+2900 val perplexity 906.7277
+2900 train 6.608460 (lr=5.8919e-05) (hash(x)=47067144)
+3200 val loss 7.0524
+3200 val perplexity 1155.6483
+3200 train 7.008028 (lr=4.0373e-05) (hash(x)=49995942)
+3000 val loss 6.7790
+3000 val perplexity 879.1960
+3000 train 6.598187 (lr=5.8140e-05) (hash(x)=45015009)
+3300 val loss 7.0304
+3300 val perplexity 1130.5018
+3300 train 6.907940 (lr=3.9775e-05) (hash(x)=52311504)
+3100 val loss 6.7659
+3100 val perplexity 867.7515
+3100 train 6.592752 (lr=5.7340e-05) (hash(x)=45245896)
+3400 val loss 6.9985
+3400 val perplexity 1094.9454
+3400 train 7.000693 (lr=3.9164e-05) (hash(x)=44332917)
+3200 val loss 6.7531
+3200 val perplexity 856.6802
+3200 train 6.716655 (lr=5.6522e-05) (hash(x)=49995942)
+3500 val loss 6.9684
+3500 val perplexity 1062.4990
+3500 train 7.082510 (lr=3.8541e-05) (hash(x)=56517159)
+3300 val loss 6.7407
+3300 val perplexity 846.1188
+3300 train 6.605199 (lr=5.5684e-05) (hash(x)=52311504)
+3600 val loss 6.9526
+3600 val perplexity 1045.8395
+3600 train 6.833664 (lr=3.7907e-05) (hash(x)=50720920)
+3400 val loss 6.7160
+3400 val perplexity 825.5479
+3400 train 6.696031 (lr=5.4829e-05) (hash(x)=44332917)
+3700 val loss 6.9372
+3700 val perplexity 1029.9263
+3700 train 7.426872 (lr=3.7262e-05) (hash(x)=62727701)
+3500 val loss 6.7101
+3500 val perplexity 820.6426
+3500 train 6.807840 (lr=5.3958e-05) (hash(x)=56517159)
+3800 val loss 6.9073
+3800 val perplexity 999.5339
+3800 train 6.755322 (lr=3.6608e-05) (hash(x)=54772539)
+3600 val loss 6.6921
+3600 val perplexity 806.0220
+3600 train 6.541726 (lr=5.3070e-05) (hash(x)=50720920)
+3900 val loss 6.8920
+3900 val perplexity 984.4092
+3900 train 6.885997 (lr=3.5944e-05) (hash(x)=52274485)
+3700 val loss 6.6747
+3700 val perplexity 792.1251
+4000 val loss 6.8666
+4000 val perplexity 959.7191
+3700 train 7.175059 (lr=5.2167e-05) (hash(x)=62727701)
+4000 train 6.580018 (lr=3.5271e-05) (hash(x)=50118307)
+4100 val loss 6.8453
+4100 val perplexity 939.4526
+4100 train 6.425397 (lr=3.4590e-05) (hash(x)=42771647)
+3800 val loss 6.6577
+3800 val perplexity 778.7675
+3800 train 6.517581 (lr=5.1251e-05) (hash(x)=54772539)
+4200 val loss 6.8241
+4200 val perplexity 919.7592
+4200 train 6.917285 (lr=3.3902e-05) (hash(x)=51748836)
+3900 val loss 6.6442
+3900 val perplexity 768.3301
+3900 train 6.636859 (lr=5.0321e-05) (hash(x)=52274485)
+4300 val loss 6.8008
+4300 val perplexity 898.5447
+4300 train 6.814617 (lr=3.3207e-05) (hash(x)=49021280)
+4000 val loss 6.6322
+4000 val perplexity 759.1318
+4000 train 6.323574 (lr=4.9379e-05) (hash(x)=50118307)
+4400 val loss 6.7751
+4400 val perplexity 875.7759
+4400 train 6.851520 (lr=3.2507e-05) (hash(x)=55200309)
+4100 val loss 6.6264
+4100 val perplexity 754.7734
+4100 train 6.207099 (lr=4.8426e-05) (hash(x)=42771647)
+4500 val loss 6.7518
+4500 val perplexity 855.5616
+4500 train 6.813970 (lr=3.1801e-05) (hash(x)=52085049)
+4200 val loss 6.6130
+4200 val perplexity 744.7159
+4200 train 6.700623 (lr=4.7463e-05) (hash(x)=51748836)
+4600 val loss 6.7426
+4600 val perplexity 847.7891
+4600 train 6.735619 (lr=3.1091e-05) (hash(x)=48935595)
+4300 val loss 6.5854
+4300 val perplexity 724.4294
+4300 train 6.597935 (lr=4.6490e-05) (hash(x)=49021280)
+4700 val loss 6.7275
+4700 val perplexity 835.0339
+4700 train 6.922513 (lr=3.0377e-05) (hash(x)=49182380)
+4400 val loss 6.5741
+4400 val perplexity 716.2935
+4400 train 6.621936 (lr=4.5509e-05) (hash(x)=55200309)
+4800 val loss 6.7063
+4800 val perplexity 817.5575
+4800 train 6.516437 (lr=2.9661e-05) (hash(x)=43941929)
+4500 val loss 6.5614
+4500 val perplexity 707.2348
+4500 train 6.612576 (lr=4.4521e-05) (hash(x)=52085049)
+4900 val loss 6.6963
+4900 val perplexity 809.4009
+4900 train 6.859163 (lr=2.8942e-05) (hash(x)=51852773)
+4600 val loss 6.5465
+4600 val perplexity 696.8232
+4600 train 6.553600 (lr=4.3527e-05) (hash(x)=48935595)
+5000 val loss 6.6846
+5000 val perplexity 800.0298
+5000 train 6.301474 (lr=2.8221e-05) (hash(x)=40509616)
+4700 val loss 6.5361
+4700 val perplexity 689.5744
+4700 train 6.776329 (lr=4.2528e-05) (hash(x)=49182380)
+5100 val loss 6.6730
+5100 val perplexity 790.7288
+5100 train 6.920803 (lr=2.7500e-05) (hash(x)=57585369)
+4800 val loss 6.5219
+4800 val perplexity 679.8577
+4800 train 6.354631 (lr=4.1525e-05) (hash(x)=43941929)
+5200 val loss 6.6732
+5200 val perplexity 790.8970
+5200 train 6.662302 (lr=2.6779e-05) (hash(x)=51042313)
+5300 val loss 6.6516
+5300 val perplexity 774.0305
+5300 train 6.798846 (lr=2.6058e-05) (hash(x)=52001684)
+4900 val loss 6.5276
+4900 val perplexity 683.7730
+4900 train 6.678216 (lr=4.0518e-05) (hash(x)=51852773)
+5400 val loss 6.6401
+5400 val perplexity 765.1759
+5400 train 6.575788 (lr=2.5339e-05) (hash(x)=48831647)
+5000 val loss 6.5124
+5000 val perplexity 673.4216
+5000 train 6.107291 (lr=3.9510e-05) (hash(x)=40509616)
+5500 val loss 6.6363
+5500 val perplexity 762.2680
+5500 train 7.035235 (lr=2.4623e-05) (hash(x)=50192069)
+5100 val loss 6.5035
+5100 val perplexity 667.4753
+5100 train 6.713647 (lr=3.8500e-05) (hash(x)=57585369)
+5600 val loss 6.6367
+5600 val perplexity 762.5941
+5600 train 6.465759 (lr=2.3909e-05) (hash(x)=47208852)
+5200 val loss 6.4972
+5200 val perplexity 663.2990
+5200 train 6.471662 (lr=3.7490e-05) (hash(x)=51042313)
+5700 val loss 6.6367
+5700 val perplexity 762.5676
+5700 train 6.232710 (lr=2.3199e-05) (hash(x)=44061694)
+5300 val loss 6.4865
+5300 val perplexity 656.2524
+5300 train 6.634539 (lr=3.6482e-05) (hash(x)=52001684)
+5800 val loss 6.6225
+5800 val perplexity 751.8287
+5800 train 7.010595 (lr=2.2493e-05) (hash(x)=56513279)
+5400 val loss 6.4770
+5400 val perplexity 650.0268
+5400 train 6.397245 (lr=3.5475e-05) (hash(x)=48831647)
+5900 val loss 6.6089
+5900 val perplexity 741.6647
+5900 train 6.761731 (lr=2.1793e-05) (hash(x)=50412818)
+5500 val loss 6.4741
+5500 val perplexity 648.1672
+5500 train 6.909862 (lr=3.4472e-05) (hash(x)=50192069)
+6000 val loss 6.6003
+6000 val perplexity 735.2872
+6000 train 6.414360 (lr=2.1098e-05) (hash(x)=47159634)
+5600 val loss 6.4683
+5600 val perplexity 644.4017
+5600 train 6.314424 (lr=3.3473e-05) (hash(x)=47208852)
+6100 val loss 6.5934
+6100 val perplexity 730.2899
+6100 train 6.695201 (lr=2.0410e-05) (hash(x)=54312795)
+5700 val loss 6.4732
+5700 val perplexity 647.5825
+5700 train 6.043439 (lr=3.2479e-05) (hash(x)=44061694)
+6200 val loss 6.5796
+6200 val perplexity 720.2858
+6200 train 6.707597 (lr=1.9729e-05) (hash(x)=54187587)
+5800 val loss 6.4643
+5800 val perplexity 641.8052
+5800 train 6.857479 (lr=3.1491e-05) (hash(x)=56513279)
+6300 val loss 6.5726
+6300 val perplexity 715.2604
+6300 train 6.670837 (lr=1.9056e-05) (hash(x)=53620387)
+6400 val loss 6.5608
+6400 val perplexity 706.8127
+6400 train 6.493462 (lr=1.8392e-05) (hash(x)=48761774)
+5900 val loss 6.4570
+5900 val perplexity 637.1591
+5900 train 6.608946 (lr=3.0510e-05) (hash(x)=50412818)
+6500 val loss 6.5542
+6500 val perplexity 702.1666
+6500 train 6.777394 (lr=1.7738e-05) (hash(x)=56690281)
+6000 val loss 6.4471
+6000 val perplexity 630.8537
+6000 train 6.271699 (lr=2.9537e-05) (hash(x)=47159634)
+6600 val loss 6.5486
+6600 val perplexity 698.2621
+6600 train 6.347186 (lr=1.7093e-05) (hash(x)=42985269)
+6100 val loss 6.4398
+6100 val perplexity 626.2506
+6100 train 6.526561 (lr=2.8574e-05) (hash(x)=54312795)
+6700 val loss 6.5654
+6700 val perplexity 710.0743
+6700 train 6.615873 (lr=1.6459e-05) (hash(x)=53315447)
+6200 val loss 6.4319
+6200 val perplexity 621.3448
+6200 train 6.558974 (lr=2.7621e-05) (hash(x)=54187587)
+6800 val loss 6.5371
+6800 val perplexity 690.3044
+6800 train 7.013281 (lr=1.5836e-05) (hash(x)=61577166)
+6300 val loss 6.4216
+6300 val perplexity 615.0062
+6300 train 6.523766 (lr=2.6679e-05) (hash(x)=53620387)
+6900 val loss 6.5317
+6900 val perplexity 686.5308
+6900 train 6.668251 (lr=1.5225e-05) (hash(x)=54641005)
+6400 val loss 6.4139
+6400 val perplexity 610.2687
+6400 train 6.355747 (lr=2.5749e-05) (hash(x)=48761774)
+7000 val loss 6.5290
+7000 val perplexity 684.6970
+7000 train 7.082861 (lr=1.4627e-05) (hash(x)=60579512)
+6500 val loss 6.4031
+6500 val perplexity 603.6854
+6500 train 6.621194 (lr=2.4833e-05) (hash(x)=56690281)
+7100 val loss 6.5275
+7100 val perplexity 683.7108
+7100 train 6.407035 (lr=1.4043e-05) (hash(x)=53151549)
+6600 val loss 6.4005
+6600 val perplexity 602.1273
+6600 train 6.214245 (lr=2.3930e-05) (hash(x)=42985269)
+7200 val loss 6.5212
+7200 val perplexity 679.3991
+7200 train 7.457515 (lr=1.3471e-05) (hash(x)=71842455)
+6700 val loss 6.4037
+6700 val perplexity 604.0975
+6700 train 6.470212 (lr=2.3042e-05) (hash(x)=53315447)
+7300 val loss 6.5188
+7300 val perplexity 677.7803
+7300 train 6.291944 (lr=1.2915e-05) (hash(x)=44516452)
+6800 val loss 6.3889
+6800 val perplexity 595.2116
+6800 train 6.838380 (lr=2.2171e-05) (hash(x)=61577166)
+7400 val loss 6.5163
+7400 val perplexity 676.0811
+7400 train 6.169219 (lr=1.2373e-05) (hash(x)=42667710)
+6900 val loss 6.3828
+6900 val perplexity 591.6084
+6900 train 6.541791 (lr=2.1316e-05) (hash(x)=54641005)
+7500 val loss 6.5125
+7500 val perplexity 673.5272
+7500 train 6.262584 (lr=1.1847e-05) (hash(x)=47050797)
+7600 val loss 6.5128
+7600 val perplexity 673.7106
+7000 val loss 6.3797
+7000 val perplexity 589.7284
+7600 train 6.422743 (lr=1.1337e-05) (hash(x)=49785056)
+7000 train 6.912982 (lr=2.0478e-05) (hash(x)=60579512)
+7700 val loss 6.5022
+7700 val perplexity 666.5777
+7700 train 6.310266 (lr=1.0844e-05) (hash(x)=53232030)
+7100 val loss 6.3786
+7100 val perplexity 589.1092
+7100 train 6.265176 (lr=1.9660e-05) (hash(x)=53151549)
+7800 val loss 6.4957
+7800 val perplexity 662.2895
+7800 train 6.329096 (lr=1.0367e-05) (hash(x)=48049749)
+7200 val loss 6.3746
+7200 val perplexity 586.7686
+7200 train 7.320697 (lr=1.8860e-05) (hash(x)=71842455)
+7900 val loss 6.4906
+7900 val perplexity 658.9155
+7900 train 6.314969 (lr=9.9088e-06) (hash(x)=44768513)
+7300 val loss 6.3713
+7300 val perplexity 584.7982
+7300 train 6.152587 (lr=1.8081e-05) (hash(x)=44516452)
+8000 val loss 6.4841
+8000 val perplexity 654.6609
+8000 train 6.349458 (lr=9.4682e-06) (hash(x)=46228039)
+7400 val loss 6.3714
+7400 val perplexity 584.8604
+7400 train 6.004220 (lr=1.7323e-05) (hash(x)=42667710)
+8100 val loss 6.4850
+8100 val perplexity 655.2425
+8100 train 6.849713 (lr=9.0461e-06) (hash(x)=60017091)
+7500 val loss 6.3709
+7500 val perplexity 584.6019
+7500 train 6.115476 (lr=1.6586e-05) (hash(x)=47050797)
+8200 val loss 6.4761
+8200 val perplexity 649.4072
+8200 train 6.484280 (lr=8.6430e-06) (hash(x)=49910198)
+7600 val loss 6.3649
+7600 val perplexity 581.0688
+7600 train 6.270297 (lr=1.5872e-05) (hash(x)=49785056)
+8300 val loss 6.4709
+8300 val perplexity 646.0813
+8300 train 6.789174 (lr=8.2593e-06) (hash(x)=57919055)
+7700 val loss 6.3568
+7700 val perplexity 576.3995
+7700 train 6.166145 (lr=1.5181e-05) (hash(x)=53232030)
+8400 val loss 6.4732
+8400 val perplexity 647.5664
+8400 train 6.565013 (lr=7.8953e-06) (hash(x)=49694964)
+7800 val loss 6.3557
+7800 val perplexity 575.7751
+7800 train 6.184354 (lr=1.4514e-05) (hash(x)=48049749)
+8500 val loss 6.4678
+8500 val perplexity 644.0825
+8500 train 6.496820 (lr=7.5515e-06) (hash(x)=53762585)
+7900 val loss 6.3477
+7900 val perplexity 571.2045
+7900 train 6.176773 (lr=1.3872e-05) (hash(x)=44768513)
+8600 val loss 6.4652
+8600 val perplexity 642.3760
+8600 train 6.497360 (lr=7.2282e-06) (hash(x)=51166973)
+8000 val loss 6.3411
+8000 val perplexity 567.4194
+8000 train 6.197558 (lr=1.3255e-05) (hash(x)=46228039)
+8700 val loss 6.4624
+8700 val perplexity 640.6073
+8700 train 6.541679 (lr=6.9257e-06) (hash(x)=53968049)
+8100 val loss 6.3394
+8100 val perplexity 566.4333
+8100 train 6.685860 (lr=1.2665e-05) (hash(x)=60017091)
+8800 val loss 6.4577
+8800 val perplexity 637.6141
+8800 train 6.545799 (lr=6.6444e-06) (hash(x)=59231056)
+8200 val loss 6.3328
+8200 val perplexity 562.7375
+8200 train 6.369884 (lr=1.2100e-05) (hash(x)=49910198)
+8900 val loss 6.4566
+8900 val perplexity 636.8945
+8900 train 6.358093 (lr=6.3845e-06) (hash(x)=50488048)
+8300 val loss 6.3289
+8300 val perplexity 560.5214
+8300 train 6.634696 (lr=1.1563e-05) (hash(x)=57919055)
+9000 val loss 6.4566
+9000 val perplexity 636.9036
+9000 train 6.185684 (lr=6.1462e-06) (hash(x)=44492956)
+8400 val loss 6.3304
+8400 val perplexity 561.3581
+8400 train 6.415321 (lr=1.1053e-05) (hash(x)=49694964)
+9100 val loss 6.4624
+9100 val perplexity 640.5786
+9100 train 6.489594 (lr=5.9300e-06) (hash(x)=51134989)
+8500 val loss 6.3274
+8500 val perplexity 559.7225
+8500 train 6.364176 (lr=1.0572e-05) (hash(x)=53762585)
+9200 val loss 6.4543
+9200 val perplexity 635.4424
+9200 train 6.258815 (lr=5.7359e-06) (hash(x)=48636056)
+9300 val loss 6.4538
+9300 val perplexity 635.1309
+9300 train 6.410451 (lr=5.5641e-06) (hash(x)=50200551)
+8600 val loss 6.3260
+8600 val perplexity 558.9059
+8600 train 6.361839 (lr=1.0119e-05) (hash(x)=51166973)
+9400 val loss 6.4542
+9400 val perplexity 635.3648
+9400 train 6.281915 (lr=5.4149e-06) (hash(x)=48057228)
+8700 val loss 6.3224
+8700 val perplexity 556.8817
+8700 train 6.396859 (lr=9.6960e-06) (hash(x)=53968049)
+9500 val loss 6.4501
+9500 val perplexity 632.7894
+9500 train 6.230661 (lr=5.2884e-06) (hash(x)=48125171)
+8800 val loss 6.3167
+8800 val perplexity 553.7402
+8800 train 6.420463 (lr=9.3021e-06) (hash(x)=59231056)
+9600 val loss 6.4492
+9600 val perplexity 632.1690
+9600 train 6.376224 (lr=5.1847e-06) (hash(x)=53375853)
+8900 val loss 6.3177
+8900 val perplexity 554.2752
+8900 train 6.200024 (lr=8.9382e-06) (hash(x)=50488048)
+9700 val loss 6.4451
+9700 val perplexity 629.6221
+9700 train 7.265676 (lr=5.1040e-06) (hash(x)=53924631)
+9000 val loss 6.3171
+9000 val perplexity 553.9613
+9000 train 6.043607 (lr=8.6047e-06) (hash(x)=44492956)
+9800 val loss 6.4416
+9800 val perplexity 627.3945
+9800 train 6.482964 (lr=5.0462e-06) (hash(x)=48895047)
+9100 val loss 6.3195
+9100 val perplexity 555.2841
+9100 train 6.320802 (lr=8.3020e-06) (hash(x)=51134989)
+9900 val loss 6.4396
+9900 val perplexity 626.1697
+9900 train 6.285259 (lr=5.0116e-06) (hash(x)=44269923)
+9200 val loss 6.3156
+9200 val perplexity 553.1354
+9200 train 6.110624 (lr=8.0302e-06) (hash(x)=48636056)
+9999 val loss 6.4422
+9999 val perplexity 627.8090
+9300 val loss 6.3171
+9300 val perplexity 553.9850
+9300 train 6.267875 (lr=7.7898e-06) (hash(x)=50200551)
+9400 val loss 6.3169
+9400 val perplexity 553.8564
+9400 train 6.124255 (lr=7.5809e-06) (hash(x)=48057228)
+9500 val loss 6.3137
+9500 val perplexity 552.0832
+9500 train 6.094636 (lr=7.4038e-06) (hash(x)=48125171)

attention_kindselective_n_heads2_seed1341/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:61917b2061d550c01c6a13d915f7eddbd256ab6e650148187f75cdce6cacbc7b
+oid sha256:770772d03e0266a5c13e9dd699e1d8542d7e85d6fc0ca58c6198e7ffb1fd50d5
 size 38587970

attention_kindselective_n_heads2_seed1341/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4cf822d46768dc68a2e287a6477f6a972e531f4ac2b6bee3659c053eddf6833b
+oid sha256:eae1983e9e16868647435651d905f85497a66f8a826b7be5246cb3a2f33b91d6
 size 38587970

attention_kindselective_n_heads2_seed1341/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d74f40d28c5bcaeacec475043051481eb3c1cac5909bf8266749547ca031e5c
+oid sha256:cb6c178c3fdcbf43ee518d9bd77f7a3e5de3634668cee9c5492f260f40c7ec9c
 size 38587970

attention_kindselective_n_heads2_seed1341/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:92515d22afc83810717ffa953962bfbf259bbbb4453961f59c3ddb677c9b5e0e
+oid sha256:0927621769e86865fd11c7c2ac120514613ed33d559219ce56792a9196b3a0e8
 size 38587970

attention_kindselective_n_heads2_seed1341/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0691eaba89da72b0b9e028c1b0f83917e0f6c2436c82b31b0d7e187f615e83e6
+oid sha256:5da108cf7f292280f70d1ac7c1ea29e4f23fdd5281ff47d321797c160fd27319
 size 70895430

attention_kindselective_n_heads2_seed1341/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:302fd87b96897189d938f9764920b673f8b54755f7112d2a04192fea75f999e4
+oid sha256:d305362d87fc48b343eba618199eed878c187e6eeeaf203e0d43d014df56b50d
 size 70895430

attention_kindselective_n_heads2_seed1341/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3bf07f379520069eceb45d10e1bc257b623074ee5e9d9c913d6d34a09b07f476
+oid sha256:25ca1280e09913927209fd304d196bcd3f957063396e0e06ee1f2b6e0d41af19
 size 70895430

attention_kindselective_n_heads2_seed1341/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03daeb14bf6ca8dcc9eded806955bb32da689c8c248d32e57fe4b5e2d430be28
+oid sha256:8cade60da08fc83cbddf22af2ab6080ef30ec634d051725fd7c91acbd24a7a43
 size 70895430