Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_2_1340", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 7e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "7e-5_10240_2_1340", "n_embd": 128}

attention_kindselective_n_heads2_seed1340/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
+oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
 size 964

attention_kindselective_n_heads2_seed1340/dataloader_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+oid sha256:7f03ed2ebf741f15e13c79e6cc1e9a19b308450d81cc3b4d8d0338c63d77ca59
 size 964

attention_kindselective_n_heads2_seed1340/dataloader_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+oid sha256:82590037fb2eecbec961f7967a8dd1b8d85515d31a252f66b92b8139858a8b7c
 size 964

attention_kindselective_n_heads2_seed1340/dataloader_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+oid sha256:c544303717d12355a69b8ffb1eb109434e4fdccfd5a61279b6e8ba2e870d6700
 size 964

attention_kindselective_n_heads2_seed1340/log2.txt

@@ -1,303 +1,603 @@
 max_steps: 10000
-0 val loss 11.8210
-0 val perplexity 136085.8281
-0 train 11.833096 (lr=1.5000e-07) (hash(x)=164406924)
-100 val loss 10.2860
-100 val perplexity 29318.0957
-100 train 10.581880 (lr=1.5150e-05) (hash(x)=177407419)
-200 val loss 9.2449
-200 val perplexity 10351.6846
-200 train 9.195401 (lr=3.0000e-05) (hash(x)=144903932)
-300 val loss 8.1777
-300 val perplexity 3560.5066
-300 train 8.432213 (lr=2.9993e-05) (hash(x)=173839165)
-400 val loss 7.8301
-400 val perplexity 2515.1199
-400 train 7.897438 (lr=2.9972e-05) (hash(x)=167734596)
-500 val loss 7.6721
-500 val perplexity 2147.6790
-500 train 7.686492 (lr=2.9938e-05) (hash(x)=153224076)
-600 val loss 7.6069
-600 val perplexity 2012.0172
-600 train 7.531812 (lr=2.9889e-05) (hash(x)=149619098)
-700 val loss 7.5730
-700 val perplexity 1944.9812
-700 train 7.526173 (lr=2.9827e-05) (hash(x)=146539909)
-800 val loss 7.5481
-800 val perplexity 1897.0706
-800 train 7.518998 (lr=2.9751e-05) (hash(x)=153710890)
-900 val loss 7.5388
-900 val perplexity 1879.5011
-900 train 7.499132 (lr=2.9662e-05) (hash(x)=155873620)
-1000 val loss 7.5291
-1000 val perplexity 1861.4382
-1000 train 7.460474 (lr=2.9558e-05) (hash(x)=145450636)
-1100 val loss 7.5105
-1100 val perplexity 1827.0963
-1100 train 7.569565 (lr=2.9442e-05) (hash(x)=154123388)
-1200 val loss 7.4993
-1200 val perplexity 1806.8583
-1200 train 7.362889 (lr=2.9312e-05) (hash(x)=145249251)
-1300 val loss 7.4861
-1300 val perplexity 1783.1503
-1300 train 7.407315 (lr=2.9169e-05) (hash(x)=148937127)
-1400 val loss 7.4802
-1400 val perplexity 1772.6321
-1400 train 7.583092 (lr=2.9013e-05) (hash(x)=150475545)
-1500 val loss 7.4730
-1500 val perplexity 1759.8701
-1500 train 7.411776 (lr=2.8845e-05) (hash(x)=154653428)
-1600 val loss 7.4660
-1600 val perplexity 1747.5458
-1600 train 7.437787 (lr=2.8663e-05) (hash(x)=144483776)
-1700 val loss 7.4535
-1700 val perplexity 1725.8730
-1700 train 7.627961 (lr=2.8469e-05) (hash(x)=157395496)
-1800 val loss 7.4387
-1800 val perplexity 1700.5133
-1800 train 7.462626 (lr=2.8263e-05) (hash(x)=157916369)
-1900 val loss 7.4201
-1900 val perplexity 1669.2788
-1900 train 7.636841 (lr=2.8044e-05) (hash(x)=166073923)
-2000 val loss 7.4135
-2000 val perplexity 1658.2711
-2000 train 7.522213 (lr=2.7814e-05) (hash(x)=154856891)
-2100 val loss 7.4018
-2100 val perplexity 1638.9025
-2100 train 7.377049 (lr=2.7572e-05) (hash(x)=151925203)
-2200 val loss 7.3915
-2200 val perplexity 1622.1089
-2200 train 7.193004 (lr=2.7319e-05) (hash(x)=136191502)
-2300 val loss 7.3719
-2300 val perplexity 1590.5979
-2300 train 7.487554 (lr=2.7055e-05) (hash(x)=153273362)
-2400 val loss 7.3512
-2400 val perplexity 1557.9932
-2400 train 7.283143 (lr=2.6780e-05) (hash(x)=148021541)
-2500 val loss 7.3460
-2500 val perplexity 1550.0237
-2500 train 7.250226 (lr=2.6494e-05) (hash(x)=141356608)
-2600 val loss 7.3185
-2600 val perplexity 1507.8693
-2600 train 7.283971 (lr=2.6198e-05) (hash(x)=146005217)
-2700 val loss 7.3159
-2700 val perplexity 1504.0547
-2700 train 7.160429 (lr=2.5892e-05) (hash(x)=144511718)
-2800 val loss 7.2898
-2800 val perplexity 1465.3499
-2800 train 7.166070 (lr=2.5576e-05) (hash(x)=146019502)
-2900 val loss 7.2652
-2900 val perplexity 1429.6077
-2900 train 7.165171 (lr=2.5251e-05) (hash(x)=146496200)
-3000 val loss 7.2538
-3000 val perplexity 1413.4381
-3000 train 7.196369 (lr=2.4917e-05) (hash(x)=150127281)
-3100 val loss 7.2288
-3100 val perplexity 1378.5229
-3100 train 7.172277 (lr=2.4574e-05) (hash(x)=142022255)
-3200 val loss 7.2119
-3200 val perplexity 1355.4843
-3200 train 7.291126 (lr=2.4224e-05) (hash(x)=154120875)
-3300 val loss 7.1995
-3300 val perplexity 1338.7019
-3300 train 7.225865 (lr=2.3865e-05) (hash(x)=153999717)
-3400 val loss 7.1853
-3400 val perplexity 1319.9038
-3400 train 6.986575 (lr=2.3498e-05) (hash(x)=139694097)
-3500 val loss 7.1647
-3500 val perplexity 1292.9854
-3500 train 7.357235 (lr=2.3125e-05) (hash(x)=162992732)
-3600 val loss 7.1354
-3600 val perplexity 1255.6628
-3600 train 7.120846 (lr=2.2744e-05) (hash(x)=147574101)
-3700 val loss 7.1180
-3700 val perplexity 1233.9659
-3700 train 7.231690 (lr=2.2357e-05) (hash(x)=157763099)
-3800 val loss 7.0878
-3800 val perplexity 1197.3303
-3800 train 7.244596 (lr=2.1965e-05) (hash(x)=170800034)
-3900 val loss 7.0743
-3900 val perplexity 1181.2009
-3900 train 7.151352 (lr=2.1566e-05) (hash(x)=164984528)
-4000 val loss 7.0556
-4000 val perplexity 1159.3071
-4000 train 6.946763 (lr=2.1162e-05) (hash(x)=141743323)
-4100 val loss 7.0423
-4100 val perplexity 1143.9823
-4100 train 7.050414 (lr=2.0754e-05) (hash(x)=153392872)
-4200 val loss 7.0286
-4200 val perplexity 1128.3999
-4200 train 6.904306 (lr=2.0341e-05) (hash(x)=149074933)
-4300 val loss 7.0063
-4300 val perplexity 1103.5922
-4300 train 7.419280 (lr=1.9924e-05) (hash(x)=167823423)
-4400 val loss 6.9891
-4400 val perplexity 1084.7200
-4400 train 6.721108 (lr=1.9504e-05) (hash(x)=141203114)
-4500 val loss 6.9697
-4500 val perplexity 1063.9399
-4500 train 6.962101 (lr=1.9081e-05) (hash(x)=146284780)
-4600 val loss 6.9558
-4600 val perplexity 1049.2501
-4600 train 6.758272 (lr=1.8655e-05) (hash(x)=141126464)
-4700 val loss 6.9420
-4700 val perplexity 1034.8446
-4700 train 6.943596 (lr=1.8226e-05) (hash(x)=154751926)
-4800 val loss 6.9285
-4800 val perplexity 1020.9679
-4800 train 7.011858 (lr=1.7796e-05) (hash(x)=154793198)
-4900 val loss 6.9213
-4900 val perplexity 1013.5986
-4900 train 6.667045 (lr=1.7365e-05) (hash(x)=139406392)
-5000 val loss 6.9092
-5000 val perplexity 1001.4541
-5000 train 6.710131 (lr=1.6933e-05) (hash(x)=153548741)
-5100 val loss 6.8940
-5100 val perplexity 986.2919
-5100 train 6.959525 (lr=1.6500e-05) (hash(x)=160488568)
-5200 val loss 6.8850
-5200 val perplexity 977.5457
-5200 train 6.886682 (lr=1.6067e-05) (hash(x)=149645053)
-5300 val loss 6.8517
-5300 val perplexity 945.4849
-5300 train 6.939456 (lr=1.5635e-05) (hash(x)=155820556)
-5400 val loss 6.8374
-5400 val perplexity 932.0350
-5400 train 6.763284 (lr=1.5204e-05) (hash(x)=147538134)
-5500 val loss 6.8217
-5500 val perplexity 917.5742
-5500 train 7.024829 (lr=1.4774e-05) (hash(x)=166889307)
-5600 val loss 6.8077
-5600 val perplexity 904.7902
-5600 train 6.529876 (lr=1.4345e-05) (hash(x)=139516699)
-5700 val loss 6.7965
-5700 val perplexity 894.6815
-5700 train 6.495327 (lr=1.3919e-05) (hash(x)=140453511)
-5800 val loss 6.7922
-5800 val perplexity 890.8710
-5800 train 6.750346 (lr=1.3496e-05) (hash(x)=162964847)
-5900 val loss 6.7734
-5900 val perplexity 874.2501
-5900 train 6.787580 (lr=1.3076e-05) (hash(x)=150606634)
-6000 val loss 6.7546
-6000 val perplexity 857.9982
-6000 train 6.869712 (lr=1.2659e-05) (hash(x)=149890857)
-6100 val loss 6.7394
-6100 val perplexity 845.0160
-6100 train 6.851167 (lr=1.2246e-05) (hash(x)=173884145)
-6200 val loss 6.7288
-6200 val perplexity 836.1551
-6200 train 6.760945 (lr=1.1838e-05) (hash(x)=151987098)
-6300 val loss 6.7160
-6300 val perplexity 825.5145
-6300 train 6.630625 (lr=1.1434e-05) (hash(x)=148853562)
-6400 val loss 6.7048
-6400 val perplexity 816.2783
-6400 train 6.468220 (lr=1.1035e-05) (hash(x)=141530101)
-6500 val loss 6.6959
-6500 val perplexity 809.1172
-6500 train 6.547406 (lr=1.0643e-05) (hash(x)=142297809)
-6600 val loss 6.6835
-6600 val perplexity 799.1030
-6600 train 6.547137 (lr=1.0256e-05) (hash(x)=142447782)
-6700 val loss 6.6707
-6700 val perplexity 788.9225
-6700 train 6.593633 (lr=9.8753e-06) (hash(x)=147004686)
-6800 val loss 6.6594
-6800 val perplexity 780.0958
-6800 train 6.405754 (lr=9.5017e-06) (hash(x)=133438702)
-6900 val loss 6.6530
-6900 val perplexity 775.0946
-6900 train 6.670481 (lr=9.1353e-06) (hash(x)=157085143)
-7000 val loss 6.6447
-7000 val perplexity 768.6899
-7000 train 6.533543 (lr=8.7764e-06) (hash(x)=139437666)
-7100 val loss 6.6337
-7100 val perplexity 760.2664
-7100 train 6.748162 (lr=8.4255e-06) (hash(x)=159792986)
-7200 val loss 6.6230
-7200 val perplexity 752.1608
-7200 train 6.588307 (lr=8.0829e-06) (hash(x)=144930687)
-7300 val loss 6.6153
-7300 val perplexity 746.4230
-7300 train 6.699518 (lr=7.7489e-06) (hash(x)=156242690)
-7400 val loss 6.6093
-7400 val perplexity 741.9413
-7400 train 6.480586 (lr=7.4239e-06) (hash(x)=148183719)
-7500 val loss 6.6042
-7500 val perplexity 738.1967
-7500 train 6.624214 (lr=7.1083e-06) (hash(x)=152494758)
-7600 val loss 6.5976
-7600 val perplexity 733.3229
-7600 train 6.321466 (lr=6.8023e-06) (hash(x)=142485027)
-7700 val loss 6.5898
-7700 val perplexity 727.6033
-7700 train 6.445391 (lr=6.5062e-06) (hash(x)=147512165)
-7800 val loss 6.5780
-7800 val perplexity 719.1275
-7800 train 6.539083 (lr=6.2205e-06) (hash(x)=160346994)
-7900 val loss 6.5764
-7900 val perplexity 717.9341
-7900 train 6.418441 (lr=5.9453e-06) (hash(x)=144488254)
-8000 val loss 6.5700
-8000 val perplexity 713.3625
-8000 train 6.375028 (lr=5.6809e-06) (hash(x)=147637019)
-8100 val loss 6.5666
-8100 val perplexity 710.9796
-8100 train 6.427360 (lr=5.4277e-06) (hash(x)=147340534)
-8200 val loss 6.5623
-8200 val perplexity 707.9001
-8200 train 6.574669 (lr=5.1858e-06) (hash(x)=151630665)
-8300 val loss 6.5526
-8300 val perplexity 701.0833
-8300 train 6.621968 (lr=4.9556e-06) (hash(x)=149747064)
-8400 val loss 6.5468
-8400 val perplexity 696.9837
-8400 train 6.719865 (lr=4.7372e-06) (hash(x)=154245770)
-8500 val loss 6.5429
-8500 val perplexity 694.2892
-8500 train 6.417419 (lr=4.5309e-06) (hash(x)=152559100)
-8600 val loss 6.5389
-8600 val perplexity 691.4964
-8600 train 7.097158 (lr=4.3369e-06) (hash(x)=181365926)
-8700 val loss 6.5363
-8700 val perplexity 689.7556
-8700 train 6.383374 (lr=4.1554e-06) (hash(x)=154405991)
-8800 val loss 6.5326
-8800 val perplexity 687.1642
-8800 train 6.495862 (lr=3.9866e-06) (hash(x)=153755904)
-8900 val loss 6.5266
-8900 val perplexity 683.0411
-8900 train 6.463516 (lr=3.8307e-06) (hash(x)=152120568)
-9000 val loss 6.5208
-9000 val perplexity 679.1530
-9000 train 6.330629 (lr=3.6877e-06) (hash(x)=142797279)
-9100 val loss 6.5182
-9100 val perplexity 677.3309
-9100 train 6.386018 (lr=3.5580e-06) (hash(x)=143037503)
-9200 val loss 6.5147
-9200 val perplexity 674.9849
-9200 train 6.315207 (lr=3.4415e-06) (hash(x)=113690273)
-9300 val loss 6.5132
-9300 val perplexity 673.9931
-9300 train 6.458709 (lr=3.3385e-06) (hash(x)=158025077)
-9400 val loss 6.5109
-9400 val perplexity 672.4445
-9400 train 6.597129 (lr=3.2490e-06) (hash(x)=158251718)
-9500 val loss 6.5048
-9500 val perplexity 668.3381
-9500 train 6.567007 (lr=3.1730e-06) (hash(x)=154752610)
-9600 val loss 6.5019
-9600 val perplexity 666.3765
-9600 train 6.378345 (lr=3.1108e-06) (hash(x)=146889093)
-9700 val loss 6.4992
-9700 val perplexity 664.6182
-9700 train 6.483963 (lr=3.0624e-06) (hash(x)=156906516)
-9800 val loss 6.4974
-9800 val perplexity 663.4178
-9800 train 6.365656 (lr=3.0277e-06) (hash(x)=153841927)
-9900 val loss 6.4961
-9900 val perplexity 662.5817
-9900 train 6.691042 (lr=3.0069e-06) (hash(x)=163514334)
-9999 val loss 6.4912
-9999 val perplexity 659.2943
+0 val loss 11.8201
+0 val perplexity 135955.4688
+0 val loss 11.8201
+0 val perplexity 135955.4688
+0 train 11.815376 (lr=2.5000e-07) (hash(x)=57791809)
+0 train 11.815376 (lr=3.5000e-07) (hash(x)=57791809)
+100 val loss 10.1415
+100 val perplexity 25373.9316
+100 train 10.167628 (lr=2.5250e-05) (hash(x)=48211824)
+100 val loss 9.9846
+100 val perplexity 21690.5820
+100 train 10.007619 (lr=3.5350e-05) (hash(x)=48211824)
+200 val loss 8.9560
+200 val perplexity 7754.3325
+200 train 8.916373 (lr=5.0000e-05) (hash(x)=50375849)
+200 val loss 8.3972
+200 val perplexity 4434.5674
+200 train 8.351953 (lr=7.0000e-05) (hash(x)=50375849)
+300 val loss 8.1359
+300 val perplexity 3414.7334
+300 train 8.400735 (lr=4.9988e-05) (hash(x)=57250808)
+300 val loss 7.9232
+300 val perplexity 2760.7124
+300 train 8.212564 (lr=6.9984e-05) (hash(x)=57250808)
+400 val loss 7.8720
+400 val perplexity 2622.8079
+400 train 8.402547 (lr=4.9954e-05) (hash(x)=62519858)
+400 val loss 7.7554
+400 val perplexity 2334.1978
+400 train 8.325225 (lr=6.9935e-05) (hash(x)=62519858)
+500 val loss 7.7049
+500 val perplexity 2219.0955
+500 train 7.604552 (lr=4.9896e-05) (hash(x)=47226806)
+500 val loss 7.6750
+500 val perplexity 2153.8015
+500 train 7.583454 (lr=6.9854e-05) (hash(x)=47226806)
+600 val loss 7.6373
+600 val perplexity 2074.1904
+600 train 7.663011 (lr=4.9815e-05) (hash(x)=51149322)
+600 val loss 7.6327
+600 val perplexity 2064.5627
+600 train 7.648909 (lr=6.9741e-05) (hash(x)=51149322)
+700 val loss 7.6224
+700 val perplexity 2043.4777
+700 train 7.632802 (lr=4.9712e-05) (hash(x)=51564551)
+700 val loss 7.6209
+700 val perplexity 2040.4641
+700 train 7.626678 (lr=6.9596e-05) (hash(x)=51564551)
+800 val loss 7.6059
+800 val perplexity 2010.0964
+800 train 7.368664 (lr=4.9585e-05) (hash(x)=45093459)
+800 val loss 7.6051
+800 val perplexity 2008.5107
+800 train 7.361159 (lr=6.9419e-05) (hash(x)=45093459)
+900 val loss 7.6009
+900 val perplexity 1999.9039
+900 train 7.899217 (lr=4.9436e-05) (hash(x)=54988361)
+900 val loss 7.5850
+900 val perplexity 1968.4664
+900 train 7.888000 (lr=6.9210e-05) (hash(x)=54988361)
+1000 val loss 7.5910
+1000 val perplexity 1980.3516
+1000 train 7.476419 (lr=4.9264e-05) (hash(x)=47588648)
+1000 val loss 7.5780
+1000 val perplexity 1954.8005
+1000 train 7.459035 (lr=6.8970e-05) (hash(x)=47588648)
+1100 val loss 7.5702
+1100 val perplexity 1939.6012
+1100 train 7.172775 (lr=4.9070e-05) (hash(x)=37984588)
+1100 val loss 7.5466
+1100 val perplexity 1894.2449
+1100 train 7.153616 (lr=6.8698e-05) (hash(x)=37984588)
+1200 val loss 7.5385
+1200 val perplexity 1878.9194
+1200 train 7.705000 (lr=4.8854e-05) (hash(x)=56333817)
+1300 val loss 7.5312
+1300 val perplexity 1865.3256
+1200 val loss 7.5242
+1200 val perplexity 1852.2689
+1300 train 7.666130 (lr=4.8616e-05) (hash(x)=53454056)
+1200 train 7.697269 (lr=6.8395e-05) (hash(x)=56333817)
+1400 val loss 7.4924
+1400 val perplexity 1794.3845
+1400 train 7.599366 (lr=4.8356e-05) (hash(x)=55284163)
+1300 val loss 7.5314
+1300 val perplexity 1865.7615
+1300 train 7.681477 (lr=6.8062e-05) (hash(x)=53454056)
+1500 val loss 7.4758
+1500 val perplexity 1764.8745
+1500 train 7.364212 (lr=4.8074e-05) (hash(x)=48162598)
+1400 val loss 7.4858
+1400 val perplexity 1782.5961
+1400 train 7.581354 (lr=6.7698e-05) (hash(x)=55284163)
+1600 val loss 7.4609
+1600 val perplexity 1738.7384
+1600 train 7.523167 (lr=4.7772e-05) (hash(x)=54214535)
+1500 val loss 7.4677
+1500 val perplexity 1750.5398
+1500 train 7.366483 (lr=6.7304e-05) (hash(x)=48162598)
+1700 val loss 7.4489
+1700 val perplexity 1718.0170
+1700 train 7.658994 (lr=4.7448e-05) (hash(x)=53525003)
+1600 val loss 7.4239
+1600 val perplexity 1675.5126
+1600 train 7.474539 (lr=6.6881e-05) (hash(x)=54214535)
+1800 val loss 7.4343
+1800 val perplexity 1693.1310
+1800 train 7.507501 (lr=4.7105e-05) (hash(x)=51848994)
+1700 val loss 7.3955
+1700 val perplexity 1628.6207
+1700 train 7.592460 (lr=6.6428e-05) (hash(x)=53525003)
+1900 val loss 7.4327
+1900 val perplexity 1690.2996
+1900 train 7.251468 (lr=4.6741e-05) (hash(x)=48405987)
+1800 val loss 7.3557
+1800 val perplexity 1565.1370
+1800 train 7.437174 (lr=6.5947e-05) (hash(x)=51848994)
+2000 val loss 7.4107
+2000 val perplexity 1653.5981
+2000 train 7.784187 (lr=4.6357e-05) (hash(x)=58592291)
+1900 val loss 7.3525
+1900 val perplexity 1560.1082
+1900 train 7.168375 (lr=6.5437e-05) (hash(x)=48405987)
+2100 val loss 7.4144
+2100 val perplexity 1659.6460
+2100 train 7.489458 (lr=4.5954e-05) (hash(x)=51167081)
+2000 val loss 7.3089
+2000 val perplexity 1493.5795
+2000 train 7.667870 (lr=6.4900e-05) (hash(x)=58592291)
+2200 val loss 7.3779
+2200 val perplexity 1600.2672
+2200 train 7.367453 (lr=4.5532e-05) (hash(x)=47994988)
+2100 val loss 7.3073
+2100 val perplexity 1491.0868
+2100 train 7.386800 (lr=6.4335e-05) (hash(x)=51167081)
+2300 val loss 7.3693
+2300 val perplexity 1586.4584
+2300 train 7.348310 (lr=4.5091e-05) (hash(x)=47377604)
+2200 val loss 7.3030
+2200 val perplexity 1484.7701
+2200 train 7.295847 (lr=6.3745e-05) (hash(x)=47994988)
+2400 val loss 7.3560
+2400 val perplexity 1565.5953
+2400 train 7.408455 (lr=4.4633e-05) (hash(x)=53554323)
+2300 val loss 7.2556
+2300 val perplexity 1416.0649
+2300 train 7.237532 (lr=6.3128e-05) (hash(x)=47377604)
+2500 val loss 7.3607
+2500 val perplexity 1573.0103
+2500 train 7.371253 (lr=4.4156e-05) (hash(x)=50780417)
+2400 val loss 7.2298
+2400 val perplexity 1379.9376
+2400 train 7.275304 (lr=6.2486e-05) (hash(x)=53554323)
+2600 val loss 7.3246
+2600 val perplexity 1517.1007
+2600 train 7.223897 (lr=4.3663e-05) (hash(x)=46453562)
+2500 val loss 7.2124
+2500 val perplexity 1356.2070
+2500 train 7.234367 (lr=6.1819e-05) (hash(x)=50780417)
+2700 val loss 7.3181
+2700 val perplexity 1507.3157
+2700 train 7.332546 (lr=4.3153e-05) (hash(x)=54404221)
+2600 val loss 7.1852
+2600 val perplexity 1319.7157
+2600 train 7.078228 (lr=6.1128e-05) (hash(x)=46453562)
+2800 val loss 7.3003
+2800 val perplexity 1480.7711
+2800 train 8.040918 (lr=4.2627e-05) (hash(x)=59318895)
+2700 val loss 7.1844
+2700 val perplexity 1318.7426
+2700 train 7.171799 (lr=6.0414e-05) (hash(x)=54404221)
+2900 val loss 7.3080
+2900 val perplexity 1492.1636
+2900 train 7.285590 (lr=4.2085e-05) (hash(x)=47845760)
+2800 val loss 7.1699
+2800 val perplexity 1299.7759
+2800 train 7.846885 (lr=5.9677e-05) (hash(x)=59318895)
+3000 val loss 7.2772
+3000 val perplexity 1446.9222
+3000 train 6.974861 (lr=4.1529e-05) (hash(x)=44336167)
+2900 val loss 7.1458
+2900 val perplexity 1268.7697
+2900 train 7.136559 (lr=5.8919e-05) (hash(x)=47845760)
+3100 val loss 7.2409
+3100 val perplexity 1395.3539
+3100 train 7.539340 (lr=4.0957e-05) (hash(x)=44479330)
+3000 val loss 7.1352
+3000 val perplexity 1255.3414
+3000 train 6.846057 (lr=5.8140e-05) (hash(x)=44336167)
+3200 val loss 7.2380
+3200 val perplexity 1391.3442
+3200 train 7.291301 (lr=4.0373e-05) (hash(x)=54593096)
+3100 val loss 7.1081
+3100 val perplexity 1221.8489
+3100 train 7.400929 (lr=5.7340e-05) (hash(x)=44479330)
+3300 val loss 7.2317
+3300 val perplexity 1382.6051
+3300 train 7.120160 (lr=3.9775e-05) (hash(x)=45347643)
+3200 val loss 7.1089
+3200 val perplexity 1222.8351
+3200 train 7.181127 (lr=5.6522e-05) (hash(x)=54593096)
+3400 val loss 7.1952
+3400 val perplexity 1332.9685
+3400 train 7.318199 (lr=3.9164e-05) (hash(x)=47797247)
+3300 val loss 7.0572
+3300 val perplexity 1161.2457
+3300 train 6.946620 (lr=5.5684e-05) (hash(x)=45347643)
+3500 val loss 7.1690
+3500 val perplexity 1298.5449
+3500 train 7.078238 (lr=3.8541e-05) (hash(x)=46115683)
+3400 val loss 7.0117
+3400 val perplexity 1109.5795
+3400 train 7.125251 (lr=5.4829e-05) (hash(x)=47797247)
+3600 val loss 7.1592
+3600 val perplexity 1285.8961
+3600 train 7.025664 (lr=3.7907e-05) (hash(x)=44502074)
+3500 val loss 6.9839
+3500 val perplexity 1079.1410
+3500 train 6.878851 (lr=5.3958e-05) (hash(x)=46115683)
+3700 val loss 7.1425
+3700 val perplexity 1264.6299
+3700 train 7.283512 (lr=3.7262e-05) (hash(x)=55388443)
+3600 val loss 6.9588
+3600 val perplexity 1052.3683
+3600 train 6.803612 (lr=5.3070e-05) (hash(x)=44502074)
+3800 val loss 7.1436
+3800 val perplexity 1266.0127
+3800 train 6.931653 (lr=3.6608e-05) (hash(x)=43790341)
+3900 val loss 7.1233
+3900 val perplexity 1240.5541
+3900 train 7.126774 (lr=3.5944e-05) (hash(x)=50013318)
+3700 val loss 6.9234
+3700 val perplexity 1015.7560
+3700 train 7.075602 (lr=5.2167e-05) (hash(x)=55388443)
+4000 val loss 7.1181
+4000 val perplexity 1234.0818
+4000 train 7.197229 (lr=3.5271e-05) (hash(x)=51704787)
+3800 val loss 6.9045
+3800 val perplexity 996.7182
+3800 train 6.700143 (lr=5.1251e-05) (hash(x)=43790341)
+4100 val loss 7.1075
+4100 val perplexity 1221.0736
+4100 train 7.178077 (lr=3.4590e-05) (hash(x)=50821964)
+3900 val loss 6.8876
+3900 val perplexity 980.0497
+3900 train 6.885145 (lr=5.0321e-05) (hash(x)=50013318)
+4200 val loss 7.1000
+4200 val perplexity 1211.9559
+4200 train 7.117894 (lr=3.3902e-05) (hash(x)=49675080)
+4000 val loss 6.8554
+4000 val perplexity 948.9904
+4000 train 6.929147 (lr=4.9379e-05) (hash(x)=51704787)
+4300 val loss 7.0877
+4300 val perplexity 1197.1761
+4300 train 6.751846 (lr=3.3207e-05) (hash(x)=43239281)
+4100 val loss 6.8422
+4100 val perplexity 936.5167
+4100 train 6.914855 (lr=4.8426e-05) (hash(x)=50821964)
+4400 val loss 7.0756
+4400 val perplexity 1182.7396
+4400 train 6.766150 (lr=3.2507e-05) (hash(x)=45076737)
+4200 val loss 6.8086
+4200 val perplexity 905.6094
+4200 train 6.836856 (lr=4.7463e-05) (hash(x)=49675080)
+4500 val loss 7.0691
+4500 val perplexity 1175.0457
+4500 train 7.197128 (lr=3.1801e-05) (hash(x)=57930262)
+4300 val loss 6.7909
+4300 val perplexity 889.7120
+4300 train 6.455951 (lr=4.6490e-05) (hash(x)=43239281)
+4600 val loss 7.0560
+4600 val perplexity 1159.7599
+4600 train 6.852347 (lr=3.1091e-05) (hash(x)=46721614)
+4400 val loss 6.7779
+4400 val perplexity 878.2114
+4400 train 6.462660 (lr=4.5509e-05) (hash(x)=45076737)
+4700 val loss 7.0485
+4700 val perplexity 1151.1281
+4700 train 6.876356 (lr=3.0377e-05) (hash(x)=49837920)
+4500 val loss 6.7576
+4500 val perplexity 860.5627
+4500 train 6.874951 (lr=4.4521e-05) (hash(x)=57930262)
+4800 val loss 7.0353
+4800 val perplexity 1135.9937
+4800 train 7.160434 (lr=2.9661e-05) (hash(x)=48380045)
+4600 val loss 6.7361
+4600 val perplexity 842.2592
+4600 train 6.513708 (lr=4.3527e-05) (hash(x)=46721614)
+4900 val loss 7.0306
+4900 val perplexity 1130.7338
+4900 train 6.909075 (lr=2.8942e-05) (hash(x)=44202577)
+4700 val loss 6.7285
+4700 val perplexity 835.9263
+4700 train 6.564556 (lr=4.2528e-05) (hash(x)=49837920)
+5000 val loss 7.0021
+5000 val perplexity 1098.8903
+5000 train 7.005834 (lr=2.8221e-05) (hash(x)=52038024)
+4800 val loss 6.7083
+4800 val perplexity 819.1469
+4800 train 6.879846 (lr=4.1525e-05) (hash(x)=48380045)
+5100 val loss 6.9806
+5100 val perplexity 1075.6146
+5100 train 7.158219 (lr=2.7500e-05) (hash(x)=53700038)
+4900 val loss 6.6987
+4900 val perplexity 811.3144
+4900 train 6.585225 (lr=4.0518e-05) (hash(x)=44202577)
+5200 val loss 6.9704
+5200 val perplexity 1064.6042
+5200 train 6.970703 (lr=2.6779e-05) (hash(x)=48137625)
+5000 val loss 6.6739
+5000 val perplexity 791.4561
+5000 train 6.682315 (lr=3.9510e-05) (hash(x)=52038024)
+5300 val loss 6.9584
+5300 val perplexity 1051.9539
+5300 train 6.836740 (lr=2.6058e-05) (hash(x)=43161573)
+5100 val loss 6.6505
+5100 val perplexity 773.1574
+5100 train 6.805350 (lr=3.8500e-05) (hash(x)=53700038)
+5400 val loss 6.9467
+5400 val perplexity 1039.6730
+5400 train 7.132883 (lr=2.5339e-05) (hash(x)=56673322)
+5200 val loss 6.6534
+5200 val perplexity 775.4347
+5200 train 6.657163 (lr=3.7490e-05) (hash(x)=48137625)
+5500 val loss 6.9299
+5500 val perplexity 1022.4167
+5500 train 7.111983 (lr=2.4623e-05) (hash(x)=53468295)
+5300 val loss 6.6340
+5300 val perplexity 760.5112
+5300 train 6.477094 (lr=3.6482e-05) (hash(x)=43161573)
+5600 val loss 6.9158
+5600 val perplexity 1008.0984
+5600 train 7.229111 (lr=2.3909e-05) (hash(x)=59287280)
+5400 val loss 6.6298
+5400 val perplexity 757.3374
+5400 train 6.820119 (lr=3.5475e-05) (hash(x)=56673322)
+5700 val loss 6.9053
+5700 val perplexity 997.5583
+5700 train 7.143720 (lr=2.3199e-05) (hash(x)=57575806)
+5500 val loss 6.6147
+5500 val perplexity 746.0064
+5500 train 6.824338 (lr=3.4472e-05) (hash(x)=53468295)
+5800 val loss 6.8949
+5800 val perplexity 987.2734
+5800 train 6.836562 (lr=2.2493e-05) (hash(x)=46897279)
+5600 val loss 6.6057
+5600 val perplexity 739.3046
+5600 train 6.898512 (lr=3.3473e-05) (hash(x)=59287280)
+5900 val loss 6.8898
+5900 val perplexity 982.2181
+5900 train 6.771390 (lr=2.1793e-05) (hash(x)=47565679)
+5700 val loss 6.5965
+5700 val perplexity 732.5139
+5700 train 6.830234 (lr=3.2479e-05) (hash(x)=57575806)
+6000 val loss 6.8769
+6000 val perplexity 969.5782
+6000 train 6.764159 (lr=2.1098e-05) (hash(x)=51590090)
+5800 val loss 6.5881
+5800 val perplexity 726.3762
+5800 train 6.606775 (lr=3.1491e-05) (hash(x)=46897279)
+6100 val loss 6.8705
+6100 val perplexity 963.4629
+6100 train 7.256005 (lr=2.0410e-05) (hash(x)=59732271)
+5900 val loss 6.5897
+5900 val perplexity 727.5562
+5900 train 6.480405 (lr=3.0510e-05) (hash(x)=47565679)
+6200 val loss 6.8648
+6200 val perplexity 957.9373
+6200 train 6.838703 (lr=1.9729e-05) (hash(x)=46394422)
+6000 val loss 6.5795
+6000 val perplexity 720.1879
+6000 train 6.441273 (lr=2.9537e-05) (hash(x)=51590090)
+6300 val loss 6.8538
+6300 val perplexity 947.4933
+6300 train 6.875601 (lr=1.9056e-05) (hash(x)=53748145)
+6100 val loss 6.5637
+6100 val perplexity 708.8743
+6100 train 7.011058 (lr=2.8574e-05) (hash(x)=59732271)
+6400 val loss 6.8485
+6400 val perplexity 942.4641
+6400 train 6.731420 (lr=1.8392e-05) (hash(x)=46054751)
+6200 val loss 6.5600
+6200 val perplexity 706.2467
+6200 train 6.588350 (lr=2.7621e-05) (hash(x)=46394422)
+6500 val loss 6.8386
+6500 val perplexity 933.1525
+6500 train 7.054920 (lr=1.7738e-05) (hash(x)=51816809)
+6300 val loss 6.5540
+6300 val perplexity 702.0677
+6300 train 6.569966 (lr=2.6679e-05) (hash(x)=53748145)
+6600 val loss 6.8378
+6600 val perplexity 932.3933
+6600 train 6.719009 (lr=1.7093e-05) (hash(x)=52453336)
+6400 val loss 6.5467
+6400 val perplexity 696.9398
+6400 train 6.418003 (lr=2.5749e-05) (hash(x)=46054751)
+6700 val loss 6.8274
+6700 val perplexity 922.7719
+6700 train 6.842637 (lr=1.6459e-05) (hash(x)=49108775)
+6800 val loss 6.8191
+6800 val perplexity 915.1290
+6800 train 6.744590 (lr=1.5836e-05) (hash(x)=46745396)
+6500 val loss 6.5368
+6500 val perplexity 690.0514
+6500 train 6.811157 (lr=2.4833e-05) (hash(x)=51816809)
+6900 val loss 6.8036
+6900 val perplexity 901.1028
+6900 train 6.938798 (lr=1.5225e-05) (hash(x)=46534986)
+6600 val loss 6.5317
+6600 val perplexity 686.5802
+6600 train 6.414201 (lr=2.3930e-05) (hash(x)=52453336)
+7000 val loss 6.7982
+7000 val perplexity 896.2391
+7000 train 7.138734 (lr=1.4627e-05) (hash(x)=49317888)
+6700 val loss 6.5316
+6700 val perplexity 686.5275
+6700 train 6.547672 (lr=2.3042e-05) (hash(x)=49108775)
+7100 val loss 6.7898
+7100 val perplexity 888.7249
+7100 train 6.800003 (lr=1.4043e-05) (hash(x)=50360484)
+6800 val loss 6.5200
+6800 val perplexity 678.6013
+6800 train 6.438734 (lr=2.2171e-05) (hash(x)=46745396)
+7200 val loss 6.7843
+7200 val perplexity 883.8223
+7200 train 6.675083 (lr=1.3471e-05) (hash(x)=49515094)
+6900 val loss 6.5101
+6900 val perplexity 671.8820
+6900 train 6.690259 (lr=2.1316e-05) (hash(x)=46534986)
+7300 val loss 6.7862
+7300 val perplexity 885.5561
+7300 train 6.775707 (lr=1.2915e-05) (hash(x)=51546861)
+7000 val loss 6.4946
+7000 val perplexity 661.5610
+7000 train 6.878696 (lr=2.0478e-05) (hash(x)=49317888)
+7400 val loss 6.7751
+7400 val perplexity 875.7767
+7400 train 6.777234 (lr=1.2373e-05) (hash(x)=48320948)
+7100 val loss 6.4876
+7100 val perplexity 656.9729
+7100 train 6.499301 (lr=1.9660e-05) (hash(x)=50360484)
+7500 val loss 6.7680
+7500 val perplexity 869.6076
+7500 train 6.685040 (lr=1.1847e-05) (hash(x)=40167457)
+7200 val loss 6.4838
+7200 val perplexity 654.4562
+7200 train 6.345729 (lr=1.8860e-05) (hash(x)=49515094)
+7600 val loss 6.7610
+7600 val perplexity 863.4881
+7600 train 6.716415 (lr=1.1337e-05) (hash(x)=49942165)
+7300 val loss 6.4852
+7300 val perplexity 655.3403
+7300 train 6.473186 (lr=1.8081e-05) (hash(x)=51546861)
+7700 val loss 6.7578
+7700 val perplexity 860.7277
+7700 train 6.508514 (lr=1.0844e-05) (hash(x)=48853311)
+7400 val loss 6.4816
+7400 val perplexity 653.0450
+7400 train 6.474329 (lr=1.7323e-05) (hash(x)=48320948)
+7800 val loss 6.7528
+7800 val perplexity 856.4212
+7800 train 6.658655 (lr=1.0367e-05) (hash(x)=48510117)
+7500 val loss 6.4817
+7500 val perplexity 653.0565
+7500 train 6.411730 (lr=1.6586e-05) (hash(x)=40167457)
+7900 val loss 6.7509
+7900 val perplexity 854.8618
+7900 train 6.762117 (lr=9.9088e-06) (hash(x)=48339781)
+7600 val loss 6.4654
+7600 val perplexity 642.5402
+7600 train 6.415967 (lr=1.5872e-05) (hash(x)=49942165)
+8000 val loss 6.7459
+8000 val perplexity 850.5717
+8000 train 6.878915 (lr=9.4682e-06) (hash(x)=54927320)
+7700 val loss 6.4599
+7700 val perplexity 638.9849
+7700 train 6.188712 (lr=1.5181e-05) (hash(x)=48853311)
+8100 val loss 6.7406
+8100 val perplexity 846.0574
+8100 train 6.488393 (lr=9.0461e-06) (hash(x)=46461786)
+7800 val loss 6.4578
+7800 val perplexity 637.6554
+7800 train 6.345714 (lr=1.4514e-05) (hash(x)=48510117)
+8200 val loss 6.7377
+8200 val perplexity 843.5936
+8200 train 6.667369 (lr=8.6430e-06) (hash(x)=51536260)
+7900 val loss 6.4579
+7900 val perplexity 637.7202
+7900 train 6.452045 (lr=1.3872e-05) (hash(x)=48339781)
+8300 val loss 6.7352
+8300 val perplexity 841.4913
+8300 train 6.494115 (lr=8.2593e-06) (hash(x)=44770722)
+8000 val loss 6.4506
+8000 val perplexity 633.1036
+8000 train 6.566749 (lr=1.3255e-05) (hash(x)=54927320)
+8400 val loss 6.7321
+8400 val perplexity 838.9099
+8400 train 6.740037 (lr=7.8953e-06) (hash(x)=50104957)
+8100 val loss 6.4473
+8100 val perplexity 631.0035
+8100 train 6.158883 (lr=1.2665e-05) (hash(x)=46461786)
+8500 val loss 6.7283
+8500 val perplexity 835.6963
+8500 train 6.834835 (lr=7.5515e-06) (hash(x)=50132971)
+8200 val loss 6.4479
+8200 val perplexity 631.3887
+8200 train 6.372143 (lr=1.2100e-05) (hash(x)=51536260)
+8600 val loss 6.7235
+8600 val perplexity 831.6856
+8600 train 6.676203 (lr=7.2282e-06) (hash(x)=52193699)
+8300 val loss 6.4479
+8300 val perplexity 631.3722
+8300 train 6.184940 (lr=1.1563e-05) (hash(x)=44770722)
+8700 val loss 6.7189
+8700 val perplexity 827.9468
+8700 train 6.706408 (lr=6.9257e-06) (hash(x)=47902319)
+8400 val loss 6.4421
+8400 val perplexity 627.6946
+8400 train 6.443494 (lr=1.1053e-05) (hash(x)=50104957)
+8800 val loss 6.7133
+8800 val perplexity 823.3087
+8800 train 6.985338 (lr=6.6444e-06) (hash(x)=54904230)
+8500 val loss 6.4391
+8500 val perplexity 625.8142
+8500 train 6.534442 (lr=1.0572e-05) (hash(x)=50132971)
+8900 val loss 6.7132
+8900 val perplexity 823.1639
+8900 train 6.608115 (lr=6.3845e-06) (hash(x)=46311615)
+8600 val loss 6.4317
+8600 val perplexity 621.2293
+8600 train 6.393053 (lr=1.0119e-05) (hash(x)=52193699)
+9000 val loss 6.7123
+9000 val perplexity 822.4835
+9000 train 6.588591 (lr=6.1462e-06) (hash(x)=48535188)
+8700 val loss 6.4298
+8700 val perplexity 620.0411
+8700 train 6.417854 (lr=9.6960e-06) (hash(x)=47902319)
+9100 val loss 6.7054
+9100 val perplexity 816.8428
+9100 train 6.734750 (lr=5.9300e-06) (hash(x)=51757372)
+8800 val loss 6.4236
+8800 val perplexity 616.1947
+8800 train 6.713367 (lr=9.3021e-06) (hash(x)=54904230)
+9200 val loss 6.7031
+9200 val perplexity 814.9101
+9200 train 6.588158 (lr=5.7359e-06) (hash(x)=51131708)
+8900 val loss 6.4212
+8900 val perplexity 614.7217
+8900 train 6.308746 (lr=8.9382e-06) (hash(x)=46311615)
+9300 val loss 6.7031
+9300 val perplexity 814.9078
+9300 train 6.644286 (lr=5.5641e-06) (hash(x)=44784276)
+9000 val loss 6.4235
+9000 val perplexity 616.1304
+9000 train 6.292938 (lr=8.6047e-06) (hash(x)=48535188)
+9400 val loss 6.7000
+9400 val perplexity 812.3867
+9400 train 6.797466 (lr=5.4149e-06) (hash(x)=51981169)
+9100 val loss 6.4145
+9100 val perplexity 610.6580
+9100 train 6.448308 (lr=8.3020e-06) (hash(x)=51757372)
+9500 val loss 6.6971
+9500 val perplexity 810.0333
+9500 train 6.648178 (lr=5.2884e-06) (hash(x)=47232936)
+9200 val loss 6.4128
+9200 val perplexity 609.6115
+9200 train 6.280666 (lr=8.0302e-06) (hash(x)=51131708)
+9600 val loss 6.6957
+9600 val perplexity 808.9078
+9600 train 6.730829 (lr=5.1847e-06) (hash(x)=53800450)
+9300 val loss 6.4138
+9300 val perplexity 610.1989
+9300 train 6.370407 (lr=7.7898e-06) (hash(x)=44784276)
+9700 val loss 6.6942
+9700 val perplexity 807.7153
+9700 train 6.812382 (lr=5.1040e-06) (hash(x)=55768123)
+9400 val loss 6.4115
+9400 val perplexity 608.8001
+9400 train 6.524776 (lr=7.5809e-06) (hash(x)=51981169)
+9800 val loss 6.6956
+9800 val perplexity 808.8314
+9800 train 6.635161 (lr=5.0462e-06) (hash(x)=47745177)
+9500 val loss 6.4100
+9500 val perplexity 607.8681
+9500 train 6.373538 (lr=7.4038e-06) (hash(x)=47232936)
+9900 val loss 6.6930
+9900 val perplexity 806.7683
+9900 train 6.937419 (lr=5.0116e-06) (hash(x)=56592246)
+9600 val loss 6.4066
+9600 val perplexity 605.8401
+9600 train 6.441170 (lr=7.2586e-06) (hash(x)=53800450)
+9999 val loss 6.6932
+9999 val perplexity 806.9299
+9700 val loss 6.4071
+9700 val perplexity 606.1292
+9700 train 6.529077 (lr=7.1456e-06) (hash(x)=55768123)
+9800 val loss 6.4078
+9800 val perplexity 606.5295
+9800 train 6.343826 (lr=7.0647e-06) (hash(x)=47745177)
+9900 val loss 6.4075
+9900 val perplexity 606.3977
+9900 train 6.676889 (lr=7.0162e-06) (hash(x)=56592246)

attention_kindselective_n_heads2_seed1340/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c6723edebcb92ca2e72054a32701bc25d1fc927f93c9e3ffc3e865f8b7415e5
+oid sha256:8c94c460d7c2e918bd19dd6c7d0a40fbf731522e2be5d922fe6d210895f4b6cd
 size 38587970

attention_kindselective_n_heads2_seed1340/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03f9fd53f979fe8cf9bf929f3f9ccb04af6199807a56cf30e6401e200e0c1bf4
+oid sha256:99967d2565438966d9e46b6de33f253f4b8e326c9f4cfe096170054f1ecb7401
 size 38587970

attention_kindselective_n_heads2_seed1340/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc16680a7b7cd0eb8bab693bc1270640af820af29ff05c8484576c829dca657b
+oid sha256:3fc8260bce2fd1ee0a25bb0aefdf1b927435a4e83a5f3001b7cd9a010becef3e
 size 38587970

attention_kindselective_n_heads2_seed1340/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bee304b6d21be05c3175629877a7130ec354483d6f47805582d72d4e56b8b5f5
+oid sha256:ffa02ec9ceef9ae047bb6c9f2df20bb20a78f2a2d95be921f8573aad83aa4f34
 size 38587970

attention_kindselective_n_heads2_seed1340/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eaf36019f143c8ca2d052bd8495bde7f6853829697fcff4a1af930a2ed55ee8a
+oid sha256:e00018265047cc47ecf529e2cb77c9da38ab5ffdb63eb4dd62a77f724a4e61f1
 size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a683404880ddecba30f3c5ded4e51f96de7555ad774fd34351ff9d5bc661c4b
+oid sha256:f1e45fc5874ff2b4f6e33851010911bf7688145d2c6254585473d064c73335ec
 size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:51bfb9ae26523dc1ea5f3a0fad4a6e93f8c6ce10ef08bbcd9d25e43431a4005b
+oid sha256:3fa9a215741606396ff07a59b06d77363e8598933439c4e40b9445a216a12bff
 size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8966865e8fa15c38fb89509c163897db98961bbf4dcf7cb885b3b3ff5e1a6ddd
+oid sha256:0b894c3c084b1f0ca1c18954ebede742cd1b3528fd4d2b7daeef15cd86b6a87b
 size 70895430