Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 7e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "7e-5_10240_4_1339", "n_embd": 256}

attention_kindselective_n_heads4_seed1339/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
+oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
 size 964

attention_kindselective_n_heads4_seed1339/dataloader_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+oid sha256:7f03ed2ebf741f15e13c79e6cc1e9a19b308450d81cc3b4d8d0338c63d77ca59
 size 964

attention_kindselective_n_heads4_seed1339/dataloader_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+oid sha256:82590037fb2eecbec961f7967a8dd1b8d85515d31a252f66b92b8139858a8b7c
 size 964

attention_kindselective_n_heads4_seed1339/dataloader_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+oid sha256:c544303717d12355a69b8ffb1eb109434e4fdccfd5a61279b6e8ba2e870d6700
 size 964

attention_kindselective_n_heads4_seed1339/log2.txt

@@ -1,303 +1,606 @@
 max_steps: 10000
-0 val loss 11.2810
-0 val perplexity 79298.4922
-0 train 11.295413 (lr=1.5000e-07) (hash(x)=150724848)
-100 val loss 9.9459
-100 val perplexity 20867.0703
-100 train 9.994777 (lr=1.5150e-05) (hash(x)=149217061)
-200 val loss 8.9447
-200 val perplexity 7666.8354
-200 train 8.962429 (lr=3.0000e-05) (hash(x)=149910534)
-300 val loss 8.1110
-300 val perplexity 3330.9646
-300 train 8.046171 (lr=2.9993e-05) (hash(x)=142185643)
-400 val loss 7.8325
-400 val perplexity 2521.1560
-400 train 7.894480 (lr=2.9972e-05) (hash(x)=148123706)
-500 val loss 7.6880
-500 val perplexity 2182.0981
-500 train 7.583099 (lr=2.9938e-05) (hash(x)=149952383)
-600 val loss 7.6161
-600 val perplexity 2030.5824
-600 train 7.453497 (lr=2.9889e-05) (hash(x)=146678221)
-700 val loss 7.5561
-700 val perplexity 1912.4199
-700 train 7.585546 (lr=2.9827e-05) (hash(x)=156180736)
-800 val loss 7.5147
-800 val perplexity 1834.8335
-800 train 7.512384 (lr=2.9751e-05) (hash(x)=151700982)
-900 val loss 7.4737
-900 val perplexity 1761.1957
-900 train 7.274730 (lr=2.9662e-05) (hash(x)=147288467)
-1000 val loss 7.4296
-1000 val perplexity 1685.1378
-1000 train 7.474689 (lr=2.9558e-05) (hash(x)=156182087)
-1100 val loss 7.3952
-1100 val perplexity 1628.2239
-1100 train 7.274476 (lr=2.9442e-05) (hash(x)=147861550)
-1200 val loss 7.3647
-1200 val perplexity 1579.2979
-1200 train 7.235832 (lr=2.9312e-05) (hash(x)=149318660)
-1300 val loss 7.3459
-1300 val perplexity 1549.8248
-1300 train 7.192342 (lr=2.9169e-05) (hash(x)=151631103)
-1400 val loss 7.3127
-1400 val perplexity 1499.1851
-1400 train 7.358240 (lr=2.9013e-05) (hash(x)=150482428)
-1500 val loss 7.2705
-1500 val perplexity 1437.3295
-1500 train 7.126445 (lr=2.8845e-05) (hash(x)=144458617)
-1600 val loss 7.2328
-1600 val perplexity 1384.0747
-1600 train 7.107043 (lr=2.8663e-05) (hash(x)=143268605)
-1700 val loss 7.1955
-1700 val perplexity 1333.4301
-1700 train 7.124366 (lr=2.8469e-05) (hash(x)=150817011)
-1800 val loss 7.1705
-1800 val perplexity 1300.5093
-1800 train 7.149174 (lr=2.8263e-05) (hash(x)=152322423)
-1900 val loss 7.1346
-1900 val perplexity 1254.6371
-1900 train 6.945065 (lr=2.8044e-05) (hash(x)=143574126)
-2000 val loss 7.1080
-2000 val perplexity 1221.6805
-2000 train 6.863167 (lr=2.7814e-05) (hash(x)=147904298)
-2100 val loss 7.0765
-2100 val perplexity 1183.7659
-2100 train 6.827906 (lr=2.7572e-05) (hash(x)=144719400)
-2200 val loss 7.0363
-2200 val perplexity 1137.2002
-2200 train 7.275653 (lr=2.7319e-05) (hash(x)=154343147)
-2300 val loss 6.9925
-2300 val perplexity 1088.4722
-2300 train 7.036984 (lr=2.7055e-05) (hash(x)=154372635)
-2400 val loss 6.9563
-2400 val perplexity 1049.7842
-2400 train 6.786735 (lr=2.6780e-05) (hash(x)=141843115)
-2500 val loss 6.9160
-2500 val perplexity 1008.2984
-2500 train 6.819649 (lr=2.6494e-05) (hash(x)=149417679)
-2600 val loss 6.8807
-2600 val perplexity 973.2830
-2600 train 6.748271 (lr=2.6198e-05) (hash(x)=145279030)
-2700 val loss 6.8496
-2700 val perplexity 943.5231
-2700 train 6.754533 (lr=2.5892e-05) (hash(x)=146363911)
-2800 val loss 6.8212
-2800 val perplexity 917.1302
-2800 train 6.784424 (lr=2.5576e-05) (hash(x)=152507639)
-2900 val loss 6.7822
-2900 val perplexity 882.0267
-2900 train 6.574083 (lr=2.5251e-05) (hash(x)=139296191)
-3000 val loss 6.7505
-3000 val perplexity 854.4930
-3000 train 6.765373 (lr=2.4917e-05) (hash(x)=148473774)
-3100 val loss 6.6986
-3100 val perplexity 811.3074
-3100 train 6.792789 (lr=2.4574e-05) (hash(x)=154130527)
-3200 val loss 6.6650
-3200 val perplexity 784.5016
-3200 train 6.606997 (lr=2.4224e-05) (hash(x)=151117002)
-3300 val loss 6.6496
-3300 val perplexity 772.4790
-3300 train 6.792391 (lr=2.3865e-05) (hash(x)=151035383)
-3400 val loss 6.6047
-3400 val perplexity 738.5643
-3400 train 6.713327 (lr=2.3498e-05) (hash(x)=150614749)
-3500 val loss 6.5760
-3500 val perplexity 717.6874
-3500 train 6.599133 (lr=2.3125e-05) (hash(x)=151431391)
-3600 val loss 6.5515
-3600 val perplexity 700.3228
-3600 train 6.488029 (lr=2.2744e-05) (hash(x)=151140797)
-3700 val loss 6.5274
-3700 val perplexity 683.6309
-3700 train 6.736988 (lr=2.2357e-05) (hash(x)=154608340)
-3800 val loss 6.5015
-3800 val perplexity 666.1614
-3800 train 6.522555 (lr=2.1965e-05) (hash(x)=169641309)
-3900 val loss 6.4751
-3900 val perplexity 648.7977
-3900 train 6.433761 (lr=2.1566e-05) (hash(x)=150592479)
-4000 val loss 6.4565
-4000 val perplexity 636.8508
-4000 train 6.442842 (lr=2.1162e-05) (hash(x)=155495317)
-4100 val loss 6.4460
-4100 val perplexity 630.1829
-4100 train 6.642754 (lr=2.0754e-05) (hash(x)=170323970)
-4200 val loss 6.4345
-4200 val perplexity 622.9916
-4200 train 6.259358 (lr=2.0341e-05) (hash(x)=140854360)
-4300 val loss 6.4055
-4300 val perplexity 605.1801
-4300 train 6.408278 (lr=1.9924e-05) (hash(x)=148926644)
-4400 val loss 6.3889
-4400 val perplexity 595.1989
-4400 train 6.426485 (lr=1.9504e-05) (hash(x)=149997871)
-4500 val loss 6.3722
-4500 val perplexity 585.3224
-4500 train 6.315977 (lr=1.9081e-05) (hash(x)=149219289)
-4600 val loss 6.3600
-4600 val perplexity 578.2376
-4600 train 6.160038 (lr=1.8655e-05) (hash(x)=139836197)
-4700 val loss 6.3449
-4700 val perplexity 569.5987
-4700 train 6.298043 (lr=1.8226e-05) (hash(x)=150722883)
-4800 val loss 6.3342
-4800 val perplexity 563.5283
-4800 train 6.542304 (lr=1.7796e-05) (hash(x)=166864648)
-4900 val loss 6.3154
-4900 val perplexity 553.0249
-4900 train 6.413557 (lr=1.7365e-05) (hash(x)=157746743)
-5000 val loss 6.3011
-5000 val perplexity 545.1600
-5000 train 6.251759 (lr=1.6933e-05) (hash(x)=153032257)
-5100 val loss 6.2911
-5100 val perplexity 539.7576
-5100 train 6.377411 (lr=1.6500e-05) (hash(x)=151862371)
-5200 val loss 6.2793
-5200 val perplexity 533.4402
-5200 train 6.614571 (lr=1.6067e-05) (hash(x)=168862995)
-5300 val loss 6.2744
-5300 val perplexity 530.7993
-5300 train 6.238945 (lr=1.5635e-05) (hash(x)=148243567)
-5400 val loss 6.2627
-5400 val perplexity 524.6243
-5400 train 6.191408 (lr=1.5204e-05) (hash(x)=145381742)
-5500 val loss 6.2534
-5500 val perplexity 519.7742
-5500 train 6.404634 (lr=1.4774e-05) (hash(x)=155602174)
-5600 val loss 6.2356
-5600 val perplexity 510.5886
-5600 train 6.321620 (lr=1.4345e-05) (hash(x)=160637672)
-5700 val loss 6.2257
-5700 val perplexity 505.5756
-5700 train 6.153231 (lr=1.3919e-05) (hash(x)=152101131)
-5800 val loss 6.2163
-5800 val perplexity 500.8534
-5800 train 6.172603 (lr=1.3496e-05) (hash(x)=152327089)
-5900 val loss 6.2131
-5900 val perplexity 499.2327
-5900 train 6.169562 (lr=1.3076e-05) (hash(x)=151094543)
-6000 val loss 6.2057
-6000 val perplexity 495.5671
-6000 train 6.303425 (lr=1.2659e-05) (hash(x)=160233315)
-6100 val loss 6.1932
-6100 val perplexity 489.3962
-6100 train 6.290517 (lr=1.2246e-05) (hash(x)=158295337)
-6200 val loss 6.1835
-6200 val perplexity 484.6647
-6200 train 6.134736 (lr=1.1838e-05) (hash(x)=151269817)
-6300 val loss 6.1760
-6300 val perplexity 481.0584
-6300 train 6.316322 (lr=1.1434e-05) (hash(x)=156912756)
-6400 val loss 6.1698
-6400 val perplexity 478.1116
-6400 train 6.093508 (lr=1.1035e-05) (hash(x)=125779637)
-6500 val loss 6.1661
-6500 val perplexity 476.3085
-6500 train 6.062563 (lr=1.0643e-05) (hash(x)=148157661)
-6600 val loss 6.1610
-6600 val perplexity 473.9046
-6600 train 6.199767 (lr=1.0256e-05) (hash(x)=152820049)
-6700 val loss 6.1536
-6700 val perplexity 470.4144
-6700 train 6.092970 (lr=9.8753e-06) (hash(x)=155818717)
-6800 val loss 6.1432
-6800 val perplexity 465.5320
-6800 train 6.174351 (lr=9.5017e-06) (hash(x)=158833881)
-6900 val loss 6.1412
-6900 val perplexity 464.6119
-6900 train 6.079893 (lr=9.1353e-06) (hash(x)=164984909)
-7000 val loss 6.1353
-7000 val perplexity 461.8669
-7000 train 6.026324 (lr=8.7764e-06) (hash(x)=151172493)
-7100 val loss 6.1325
-7100 val perplexity 460.5665
-7100 train 6.166331 (lr=8.4255e-06) (hash(x)=154023633)
-7200 val loss 6.1241
-7200 val perplexity 456.7562
-7200 train 5.977105 (lr=8.0829e-06) (hash(x)=139351250)
-7300 val loss 6.1208
-7300 val perplexity 455.2353
-7300 train 6.070672 (lr=7.7489e-06) (hash(x)=158719458)
-7400 val loss 6.1134
-7400 val perplexity 451.8548
-7400 train 6.197914 (lr=7.4239e-06) (hash(x)=162250810)
-7500 val loss 6.1050
-7500 val perplexity 448.1147
-7500 train 5.971425 (lr=7.1083e-06) (hash(x)=142827557)
-7600 val loss 6.1044
-7600 val perplexity 447.8327
-7600 train 6.001914 (lr=6.8023e-06) (hash(x)=147830801)
-7700 val loss 6.1000
-7700 val perplexity 445.8545
-7700 train 6.213936 (lr=6.5062e-06) (hash(x)=165233314)
-7800 val loss 6.0973
-7800 val perplexity 444.6605
-7800 train 6.092710 (lr=6.2205e-06) (hash(x)=160654717)
-7900 val loss 6.0936
-7900 val perplexity 443.0253
-7900 train 5.945004 (lr=5.9453e-06) (hash(x)=140712586)
-8000 val loss 6.0883
-8000 val perplexity 440.6558
-8000 train 6.236843 (lr=5.6809e-06) (hash(x)=150309061)
-8100 val loss 6.0839
-8100 val perplexity 438.7413
-8100 train 6.095557 (lr=5.4277e-06) (hash(x)=158805466)
-8200 val loss 6.0804
-8200 val perplexity 437.2005
-8200 train 5.950284 (lr=5.1858e-06) (hash(x)=142814078)
-8300 val loss 6.0784
-8300 val perplexity 436.3120
-8300 train 6.033829 (lr=4.9556e-06) (hash(x)=148340417)
-8400 val loss 6.0766
-8400 val perplexity 435.5259
-8400 train 6.016440 (lr=4.7372e-06) (hash(x)=157132639)
-8500 val loss 6.0766
-8500 val perplexity 435.5307
-8500 train 6.019042 (lr=4.5309e-06) (hash(x)=152994240)
-8600 val loss 6.0692
-8600 val perplexity 432.3297
-8600 train 6.012918 (lr=4.3369e-06) (hash(x)=152210154)
-8700 val loss 6.0673
-8700 val perplexity 431.4998
-8700 train 6.135266 (lr=4.1554e-06) (hash(x)=149631247)
-8800 val loss 6.0635
-8800 val perplexity 429.8681
-8800 train 5.918789 (lr=3.9866e-06) (hash(x)=145617784)
-8900 val loss 6.0627
-8900 val perplexity 429.5137
-8900 train 6.162240 (lr=3.8307e-06) (hash(x)=155984970)
-9000 val loss 6.0619
-9000 val perplexity 429.1812
-9000 train 5.790365 (lr=3.6877e-06) (hash(x)=143433013)
-9100 val loss 6.0604
-9100 val perplexity 428.5354
-9100 train 5.963179 (lr=3.5580e-06) (hash(x)=152507533)
-9200 val loss 6.0564
-9200 val perplexity 426.8368
-9200 train 6.185226 (lr=3.4415e-06) (hash(x)=162400650)
-9300 val loss 6.0525
-9300 val perplexity 425.1668
-9300 train 6.100528 (lr=3.3385e-06) (hash(x)=148419068)
-9400 val loss 6.0515
-9400 val perplexity 424.7288
-9400 train 6.007780 (lr=3.2490e-06) (hash(x)=145025169)
-9500 val loss 6.0493
-9500 val perplexity 423.8244
-9500 train 6.020970 (lr=3.1730e-06) (hash(x)=145104235)
-9600 val loss 6.0494
-9600 val perplexity 423.8671
-9600 train 5.979624 (lr=3.1108e-06) (hash(x)=150499797)
-9700 val loss 6.0480
-9700 val perplexity 423.2599
-9700 train 5.853704 (lr=3.0624e-06) (hash(x)=139690022)
-9800 val loss 6.0438
-9800 val perplexity 421.4906
-9800 train 6.032509 (lr=3.0277e-06) (hash(x)=145645279)
-9900 val loss 6.0413
-9900 val perplexity 420.4520
-9900 train 5.953890 (lr=3.0069e-06) (hash(x)=150103432)
-9999 val loss 6.0409
-9999 val perplexity 420.2706
+max_steps: 10000
+0 val loss 11.2766
+0 val perplexity 78951.9062
+0 val loss 11.2766
+0 val perplexity 78951.9062
+0 train 11.297240 (lr=2.5000e-07) (hash(x)=55241167)
+0 train 11.297240 (lr=3.5000e-07) (hash(x)=55241167)
+100 val loss 9.8269
+100 val perplexity 18526.1348
+100 train 9.819736 (lr=2.5250e-05) (hash(x)=46387919)
+100 val loss 9.6928
+100 val perplexity 16200.6123
+100 train 9.682398 (lr=3.5350e-05) (hash(x)=46387919)
+200 val loss 8.7461
+200 val perplexity 6286.3667
+200 train 8.832648 (lr=5.0000e-05) (hash(x)=51276369)
+200 val loss 8.1592
+200 val perplexity 3495.5115
+200 train 8.252799 (lr=7.0000e-05) (hash(x)=51276369)
+300 val loss 8.0583
+300 val perplexity 3160.0583
+300 train 7.995393 (lr=4.9988e-05) (hash(x)=48572728)
+300 val loss 7.6698
+300 val perplexity 2142.7515
+300 train 7.577228 (lr=6.9984e-05) (hash(x)=48572728)
+400 val loss 7.7693
+400 val perplexity 2366.7744
+400 train 7.522111 (lr=4.9954e-05) (hash(x)=44943400)
+400 val loss 7.6067
+400 val perplexity 2011.6699
+400 train 7.332860 (lr=6.9935e-05) (hash(x)=44943400)
+500 val loss 7.6528
+500 val perplexity 2106.5469
+500 train 7.579549 (lr=4.9896e-05) (hash(x)=48434590)
+500 val loss 7.5459
+500 val perplexity 1892.9581
+500 train 7.433667 (lr=6.9854e-05) (hash(x)=48434590)
+600 val loss 7.6072
+600 val perplexity 2012.7042
+600 train 7.477727 (lr=4.9815e-05) (hash(x)=47442266)
+600 val loss 7.4994
+600 val perplexity 1806.9426
+600 train 7.327074 (lr=6.9741e-05) (hash(x)=47442266)
+700 val loss 7.5548
+700 val perplexity 1909.8828
+700 train 7.611083 (lr=4.9712e-05) (hash(x)=56495712)
+700 val loss 7.4660
+700 val perplexity 1747.5291
+700 train 7.532721 (lr=6.9596e-05) (hash(x)=56495712)
+800 val loss 7.5249
+800 val perplexity 1853.5676
+800 train 7.523900 (lr=4.9585e-05) (hash(x)=50093738)
+800 val loss 7.4516
+800 val perplexity 1722.6797
+800 train 7.450620 (lr=6.9419e-05) (hash(x)=50093738)
+900 val loss 7.5121
+900 val perplexity 1830.0836
+900 train 7.484754 (lr=4.9436e-05) (hash(x)=49436061)
+900 val loss 7.4213
+900 val perplexity 1671.1631
+900 train 7.381460 (lr=6.9210e-05) (hash(x)=49436061)
+1000 val loss 7.5026
+1000 val perplexity 1812.8086
+1000 train 7.425063 (lr=4.9264e-05) (hash(x)=48818282)
+1000 val loss 7.3975
+1000 val perplexity 1631.8879
+1000 train 7.324411 (lr=6.8970e-05) (hash(x)=48818282)
+1100 val loss 7.4779
+1100 val perplexity 1768.5020
+1100 train 7.553754 (lr=4.9070e-05) (hash(x)=55387425)
+1100 val loss 7.3392
+1100 val perplexity 1539.5417
+1100 train 7.412958 (lr=6.8698e-05) (hash(x)=55387425)
+1200 val loss 7.4630
+1200 val perplexity 1742.3455
+1200 train 7.370140 (lr=4.8854e-05) (hash(x)=48328378)
+1200 val loss 7.3034
+1200 val perplexity 1485.2927
+1200 train 7.183446 (lr=6.8395e-05) (hash(x)=48328378)
+1300 val loss 7.4402
+1300 val perplexity 1703.0768
+1300 train 7.354329 (lr=4.8616e-05) (hash(x)=48956717)
+1300 val loss 7.2614
+1300 val perplexity 1424.2318
+1300 train 7.164160 (lr=6.8062e-05) (hash(x)=48956717)
+1400 val loss 7.4390
+1400 val perplexity 1701.0858
+1400 train 7.325079 (lr=4.8356e-05) (hash(x)=49782571)
+1400 val loss 7.2354
+1400 val perplexity 1387.6724
+1400 train 7.128095 (lr=6.7698e-05) (hash(x)=49782571)
+1500 val loss 7.4168
+1500 val perplexity 1663.6846
+1500 train 7.217878 (lr=4.8074e-05) (hash(x)=51081994)
+1500 val loss 7.2017
+1500 val perplexity 1341.7664
+1500 train 6.971539 (lr=6.7304e-05) (hash(x)=51081994)
+1600 val loss 7.4079
+1600 val perplexity 1649.0146
+1600 train 7.481792 (lr=4.7772e-05) (hash(x)=48791204)
+1600 val loss 7.1565
+1600 val perplexity 1282.4695
+1600 train 7.212631 (lr=6.6881e-05) (hash(x)=48791204)
+1700 val loss 7.3899
+1700 val perplexity 1619.6024
+1700 train 7.112850 (lr=4.7448e-05) (hash(x)=46425011)
+1700 val loss 7.1197
+1700 val perplexity 1236.1090
+1700 train 6.791309 (lr=6.6428e-05) (hash(x)=46425011)
+1800 val loss 7.0838
+1800 val perplexity 1192.4474
+1800 train 6.699710 (lr=6.5947e-05) (hash(x)=44919141)
+1800 val loss 7.3832
+1800 val perplexity 1608.7535
+1800 train 7.020766 (lr=4.7105e-05) (hash(x)=44919141)
+1900 val loss 7.0346
+1900 val perplexity 1135.2908
+1900 train 7.144853 (lr=6.5437e-05) (hash(x)=48299675)
+1900 val loss 7.3649
+1900 val perplexity 1579.5735
+1900 train 7.448961 (lr=4.6741e-05) (hash(x)=48299675)
+2000 val loss 6.9733
+2000 val perplexity 1067.7863
+2000 train 7.800293 (lr=6.4900e-05) (hash(x)=61804797)
+2000 val loss 7.3407
+2000 val perplexity 1541.7869
+2000 train 8.138843 (lr=4.6357e-05) (hash(x)=61804797)
+2100 val loss 6.9329
+2100 val perplexity 1025.4562
+2100 train 7.034851 (lr=6.4335e-05) (hash(x)=53638030)
+2100 val loss 7.3219
+2100 val perplexity 1513.0824
+2100 train 7.405197 (lr=4.5954e-05) (hash(x)=53638030)
+2200 val loss 6.9051
+2200 val perplexity 997.3262
+2200 train 7.128452 (lr=6.3745e-05) (hash(x)=56364593)
+2200 val loss 7.3057
+2200 val perplexity 1488.7239
+2200 train 7.551863 (lr=4.5532e-05) (hash(x)=56364593)
+2300 val loss 6.8697
+2300 val perplexity 962.6473
+2300 train 6.986196 (lr=6.3128e-05) (hash(x)=52945580)
+2300 val loss 7.3142
+2300 val perplexity 1501.4471
+2300 train 7.455043 (lr=4.5091e-05) (hash(x)=52945580)
+2400 val loss 6.8283
+2400 val perplexity 923.5925
+2400 train 6.899657 (lr=6.2486e-05) (hash(x)=54595211)
+2400 val loss 7.2843
+2400 val perplexity 1457.2745
+2400 train 7.338539 (lr=4.4633e-05) (hash(x)=54595211)
+2500 val loss 6.8054
+2500 val perplexity 902.7001
+2500 train 6.809216 (lr=6.1819e-05) (hash(x)=52323363)
+2500 val loss 7.2856
+2500 val perplexity 1459.1227
+2500 train 7.312573 (lr=4.4156e-05) (hash(x)=52323363)
+2600 val loss 6.7748
+2600 val perplexity 875.4703
+2600 train 6.542181 (lr=6.1128e-05) (hash(x)=42652707)
+2600 val loss 7.2808
+2600 val perplexity 1452.1907
+2600 train 7.076187 (lr=4.3663e-05) (hash(x)=42652707)
+2700 val loss 6.7567
+2700 val perplexity 859.7908
+2700 train 6.453963 (lr=6.0414e-05) (hash(x)=45687855)
+2700 val loss 7.2677
+2700 val perplexity 1433.2026
+2700 train 6.959708 (lr=4.3153e-05) (hash(x)=45687855)
+2800 val loss 6.7146
+2800 val perplexity 824.3206
+2800 train 6.724705 (lr=5.9677e-05) (hash(x)=49962830)
+2800 val loss 7.2714
+2800 val perplexity 1438.6337
+2800 train 7.254436 (lr=4.2627e-05) (hash(x)=49962830)
+2900 val loss 6.6880
+2900 val perplexity 802.7047
+2900 train 6.716569 (lr=5.8919e-05) (hash(x)=52958451)
+2900 val loss 7.2474
+2900 val perplexity 1404.4911
+2900 train 7.309949 (lr=4.2085e-05) (hash(x)=52958451)
+3000 val loss 6.6736
+3000 val perplexity 791.2670
+3000 train 6.612247 (lr=5.8140e-05) (hash(x)=48896990)
+3000 val loss 7.2392
+3000 val perplexity 1393.0072
+3000 train 7.165527 (lr=4.1529e-05) (hash(x)=48896990)
+3100 val loss 6.6638
+3100 val perplexity 783.4915
+3100 train 6.662642 (lr=5.7340e-05) (hash(x)=47064930)
+3100 val loss 7.2448
+3100 val perplexity 1400.7638
+3100 train 7.213631 (lr=4.0957e-05) (hash(x)=47064930)
+3200 val loss 6.6385
+3200 val perplexity 763.9189
+3200 train 6.342895 (lr=5.6522e-05) (hash(x)=45145921)
+3200 val loss 7.2305
+3200 val perplexity 1380.9763
+3200 train 6.985227 (lr=4.0373e-05) (hash(x)=45145921)
+3300 val loss 6.6241
+3300 val perplexity 753.0123
+3300 train 6.447469 (lr=5.5684e-05) (hash(x)=48085018)
+3300 val loss 7.2399
+3300 val perplexity 1393.9142
+3300 train 7.104490 (lr=3.9775e-05) (hash(x)=48085018)
+3400 val loss 6.6132
+3400 val perplexity 744.8303
+3400 train 6.614527 (lr=5.4829e-05) (hash(x)=49596053)
+3400 val loss 7.2185
+3400 val perplexity 1364.4150
+3400 train 7.253217 (lr=3.9164e-05) (hash(x)=49596053)
+3500 val loss 6.6042
+3500 val perplexity 738.2034
+3500 train 6.221120 (lr=5.3958e-05) (hash(x)=41135634)
+3500 val loss 7.2577
+3500 val perplexity 1418.9370
+3500 train 6.842973 (lr=3.8541e-05) (hash(x)=41135634)
+3600 val loss 6.5929
+3600 val perplexity 729.9281
+3600 train 6.570555 (lr=5.3070e-05) (hash(x)=54286330)
+3600 val loss 7.2149
+3600 val perplexity 1359.6013
+3600 train 7.226971 (lr=3.7907e-05) (hash(x)=54286330)
+3700 val loss 6.5851
+3700 val perplexity 724.2557
+3700 train 6.428621 (lr=5.2167e-05) (hash(x)=57061993)
+3700 val loss 7.2117
+3700 val perplexity 1355.2239
+3700 train 7.127274 (lr=3.7262e-05) (hash(x)=57061993)
+3800 val loss 6.5726
+3800 val perplexity 715.2515
+3800 train 6.137836 (lr=5.1251e-05) (hash(x)=46544884)
+3800 val loss 7.2026
+3800 val perplexity 1342.9222
+3800 train 6.810471 (lr=3.6608e-05) (hash(x)=46544884)
+3900 val loss 6.5798
+3900 val perplexity 720.3782
+3900 train 6.413338 (lr=5.0321e-05) (hash(x)=48937435)
+3900 val loss 7.2019
+3900 val perplexity 1341.9429
+3900 train 7.051498 (lr=3.5944e-05) (hash(x)=48937435)
+4000 val loss 6.5694
+4000 val perplexity 712.9241
+4000 train 6.600487 (lr=4.9379e-05) (hash(x)=54466186)
+4000 val loss 7.1906
+4000 val perplexity 1326.9261
+4000 train 7.239226 (lr=3.5271e-05) (hash(x)=54466186)
+4100 val loss 6.5491
+4100 val perplexity 698.6298
+4100 train 6.357440 (lr=4.8426e-05) (hash(x)=51079995)
+4100 val loss 7.1948
+4100 val perplexity 1332.4652
+4100 train 7.051948 (lr=3.4590e-05) (hash(x)=51079995)
+4200 val loss 6.5377
+4200 val perplexity 690.7164
+4200 train 6.665975 (lr=4.7463e-05) (hash(x)=56495581)
+4200 val loss 7.1759
+4200 val perplexity 1307.5887
+4200 train 7.325971 (lr=3.3902e-05) (hash(x)=56495581)
+4300 val loss 6.5119
+4300 val perplexity 673.0989
+4300 train 6.342508 (lr=4.6490e-05) (hash(x)=45095478)
+4300 val loss 7.1620
+4300 val perplexity 1289.4537
+4300 train 7.022624 (lr=3.3207e-05) (hash(x)=45095478)
+4400 val loss 6.4969
+4400 val perplexity 663.0798
+4400 train 6.514081 (lr=4.5509e-05) (hash(x)=49954028)
+4400 val loss 7.1656
+4400 val perplexity 1294.1129
+4400 train 7.113908 (lr=3.2507e-05) (hash(x)=49954028)
+4500 val loss 6.4922
+4500 val perplexity 659.9440
+4500 train 6.422389 (lr=4.4521e-05) (hash(x)=48975821)
+4500 val loss 7.1499
+4500 val perplexity 1274.0404
+4500 train 7.073869 (lr=3.1801e-05) (hash(x)=48975821)
+4600 val loss 6.4620
+4600 val perplexity 640.3523
+4600 train 6.557837 (lr=4.3527e-05) (hash(x)=49293028)
+4600 val loss 7.1385
+4600 val perplexity 1259.4857
+4600 train 7.196972 (lr=3.1091e-05) (hash(x)=49293028)
+4700 val loss 6.4616
+4700 val perplexity 640.1035
+4700 train 6.356697 (lr=4.2528e-05) (hash(x)=48077753)
+4700 val loss 7.1326
+4700 val perplexity 1252.1656
+4700 train 7.069026 (lr=3.0377e-05) (hash(x)=48077753)
+4800 val loss 6.4458
+4800 val perplexity 630.0588
+4800 train 6.352581 (lr=4.1525e-05) (hash(x)=45037879)
+4800 val loss 7.1220
+4800 val perplexity 1238.9526
+4800 train 6.939941 (lr=2.9661e-05) (hash(x)=45037879)
+4900 val loss 6.4401
+4900 val perplexity 626.4606
+4900 train 6.359092 (lr=4.0518e-05) (hash(x)=50320154)
+4900 val loss 7.1202
+4900 val perplexity 1236.6737
+4900 train 7.031347 (lr=2.8942e-05) (hash(x)=50320154)
+5000 val loss 6.4270
+5000 val perplexity 618.3121
+5000 train 6.702835 (lr=3.9510e-05) (hash(x)=55024523)
+5000 val loss 7.1210
+5000 val perplexity 1237.6714
+5000 train 7.321143 (lr=2.8221e-05) (hash(x)=55024523)
+5100 val loss 6.4187
+5100 val perplexity 613.1933
+5100 train 6.337059 (lr=3.8500e-05) (hash(x)=47570607)
+5100 val loss 7.0966
+5100 val perplexity 1207.8408
+5100 train 7.021636 (lr=2.7500e-05) (hash(x)=47570607)
+5200 val loss 6.4092
+5200 val perplexity 607.4013
+5200 train 6.531659 (lr=3.7490e-05) (hash(x)=54311349)
+5200 val loss 7.0982
+5200 val perplexity 1209.7371
+5200 train 7.204744 (lr=2.6779e-05) (hash(x)=54311349)
+5300 val loss 6.4037
+5300 val perplexity 604.0920
+5300 train 6.300325 (lr=3.6482e-05) (hash(x)=47178524)
+5300 val loss 7.0937
+5300 val perplexity 1204.3757
+5300 train 7.004245 (lr=2.6058e-05) (hash(x)=47178524)
+5400 val loss 6.4010
+5400 val perplexity 602.4624
+5400 train 6.421272 (lr=3.5475e-05) (hash(x)=49425088)
+5400 val loss 7.0886
+5400 val perplexity 1198.2756
+5400 train 7.084049 (lr=2.5339e-05) (hash(x)=49425088)
+5500 val loss 6.3854
+5500 val perplexity 593.1009
+5500 train 6.211343 (lr=3.4472e-05) (hash(x)=46383189)
+5500 val loss 7.0866
+5500 val perplexity 1195.8286
+5500 train 6.919466 (lr=2.4623e-05) (hash(x)=46383189)
+5600 val loss 6.3941
+5600 val perplexity 598.3093
+5600 train 6.562718 (lr=3.3473e-05) (hash(x)=54084990)
+5600 val loss 7.0990
+5600 val perplexity 1210.7424
+5600 train 7.309231 (lr=2.3909e-05) (hash(x)=54084990)
+5700 val loss 6.3745
+5700 val perplexity 586.6678
+5700 train 6.056097 (lr=3.2479e-05) (hash(x)=47384182)
+5700 val loss 7.0692
+5700 val perplexity 1175.1588
+5700 train 6.801169 (lr=2.3199e-05) (hash(x)=47384182)
+5800 val loss 6.3737
+5800 val perplexity 586.2322
+5800 train 6.243022 (lr=3.1491e-05) (hash(x)=51683744)
+5800 val loss 7.0651
+5800 val perplexity 1170.3546
+5800 train 6.965415 (lr=2.2493e-05) (hash(x)=51683744)
+5900 val loss 6.3628
+5900 val perplexity 579.8835
+5900 train 6.376868 (lr=3.0510e-05) (hash(x)=52798836)
+5900 val loss 7.0753
+5900 val perplexity 1182.4492
+5900 train 7.069267 (lr=2.1793e-05) (hash(x)=52798836)
+6000 val loss 6.3711
+6000 val perplexity 584.6808
+6000 train 6.247975 (lr=2.9537e-05) (hash(x)=50945000)
+6000 val loss 7.0655
+6000 val perplexity 1170.8513
+6000 train 6.961160 (lr=2.1098e-05) (hash(x)=50945000)
+6100 val loss 6.3612
+6100 val perplexity 578.9307
+6100 train 6.045362 (lr=2.8574e-05) (hash(x)=48964427)
+6100 val loss 7.0515
+6100 val perplexity 1154.6139
+6100 train 6.805200 (lr=2.0410e-05) (hash(x)=48964427)
+6200 val loss 6.3579
+6200 val perplexity 577.0609
+6200 train 6.127436 (lr=2.7621e-05) (hash(x)=47148610)
+6200 val loss 7.0483
+6200 val perplexity 1150.8965
+6200 train 6.847069 (lr=1.9729e-05) (hash(x)=47148610)
+6300 val loss 6.3510
+6300 val perplexity 573.0493
+6300 train 6.284134 (lr=2.6679e-05) (hash(x)=50860553)
+6300 val loss 7.0491
+6300 val perplexity 1151.8496
+6300 train 6.979808 (lr=1.9056e-05) (hash(x)=50860553)
+6400 val loss 6.3487
+6400 val perplexity 571.7348
+6400 train 6.138104 (lr=2.5749e-05) (hash(x)=50553992)
+6400 val loss 7.0396
+6400 val perplexity 1140.9142
+6400 train 6.835309 (lr=1.8392e-05) (hash(x)=50553992)
+6500 val loss 6.3525
+6500 val perplexity 573.9421
+6500 train 6.159073 (lr=2.4833e-05) (hash(x)=52521332)
+6500 val loss 7.0408
+6500 val perplexity 1142.3137
+6500 train 6.891948 (lr=1.7738e-05) (hash(x)=52521332)
+6600 val loss 6.3276
+6600 val perplexity 559.8296
+6600 train 6.666481 (lr=2.3930e-05) (hash(x)=52609843)
+6600 val loss 7.0210
+6600 val perplexity 1119.9614
+6600 train 7.358459 (lr=1.7093e-05) (hash(x)=52609843)
+6700 val loss 6.3134
+6700 val perplexity 551.8937
+6700 train 6.365655 (lr=2.3042e-05) (hash(x)=53632957)
+6700 val loss 7.0149
+6700 val perplexity 1113.1040
+6700 train 7.083940 (lr=1.6459e-05) (hash(x)=53632957)
+6800 val loss 6.3112
+6800 val perplexity 550.6937
+6800 train 6.342908 (lr=2.2171e-05) (hash(x)=46927608)
+6900 val loss 6.2981
+6900 val perplexity 543.5120
+6900 train 6.319361 (lr=2.1316e-05) (hash(x)=54483057)
+6800 val loss 7.0048
+6800 val perplexity 1101.9363
+6800 train 7.046721 (lr=1.5836e-05) (hash(x)=46927608)
+7000 val loss 6.2918
+7000 val perplexity 540.1509
+7000 train 6.423561 (lr=2.0478e-05) (hash(x)=51493884)
+6900 val loss 7.0009
+6900 val perplexity 1097.6104
+6900 train 7.066904 (lr=1.5225e-05) (hash(x)=54483057)
+7100 val loss 6.2839
+7100 val perplexity 535.8787
+7100 train 6.388433 (lr=1.9660e-05) (hash(x)=53303341)
+7000 val loss 6.9954
+7000 val perplexity 1091.6002
+7000 train 7.167415 (lr=1.4627e-05) (hash(x)=51493884)
+7200 val loss 6.2875
+7200 val perplexity 537.8173
+7200 train 5.993762 (lr=1.8860e-05) (hash(x)=45272178)
+7100 val loss 6.9877
+7100 val perplexity 1083.2349
+7100 train 7.118485 (lr=1.4043e-05) (hash(x)=53303341)
+7300 val loss 6.2765
+7300 val perplexity 531.9218
+7300 train 6.216062 (lr=1.8081e-05) (hash(x)=50389872)
+7200 val loss 6.9902
+7200 val perplexity 1085.9910
+7200 train 6.663062 (lr=1.3471e-05) (hash(x)=45272178)
+7400 val loss 6.2720
+7400 val perplexity 529.5265
+7400 train 5.910453 (lr=1.7323e-05) (hash(x)=43796301)
+7300 val loss 6.9799
+7300 val perplexity 1074.8020
+7300 train 6.914957 (lr=1.2915e-05) (hash(x)=50389872)
+7500 val loss 6.2674
+7500 val perplexity 527.1163
+7500 train 6.223414 (lr=1.6586e-05) (hash(x)=47808686)
+7400 val loss 6.9701
+7400 val perplexity 1064.3251
+7400 train 6.588149 (lr=1.2373e-05) (hash(x)=43796301)
+7600 val loss 6.2662
+7600 val perplexity 526.4735
+7600 train 6.001786 (lr=1.5872e-05) (hash(x)=41936898)
+7500 val loss 6.9646
+7500 val perplexity 1058.4805
+7500 train 6.888280 (lr=1.1847e-05) (hash(x)=47808686)
+7700 val loss 6.2684
+7700 val perplexity 527.6361
+7700 train 6.526704 (lr=1.5181e-05) (hash(x)=57550318)
+7600 val loss 6.9621
+7600 val perplexity 1055.8838
+7600 train 6.685042 (lr=1.1337e-05) (hash(x)=41936898)
+7800 val loss 6.2607
+7800 val perplexity 523.5745
+7800 train 6.117768 (lr=1.4514e-05) (hash(x)=47485210)
+7700 val loss 6.9577
+7700 val perplexity 1051.2378
+7700 train 7.145366 (lr=1.0844e-05) (hash(x)=57550318)
+7900 val loss 6.2583
+7900 val perplexity 522.3495
+7900 train 6.632966 (lr=1.3872e-05) (hash(x)=53228688)
+7800 val loss 6.9528
+7800 val perplexity 1046.0869
+7800 train 6.833837 (lr=1.0367e-05) (hash(x)=47485210)
+8000 val loss 6.2552
+8000 val perplexity 520.6916
+8000 train 6.333637 (lr=1.3255e-05) (hash(x)=52018673)
+7900 val loss 6.9490
+7900 val perplexity 1042.1179
+7900 train 7.275203 (lr=9.9088e-06) (hash(x)=53228688)
+8100 val loss 6.2521
+8100 val perplexity 519.1161
+8100 train 6.090673 (lr=1.2665e-05) (hash(x)=47079349)
+8000 val loss 6.9462
+8000 val perplexity 1039.2284
+8000 train 6.982090 (lr=9.4682e-06) (hash(x)=52018673)
+8200 val loss 6.2518
+8200 val perplexity 518.9553
+8200 train 6.302469 (lr=1.2100e-05) (hash(x)=57921563)
+8100 val loss 6.9412
+8100 val perplexity 1033.9647
+8100 train 6.762253 (lr=9.0461e-06) (hash(x)=47079349)
+8300 val loss 6.2460
+8300 val perplexity 515.9507
+8300 train 5.783485 (lr=1.1563e-05) (hash(x)=45038933)
+8200 val loss 6.9394
+8200 val perplexity 1032.1396
+8200 train 6.955695 (lr=8.6430e-06) (hash(x)=57921563)
+8400 val loss 6.2587
+8400 val perplexity 522.5202
+8400 train 6.118326 (lr=1.1053e-05) (hash(x)=47763246)
+8300 val loss 6.9366
+8300 val perplexity 1029.2189
+8300 train 6.503049 (lr=8.2593e-06) (hash(x)=45038933)
+8500 val loss 6.2517
+8500 val perplexity 518.8879
+8500 train 6.352527 (lr=1.0572e-05) (hash(x)=56176595)
+8400 val loss 6.9368
+8400 val perplexity 1029.5016
+8400 train 6.823294 (lr=7.8953e-06) (hash(x)=47763246)
+8600 val loss 6.2449
+8600 val perplexity 515.3643
+8600 train 6.332339 (lr=1.0119e-05) (hash(x)=55184249)
+8500 val loss 6.9325
+8500 val perplexity 1025.0548
+8500 train 6.934601 (lr=7.5515e-06) (hash(x)=56176595)
+8700 val loss 6.2427
+8700 val perplexity 514.2684
+8700 train 6.192267 (lr=9.6960e-06) (hash(x)=46471646)
+8600 val loss 6.9311
+8600 val perplexity 1023.5968
+8600 train 6.992166 (lr=7.2282e-06) (hash(x)=55184249)
+8800 val loss 6.2414
+8800 val perplexity 513.5671
+8800 train 5.959139 (lr=9.3021e-06) (hash(x)=46233162)
+8700 val loss 6.9270
+8700 val perplexity 1019.4482
+8700 train 6.903887 (lr=6.9257e-06) (hash(x)=46471646)
+8800 val loss 6.9308
+8800 val perplexity 1023.3308
+8800 train 6.686684 (lr=6.6444e-06) (hash(x)=46233162)
+8900 val loss 6.2401
+8900 val perplexity 512.9296
+8900 train 6.172048 (lr=8.9382e-06) (hash(x)=47233684)
+8900 val loss 6.9445
+8900 val perplexity 1037.4430
+8900 train 6.853402 (lr=6.3845e-06) (hash(x)=47233684)
+9000 val loss 6.2341
+9000 val perplexity 509.8349
+9000 train 6.103183 (lr=8.6047e-06) (hash(x)=48374529)
+9000 val loss 6.9224
+9000 val perplexity 1014.7220
+9000 train 6.834804 (lr=6.1462e-06) (hash(x)=48374529)
+9100 val loss 6.2260
+9100 val perplexity 505.7538
+9100 train 6.227113 (lr=8.3020e-06) (hash(x)=48065371)
+9100 val loss 6.9177
+9100 val perplexity 1010.0148
+9100 train 6.851860 (lr=5.9300e-06) (hash(x)=48065371)
+9200 val loss 6.2213
+9200 val perplexity 503.3779
+9200 train 6.224229 (lr=8.0302e-06) (hash(x)=47408078)
+9200 val loss 6.9111
+9200 val perplexity 1003.3564
+9200 train 6.844304 (lr=5.7359e-06) (hash(x)=47408078)
+9300 val loss 6.2190
+9300 val perplexity 502.2050
+9300 train 6.201622 (lr=7.7898e-06) (hash(x)=50749781)
+9300 val loss 6.9071
+9300 val perplexity 999.3813
+9300 train 6.925474 (lr=5.5641e-06) (hash(x)=50749781)
+9400 val loss 6.2138
+9400 val perplexity 499.5949
+9400 train 6.430583 (lr=7.5809e-06) (hash(x)=48560169)
+9400 val loss 6.9031
+9400 val perplexity 995.3218
+9400 train 7.070158 (lr=5.4149e-06) (hash(x)=48560169)
+9500 val loss 6.2137
+9500 val perplexity 499.5220
+9500 train 6.352744 (lr=7.4038e-06) (hash(x)=50936392)
+9500 val loss 6.9017
+9500 val perplexity 993.9194
+9500 train 6.979595 (lr=5.2884e-06) (hash(x)=50936392)
+9600 val loss 6.2106
+9600 val perplexity 497.9799
+9600 train 6.226837 (lr=7.2586e-06) (hash(x)=50651714)
+9600 val loss 6.8968
+9600 val perplexity 989.1498
+9600 train 6.905745 (lr=5.1847e-06) (hash(x)=50651714)
+9700 val loss 6.2115
+9700 val perplexity 498.4665
+9700 train 6.296004 (lr=7.1456e-06) (hash(x)=47311384)
+9700 val loss 6.8969
+9700 val perplexity 989.1899
+9700 train 6.926633 (lr=5.1040e-06) (hash(x)=47311384)
+9800 val loss 6.2085
+9800 val perplexity 496.9543
+9800 train 6.317750 (lr=7.0647e-06) (hash(x)=50921139)
+9800 val loss 6.8953
+9800 val perplexity 987.6577
+9800 train 6.967777 (lr=5.0462e-06) (hash(x)=50921139)
+9900 val loss 6.2224
+9900 val perplexity 503.9033
+9900 train 6.376268 (lr=7.0162e-06) (hash(x)=48142455)
+9900 val loss 6.8939
+9900 val perplexity 986.2411
+9900 train 6.922864 (lr=5.0116e-06) (hash(x)=48142455)
+9999 val loss 6.2055
+9999 val perplexity 495.4679
+9999 val loss 6.8894
+9999 val perplexity 981.8267

attention_kindselective_n_heads4_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ffaffcf5c0f6efcc94da68cdc405e6e9862dd0790ef6e99afe7668125fbccdce
+oid sha256:bf077b4c7d20c894d8cf19edf6e71253f1c6da1d6da3ddf7a45dfdc81c7786f4
 size 92843394

attention_kindselective_n_heads4_seed1339/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db5b06d727a530bfc9f8d978f75c6c949cca378fb6fbf2ad60ab2c17a58c0ec9
+oid sha256:bcc6cba1e8aba35cfffd617b9865dd991d56d86cd65fc851959ab0cc5158369f
 size 92843394

attention_kindselective_n_heads4_seed1339/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:748e5f7bfc76da340284ca983de81118dd5f42a18f1f1b8f4f780ff779d13549
+oid sha256:cc8a63ddb0b4127e1cd6e71226f2e5d191a1223a81af5752764c496f77640ec9
 size 92843394

attention_kindselective_n_heads4_seed1339/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bbfb35a4b6f2bc6db9def860938d3a0628e0888b50937b40c4e0687cd2316940
+oid sha256:a017e4f0db2e8d2e97fe162e68ca9b3e6ceb4b94621a0a854783655339fe8482
 size 92843394

attention_kindselective_n_heads4_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:baff6d85d9201e86380a48a9008e3ec6f8a1d6910ac6e49f3ad62464aedac188
+oid sha256:4be917af745db25f4dbcccc3b0396860aa3f5a668f97463af1fdf11b774e3b27
 size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:249f2cee3d0c60145e854ec28f58efd8a1ab7615acee3baee7a18e30dc03a9c9
+oid sha256:f2a4ed44be047138dcaf6fa604ef8834bd84327b45720eeceeb37c7f6ec0749f
 size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:edd2d28c22022d78b3ba5c4a7fe91806761d53475756f99a6074fc3988bc001e
+oid sha256:1c2105389bd849202e620f1820ffe080b1744d6a2dbede4772ef73b165d137c6
 size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9b87a8ea80a58a13c81dc931e7a310144b15c4825ce7db1065749753691aec4
+oid sha256:7729c1303128128ddc169e9909211856111356cb9371ca7f1b11f22b05547fff
 size 179406214