Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 7e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "7e-5_10240_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
+oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
 size 964

attention_kindselective_n_heads4_seed1338/dataloader_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+oid sha256:7f03ed2ebf741f15e13c79e6cc1e9a19b308450d81cc3b4d8d0338c63d77ca59
 size 964

attention_kindselective_n_heads4_seed1338/dataloader_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+oid sha256:82590037fb2eecbec961f7967a8dd1b8d85515d31a252f66b92b8139858a8b7c
 size 964

attention_kindselective_n_heads4_seed1338/dataloader_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+oid sha256:c544303717d12355a69b8ffb1eb109434e4fdccfd5a61279b6e8ba2e870d6700
 size 964

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,303 +1,605 @@
 max_steps: 10000
-0 val loss 11.2663
-0 val perplexity 78147.0000
-0 train 11.263341 (lr=1.5000e-07) (hash(x)=150327452)
-100 val loss 9.9038
-100 val perplexity 20006.5938
-100 train 9.914461 (lr=1.5150e-05) (hash(x)=166441190)
-200 val loss 8.9665
-200 val perplexity 7836.3984
-200 train 9.079846 (lr=3.0000e-05) (hash(x)=166780046)
-300 val loss 8.0400
-300 val perplexity 3102.7432
-300 train 7.989579 (lr=2.9993e-05) (hash(x)=159835303)
-400 val loss 7.7330
-400 val perplexity 2282.3311
-400 train 7.532744 (lr=2.9972e-05) (hash(x)=155040610)
-500 val loss 7.6145
-500 val perplexity 2027.4534
-500 train 7.370183 (lr=2.9938e-05) (hash(x)=130190460)
-600 val loss 7.5275
-600 val perplexity 1858.4928
-600 train 7.548462 (lr=2.9889e-05) (hash(x)=155504036)
-700 val loss 7.4642
-700 val perplexity 1744.3755
-700 train 7.321239 (lr=2.9827e-05) (hash(x)=137347213)
-800 val loss 7.4136
-800 val perplexity 1658.3201
-800 train 7.269616 (lr=2.9751e-05) (hash(x)=143823248)
-900 val loss 7.3688
-900 val perplexity 1585.6975
-900 train 7.486793 (lr=2.9662e-05) (hash(x)=156260416)
-1000 val loss 7.3444
-1000 val perplexity 1547.5503
-1000 train 7.314602 (lr=2.9558e-05) (hash(x)=143734685)
-1100 val loss 7.3103
-1100 val perplexity 1495.6113
-1100 train 7.249287 (lr=2.9442e-05) (hash(x)=160013925)
-1200 val loss 7.2809
-1200 val perplexity 1452.2426
-1200 train 7.030381 (lr=2.9312e-05) (hash(x)=150678249)
-1300 val loss 7.2499
-1300 val perplexity 1407.9894
-1300 train 7.035954 (lr=2.9169e-05) (hash(x)=149073315)
-1400 val loss 7.1976
-1400 val perplexity 1336.1791
-1400 train 7.560995 (lr=2.9013e-05) (hash(x)=175802021)
-1500 val loss 7.1596
-1500 val perplexity 1286.3782
-1500 train 7.414792 (lr=2.8845e-05) (hash(x)=171034639)
-1600 val loss 7.1100
-1600 val perplexity 1224.1091
-1600 train 7.318711 (lr=2.8663e-05) (hash(x)=158681215)
-1700 val loss 7.0716
-1700 val perplexity 1178.0487
-1700 train 7.011314 (lr=2.8469e-05) (hash(x)=152116061)
-1800 val loss 7.0428
-1800 val perplexity 1144.6343
-1800 train 6.981174 (lr=2.8263e-05) (hash(x)=146108145)
-1900 val loss 7.0107
-1900 val perplexity 1108.4087
-1900 train 6.845355 (lr=2.8044e-05) (hash(x)=147598108)
-2000 val loss 6.9936
-2000 val perplexity 1089.6827
-2000 train 6.852312 (lr=2.7814e-05) (hash(x)=154996086)
-2100 val loss 6.9603
-2100 val perplexity 1053.9778
-2100 train 6.747998 (lr=2.7572e-05) (hash(x)=153396183)
-2200 val loss 6.9095
-2200 val perplexity 1001.7344
-2200 train 6.897384 (lr=2.7319e-05) (hash(x)=153885445)
-2300 val loss 6.8842
-2300 val perplexity 976.7466
-2300 train 6.883799 (lr=2.7055e-05) (hash(x)=159666385)
-2400 val loss 6.8381
-2400 val perplexity 932.6805
-2400 train 6.814809 (lr=2.6780e-05) (hash(x)=142353087)
-2500 val loss 6.8053
-2500 val perplexity 902.6398
-2500 train 6.818474 (lr=2.6494e-05) (hash(x)=146491718)
-2600 val loss 6.7792
-2600 val perplexity 879.3629
-2600 train 6.706415 (lr=2.6198e-05) (hash(x)=150750353)
-2700 val loss 6.7435
-2700 val perplexity 848.5612
-2700 train 6.450558 (lr=2.5892e-05) (hash(x)=129849193)
-2800 val loss 6.7261
-2800 val perplexity 833.9042
-2800 train 6.579156 (lr=2.5576e-05) (hash(x)=152767913)
-2900 val loss 6.6977
-2900 val perplexity 810.5140
-2900 train 6.479834 (lr=2.5251e-05) (hash(x)=146531140)
-3000 val loss 6.6654
-3000 val perplexity 784.7470
-3000 train 6.737319 (lr=2.4917e-05) (hash(x)=151562048)
-3100 val loss 6.6276
-3100 val perplexity 755.6806
-3100 train 6.601431 (lr=2.4574e-05) (hash(x)=146001424)
-3200 val loss 6.6050
-3200 val perplexity 738.7577
-3200 train 6.661371 (lr=2.4224e-05) (hash(x)=166486165)
-3300 val loss 6.5734
-3300 val perplexity 715.8162
-3300 train 6.504717 (lr=2.3865e-05) (hash(x)=150866680)
-3400 val loss 6.5584
-3400 val perplexity 705.1649
-3400 train 6.459944 (lr=2.3498e-05) (hash(x)=143900419)
-3500 val loss 6.5325
-3500 val perplexity 687.0813
-3500 train 6.343013 (lr=2.3125e-05) (hash(x)=148845794)
-3600 val loss 6.5133
-3600 val perplexity 674.0522
-3600 train 6.354141 (lr=2.2744e-05) (hash(x)=145667796)
-3700 val loss 6.4988
-3700 val perplexity 664.3641
-3700 train 6.519285 (lr=2.2357e-05) (hash(x)=163563851)
-3800 val loss 6.4812
-3800 val perplexity 652.7576
-3800 train 6.425587 (lr=2.1965e-05) (hash(x)=147488689)
-3900 val loss 6.4513
-3900 val perplexity 633.5565
-3900 train 6.449866 (lr=2.1566e-05) (hash(x)=148186608)
-4000 val loss 6.4286
-4000 val perplexity 619.2852
-4000 train 6.293235 (lr=2.1162e-05) (hash(x)=142970187)
-4100 val loss 6.4098
-4100 val perplexity 607.7936
-4100 train 6.482577 (lr=2.0754e-05) (hash(x)=141584883)
-4200 val loss 6.3939
-4200 val perplexity 598.1957
-4200 train 6.192812 (lr=2.0341e-05) (hash(x)=145664585)
-4300 val loss 6.3816
-4300 val perplexity 590.8762
-4300 train 6.243621 (lr=1.9924e-05) (hash(x)=143736499)
-4400 val loss 6.3786
-4400 val perplexity 589.0906
-4400 train 6.205530 (lr=1.9504e-05) (hash(x)=151883322)
-4500 val loss 6.3650
-4500 val perplexity 581.1219
-4500 train 6.248657 (lr=1.9081e-05) (hash(x)=153904871)
-4600 val loss 6.3499
-4600 val perplexity 572.4113
-4600 train 6.429447 (lr=1.8655e-05) (hash(x)=154893521)
-4700 val loss 6.3303
-4700 val perplexity 561.3388
-4700 train 6.389878 (lr=1.8226e-05) (hash(x)=152323949)
-4800 val loss 6.3137
-4800 val perplexity 552.0685
-4800 train 6.247524 (lr=1.7796e-05) (hash(x)=154104619)
-4900 val loss 6.3041
-4900 val perplexity 546.8019
-4900 train 6.426771 (lr=1.7365e-05) (hash(x)=146311426)
-5000 val loss 6.2913
-5000 val perplexity 539.8301
-5000 train 6.296994 (lr=1.6933e-05) (hash(x)=156741847)
-5100 val loss 6.2820
-5100 val perplexity 534.8316
-5100 train 6.057541 (lr=1.6500e-05) (hash(x)=142086346)
-5200 val loss 6.2774
-5200 val perplexity 532.4290
-5200 train 6.050554 (lr=1.6067e-05) (hash(x)=150265428)
-5300 val loss 6.2747
-5300 val perplexity 530.9745
-5300 train 6.163742 (lr=1.5635e-05) (hash(x)=151339108)
-5400 val loss 6.2586
-5400 val perplexity 522.4918
-5400 train 6.242078 (lr=1.5204e-05) (hash(x)=154654372)
-5500 val loss 6.2442
-5500 val perplexity 515.0331
-5500 train 6.253026 (lr=1.4774e-05) (hash(x)=150575051)
-5600 val loss 6.2343
-5600 val perplexity 509.9215
-5600 train 6.134394 (lr=1.4345e-05) (hash(x)=140396423)
-5700 val loss 6.2225
-5700 val perplexity 503.9442
-5700 train 6.086608 (lr=1.3919e-05) (hash(x)=144678758)
-5800 val loss 6.2157
-5800 val perplexity 500.5370
-5800 train 6.277523 (lr=1.3496e-05) (hash(x)=151992743)
-5900 val loss 6.2089
-5900 val perplexity 497.1764
-5900 train 5.998001 (lr=1.3076e-05) (hash(x)=144396927)
-6000 val loss 6.2031
-6000 val perplexity 494.2646
-6000 train 6.175484 (lr=1.2659e-05) (hash(x)=165478625)
-6100 val loss 6.2044
-6100 val perplexity 494.9267
-6100 train 5.943242 (lr=1.2246e-05) (hash(x)=147088621)
-6200 val loss 6.1952
-6200 val perplexity 490.3867
-6200 train 5.959939 (lr=1.1838e-05) (hash(x)=140794994)
-6300 val loss 6.1857
-6300 val perplexity 485.7651
-6300 train 6.112713 (lr=1.1434e-05) (hash(x)=134780906)
-6400 val loss 6.1726
-6400 val perplexity 479.4460
-6400 train 6.158383 (lr=1.1035e-05) (hash(x)=149023655)
-6500 val loss 6.1676
-6500 val perplexity 477.0429
-6500 train 6.019427 (lr=1.0643e-05) (hash(x)=147497796)
-6600 val loss 6.1663
-6600 val perplexity 476.4064
-6600 train 6.022311 (lr=1.0256e-05) (hash(x)=152902689)
-6700 val loss 6.1598
-6700 val perplexity 473.3375
-6700 train 6.199059 (lr=9.8753e-06) (hash(x)=153846046)
-6800 val loss 6.1487
-6800 val perplexity 468.0990
-6800 train 6.287767 (lr=9.5017e-06) (hash(x)=158512738)
-6900 val loss 6.1438
-6900 val perplexity 465.8420
-6900 train 6.667380 (lr=9.1353e-06) (hash(x)=156849968)
-7000 val loss 6.1404
-7000 val perplexity 464.2478
-7000 train 5.972945 (lr=8.7764e-06) (hash(x)=142395855)
-7100 val loss 6.1347
-7100 val perplexity 461.6042
-7100 train 5.991301 (lr=8.4255e-06) (hash(x)=147114884)
-7200 val loss 6.1303
-7200 val perplexity 459.5841
-7200 train 6.150269 (lr=8.0829e-06) (hash(x)=156979839)
-7300 val loss 6.1279
-7300 val perplexity 458.4534
-7300 train 5.902767 (lr=7.7489e-06) (hash(x)=145584373)
-7400 val loss 6.1273
-7400 val perplexity 458.1935
-7400 train 5.868430 (lr=7.4239e-06) (hash(x)=141508204)
-7500 val loss 6.1193
-7500 val perplexity 454.5416
-7500 train 6.157444 (lr=7.1083e-06) (hash(x)=148803965)
-7600 val loss 6.1136
-7600 val perplexity 451.9759
-7600 train 6.178225 (lr=6.8023e-06) (hash(x)=151019676)
-7700 val loss 6.1085
-7700 val perplexity 449.6451
-7700 train 6.151628 (lr=6.5062e-06) (hash(x)=143155750)
-7800 val loss 6.1058
-7800 val perplexity 448.4541
-7800 train 6.123940 (lr=6.2205e-06) (hash(x)=152569653)
-7900 val loss 6.1027
-7900 val perplexity 447.0435
-7900 train 5.969548 (lr=5.9453e-06) (hash(x)=143519455)
-8000 val loss 6.1011
-8000 val perplexity 446.3461
-8000 train 6.265728 (lr=5.6809e-06) (hash(x)=161180944)
-8100 val loss 6.1003
-8100 val perplexity 445.9847
-8100 train 6.132887 (lr=5.4277e-06) (hash(x)=154107345)
-8200 val loss 6.1007
-8200 val perplexity 446.1861
-8200 train 6.124985 (lr=5.1858e-06) (hash(x)=152486517)
-8300 val loss 6.0919
-8300 val perplexity 442.2632
-8300 train 6.113282 (lr=4.9556e-06) (hash(x)=156167749)
-8400 val loss 6.0884
-8400 val perplexity 440.7003
-8400 train 6.169589 (lr=4.7372e-06) (hash(x)=149155006)
-8500 val loss 6.0843
-8500 val perplexity 438.8997
-8500 train 6.044793 (lr=4.5309e-06) (hash(x)=147844390)
-8600 val loss 6.0825
-8600 val perplexity 438.1269
-8600 train 6.348199 (lr=4.3369e-06) (hash(x)=165753320)
-8700 val loss 6.0821
-8700 val perplexity 437.9322
-8700 train 5.860587 (lr=4.1554e-06) (hash(x)=146079979)
-8800 val loss 6.0813
-8800 val perplexity 437.6174
-8800 train 6.286478 (lr=3.9866e-06) (hash(x)=172259509)
-8900 val loss 6.0796
-8900 val perplexity 436.8750
-8900 train 5.752324 (lr=3.8307e-06) (hash(x)=145148314)
-9000 val loss 6.0798
-9000 val perplexity 436.9346
-9000 train 5.990164 (lr=3.6877e-06) (hash(x)=144250633)
-9100 val loss 6.0743
-9100 val perplexity 434.5333
-9100 train 6.182425 (lr=3.5580e-06) (hash(x)=157219797)
-9200 val loss 6.0697
-9200 val perplexity 432.5594
-9200 train 6.088066 (lr=3.4415e-06) (hash(x)=142743778)
-9300 val loss 6.0686
-9300 val perplexity 432.0853
-9300 train 5.967726 (lr=3.3385e-06) (hash(x)=139669771)
-9400 val loss 6.0656
-9400 val perplexity 430.7808
-9400 train 6.073001 (lr=3.2490e-06) (hash(x)=145916843)
-9500 val loss 6.0646
-9500 val perplexity 430.3534
-9500 train 5.989102 (lr=3.1730e-06) (hash(x)=150196125)
-9600 val loss 6.0638
-9600 val perplexity 429.9952
-9600 train 6.368902 (lr=3.1108e-06) (hash(x)=160041419)
-9700 val loss 6.0641
-9700 val perplexity 430.1185
-9700 train 5.840649 (lr=3.0624e-06) (hash(x)=139931627)
-9800 val loss 6.0617
-9800 val perplexity 429.1059
-9800 train 6.026587 (lr=3.0277e-06) (hash(x)=150370792)
-9900 val loss 6.0607
-9900 val perplexity 428.6625
-9900 train 5.908560 (lr=3.0069e-06) (hash(x)=153014886)
-9999 val loss 6.0601
-9999 val perplexity 428.4363
+0 val loss 11.2672
+0 val perplexity 78216.1172
+0 val loss 11.2672
+0 val perplexity 78216.1172
+0 train 11.243532 (lr=2.5000e-07) (hash(x)=50671684)
+0 train 11.243532 (lr=3.5000e-07) (hash(x)=50671684)
+100 val loss 9.7863
+100 val perplexity 17788.1406
+100 train 9.748065 (lr=2.5250e-05) (hash(x)=52740221)
+100 val loss 9.6966
+100 val perplexity 16261.9414
+100 train 9.656058 (lr=3.5350e-05) (hash(x)=52740221)
+200 val loss 8.6243
+200 val perplexity 5565.2393
+200 train 8.645584 (lr=5.0000e-05) (hash(x)=49034180)
+200 val loss 8.5884
+200 val perplexity 5369.2666
+200 train 8.609220 (lr=7.0000e-05) (hash(x)=49034180)
+300 val loss 7.8432
+300 val perplexity 2548.3008
+300 train 8.034177 (lr=4.9988e-05) (hash(x)=63180688)
+300 val loss 7.9184
+300 val perplexity 2747.3516
+300 train 8.088190 (lr=6.9984e-05) (hash(x)=63180688)
+400 val loss 7.6322
+400 val perplexity 2063.5647
+400 train 7.646658 (lr=4.9954e-05) (hash(x)=50373500)
+400 val loss 7.6588
+400 val perplexity 2119.1797
+400 train 7.679573 (lr=6.9935e-05) (hash(x)=50373500)
+500 val loss 7.5570
+500 val perplexity 1914.0950
+500 train 7.449134 (lr=4.9896e-05) (hash(x)=44547422)
+500 val loss 7.5919
+500 val perplexity 1982.1578
+500 train 7.476151 (lr=6.9854e-05) (hash(x)=44547422)
+600 val loss 7.5231
+600 val perplexity 1850.2341
+600 train 7.392508 (lr=4.9815e-05) (hash(x)=47184699)
+600 val loss 7.5484
+600 val perplexity 1897.6368
+600 train 7.411293 (lr=6.9741e-05) (hash(x)=47184699)
+700 val loss 7.4862
+700 val perplexity 1783.2515
+700 train 7.382556 (lr=4.9712e-05) (hash(x)=51374582)
+700 val loss 7.5192
+700 val perplexity 1843.0841
+700 train 7.419620 (lr=6.9596e-05) (hash(x)=51374582)
+800 val loss 7.4512
+800 val perplexity 1721.9939
+800 train 7.184936 (lr=4.9585e-05) (hash(x)=46264805)
+800 val loss 7.5009
+800 val perplexity 1809.6692
+800 train 7.231192 (lr=6.9419e-05) (hash(x)=46264805)
+900 val loss 7.4239
+900 val perplexity 1675.6316
+900 train 7.616142 (lr=4.9436e-05) (hash(x)=61178712)
+900 val loss 7.4928
+900 val perplexity 1795.0725
+900 train 7.696317 (lr=6.9210e-05) (hash(x)=61178712)
+1000 val loss 7.3880
+1000 val perplexity 1616.4591
+1000 train 7.320238 (lr=4.9264e-05) (hash(x)=50886520)
+1000 val loss 7.4503
+1000 val perplexity 1720.3885
+1000 train 7.406900 (lr=6.8970e-05) (hash(x)=50886520)
+1100 val loss 7.3679
+1100 val perplexity 1584.2887
+1100 train 7.124536 (lr=4.9070e-05) (hash(x)=48600099)
+1100 val loss 7.4293
+1100 val perplexity 1684.7103
+1100 train 7.190936 (lr=6.8698e-05) (hash(x)=48600099)
+1200 val loss 7.3450
+1200 val perplexity 1548.3999
+1200 train 7.066013 (lr=4.8854e-05) (hash(x)=50146792)
+1200 val loss 7.4227
+1200 val perplexity 1673.5076
+1200 train 7.144311 (lr=6.8395e-05) (hash(x)=50146792)
+1300 val loss 7.3218
+1300 val perplexity 1512.8927
+1300 train 7.209438 (lr=4.8616e-05) (hash(x)=52617313)
+1300 val loss 7.4052
+1300 val perplexity 1644.5828
+1300 train 7.299134 (lr=6.8062e-05) (hash(x)=52617313)
+1400 val loss 7.2835
+1400 val perplexity 1456.0742
+1400 train 6.942636 (lr=4.8356e-05) (hash(x)=49794446)
+1400 val loss 7.3959
+1400 val perplexity 1629.2686
+1400 train 7.067560 (lr=6.7698e-05) (hash(x)=49794446)
+1500 val loss 7.2511
+1500 val perplexity 1409.6560
+1500 train 6.871803 (lr=4.8074e-05) (hash(x)=50766317)
+1500 val loss 7.3682
+1500 val perplexity 1584.7428
+1500 train 7.009883 (lr=6.7304e-05) (hash(x)=50766317)
+1600 val loss 7.2362
+1600 val perplexity 1388.8374
+1600 train 7.086946 (lr=4.7772e-05) (hash(x)=55551175)
+1600 val loss 7.3618
+1600 val perplexity 1574.6808
+1600 train 7.228473 (lr=6.6881e-05) (hash(x)=55551175)
+1700 val loss 7.1878
+1700 val perplexity 1323.1958
+1700 train 7.377913 (lr=4.7448e-05) (hash(x)=56717172)
+1700 val loss 7.3199
+1700 val perplexity 1510.0026
+1700 train 7.497116 (lr=6.6428e-05) (hash(x)=56717172)
+1800 val loss 7.1270
+1800 val perplexity 1245.1523
+1800 train 7.316638 (lr=4.7105e-05) (hash(x)=55376447)
+1800 val loss 7.2595
+1800 val perplexity 1421.4961
+1800 train 7.487352 (lr=6.5947e-05) (hash(x)=55376447)
+1900 val loss 7.1062
+1900 val perplexity 1219.4828
+1900 train 6.827673 (lr=4.6741e-05) (hash(x)=43810837)
+1900 val loss 7.2502
+1900 val perplexity 1408.3661
+1900 train 6.984783 (lr=6.5437e-05) (hash(x)=43810837)
+2000 val loss 7.0607
+2000 val perplexity 1165.2810
+2000 train 7.099247 (lr=4.6357e-05) (hash(x)=50881655)
+2000 val loss 7.2211
+2000 val perplexity 1368.0502
+2000 train 7.268182 (lr=6.4900e-05) (hash(x)=50881655)
+2100 val loss 7.0425
+2100 val perplexity 1144.2305
+2100 train 6.927150 (lr=4.5954e-05) (hash(x)=49386015)
+2100 val loss 7.1989
+2100 val perplexity 1337.9802
+2100 train 7.097069 (lr=6.4335e-05) (hash(x)=49386015)
+2200 val loss 7.0278
+2200 val perplexity 1127.5027
+2200 train 6.943182 (lr=4.5532e-05) (hash(x)=48572079)
+2200 val loss 7.1965
+2200 val perplexity 1334.7056
+2200 train 7.109676 (lr=6.3745e-05) (hash(x)=48572079)
+2300 val loss 7.0047
+2300 val perplexity 1101.8270
+2300 train 7.066078 (lr=4.5091e-05) (hash(x)=54950719)
+2300 val loss 7.1867
+2300 val perplexity 1321.7347
+2300 train 7.240222 (lr=6.3128e-05) (hash(x)=54950719)
+2400 val loss 6.9956
+2400 val perplexity 1091.7751
+2400 train 6.643163 (lr=4.4633e-05) (hash(x)=42190240)
+2400 val loss 7.1732
+2400 val perplexity 1303.9774
+2400 train 6.820462 (lr=6.2486e-05) (hash(x)=42190240)
+2500 val loss 6.9921
+2500 val perplexity 1088.0327
+2500 train 7.077521 (lr=4.4156e-05) (hash(x)=45223539)
+2500 val loss 7.1472
+2500 val perplexity 1270.5563
+2500 train 7.217351 (lr=6.1819e-05) (hash(x)=45223539)
+2600 val loss 6.9545
+2600 val perplexity 1047.8682
+2600 train 6.976867 (lr=4.3663e-05) (hash(x)=54037353)
+2600 val loss 7.1369
+2600 val perplexity 1257.5035
+2600 train 7.166002 (lr=6.1128e-05) (hash(x)=54037353)
+2700 val loss 6.9415
+2700 val perplexity 1034.3434
+2700 train 7.436725 (lr=4.3153e-05) (hash(x)=59131616)
+2700 val loss 7.1107
+2700 val perplexity 1224.9740
+2700 train 7.566201 (lr=6.0414e-05) (hash(x)=59131616)
+2800 val loss 6.9349
+2800 val perplexity 1027.5295
+2800 train 6.777809 (lr=4.2627e-05) (hash(x)=45882743)
+2800 val loss 7.1161
+2800 val perplexity 1231.6880
+2800 train 6.949253 (lr=5.9677e-05) (hash(x)=45882743)
+2900 val loss 6.9195
+2900 val perplexity 1011.8037
+2900 train 6.530170 (lr=4.2085e-05) (hash(x)=43758910)
+2900 val loss 7.1009
+2900 val perplexity 1213.0134
+2900 train 6.715117 (lr=5.8919e-05) (hash(x)=43758910)
+3000 val loss 6.9221
+3000 val perplexity 1014.4709
+3000 train 6.823209 (lr=4.1529e-05) (hash(x)=47965974)
+3000 val loss 7.1030
+3000 val perplexity 1215.6672
+3000 train 7.019185 (lr=5.8140e-05) (hash(x)=47965974)
+3100 val loss 6.8896
+3100 val perplexity 982.0345
+3100 train 6.756184 (lr=4.0957e-05) (hash(x)=48205243)
+3100 val loss 7.0986
+3100 val perplexity 1210.2229
+3100 train 6.971920 (lr=5.7340e-05) (hash(x)=48205243)
+3200 val loss 6.8666
+3200 val perplexity 959.7057
+3200 train 6.943860 (lr=4.0373e-05) (hash(x)=54511383)
+3200 val loss 7.0723
+3200 val perplexity 1178.8484
+3200 train 7.163098 (lr=5.6522e-05) (hash(x)=54511383)
+3300 val loss 6.8515
+3300 val perplexity 945.2843
+3300 train 6.816091 (lr=3.9775e-05) (hash(x)=54428388)
+3300 val loss 7.0479
+3300 val perplexity 1150.4756
+3300 train 7.016342 (lr=5.5684e-05) (hash(x)=54428388)
+3400 val loss 6.8383
+3400 val perplexity 932.9412
+3400 train 6.837810 (lr=3.9164e-05) (hash(x)=48115990)
+3400 val loss 7.0391
+3400 val perplexity 1140.3251
+3400 train 7.041271 (lr=5.4829e-05) (hash(x)=48115990)
+3500 val loss 6.8256
+3500 val perplexity 921.1580
+3500 train 6.416183 (lr=3.8541e-05) (hash(x)=41137345)
+3500 val loss 7.0312
+3500 val perplexity 1131.3469
+3500 train 6.639562 (lr=5.3958e-05) (hash(x)=41137345)
+3600 val loss 6.8099
+3600 val perplexity 906.7393
+3600 train 6.700560 (lr=3.7907e-05) (hash(x)=55186224)
+3600 val loss 7.0239
+3600 val perplexity 1123.1450
+3600 train 6.912842 (lr=5.3070e-05) (hash(x)=55186224)
+3700 val loss 6.8071
+3700 val perplexity 904.2829
+3700 train 6.704582 (lr=3.7262e-05) (hash(x)=54990049)
+3700 val loss 7.0245
+3700 val perplexity 1123.8029
+3700 train 6.886618 (lr=5.2167e-05) (hash(x)=54990049)
+3800 val loss 6.7851
+3800 val perplexity 884.5947
+3800 train 6.510171 (lr=3.6608e-05) (hash(x)=46288812)
+3800 val loss 7.0206
+3800 val perplexity 1119.4424
+3800 train 6.773824 (lr=5.1251e-05) (hash(x)=46288812)
+3900 val loss 6.7655
+3900 val perplexity 867.3606
+3900 train 6.406036 (lr=3.5944e-05) (hash(x)=45829773)
+3900 val loss 7.0133
+3900 val perplexity 1111.2819
+3900 train 6.649640 (lr=5.0321e-05) (hash(x)=45829773)
+4000 val loss 6.7527
+4000 val perplexity 856.3824
+4000 train 6.514423 (lr=3.5271e-05) (hash(x)=52499943)
+4000 val loss 6.9932
+4000 val perplexity 1089.1742
+4000 train 6.779234 (lr=4.9379e-05) (hash(x)=52499943)
+4100 val loss 6.7186
+4100 val perplexity 827.6704
+4100 train 6.617018 (lr=3.4590e-05) (hash(x)=48563796)
+4100 val loss 6.9636
+4100 val perplexity 1057.4326
+4100 train 6.877402 (lr=4.8426e-05) (hash(x)=48563796)
+4200 val loss 6.6921
+4200 val perplexity 806.0439
+4200 train 6.658999 (lr=3.3902e-05) (hash(x)=49165143)
+4200 val loss 6.9332
+4200 val perplexity 1025.7916
+4200 train 6.899343 (lr=4.7463e-05) (hash(x)=49165143)
+4300 val loss 6.6656
+4300 val perplexity 784.9682
+4300 train 6.739098 (lr=3.3207e-05) (hash(x)=50973176)
+4300 val loss 6.9141
+4300 val perplexity 1006.3737
+4300 train 6.983593 (lr=4.6490e-05) (hash(x)=50973176)
+4400 val loss 6.6427
+4400 val perplexity 767.1820
+4400 train 6.686252 (lr=3.2507e-05) (hash(x)=55275124)
+4400 val loss 6.8895
+4400 val perplexity 981.9423
+4400 train 6.929132 (lr=4.5509e-05) (hash(x)=55275124)
+4500 val loss 6.6312
+4500 val perplexity 758.4039
+4500 train 6.999173 (lr=3.1801e-05) (hash(x)=58646505)
+4500 val loss 6.8767
+4500 val perplexity 969.4566
+4500 train 7.242509 (lr=4.4521e-05) (hash(x)=58646505)
+4600 val loss 6.6130
+4600 val perplexity 744.7347
+4600 train 6.412494 (lr=3.1091e-05) (hash(x)=42554666)
+4600 val loss 6.8488
+4600 val perplexity 942.7289
+4600 train 6.660697 (lr=4.3527e-05) (hash(x)=42554666)
+4700 val loss 6.6027
+4700 val perplexity 737.1144
+4700 train 6.528410 (lr=3.0377e-05) (hash(x)=47846764)
+4700 val loss 6.8299
+4700 val perplexity 925.0769
+4700 train 6.747003 (lr=4.2528e-05) (hash(x)=47846764)
+4800 val loss 6.5888
+4800 val perplexity 726.8744
+4800 train 7.141794 (lr=2.9661e-05) (hash(x)=58239019)
+4800 val loss 6.8121
+4800 val perplexity 908.7399
+4800 train 7.329013 (lr=4.1525e-05) (hash(x)=58239019)
+4900 val loss 6.5791
+4900 val perplexity 719.8779
+4900 train 6.642244 (lr=2.8942e-05) (hash(x)=50711220)
+4900 val loss 6.7941
+4900 val perplexity 892.5339
+4900 train 6.849504 (lr=4.0518e-05) (hash(x)=50711220)
+5000 val loss 6.5743
+5000 val perplexity 716.4384
+5000 train 6.507086 (lr=2.8221e-05) (hash(x)=45994194)
+5000 val loss 6.7812
+5000 val perplexity 881.1363
+5000 train 6.706810 (lr=3.9510e-05) (hash(x)=45994194)
+5100 val loss 6.5611
+5100 val perplexity 707.0463
+5100 train 6.430910 (lr=2.7500e-05) (hash(x)=48659050)
+5100 val loss 6.7622
+5100 val perplexity 864.5593
+5100 train 6.633748 (lr=3.8500e-05) (hash(x)=48659050)
+5200 val loss 6.5536
+5200 val perplexity 701.7662
+5200 train 6.456882 (lr=2.6779e-05) (hash(x)=49369682)
+5200 val loss 6.7529
+5200 val perplexity 856.5147
+5200 train 6.673459 (lr=3.7490e-05) (hash(x)=49369682)
+5300 val loss 6.5406
+5300 val perplexity 692.6973
+5300 train 6.962357 (lr=2.6058e-05) (hash(x)=57787700)
+5300 val loss 6.7406
+5300 val perplexity 846.0259
+5300 train 7.159966 (lr=3.6482e-05) (hash(x)=57787700)
+5400 val loss 6.5395
+5400 val perplexity 691.9344
+5400 train 6.458861 (lr=2.5339e-05) (hash(x)=49365400)
+5400 val loss 6.7261
+5400 val perplexity 833.8752
+5500 val loss 6.5243
+5500 val perplexity 681.5267
+5400 train 6.657549 (lr=3.5475e-05) (hash(x)=49365400)
+5500 train 6.480797 (lr=2.4623e-05) (hash(x)=48720412)
+5500 val loss 6.7196
+5500 val perplexity 828.4913
+5600 val loss 6.5153
+5600 val perplexity 675.3881
+5500 train 6.674323 (lr=3.4472e-05) (hash(x)=48720412)
+5600 train 6.858240 (lr=2.3909e-05) (hash(x)=55784800)
+5700 val loss 6.5099
+5700 val perplexity 671.7321
+5700 train 6.328226 (lr=2.3199e-05) (hash(x)=50073634)
+5600 val loss 6.7044
+5600 val perplexity 815.9836
+5600 train 7.043695 (lr=3.3473e-05) (hash(x)=55784800)
+5800 val loss 6.5062
+5800 val perplexity 669.2766
+5800 train 6.321751 (lr=2.2493e-05) (hash(x)=50170324)
+5700 val loss 6.6907
+5700 val perplexity 804.8875
+5700 train 6.527936 (lr=3.2479e-05) (hash(x)=50073634)
+5900 val loss 6.5102
+5900 val perplexity 671.9464
+5900 train 6.234162 (lr=2.1793e-05) (hash(x)=48410268)
+5800 val loss 6.6803
+5800 val perplexity 796.5750
+5800 train 6.504085 (lr=3.1491e-05) (hash(x)=50170324)
+6000 val loss 6.5055
+6000 val perplexity 668.7971
+6000 train 6.368552 (lr=2.1098e-05) (hash(x)=49527342)
+5900 val loss 6.6904
+5900 val perplexity 804.6039
+5900 train 6.427372 (lr=3.0510e-05) (hash(x)=48410268)
+6100 val loss 6.4971
+6100 val perplexity 663.2069
+6100 train 6.286675 (lr=2.0410e-05) (hash(x)=49550294)
+6000 val loss 6.6704
+6000 val perplexity 788.7175
+6000 train 6.565147 (lr=2.9537e-05) (hash(x)=49527342)
+6200 val loss 6.4929
+6200 val perplexity 660.4078
+6200 train 6.008740 (lr=1.9729e-05) (hash(x)=42126106)
+6100 val loss 6.6618
+6100 val perplexity 781.9240
+6100 train 6.465837 (lr=2.8574e-05) (hash(x)=49550294)
+6300 val loss 6.4897
+6300 val perplexity 658.3044
+6300 train 6.224653 (lr=1.9056e-05) (hash(x)=49608772)
+6200 val loss 6.6584
+6200 val perplexity 779.2789
+6200 train 6.193086 (lr=2.7621e-05) (hash(x)=42126106)
+6400 val loss 6.4802
+6400 val perplexity 652.1339
+6400 train 6.016216 (lr=1.8392e-05) (hash(x)=52324417)
+6300 val loss 6.6522
+6300 val perplexity 774.4972
+6300 train 6.393860 (lr=2.6679e-05) (hash(x)=49608772)
+6500 val loss 6.4586
+6500 val perplexity 638.1494
+6500 train 6.503439 (lr=1.7738e-05) (hash(x)=46207215)
+6400 val loss 6.6367
+6400 val perplexity 762.5599
+6400 train 6.191780 (lr=2.5749e-05) (hash(x)=52324417)
+6600 val loss 6.4430
+6600 val perplexity 628.3019
+6600 train 6.401751 (lr=1.7093e-05) (hash(x)=49027014)
+6500 val loss 6.6079
+6500 val perplexity 740.8976
+6500 train 6.642453 (lr=2.4833e-05) (hash(x)=46207215)
+6700 val loss 6.4321
+6700 val perplexity 621.4651
+6700 train 6.459781 (lr=1.6459e-05) (hash(x)=46232513)
+6600 val loss 6.5917
+6600 val perplexity 729.0474
+6600 train 6.547888 (lr=2.3930e-05) (hash(x)=49027014)
+6800 val loss 6.4318
+6800 val perplexity 621.2779
+6800 train 6.339231 (lr=1.5836e-05) (hash(x)=47348403)
+6700 val loss 6.5817
+6700 val perplexity 721.7724
+6700 train 6.570748 (lr=2.3042e-05) (hash(x)=46232513)
+6900 val loss 6.4355
+6900 val perplexity 623.5914
+6900 train 6.444074 (lr=1.5225e-05) (hash(x)=49806647)
+6800 val loss 6.5738
+6800 val perplexity 716.0906
+6800 train 6.504720 (lr=2.2171e-05) (hash(x)=47348403)
+7000 val loss 6.4125
+7000 val perplexity 609.4304
+7000 train 6.457445 (lr=1.4627e-05) (hash(x)=50893018)
+6900 val loss 6.5837
+6900 val perplexity 723.2028
+6900 train 6.578163 (lr=2.1316e-05) (hash(x)=49806647)
+7100 val loss 6.4138
+7100 val perplexity 610.1981
+7100 train 6.440397 (lr=1.4043e-05) (hash(x)=49157639)
+7000 val loss 6.5469
+7000 val perplexity 697.0728
+7000 train 6.610025 (lr=2.0478e-05) (hash(x)=50893018)
+7200 val loss 6.4047
+7200 val perplexity 604.6961
+7200 train 6.387370 (lr=1.3471e-05) (hash(x)=47014759)
+7100 val loss 6.5489
+7100 val perplexity 698.4415
+7100 train 6.581534 (lr=1.9660e-05) (hash(x)=49157639)
+7300 val loss 6.4001
+7300 val perplexity 601.9218
+7300 train 6.383959 (lr=1.2915e-05) (hash(x)=47325591)
+7200 val loss 6.5344
+7200 val perplexity 688.4330
+7200 train 6.524500 (lr=1.8860e-05) (hash(x)=47014759)
+7400 val loss 6.3930
+7400 val perplexity 597.6287
+7400 train 6.285221 (lr=1.2373e-05) (hash(x)=49184604)
+7300 val loss 6.5278
+7300 val perplexity 683.9073
+7300 train 6.516582 (lr=1.8081e-05) (hash(x)=47325591)
+7500 val loss 6.3856
+7500 val perplexity 593.2499
+7500 train 6.649272 (lr=1.1847e-05) (hash(x)=55053584)
+7400 val loss 6.5201
+7400 val perplexity 678.6690
+7400 train 6.414679 (lr=1.7323e-05) (hash(x)=49184604)
+7600 val loss 6.3827
+7600 val perplexity 591.5508
+7600 train 6.325932 (lr=1.1337e-05) (hash(x)=48693923)
+7500 val loss 6.5144
+7500 val perplexity 674.8160
+7500 train 6.811224 (lr=1.6586e-05) (hash(x)=55053584)
+7700 val loss 6.3809
+7700 val perplexity 590.4791
+7700 train 5.916291 (lr=1.0844e-05) (hash(x)=40952882)
+7600 val loss 6.5111
+7600 val perplexity 672.5692
+7600 train 6.471277 (lr=1.5872e-05) (hash(x)=48693923)
+7800 val loss 6.3779
+7800 val perplexity 588.6711
+7800 train 6.476369 (lr=1.0367e-05) (hash(x)=52487845)
+7700 val loss 6.5067
+7700 val perplexity 669.5911
+7700 train 6.060650 (lr=1.5181e-05) (hash(x)=40952882)
+7900 val loss 6.3770
+7900 val perplexity 588.1319
+7900 train 6.461136 (lr=9.9088e-06) (hash(x)=50221547)
+7800 val loss 6.5008
+7800 val perplexity 665.6864
+7800 train 6.590390 (lr=1.4514e-05) (hash(x)=52487845)
+8000 val loss 6.3713
+8000 val perplexity 584.8088
+8000 train 6.648645 (lr=9.4682e-06) (hash(x)=62294204)
+7900 val loss 6.4972
+7900 val perplexity 663.2847
+7900 train 6.574942 (lr=1.3872e-05) (hash(x)=50221547)
+8100 val loss 6.3669
+8100 val perplexity 582.2661
+8100 train 6.040726 (lr=9.0461e-06) (hash(x)=44401967)
+8000 val loss 6.4938
+8000 val perplexity 661.0527
+8000 train 6.781792 (lr=1.3255e-05) (hash(x)=62294204)
+8200 val loss 6.3676
+8200 val perplexity 582.6852
+8200 train 6.291852 (lr=8.6430e-06) (hash(x)=52769095)
+8100 val loss 6.4890
+8100 val perplexity 657.8419
+8100 train 6.187840 (lr=1.2665e-05) (hash(x)=44401967)
+8300 val loss 6.3675
+8300 val perplexity 582.5966
+8300 train 6.317422 (lr=8.2593e-06) (hash(x)=56829883)
+8200 val loss 6.4881
+8200 val perplexity 657.2552
+8200 train 6.420963 (lr=1.2100e-05) (hash(x)=52769095)
+8400 val loss 6.3669
+8400 val perplexity 582.2473
+8400 train 6.269656 (lr=7.8953e-06) (hash(x)=52147375)
+8300 val loss 6.4923
+8300 val perplexity 660.0215
+8300 train 6.443340 (lr=1.1563e-05) (hash(x)=56829883)
+8500 val loss 6.3666
+8500 val perplexity 582.0471
+8500 train 6.658837 (lr=7.5515e-06) (hash(x)=60197820)
+8400 val loss 6.4939
+8400 val perplexity 661.1094
+8400 train 6.411846 (lr=1.1053e-05) (hash(x)=52147375)
+8600 val loss 6.3644
+8600 val perplexity 580.8034
+8600 train 6.102660 (lr=7.2282e-06) (hash(x)=49377068)
+8700 val loss 6.3617
+8700 val perplexity 579.2385
+8700 train 6.333673 (lr=6.9257e-06) (hash(x)=51092724)
+8500 val loss 6.4915
+8500 val perplexity 659.5043
+8500 train 6.751138 (lr=1.0572e-05) (hash(x)=60197820)
+8800 val loss 6.3632
+8800 val perplexity 580.0929
+8800 train 6.343732 (lr=6.6444e-06) (hash(x)=48642928)
+8600 val loss 6.4865
+8600 val perplexity 656.2393
+8600 train 6.233767 (lr=1.0119e-05) (hash(x)=49377068)
+8900 val loss 6.3662
+8900 val perplexity 581.8434
+8900 train 6.577977 (lr=6.3845e-06) (hash(x)=55342246)
+8700 val loss 6.4884
+8700 val perplexity 657.4505
+8700 train 6.452022 (lr=9.6960e-06) (hash(x)=51092724)
+9000 val loss 6.3454
+9000 val perplexity 569.8459
+9000 train 6.334527 (lr=6.1462e-06) (hash(x)=48093368)
+8800 val loss 6.4842
+8800 val perplexity 654.6934
+8800 train 6.479820 (lr=9.3021e-06) (hash(x)=48642928)
+9100 val loss 6.3400
+9100 val perplexity 566.8240
+9100 train 6.413877 (lr=5.9300e-06) (hash(x)=48578183)
+8900 val loss 6.4770
+8900 val perplexity 650.0265
+8900 train 6.673745 (lr=8.9382e-06) (hash(x)=55342246)
+9200 val loss 6.3371
+9200 val perplexity 565.1450
+9200 train 6.528768 (lr=5.7359e-06) (hash(x)=50794720)
+9000 val loss 6.4624
+9000 val perplexity 640.5951
+9000 train 6.451552 (lr=8.6047e-06) (hash(x)=48093368)
+9300 val loss 6.3341
+9300 val perplexity 563.4738
+9300 train 6.083742 (lr=5.5641e-06) (hash(x)=46513190)
+9100 val loss 6.4584
+9100 val perplexity 638.0627
+9100 train 6.534726 (lr=8.3020e-06) (hash(x)=48578183)
+9400 val loss 6.3325
+9400 val perplexity 562.5811
+9400 train 6.010540 (lr=5.4149e-06) (hash(x)=43808238)
+9500 val loss 6.3304
+9500 val perplexity 561.3873
+9500 train 6.091447 (lr=5.2884e-06) (hash(x)=45021888)
+9200 val loss 6.4552
+9200 val perplexity 636.0129
+9200 train 6.642209 (lr=8.0302e-06) (hash(x)=50794720)
+9600 val loss 6.3272
+9600 val perplexity 559.5969
+9600 train 6.422736 (lr=5.1847e-06) (hash(x)=56525570)
+9300 val loss 6.4515
+9300 val perplexity 633.6569
+9300 train 6.224042 (lr=7.7898e-06) (hash(x)=46513190)
+9700 val loss 6.3240
+9700 val perplexity 557.8226
+9700 train 6.512197 (lr=5.1040e-06) (hash(x)=52585913)
+9400 val loss 6.4492
+9400 val perplexity 632.1708
+9400 train 6.119009 (lr=7.5809e-06) (hash(x)=43808238)
+9800 val loss 6.3230
+9800 val perplexity 557.2422
+9800 train 6.537199 (lr=5.0462e-06) (hash(x)=52344698)
+9500 val loss 6.4458
+9500 val perplexity 630.0528
+9900 val loss 6.3207
+9900 val perplexity 555.9839
+9500 train 6.227046 (lr=7.4038e-06) (hash(x)=45021888)
+9900 train 6.319592 (lr=5.0116e-06) (hash(x)=51740945)
+9999 val loss 6.3184
+9999 val perplexity 554.7078
+9600 val loss 6.4448
+9600 val perplexity 629.4426
+9600 train 6.529235 (lr=7.2586e-06) (hash(x)=56525570)
+9700 val loss 6.4413
+9700 val perplexity 627.2043
+9700 train 6.629733 (lr=7.1456e-06) (hash(x)=52585913)
+9800 val loss 6.4397
+9800 val perplexity 626.2005
+9800 train 6.633550 (lr=7.0647e-06) (hash(x)=52344698)
+9900 val loss 6.4373
+9900 val perplexity 624.7111
+9900 train 6.438586 (lr=7.0162e-06) (hash(x)=51740945)
+9999 val loss 6.4359
+9999 val perplexity 623.8198

attention_kindselective_n_heads4_seed1338/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cfb2160933da3c9bd20cc75e7b5e3e17efe7597e8f4ceb7e2131d2db5fe2a7be
+oid sha256:e5555ffb4b2cceb8de7a85935af9bde78a8e12f00bfe537f9a6700433d8845a8
 size 92843394

attention_kindselective_n_heads4_seed1338/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d1a68456ab6dd22f9e0fa8d9d98fa5890dc9184894790e74af0dbd1de1e7eabb
+oid sha256:c04035190bd46f9b0ed7808da2d4a12edcfe48d98592d3f25b51cfa5cebc7f75
 size 92843394

attention_kindselective_n_heads4_seed1338/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ad3c1940f30f39a067179a398d7645ba2a216761c016c202a7bd2ff59ad9146
+oid sha256:935d9743a04892562e7641457ff0d4005ceb841e7c4c4235482eaeed288493f0
 size 92843394

attention_kindselective_n_heads4_seed1338/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48cf363ca99835cf2690a96b5b649ff230b1bd43022ac3c5cc864f3dbc4ad02b
+oid sha256:ee20cfea6b75b11ea0b76d6461126f89256d5bbb4f082ae38fe67c7d47377126
 size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4b9a2b4aab41d39c6c30b36ca11463d9a9c59401a5856681c6f004e3eb2b08b
+oid sha256:43d0cb2f3cfea37b5079b99029c5c9bd80a6d7bb4746e206f8cec47096fd7833
 size 179406214

attention_kindselective_n_heads4_seed1338/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:85d3d0e3997291682cfc03365e6569ad60b2cc96084e2f466df903968cb7dad4
+oid sha256:38cc2f6202bc33ddc3a918860a93e0d1dc9ef15836049b642b28e22df331fa4c
 size 179406214

attention_kindselective_n_heads4_seed1338/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6720b9370b3c247560dd03bca887c5fd03cca9f0f8935c8f0e32f2b2d0bc882
+oid sha256:14cbf8ccb5561be3b3741e5cf9b629cfbcb429c960a08e1b812c4786d2ef9cf9
 size 179406214

attention_kindselective_n_heads4_seed1338/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f09307ada0be1dc0ce77e7e2f58b55e8fcbcd5554b4db8268178906ff5dee43
+oid sha256:f7c734804fea06b9be0d6722d9437244ede6c63f1ca46b939296e687e528677e
 size 179406214