Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.0001, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "10e-5_10240_4_1340", "n_embd": 256}

attention_kindselective_n_heads4_seed1340/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
+oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
 size 964

attention_kindselective_n_heads4_seed1340/dataloader_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+oid sha256:7f03ed2ebf741f15e13c79e6cc1e9a19b308450d81cc3b4d8d0338c63d77ca59
 size 964

attention_kindselective_n_heads4_seed1340/dataloader_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+oid sha256:82590037fb2eecbec961f7967a8dd1b8d85515d31a252f66b92b8139858a8b7c
 size 964

attention_kindselective_n_heads4_seed1340/dataloader_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+oid sha256:c544303717d12355a69b8ffb1eb109434e4fdccfd5a61279b6e8ba2e870d6700
 size 964

attention_kindselective_n_heads4_seed1340/log2.txt

@@ -1,303 +1,592 @@
 max_steps: 10000
-0 val loss 11.2714
-0 val perplexity 78545.6094
-0 train 11.276529 (lr=1.5000e-07) (hash(x)=164406924)
-100 val loss 9.9606
-100 val perplexity 21175.6562
-100 train 10.004577 (lr=1.5150e-05) (hash(x)=177407419)
-200 val loss 9.0320
-200 val perplexity 8366.1895
-200 train 8.999515 (lr=3.0000e-05) (hash(x)=144903932)
-300 val loss 7.8938
-300 val perplexity 2680.6653
-300 train 8.154392 (lr=2.9993e-05) (hash(x)=173839165)
-400 val loss 7.5861
-400 val perplexity 1970.6659
-400 train 7.666971 (lr=2.9972e-05) (hash(x)=167734596)
-500 val loss 7.4743
-500 val perplexity 1762.1843
-500 train 7.479569 (lr=2.9938e-05) (hash(x)=153224076)
-600 val loss 7.3984
-600 val perplexity 1633.3772
-600 train 7.299772 (lr=2.9889e-05) (hash(x)=149619098)
-700 val loss 7.3482
-700 val perplexity 1553.3265
-700 train 7.271634 (lr=2.9827e-05) (hash(x)=146539909)
-800 val loss 7.2867
-800 val perplexity 1460.7719
-800 train 7.243446 (lr=2.9751e-05) (hash(x)=153710890)
-900 val loss 7.2455
-900 val perplexity 1401.7346
-900 train 7.184447 (lr=2.9662e-05) (hash(x)=155873620)
-1000 val loss 7.1940
-1000 val perplexity 1331.4694
-1000 train 7.114730 (lr=2.9558e-05) (hash(x)=145450636)
-1100 val loss 7.1314
-1100 val perplexity 1250.6267
-1100 train 7.213695 (lr=2.9442e-05) (hash(x)=154123388)
-1200 val loss 7.0798
-1200 val perplexity 1187.7039
-1200 train 6.951880 (lr=2.9312e-05) (hash(x)=145249251)
-1300 val loss 7.0330
-1300 val perplexity 1133.4534
-1300 train 6.922292 (lr=2.9169e-05) (hash(x)=148937127)
-1400 val loss 6.9842
-1400 val perplexity 1079.3993
-1400 train 7.102539 (lr=2.9013e-05) (hash(x)=150475545)
-1500 val loss 6.9463
-1500 val perplexity 1039.3448
-1500 train 6.868868 (lr=2.8845e-05) (hash(x)=154653428)
-1600 val loss 6.9020
-1600 val perplexity 994.2404
-1600 train 6.883031 (lr=2.8663e-05) (hash(x)=144483776)
-1700 val loss 6.8505
-1700 val perplexity 944.3855
-1700 train 6.997636 (lr=2.8469e-05) (hash(x)=157395496)
-1800 val loss 6.8109
-1800 val perplexity 907.7036
-1800 train 6.835225 (lr=2.8263e-05) (hash(x)=157916369)
-1900 val loss 6.7660
-1900 val perplexity 867.8434
-1900 train 6.961428 (lr=2.8044e-05) (hash(x)=166073923)
-2000 val loss 6.7284
-2000 val perplexity 835.7856
-2000 train 6.798526 (lr=2.7814e-05) (hash(x)=154856891)
-2100 val loss 6.6863
-2100 val perplexity 801.3806
-2100 train 6.677713 (lr=2.7572e-05) (hash(x)=151925203)
-2200 val loss 6.6460
-2200 val perplexity 769.6996
-2200 train 6.342697 (lr=2.7319e-05) (hash(x)=136191502)
-2300 val loss 6.6026
-2300 val perplexity 736.9893
-2300 train 6.794070 (lr=2.7055e-05) (hash(x)=153273362)
-2400 val loss 6.5684
-2400 val perplexity 712.1960
-2400 train 6.527608 (lr=2.6780e-05) (hash(x)=148021541)
-2500 val loss 6.5452
-2500 val perplexity 695.8914
-2500 train 6.497287 (lr=2.6494e-05) (hash(x)=141356608)
-2600 val loss 6.5098
-2600 val perplexity 671.6882
-2600 train 6.449068 (lr=2.6198e-05) (hash(x)=146005217)
-2700 val loss 6.4881
-2700 val perplexity 657.2935
-2700 train 6.327260 (lr=2.5892e-05) (hash(x)=144511718)
-2800 val loss 6.4644
-2800 val perplexity 641.8793
-2800 train 6.342343 (lr=2.5576e-05) (hash(x)=146019502)
-2900 val loss 6.4402
-2900 val perplexity 626.5353
-2900 train 6.346152 (lr=2.5251e-05) (hash(x)=146496200)
-3000 val loss 6.4309
-3000 val perplexity 620.7094
-3000 train 6.369481 (lr=2.4917e-05) (hash(x)=150127281)
-3100 val loss 6.3944
-3100 val perplexity 598.4836
-3100 train 6.382371 (lr=2.4574e-05) (hash(x)=142022255)
-3200 val loss 6.3760
-3200 val perplexity 587.5679
-3200 train 6.428134 (lr=2.4224e-05) (hash(x)=154120875)
-3300 val loss 6.3658
-3300 val perplexity 581.6215
-3300 train 6.442893 (lr=2.3865e-05) (hash(x)=153999717)
-3400 val loss 6.3510
-3400 val perplexity 573.0490
-3400 train 6.126100 (lr=2.3498e-05) (hash(x)=139694097)
-3500 val loss 6.3252
-3500 val perplexity 558.4834
-3500 train 6.507070 (lr=2.3125e-05) (hash(x)=162992732)
-3600 val loss 6.3052
-3600 val perplexity 547.4195
-3600 train 6.278409 (lr=2.2744e-05) (hash(x)=147574101)
-3700 val loss 6.2949
-3700 val perplexity 541.7805
-3700 train 6.414897 (lr=2.2357e-05) (hash(x)=157763099)
-3800 val loss 6.2728
-3800 val perplexity 529.9534
-3800 train 6.433603 (lr=2.1965e-05) (hash(x)=170800034)
-3900 val loss 6.2618
-3900 val perplexity 524.1657
-3900 train 6.339255 (lr=2.1566e-05) (hash(x)=164984528)
-4000 val loss 6.2489
-4000 val perplexity 517.4280
-4000 train 6.104458 (lr=2.1162e-05) (hash(x)=141743323)
-4100 val loss 6.2473
-4100 val perplexity 516.5930
-4100 train 6.270818 (lr=2.0754e-05) (hash(x)=153392872)
-4200 val loss 6.2393
-4200 val perplexity 512.4744
-4200 train 6.103446 (lr=2.0341e-05) (hash(x)=149074933)
-4300 val loss 6.2196
-4300 val perplexity 502.5102
-4300 train 6.615050 (lr=1.9924e-05) (hash(x)=167823423)
-4400 val loss 6.2036
-4400 val perplexity 494.5025
-4400 train 5.975879 (lr=1.9504e-05) (hash(x)=141203114)
-4500 val loss 6.1913
-4500 val perplexity 488.4986
-4500 train 6.191016 (lr=1.9081e-05) (hash(x)=146284780)
-4600 val loss 6.1767
-4600 val perplexity 481.3957
-4600 train 5.980778 (lr=1.8655e-05) (hash(x)=141126464)
-4700 val loss 6.1681
-4700 val perplexity 477.2804
-4700 train 6.195337 (lr=1.8226e-05) (hash(x)=154751926)
-4800 val loss 6.1580
-4800 val perplexity 472.4835
-4800 train 6.241466 (lr=1.7796e-05) (hash(x)=154793198)
-4900 val loss 6.1548
-4900 val perplexity 470.9877
-4900 train 5.870466 (lr=1.7365e-05) (hash(x)=139406392)
-5000 val loss 6.1565
-5000 val perplexity 471.7581
-5000 train 5.880513 (lr=1.6933e-05) (hash(x)=153548741)
-5100 val loss 6.1398
-5100 val perplexity 463.9838
-5100 train 6.219454 (lr=1.6500e-05) (hash(x)=160488568)
-5200 val loss 6.1232
-5200 val perplexity 456.3128
-5200 train 6.138048 (lr=1.6067e-05) (hash(x)=149645053)
-5300 val loss 6.1116
-5300 val perplexity 451.0385
-5300 train 6.218238 (lr=1.5635e-05) (hash(x)=155820556)
-5400 val loss 6.1054
-5400 val perplexity 448.2636
-5400 train 6.051918 (lr=1.5204e-05) (hash(x)=147538134)
-5500 val loss 6.0996
-5500 val perplexity 445.6834
-5500 train 6.314559 (lr=1.4774e-05) (hash(x)=166889307)
-5600 val loss 6.0906
-5600 val perplexity 441.7078
-5600 train 5.787103 (lr=1.4345e-05) (hash(x)=139516699)
-5700 val loss 6.0892
-5700 val perplexity 441.0808
-5700 train 5.765007 (lr=1.3919e-05) (hash(x)=140453511)
-5800 val loss 6.0883
-5800 val perplexity 440.6656
-5800 train 6.035215 (lr=1.3496e-05) (hash(x)=162964847)
-5900 val loss 6.0759
-5900 val perplexity 435.2562
-5900 train 6.082222 (lr=1.3076e-05) (hash(x)=150606634)
-6000 val loss 6.0622
-6000 val perplexity 429.3382
-6000 train 6.153077 (lr=1.2659e-05) (hash(x)=149890857)
-6100 val loss 6.0595
-6100 val perplexity 428.1720
-6100 train 6.164173 (lr=1.2246e-05) (hash(x)=173884145)
-6200 val loss 6.0520
-6200 val perplexity 424.9789
-6200 train 6.085427 (lr=1.1838e-05) (hash(x)=151987098)
-6300 val loss 6.0481
-6300 val perplexity 423.3203
-6300 train 5.995059 (lr=1.1434e-05) (hash(x)=148853562)
-6400 val loss 6.0401
-6400 val perplexity 419.9156
-6400 train 5.790720 (lr=1.1035e-05) (hash(x)=141530101)
-6500 val loss 6.0382
-6500 val perplexity 419.1209
-6500 train 5.891243 (lr=1.0643e-05) (hash(x)=142297809)
-6600 val loss 6.0280
-6600 val perplexity 414.8662
-6600 train 5.915996 (lr=1.0256e-05) (hash(x)=142447782)
-6700 val loss 6.0262
-6700 val perplexity 414.1220
-6700 train 6.011028 (lr=9.8753e-06) (hash(x)=147004686)
-6800 val loss 6.0205
-6800 val perplexity 411.7659
-6800 train 5.761605 (lr=9.5017e-06) (hash(x)=133438702)
-6900 val loss 6.0187
-6900 val perplexity 411.0467
-6900 train 6.033222 (lr=9.1353e-06) (hash(x)=157085143)
-7000 val loss 6.0155
-7000 val perplexity 409.7354
-7000 train 5.893958 (lr=8.7764e-06) (hash(x)=139437666)
-7100 val loss 6.0085
-7100 val perplexity 406.8734
-7100 train 6.023874 (lr=8.4255e-06) (hash(x)=159792986)
-7200 val loss 6.0023
-7200 val perplexity 404.3719
-7200 train 5.971371 (lr=8.0829e-06) (hash(x)=144930687)
-7300 val loss 5.9993
-7300 val perplexity 403.1321
-7300 train 6.061211 (lr=7.7489e-06) (hash(x)=156242690)
-7400 val loss 5.9979
-7400 val perplexity 402.5811
-7400 train 5.837852 (lr=7.4239e-06) (hash(x)=148183719)
-7500 val loss 5.9952
-7500 val perplexity 401.4802
-7500 train 6.040481 (lr=7.1083e-06) (hash(x)=152494758)
-7600 val loss 5.9928
-7600 val perplexity 400.5148
-7600 train 5.688015 (lr=6.8023e-06) (hash(x)=142485027)
-7700 val loss 5.9925
-7700 val perplexity 400.4065
-7700 train 5.856967 (lr=6.5062e-06) (hash(x)=147512165)
-7800 val loss 5.9823
-7800 val perplexity 396.3450
-7800 train 5.875477 (lr=6.2205e-06) (hash(x)=160346994)
-7900 val loss 5.9835
-7900 val perplexity 396.8160
-7900 train 5.854673 (lr=5.9453e-06) (hash(x)=144488254)
-8000 val loss 5.9761
-8000 val perplexity 393.9162
-8000 train 5.791002 (lr=5.6809e-06) (hash(x)=147637019)
-8100 val loss 5.9770
-8100 val perplexity 394.2499
-8100 train 5.836208 (lr=5.4277e-06) (hash(x)=147340534)
-8200 val loss 5.9740
-8200 val perplexity 393.0701
-8200 train 6.029647 (lr=5.1858e-06) (hash(x)=151630665)
-8300 val loss 5.9693
-8300 val perplexity 391.2296
-8300 train 6.069899 (lr=4.9556e-06) (hash(x)=149747064)
-8400 val loss 5.9674
-8400 val perplexity 390.5003
-8400 train 6.151947 (lr=4.7372e-06) (hash(x)=154245770)
-8500 val loss 5.9659
-8500 val perplexity 389.9173
-8500 train 5.844897 (lr=4.5309e-06) (hash(x)=152559100)
-8600 val loss 5.9633
-8600 val perplexity 388.8792
-8600 train 6.405670 (lr=4.3369e-06) (hash(x)=181365926)
-8700 val loss 5.9625
-8700 val perplexity 388.5897
-8700 train 5.803471 (lr=4.1554e-06) (hash(x)=154405991)
-8800 val loss 5.9606
-8800 val perplexity 387.8566
-8800 train 5.906060 (lr=3.9866e-06) (hash(x)=153755904)
-8900 val loss 5.9565
-8900 val perplexity 386.2402
-8900 train 5.878875 (lr=3.8307e-06) (hash(x)=152120568)
-9000 val loss 5.9533
-9000 val perplexity 385.0145
-9000 train 5.743235 (lr=3.6877e-06) (hash(x)=142797279)
-9100 val loss 5.9519
-9100 val perplexity 384.4747
-9100 train 5.811658 (lr=3.5580e-06) (hash(x)=143037503)
-9200 val loss 5.9518
-9200 val perplexity 384.4454
-9200 train 5.815796 (lr=3.4415e-06) (hash(x)=113690273)
-9300 val loss 5.9511
-9300 val perplexity 384.1758
-9300 train 5.896542 (lr=3.3385e-06) (hash(x)=158025077)
-9400 val loss 5.9488
-9400 val perplexity 383.2877
-9400 train 6.020230 (lr=3.2490e-06) (hash(x)=158251718)
-9500 val loss 5.9458
-9500 val perplexity 382.1567
-9500 train 5.991383 (lr=3.1730e-06) (hash(x)=154752610)
-9600 val loss 5.9455
-9600 val perplexity 382.0358
-9600 train 5.821980 (lr=3.1108e-06) (hash(x)=146889093)
-9700 val loss 5.9428
-9700 val perplexity 380.9863
-9700 train 5.947937 (lr=3.0624e-06) (hash(x)=156906516)
-9800 val loss 5.9424
-9800 val perplexity 380.8622
-9800 train 5.783137 (lr=3.0277e-06) (hash(x)=153841927)
-9900 val loss 5.9426
-9900 val perplexity 380.9407
-9900 train 6.163361 (lr=3.0069e-06) (hash(x)=163514334)
-9999 val loss 5.9385
-9999 val perplexity 379.3569
+0 val loss 11.2703
+0 val perplexity 78458.0234
+0 train 11.272942 (lr=7.5000e-07) (hash(x)=57791809)
+100 val loss 9.1994
+100 val perplexity 9891.5127
+100 train 9.216334 (lr=7.5750e-05) (hash(x)=48211824)
+200 val loss 7.7349
+200 val perplexity 2286.7070
+200 train 7.673902 (lr=1.5000e-04) (hash(x)=50375849)
+0 train 11.272942 (lr=5.0000e-07) (hash(x)=57791809)
+300 val loss 7.6076
+300 val perplexity 2013.5125
+300 train 7.925827 (lr=1.4997e-04) (hash(x)=57250808)
+100 val loss 9.5575
+100 val perplexity 14150.3975
+100 train 9.571983 (lr=5.0500e-05) (hash(x)=48211824)
+400 val loss 7.5727
+400 val perplexity 1944.4767
+400 train 8.180161 (lr=1.4986e-04) (hash(x)=62519858)
+200 val loss 8.0717
+200 val perplexity 3202.6396
+200 train 8.018360 (lr=1.0000e-04) (hash(x)=50375849)
+500 val loss 7.5053
+500 val perplexity 1817.6418
+500 train 7.404214 (lr=1.4969e-04) (hash(x)=47226806)
+300 val loss 7.6542
+300 val perplexity 2109.4519
+300 train 7.955753 (lr=9.9977e-05) (hash(x)=57250808)
+600 val loss 7.4856
+600 val perplexity 1782.2722
+600 train 7.516066 (lr=1.4945e-04) (hash(x)=51149322)
+400 val loss 7.6014
+400 val perplexity 2001.0934
+400 train 8.229440 (lr=9.9908e-05) (hash(x)=62519858)
+700 val loss 7.4777
+700 val perplexity 1768.1790
+700 train 7.494768 (lr=1.4913e-04) (hash(x)=51564551)
+500 val loss 7.5660
+500 val perplexity 1931.4121
+500 train 7.453595 (lr=9.9792e-05) (hash(x)=47226806)
+800 val loss 7.4734
+800 val perplexity 1760.6666
+800 train 7.244524 (lr=1.4876e-04) (hash(x)=45093459)
+600 val loss 7.5616
+600 val perplexity 1922.8881
+600 train 7.599857 (lr=9.9631e-05) (hash(x)=51149322)
+900 val loss 7.4478
+900 val perplexity 1716.0348
+900 train 7.690024 (lr=1.4831e-04) (hash(x)=54988361)
+700 val loss 7.5504
+700 val perplexity 1901.4637
+700 train 7.545293 (lr=9.9423e-05) (hash(x)=51564551)
+1000 val loss 7.4670
+1000 val perplexity 1749.4158
+1000 train 7.367063 (lr=1.4779e-04) (hash(x)=47588648)
+800 val loss 7.5201
+800 val perplexity 1844.7793
+800 train 7.278911 (lr=9.9170e-05) (hash(x)=45093459)
+1100 val loss 7.4328
+1100 val perplexity 1690.5156
+1100 train 7.082927 (lr=1.4721e-04) (hash(x)=37984588)
+900 val loss 7.5207
+900 val perplexity 1845.7920
+900 train 7.809628 (lr=9.8872e-05) (hash(x)=54988361)
+1200 val loss 7.3818
+1200 val perplexity 1606.4569
+1200 train 7.558693 (lr=1.4656e-04) (hash(x)=56333817)
+1000 val loss 7.5099
+1000 val perplexity 1825.9989
+1000 train 7.391204 (lr=9.8528e-05) (hash(x)=47588648)
+1300 val loss 7.3642
+1300 val perplexity 1578.5209
+1300 train 7.491827 (lr=1.4585e-04) (hash(x)=53454056)
+1100 val loss 7.4699
+1100 val perplexity 1754.4908
+1100 train 7.058946 (lr=9.8140e-05) (hash(x)=37984588)
+1400 val loss 7.3266
+1400 val perplexity 1520.1290
+1400 train 7.428581 (lr=1.4507e-04) (hash(x)=55284163)
+1200 val loss 7.4695
+1200 val perplexity 1753.7841
+1200 train 7.647787 (lr=9.7708e-05) (hash(x)=56333817)
+1500 val loss 7.3234
+1500 val perplexity 1515.3085
+1500 train 7.260311 (lr=1.4422e-04) (hash(x)=48162598)
+1300 val loss 7.4999
+1300 val perplexity 1807.8933
+1300 train 7.617690 (lr=9.7231e-05) (hash(x)=53454056)
+1600 val loss 7.2709
+1600 val perplexity 1437.7860
+1600 train 7.330367 (lr=1.4332e-04) (hash(x)=54214535)
+1400 val loss 7.4034
+1400 val perplexity 1641.5452
+1400 train 7.515925 (lr=9.6711e-05) (hash(x)=55284163)
+1700 val loss 7.3486
+1700 val perplexity 1554.0712
+1700 train 7.561377 (lr=1.4235e-04) (hash(x)=53525003)
+1500 val loss 7.3976
+1500 val perplexity 1631.9938
+1500 train 7.301088 (lr=9.6149e-05) (hash(x)=48162598)
+1800 val loss 7.2820
+1800 val perplexity 1453.8403
+1800 train 7.356642 (lr=1.4131e-04) (hash(x)=51848994)
+1600 val loss 7.4057
+1600 val perplexity 1645.3561
+1600 train 7.469926 (lr=9.5544e-05) (hash(x)=54214535)
+1900 val loss 7.3030
+1900 val perplexity 1484.7439
+1900 train 7.137829 (lr=1.4022e-04) (hash(x)=48405987)
+1700 val loss 7.3512
+1700 val perplexity 1558.1410
+1700 train 7.569810 (lr=9.4897e-05) (hash(x)=53525003)
+2000 val loss 7.2506
+2000 val perplexity 1409.0001
+2000 train 7.602566 (lr=1.3907e-04) (hash(x)=58592291)
+1800 val loss 7.3285
+1800 val perplexity 1523.0544
+1800 train 7.415604 (lr=9.4209e-05) (hash(x)=51848994)
+2100 val loss 7.2532
+2100 val perplexity 1412.6349
+2100 train 7.341368 (lr=1.3786e-04) (hash(x)=51167081)
+1900 val loss 7.3569
+1900 val perplexity 1566.9419
+1900 train 7.183960 (lr=9.3481e-05) (hash(x)=48405987)
+2200 val loss 7.2590
+2200 val perplexity 1420.7731
+2200 train 7.260748 (lr=1.3660e-04) (hash(x)=47994988)
+2000 val loss 7.3472
+2000 val perplexity 1551.8170
+2000 train 7.699927 (lr=9.2714e-05) (hash(x)=58592291)
+2300 val loss 7.2521
+2300 val perplexity 1411.0697
+2300 train 7.250330 (lr=1.3527e-04) (hash(x)=47377604)
+2100 val loss 7.3025
+2100 val perplexity 1483.9901
+2100 train 7.402580 (lr=9.1908e-05) (hash(x)=51167081)
+2400 val loss 7.2556
+2400 val perplexity 1416.0265
+2400 train 7.304438 (lr=1.3390e-04) (hash(x)=53554323)
+2200 val loss 7.3241
+2200 val perplexity 1516.3789
+2200 train 7.340111 (lr=9.1064e-05) (hash(x)=47994988)
+2500 val loss 7.3073
+2500 val perplexity 1491.1608
+2500 train 7.313736 (lr=1.3247e-04) (hash(x)=50780417)
+2300 val loss 7.2707
+2300 val perplexity 1437.5735
+2300 train 7.255850 (lr=9.0182e-05) (hash(x)=47377604)
+2600 val loss 7.2764
+2600 val perplexity 1445.7822
+2600 train 7.182763 (lr=1.3099e-04) (hash(x)=46453562)
+2400 val loss 7.2498
+2400 val perplexity 1407.8685
+2400 train 7.306433 (lr=8.9265e-05) (hash(x)=53554323)
+2700 val loss 7.2792
+2700 val perplexity 1449.8409
+2700 train 7.287231 (lr=1.2946e-04) (hash(x)=54404221)
+2500 val loss 7.2853
+2500 val perplexity 1458.7178
+2500 train 7.310247 (lr=8.8313e-05) (hash(x)=50780417)
+2800 val loss 7.2837
+2800 val perplexity 1456.3575
+2800 train 7.949651 (lr=1.2788e-04) (hash(x)=59318895)
+2600 val loss 7.2457
+2600 val perplexity 1402.0936
+2600 train 7.150898 (lr=8.7326e-05) (hash(x)=46453562)
+2900 val loss 7.2333
+2900 val perplexity 1384.8267
+2900 train 7.221962 (lr=1.2626e-04) (hash(x)=47845760)
+2700 val loss 7.2470
+2700 val perplexity 1403.8638
+2700 train 7.253963 (lr=8.6306e-05) (hash(x)=54404221)
+3000 val loss 7.2471
+3000 val perplexity 1403.9862
+3000 train 6.979670 (lr=1.2459e-04) (hash(x)=44336167)
+2800 val loss 7.2707
+2800 val perplexity 1437.6050
+2800 train 7.978097 (lr=8.5254e-05) (hash(x)=59318895)
+3100 val loss 7.2154
+3100 val perplexity 1360.2776
+3100 train 7.447736 (lr=1.2287e-04) (hash(x)=44479330)
+2900 val loss 7.2657
+2900 val perplexity 1430.3938
+2900 train 7.266902 (lr=8.4170e-05) (hash(x)=47845760)
+3200 val loss 7.2325
+3200 val perplexity 1383.7072
+3200 train 7.285394 (lr=1.2112e-04) (hash(x)=54593096)
+3000 val loss 7.2402
+3000 val perplexity 1394.3390
+3000 train 6.979912 (lr=8.3057e-05) (hash(x)=44336167)
+3300 val loss 7.2123
+3300 val perplexity 1356.0752
+3300 train 7.084377 (lr=1.1932e-04) (hash(x)=45347643)
+3100 val loss 7.2144
+3100 val perplexity 1358.8521
+3100 train 7.504636 (lr=8.1915e-05) (hash(x)=44479330)
+3400 val loss 7.2006
+3400 val perplexity 1340.2240
+3400 train 7.304337 (lr=1.1749e-04) (hash(x)=47797247)
+3200 val loss 7.2444
+3200 val perplexity 1400.2949
+3200 train 7.282361 (lr=8.0745e-05) (hash(x)=54593096)
+3500 val loss 7.1473
+3500 val perplexity 1270.7017
+3500 train 7.050789 (lr=1.1562e-04) (hash(x)=46115683)
+3300 val loss 7.2259
+3300 val perplexity 1374.5964
+3300 train 7.102276 (lr=7.9549e-05) (hash(x)=45347643)
+3600 val loss 7.1462
+3600 val perplexity 1269.2581
+3600 train 6.955878 (lr=1.1372e-04) (hash(x)=44502074)
+3400 val loss 7.2068
+3400 val perplexity 1348.5294
+3400 train 7.334606 (lr=7.8328e-05) (hash(x)=47797247)
+3700 val loss 7.1355
+3700 val perplexity 1255.8024
+3700 train 7.249683 (lr=1.1179e-04) (hash(x)=55388443)
+3500 val loss 7.1718
+3500 val perplexity 1302.1456
+3500 train 7.073598 (lr=7.7082e-05) (hash(x)=46115683)
+3800 val loss 7.1516
+3800 val perplexity 1276.1625
+3800 train 6.977693 (lr=1.0982e-04) (hash(x)=43790341)
+3600 val loss 7.1680
+3600 val perplexity 1297.1829
+3600 train 7.016492 (lr=7.5814e-05) (hash(x)=44502074)
+3900 val loss 7.1724
+3900 val perplexity 1302.9469
+3900 train 7.196667 (lr=1.0783e-04) (hash(x)=50013318)
+3700 val loss 7.1518
+3700 val perplexity 1276.4393
+3700 train 7.289806 (lr=7.4525e-05) (hash(x)=55388443)
+4000 val loss 7.1715
+4000 val perplexity 1301.7539
+4000 train 7.247557 (lr=1.0581e-04) (hash(x)=51704787)
+3800 val loss 7.1593
+3800 val perplexity 1286.0034
+3800 train 6.950498 (lr=7.3215e-05) (hash(x)=43790341)
+4100 val loss 7.1315
+4100 val perplexity 1250.7228
+4100 train 7.198262 (lr=1.0377e-04) (hash(x)=50821964)
+3900 val loss 7.1414
+3900 val perplexity 1263.2340
+3900 train 7.136201 (lr=7.1887e-05) (hash(x)=50013318)
+4200 val loss 7.1552
+4200 val perplexity 1280.7772
+4200 train 7.200123 (lr=1.0171e-04) (hash(x)=49675080)
+4000 val loss 7.1388
+4000 val perplexity 1259.8586
+4000 train 7.210222 (lr=7.0541e-05) (hash(x)=51704787)
+4300 val loss 7.1326
+4300 val perplexity 1252.0898
+4300 train 6.829738 (lr=9.9622e-05) (hash(x)=43239281)
+4100 val loss 7.1296
+4100 val perplexity 1248.3210
+4100 train 7.190423 (lr=6.9180e-05) (hash(x)=50821964)
+4400 val loss 7.1795
+4400 val perplexity 1312.2640
+4400 train 6.915021 (lr=9.7520e-05) (hash(x)=45076737)
+4200 val loss 7.1301
+4200 val perplexity 1249.0374
+4200 train 7.166576 (lr=6.7804e-05) (hash(x)=49675080)
+4500 val loss 7.1361
+4500 val perplexity 1256.5530
+4500 train 7.226453 (lr=9.5403e-05) (hash(x)=57930262)
+4300 val loss 7.1375
+4300 val perplexity 1258.2251
+4300 train 6.828071 (lr=6.6414e-05) (hash(x)=43239281)
+4600 val loss 7.1293
+4600 val perplexity 1247.9628
+4600 train 6.932743 (lr=9.3273e-05) (hash(x)=46721614)
+4400 val loss 7.1356
+4400 val perplexity 1255.8755
+4400 train 6.854441 (lr=6.5013e-05) (hash(x)=45076737)
+4700 val loss 7.1447
+4700 val perplexity 1267.3850
+4700 train 7.003017 (lr=9.1132e-05) (hash(x)=49837920)
+4500 val loss 7.1212
+4500 val perplexity 1237.9576
+4500 train 7.217383 (lr=6.3602e-05) (hash(x)=57930262)
+4800 val loss 7.1396
+4800 val perplexity 1260.9735
+4800 train 7.256048 (lr=8.8982e-05) (hash(x)=48380045)
+4600 val loss 7.1148
+4600 val perplexity 1230.0862
+4600 train 6.938160 (lr=6.2182e-05) (hash(x)=46721614)
+4900 val loss 7.1166
+4900 val perplexity 1232.3048
+4900 train 7.003822 (lr=8.6825e-05) (hash(x)=44202577)
+4700 val loss 7.1160
+4700 val perplexity 1231.4736
+4700 train 6.945326 (lr=6.0754e-05) (hash(x)=49837920)
+5000 val loss 7.1251
+5000 val perplexity 1242.7483
+5000 train 7.123570 (lr=8.4663e-05) (hash(x)=52038024)
+4800 val loss 7.1030
+4800 val perplexity 1215.5854
+4800 train 7.221812 (lr=5.9321e-05) (hash(x)=48380045)
+5100 val loss 7.0881
+5100 val perplexity 1197.5981
+5100 train 7.271852 (lr=8.2500e-05) (hash(x)=53700038)
+4900 val loss 7.0901
+4900 val perplexity 1200.0150
+4900 train 6.977787 (lr=5.7883e-05) (hash(x)=44202577)
+5200 val loss 7.0702
+5200 val perplexity 1176.3390
+5200 train 7.041414 (lr=8.0337e-05) (hash(x)=48137625)
+5000 val loss 7.0778
+5000 val perplexity 1185.3610
+5000 train 7.090936 (lr=5.6442e-05) (hash(x)=52038024)
+5300 val loss 7.0610
+5300 val perplexity 1165.6478
+5300 train 6.929040 (lr=7.8175e-05) (hash(x)=43161573)
+5100 val loss 7.0503
+5100 val perplexity 1153.2191
+5100 train 7.230675 (lr=5.5000e-05) (hash(x)=53700038)
+5400 val loss 7.0882
+5400 val perplexity 1197.7939
+5400 train 7.294000 (lr=7.6018e-05) (hash(x)=56673322)
+5200 val loss 7.0469
+5200 val perplexity 1149.2523
+5200 train 7.025224 (lr=5.3558e-05) (hash(x)=48137625)
+5500 val loss 7.0674
+5500 val perplexity 1173.1265
+5500 train 7.230914 (lr=7.3868e-05) (hash(x)=53468295)
+5300 val loss 7.0382
+5300 val perplexity 1139.3370
+5300 train 6.922906 (lr=5.2117e-05) (hash(x)=43161573)
+5600 val loss 7.0512
+5600 val perplexity 1154.2776
+5600 train 7.362746 (lr=7.1727e-05) (hash(x)=59287280)
+5400 val loss 7.0431
+5400 val perplexity 1144.9668
+5400 train 7.235172 (lr=5.0679e-05) (hash(x)=56673322)
+5700 val loss 7.0557
+5700 val perplexity 1159.4276
+5700 train 7.293931 (lr=6.9597e-05) (hash(x)=57575806)
+5500 val loss 7.0248
+5500 val perplexity 1124.2102
+5500 train 7.204517 (lr=4.9246e-05) (hash(x)=53468295)
+5800 val loss 7.0477
+5800 val perplexity 1150.1986
+5800 train 6.968884 (lr=6.7480e-05) (hash(x)=46897279)
+5600 val loss 7.0278
+5600 val perplexity 1127.5576
+5600 train 7.337178 (lr=4.7818e-05) (hash(x)=59287280)
+5900 val loss 7.0581
+5900 val perplexity 1162.2921
+5900 train 6.934814 (lr=6.5378e-05) (hash(x)=47565679)
+5700 val loss 7.0104
+5700 val perplexity 1108.0684
+5700 train 7.266252 (lr=4.6398e-05) (hash(x)=57575806)
+5800 val loss 6.9975
+5800 val perplexity 1093.8611
+5800 train 6.937305 (lr=4.4987e-05) (hash(x)=46897279)
+6000 val loss 7.0583
+6000 val perplexity 1162.4668
+6000 train 6.945346 (lr=6.3294e-05) (hash(x)=51590090)
+5900 val loss 7.0068
+5900 val perplexity 1104.0801
+5900 train 6.872268 (lr=4.3586e-05) (hash(x)=47565679)
+6100 val loss 7.0384
+6100 val perplexity 1139.5461
+6100 train 7.429985 (lr=6.1230e-05) (hash(x)=59732271)
+6000 val loss 6.9759
+6000 val perplexity 1070.5658
+6000 train 6.855279 (lr=4.2196e-05) (hash(x)=51590090)
+6200 val loss 7.0536
+6200 val perplexity 1157.0466
+6200 train 7.013299 (lr=5.9188e-05) (hash(x)=46394422)
+6100 val loss 6.9696
+6100 val perplexity 1063.7852
+6100 train 7.359732 (lr=4.0820e-05) (hash(x)=59732271)
+6200 val loss 6.9557
+6200 val perplexity 1049.0975
+6200 train 6.948706 (lr=3.9459e-05) (hash(x)=46394422)
+6300 val loss 7.0336
+6300 val perplexity 1134.1621
+6300 train 7.057237 (lr=5.7169e-05) (hash(x)=53748145)
+6300 val loss 6.9495
+6300 val perplexity 1042.5876
+6300 train 6.970834 (lr=3.8113e-05) (hash(x)=53748145)
+6400 val loss 7.0264
+6400 val perplexity 1125.9877
+6400 train 6.902154 (lr=5.5177e-05) (hash(x)=46054751)
+6400 val loss 6.9335
+6400 val perplexity 1026.0587
+6400 train 6.816382 (lr=3.6785e-05) (hash(x)=46054751)
+6500 val loss 6.9250
+6500 val perplexity 1017.3809
+6500 train 7.130113 (lr=3.5475e-05) (hash(x)=51816809)
+6500 val loss 7.0105
+6500 val perplexity 1108.1730
+6500 train 7.192453 (lr=5.3213e-05) (hash(x)=51816809)
+6600 val loss 6.9167
+6600 val perplexity 1009.0242
+6600 train 6.753559 (lr=3.4186e-05) (hash(x)=52453336)
+6600 val loss 7.0129
+6600 val perplexity 1110.8755
+6600 train 6.860138 (lr=5.1279e-05) (hash(x)=52453336)
+6700 val loss 6.9028
+6700 val perplexity 995.0874
+6700 train 6.934293 (lr=3.2918e-05) (hash(x)=49108775)
+6800 val loss 6.8958
+6800 val perplexity 988.1391
+6800 train 6.812268 (lr=3.1672e-05) (hash(x)=46745396)
+6700 val loss 6.9999
+6700 val perplexity 1096.5558
+6700 train 7.014852 (lr=4.9377e-05) (hash(x)=49108775)
+6900 val loss 6.8921
+6900 val perplexity 984.4801
+6900 train 7.014542 (lr=3.0451e-05) (hash(x)=46534986)
+6800 val loss 7.0024
+6800 val perplexity 1099.2440
+6800 train 6.900960 (lr=4.7509e-05) (hash(x)=46745396)
+7000 val loss 6.8785
+7000 val perplexity 971.1417
+7000 train 7.170157 (lr=2.9255e-05) (hash(x)=49317888)
+7100 val loss 6.8711
+7100 val perplexity 964.0015
+7100 train 6.891702 (lr=2.8085e-05) (hash(x)=50360484)
+6900 val loss 6.9860
+6900 val perplexity 1081.3929
+6900 train 7.101790 (lr=4.5676e-05) (hash(x)=46534986)
+7200 val loss 6.8633
+7200 val perplexity 956.5050
+7200 train 6.759152 (lr=2.6943e-05) (hash(x)=49515094)
+7000 val loss 6.9851
+7000 val perplexity 1080.4318
+7000 train 7.236263 (lr=4.3882e-05) (hash(x)=49317888)
+7300 val loss 6.8535
+7300 val perplexity 947.2141
+7300 train 6.831221 (lr=2.5830e-05) (hash(x)=51546861)
+7400 val loss 6.8538
+7400 val perplexity 947.4413
+7400 train 6.841997 (lr=2.4746e-05) (hash(x)=48320948)
+7100 val loss 6.9737
+7100 val perplexity 1068.2135
+7100 train 6.995534 (lr=4.2128e-05) (hash(x)=50360484)
+7500 val loss 6.8490
+7500 val perplexity 942.9482
+7500 train 6.744160 (lr=2.3694e-05) (hash(x)=40167457)
+7200 val loss 6.9736
+7200 val perplexity 1068.0883
+7200 train 6.880153 (lr=4.0414e-05) (hash(x)=49515094)
+7600 val loss 6.8423
+7600 val perplexity 936.6636
+7600 train 6.787523 (lr=2.2674e-05) (hash(x)=49942165)
+7300 val loss 6.9733
+7300 val perplexity 1067.7426
+7300 train 6.945283 (lr=3.8745e-05) (hash(x)=51546861)
+7700 val loss 6.8377
+7700 val perplexity 932.3729
+7700 train 6.593207 (lr=2.1687e-05) (hash(x)=48853311)
+7800 val loss 6.8349
+7800 val perplexity 929.7131
+7800 train 6.757367 (lr=2.0735e-05) (hash(x)=48510117)
+7400 val loss 6.9641
+7400 val perplexity 1058.0121
+7400 train 6.963182 (lr=3.7120e-05) (hash(x)=48320948)
+7900 val loss 6.8322
+7900 val perplexity 927.1918
+7900 train 6.843838 (lr=1.9818e-05) (hash(x)=48339781)
+7500 val loss 6.9680
+7500 val perplexity 1062.0619
+7500 train 6.844945 (lr=3.5541e-05) (hash(x)=40167457)
+8000 val loss 6.8184
+8000 val perplexity 914.4979
+8000 train 6.935864 (lr=1.8936e-05) (hash(x)=54927320)
+8100 val loss 6.8138
+8100 val perplexity 910.3555
+8100 train 6.570134 (lr=1.8092e-05) (hash(x)=46461786)
+7600 val loss 6.9606
+7600 val perplexity 1054.3075
+7600 train 6.885594 (lr=3.4011e-05) (hash(x)=49942165)
+8200 val loss 6.8039
+8200 val perplexity 901.3185
+8200 train 6.741070 (lr=1.7286e-05) (hash(x)=51536260)
+7700 val loss 6.9648
+7700 val perplexity 1058.6656
+7700 train 6.718952 (lr=3.2531e-05) (hash(x)=48853311)
+8300 val loss 6.7965
+8300 val perplexity 894.6973
+8300 train 6.555057 (lr=1.6519e-05) (hash(x)=44770722)
+8400 val loss 6.7932
+8400 val perplexity 891.7589
+8400 train 6.784442 (lr=1.5791e-05) (hash(x)=50104957)
+7800 val loss 6.9621
+7800 val perplexity 1055.7982
+7800 train 6.868794 (lr=3.1102e-05) (hash(x)=48510117)
+8500 val loss 6.7882
+8500 val perplexity 887.2971
+8500 train 6.884346 (lr=1.5103e-05) (hash(x)=50132971)
+7900 val loss 6.9607
+7900 val perplexity 1054.3367
+7900 train 6.978685 (lr=2.9726e-05) (hash(x)=48339781)
+8600 val loss 6.7788
+8600 val perplexity 879.0137
+8600 train 6.735719 (lr=1.4456e-05) (hash(x)=52193699)
+8700 val loss 6.7764
+8700 val perplexity 876.9025
+8700 train 6.774114 (lr=1.3851e-05) (hash(x)=47902319)
+8000 val loss 6.9558
+8000 val perplexity 1049.2601
+8000 train 7.068865 (lr=2.8405e-05) (hash(x)=54927320)
+8800 val loss 6.7649
+8800 val perplexity 866.8619
+8800 train 7.029649 (lr=1.3289e-05) (hash(x)=54904230)
+8100 val loss 6.9537
+8100 val perplexity 1047.0311
+8100 train 6.719138 (lr=2.7138e-05) (hash(x)=46461786)
+8900 val loss 6.7611
+8900 val perplexity 863.6302
+8900 train 6.677911 (lr=1.2769e-05) (hash(x)=46311615)
+9000 val loss 6.7601
+9000 val perplexity 862.7271
+9000 train 6.630702 (lr=1.2292e-05) (hash(x)=48535188)
+8200 val loss 6.9487
+8200 val perplexity 1041.8079
+8200 train 6.892978 (lr=2.5929e-05) (hash(x)=51536260)
+9100 val loss 6.7496
+9100 val perplexity 853.6826
+9100 train 6.778772 (lr=1.1860e-05) (hash(x)=51757372)
+8300 val loss 6.9447
+8300 val perplexity 1037.6498
+8300 train 6.711561 (lr=2.4778e-05) (hash(x)=44770722)
+9200 val loss 6.7470
+9200 val perplexity 851.5067
+9200 train 6.636959 (lr=1.1472e-05) (hash(x)=51131708)
+8400 val loss 6.9416
+8400 val perplexity 1034.3774
+8400 train 6.938999 (lr=2.3686e-05) (hash(x)=50104957)
+9300 val loss 6.7415
+9300 val perplexity 846.8267
+9300 train 6.686966 (lr=1.1128e-05) (hash(x)=44784276)
+8500 val loss 6.9362
+8500 val perplexity 1028.8685
+8500 train 7.034618 (lr=2.2655e-05) (hash(x)=50132971)
+9400 val loss 6.7376
+9400 val perplexity 843.5647
+9400 train 6.831891 (lr=1.0830e-05) (hash(x)=51981169)
+8600 val loss 6.9281
+8600 val perplexity 1020.5430
+8600 train 6.879009 (lr=2.1685e-05) (hash(x)=52193699)
+9500 val loss 6.7318
+9500 val perplexity 838.6479
+9500 train 6.684929 (lr=1.0577e-05) (hash(x)=47232936)
+8700 val loss 6.9265
+8700 val perplexity 1018.9719
+8700 train 6.928772 (lr=2.0777e-05) (hash(x)=47902319)
+9600 val loss 6.7301
+9600 val perplexity 837.1972
+9600 train 6.762636 (lr=1.0369e-05) (hash(x)=53800450)
+8800 val loss 6.9224
+8800 val perplexity 1014.7075
+8800 train 7.177705 (lr=1.9933e-05) (hash(x)=54904230)
+9700 val loss 6.7257
+9700 val perplexity 833.5361
+9700 train 6.842899 (lr=1.0208e-05) (hash(x)=55768123)
+8900 val loss 6.9157
+8900 val perplexity 1008.0081
+8900 train 6.824413 (lr=1.9153e-05) (hash(x)=46311615)
+9800 val loss 6.7242
+9800 val perplexity 832.2679
+9800 train 6.656127 (lr=1.0092e-05) (hash(x)=47745177)
+9900 val loss 6.7249
+9900 val perplexity 832.9302
+9900 train 6.977684 (lr=1.0023e-05) (hash(x)=56592246)
+9000 val loss 6.9129
+9000 val perplexity 1005.1110
+9000 train 6.765456 (lr=1.8439e-05) (hash(x)=48535188)
+9999 val loss 6.7242
+9999 val perplexity 832.3088
+9100 val loss 6.9100
+9100 val perplexity 1002.2089
+9100 train 6.928776 (lr=1.7790e-05) (hash(x)=51757372)
+9200 val loss 6.9081
+9200 val perplexity 1000.3215
+9200 train 6.806732 (lr=1.7208e-05) (hash(x)=51131708)
+9300 val loss 6.9059
+9300 val perplexity 998.1198
+9300 train 6.841065 (lr=1.6692e-05) (hash(x)=44784276)
+9400 val loss 6.9072
+9400 val perplexity 999.4905
+9400 train 6.993570 (lr=1.6245e-05) (hash(x)=51981169)
+9500 val loss 6.9063
+9500 val perplexity 998.5016
+9500 train 6.858905 (lr=1.5865e-05) (hash(x)=47232936)
+9600 val loss 6.9042
+9600 val perplexity 996.4511
+9600 train 6.943208 (lr=1.5554e-05) (hash(x)=53800450)

attention_kindselective_n_heads4_seed1340/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e81f78916f1fcd907958b2ffa161577c66b852e8fd3c0779d687fb593da6ad3b
+oid sha256:e5d1d69356d9c198afaea93cf7ccf66f2d5809310214a3e417a834f53abe451a
 size 92843394

attention_kindselective_n_heads4_seed1340/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6018096758756901f3094eb569bb1ff11a3078e161f7128e635f37a4fe73d4ac
+oid sha256:3969de9631fa1091a1c449d2a388db43ecd8c763683bd08da56a12468a2b209a
 size 92843394

attention_kindselective_n_heads4_seed1340/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd6e8eabd0b98757525d429e9a980b400592f2326c9b6e08461fb96801fc5fa6
+oid sha256:cf4eafbd63635551ef8204a5b1e4fa01c7111a78c8eb15db22b64873e52bad4b
 size 92843394

attention_kindselective_n_heads4_seed1340/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0edff24b6add430f0202c2e51013b6a0300d2a177d1940ce2c18dd7623b89ca9
+oid sha256:de39ee085ed7a1f670eb15c44b15b334d336a74a051f9d7eee6caab827ba805b
 size 92843394

attention_kindselective_n_heads4_seed1340/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d15f93b84a2def82d5aafdfa962ee066ed3a7abc6563962369764cb0ef0b364b
+oid sha256:06179ff7fbca3cf6f6d3d1546cd7e6f27c331a63a9a71de710896c329a40b635
 size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca694617631608a39e17abec4008d3ce4640f7e1ad9e64b3ffbd590a9a2c5aaa
+oid sha256:534c88f2b95620226732b552d992f6d228437ef793d6a67da093fc7cc42ce5e8
 size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25b0c8eb9d31c8f5adf2d93370de5825f34eb19beb4933795ecf5b8f04413bd9
+oid sha256:1f9ae9b15de67f44a4e1d5dcb110a3995dd6867e2ff4a70347ffd32b561b4229
 size 179406214

attention_kindselective_n_heads4_seed1340/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b235a69300a9359b79b862673bd8470faa6c26089f65fcee66976e068cc6662e
+oid sha256:64df6be512db451ba0791b8b0f279346c9ed0ec1cdbb0a281d2a5d76a64ac0f6
 size 179406214