Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_2_1338", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_10240_2_1338", "n_embd": 128}

attention_kindselective_n_heads2_seed1338/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
+oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
 size 964

attention_kindselective_n_heads2_seed1338/dataloader_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+oid sha256:7f03ed2ebf741f15e13c79e6cc1e9a19b308450d81cc3b4d8d0338c63d77ca59
 size 964

attention_kindselective_n_heads2_seed1338/dataloader_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+oid sha256:82590037fb2eecbec961f7967a8dd1b8d85515d31a252f66b92b8139858a8b7c
 size 964

attention_kindselective_n_heads2_seed1338/dataloader_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+oid sha256:c544303717d12355a69b8ffb1eb109434e4fdccfd5a61279b6e8ba2e870d6700
 size 964

attention_kindselective_n_heads2_seed1338/log2.txt

@@ -1,303 +1,606 @@
 max_steps: 10000
-0 val loss 11.7630
-0 val perplexity 128408.9922
-0 train 11.769258 (lr=1.5000e-07) (hash(x)=150327452)
-100 val loss 10.4010
-100 val perplexity 32894.0078
-100 train 10.388559 (lr=1.5150e-05) (hash(x)=166441190)
-200 val loss 9.3220
-200 val perplexity 11180.8262
-200 train 9.420354 (lr=3.0000e-05) (hash(x)=166780046)
-300 val loss 8.1198
-300 val perplexity 3360.3025
-300 train 8.114033 (lr=2.9993e-05) (hash(x)=159835303)
-400 val loss 7.7643
-400 val perplexity 2355.1169
-400 train 7.589100 (lr=2.9972e-05) (hash(x)=155040610)
-500 val loss 7.6660
-500 val perplexity 2134.4812
-500 train 7.423435 (lr=2.9938e-05) (hash(x)=130190460)
-600 val loss 7.6109
-600 val perplexity 2020.0375
-600 train 7.641937 (lr=2.9889e-05) (hash(x)=155504036)
-700 val loss 7.5711
-700 val perplexity 1941.2490
-700 train 7.414425 (lr=2.9827e-05) (hash(x)=137347213)
-800 val loss 7.5448
-800 val perplexity 1890.9220
-800 train 7.399529 (lr=2.9751e-05) (hash(x)=143823248)
-900 val loss 7.5278
-900 val perplexity 1859.0282
-900 train 7.648718 (lr=2.9662e-05) (hash(x)=156260416)
-1000 val loss 7.5065
-1000 val perplexity 1819.8811
-1000 train 7.474764 (lr=2.9558e-05) (hash(x)=143734685)
-1100 val loss 7.4984
-1100 val perplexity 1805.1945
-1100 train 7.444295 (lr=2.9442e-05) (hash(x)=160013925)
-1200 val loss 7.4866
-1200 val perplexity 1784.0110
-1200 train 7.261100 (lr=2.9312e-05) (hash(x)=150678249)
-1300 val loss 7.4759
-1300 val perplexity 1764.9208
-1300 train 7.281197 (lr=2.9169e-05) (hash(x)=149073315)
-1400 val loss 7.4569
-1400 val perplexity 1731.7219
-1400 train 7.886369 (lr=2.9013e-05) (hash(x)=175802021)
-1500 val loss 7.4343
-1500 val perplexity 1693.1479
-1500 train 7.685257 (lr=2.8845e-05) (hash(x)=171034639)
-1600 val loss 7.4127
-1600 val perplexity 1656.9448
-1600 train 7.640902 (lr=2.8663e-05) (hash(x)=158681215)
-1700 val loss 7.3985
-1700 val perplexity 1633.6101
-1700 train 7.341957 (lr=2.8469e-05) (hash(x)=152116061)
-1800 val loss 7.3886
-1800 val perplexity 1617.4152
-1800 train 7.323990 (lr=2.8263e-05) (hash(x)=146108145)
-1900 val loss 7.3732
-1900 val perplexity 1592.7101
-1900 train 7.219169 (lr=2.8044e-05) (hash(x)=147598108)
-2000 val loss 7.3618
-2000 val perplexity 1574.7190
-2000 train 7.232282 (lr=2.7814e-05) (hash(x)=154996086)
-2100 val loss 7.3471
-2100 val perplexity 1551.6771
-2100 train 7.141953 (lr=2.7572e-05) (hash(x)=153396183)
-2200 val loss 7.3012
-2200 val perplexity 1482.1154
-2200 train 7.291854 (lr=2.7319e-05) (hash(x)=153885445)
-2300 val loss 7.2835
-2300 val perplexity 1456.0382
-2300 train 7.268628 (lr=2.7055e-05) (hash(x)=159666385)
-2400 val loss 7.2455
-2400 val perplexity 1401.8269
-2400 train 7.193364 (lr=2.6780e-05) (hash(x)=142353087)
-2500 val loss 7.2123
-2500 val perplexity 1355.9503
-2500 train 7.208140 (lr=2.6494e-05) (hash(x)=146491718)
-2600 val loss 7.1809
-2600 val perplexity 1314.0718
-2600 train 7.118464 (lr=2.6198e-05) (hash(x)=150750353)
-2700 val loss 7.1504
-2700 val perplexity 1274.6262
-2700 train 6.869782 (lr=2.5892e-05) (hash(x)=129849193)
-2800 val loss 7.1357
-2800 val perplexity 1256.0383
-2800 train 6.996274 (lr=2.5576e-05) (hash(x)=152767913)
-2900 val loss 7.1052
-2900 val perplexity 1218.3030
-2900 train 6.898202 (lr=2.5251e-05) (hash(x)=146531140)
-3000 val loss 7.0670
-3000 val perplexity 1172.6589
-3000 train 7.135310 (lr=2.4917e-05) (hash(x)=151562048)
-3100 val loss 7.0301
-3100 val perplexity 1130.1343
-3100 train 7.015220 (lr=2.4574e-05) (hash(x)=146001424)
-3200 val loss 7.0035
-3200 val perplexity 1100.5248
-3200 train 7.012419 (lr=2.4224e-05) (hash(x)=166486165)
-3300 val loss 6.9697
-3300 val perplexity 1063.8522
-3300 train 6.894574 (lr=2.3865e-05) (hash(x)=150866680)
-3400 val loss 6.9519
-3400 val perplexity 1045.1385
-3400 train 6.863424 (lr=2.3498e-05) (hash(x)=143900419)
-3500 val loss 6.9233
-3500 val perplexity 1015.6485
-3500 train 6.757936 (lr=2.3125e-05) (hash(x)=148845794)
-3600 val loss 6.9006
-3600 val perplexity 992.9057
-3600 train 6.751880 (lr=2.2744e-05) (hash(x)=145667796)
-3700 val loss 6.8779
-3700 val perplexity 970.6259
-3700 train 6.897323 (lr=2.2357e-05) (hash(x)=163563851)
-3800 val loss 6.8582
-3800 val perplexity 951.6208
-3800 train 6.815534 (lr=2.1965e-05) (hash(x)=147488689)
-3900 val loss 6.8149
-3900 val perplexity 911.3462
-3900 train 6.802123 (lr=2.1566e-05) (hash(x)=148186608)
-4000 val loss 6.7898
-4000 val perplexity 888.6940
-4000 train 6.638687 (lr=2.1162e-05) (hash(x)=142970187)
-4100 val loss 6.7610
-4100 val perplexity 863.5334
-4100 train 6.777088 (lr=2.0754e-05) (hash(x)=141584883)
-4200 val loss 6.7359
-4200 val perplexity 842.1255
-4200 train 6.537104 (lr=2.0341e-05) (hash(x)=145664585)
-4300 val loss 6.7157
-4300 val perplexity 825.2897
-4300 train 6.550407 (lr=1.9924e-05) (hash(x)=143736499)
-4400 val loss 6.7043
-4400 val perplexity 815.9366
-4400 train 6.524540 (lr=1.9504e-05) (hash(x)=151883322)
-4500 val loss 6.6853
-4500 val perplexity 800.5656
-4500 train 6.591630 (lr=1.9081e-05) (hash(x)=153904871)
-4600 val loss 6.6625
-4600 val perplexity 782.4913
-4600 train 6.722117 (lr=1.8655e-05) (hash(x)=154893521)
-4700 val loss 6.6334
-4700 val perplexity 760.0438
-4700 train 6.709569 (lr=1.8226e-05) (hash(x)=152323949)
-4800 val loss 6.6148
-4800 val perplexity 746.0839
-4800 train 6.544276 (lr=1.7796e-05) (hash(x)=154104619)
-4900 val loss 6.5995
-4900 val perplexity 734.7310
-4900 train 6.678655 (lr=1.7365e-05) (hash(x)=146311426)
-5000 val loss 6.5837
-5000 val perplexity 723.2397
-5000 train 6.608577 (lr=1.6933e-05) (hash(x)=156741847)
-5100 val loss 6.5716
-5100 val perplexity 714.5206
-5100 train 6.365155 (lr=1.6500e-05) (hash(x)=142086346)
-5200 val loss 6.5636
-5200 val perplexity 708.8334
-5200 train 6.348958 (lr=1.6067e-05) (hash(x)=150265428)
-5300 val loss 6.5557
-5300 val perplexity 703.2093
-5300 train 6.452664 (lr=1.5635e-05) (hash(x)=151339108)
-5400 val loss 6.5387
-5400 val perplexity 691.3912
-5400 train 6.534310 (lr=1.5204e-05) (hash(x)=154654372)
-5500 val loss 6.5184
-5500 val perplexity 677.4675
-5500 train 6.529605 (lr=1.4774e-05) (hash(x)=150575051)
-5600 val loss 6.5039
-5600 val perplexity 667.7398
-5600 train 6.391018 (lr=1.4345e-05) (hash(x)=140396423)
-5700 val loss 6.4901
-5700 val perplexity 658.5672
-5700 train 6.374229 (lr=1.3919e-05) (hash(x)=144678758)
-5800 val loss 6.4787
-5800 val perplexity 651.1094
-5800 train 6.583964 (lr=1.3496e-05) (hash(x)=151992743)
-5900 val loss 6.4715
-5900 val perplexity 646.4711
-5900 train 6.251740 (lr=1.3076e-05) (hash(x)=144396927)
-6000 val loss 6.4619
-6000 val perplexity 640.2845
-6000 train 6.434009 (lr=1.2659e-05) (hash(x)=165478625)
-6100 val loss 6.4656
-6100 val perplexity 642.6783
-6100 train 6.216571 (lr=1.2246e-05) (hash(x)=147088621)
-6200 val loss 6.4535
-6200 val perplexity 634.8896
-6200 train 6.240591 (lr=1.1838e-05) (hash(x)=140794994)
-6300 val loss 6.4400
-6300 val perplexity 626.4254
-6300 train 6.364601 (lr=1.1434e-05) (hash(x)=134780906)
-6400 val loss 6.4261
-6400 val perplexity 617.7345
-6400 train 6.413403 (lr=1.1035e-05) (hash(x)=149023655)
-6500 val loss 6.4153
-6500 val perplexity 611.1179
-6500 train 6.281328 (lr=1.0643e-05) (hash(x)=147497796)
-6600 val loss 6.4143
-6600 val perplexity 610.5001
-6600 train 6.261950 (lr=1.0256e-05) (hash(x)=152902689)
-6700 val loss 6.4073
-6700 val perplexity 606.2618
-6700 train 6.460744 (lr=9.8753e-06) (hash(x)=153846046)
-6800 val loss 6.3954
-6800 val perplexity 599.0741
-6800 train 6.533175 (lr=9.5017e-06) (hash(x)=158512738)
-6900 val loss 6.3901
-6900 val perplexity 595.9360
-6900 train 6.872580 (lr=9.1353e-06) (hash(x)=156849968)
-7000 val loss 6.3825
-7000 val perplexity 591.4265
-7000 train 6.211296 (lr=8.7764e-06) (hash(x)=142395855)
-7100 val loss 6.3768
-7100 val perplexity 588.0710
-7100 train 6.247732 (lr=8.4255e-06) (hash(x)=147114884)
-7200 val loss 6.3739
-7200 val perplexity 586.3677
-7200 train 6.394201 (lr=8.0829e-06) (hash(x)=156979839)
-7300 val loss 6.3695
-7300 val perplexity 583.7812
-7300 train 6.134778 (lr=7.7489e-06) (hash(x)=145584373)
-7400 val loss 6.3681
-7400 val perplexity 582.9282
-7400 train 6.108246 (lr=7.4239e-06) (hash(x)=141508204)
-7500 val loss 6.3592
-7500 val perplexity 577.7770
-7500 train 6.410224 (lr=7.1083e-06) (hash(x)=148803965)
-7600 val loss 6.3525
-7600 val perplexity 573.9427
-7600 train 6.412134 (lr=6.8023e-06) (hash(x)=151019676)
-7700 val loss 6.3469
-7700 val perplexity 570.7330
-7700 train 6.363933 (lr=6.5062e-06) (hash(x)=143155750)
-7800 val loss 6.3412
-7800 val perplexity 567.4922
-7800 train 6.365192 (lr=6.2205e-06) (hash(x)=152569653)
-7900 val loss 6.3391
-7900 val perplexity 566.2675
-7900 train 6.198283 (lr=5.9453e-06) (hash(x)=143519455)
-8000 val loss 6.3370
-8000 val perplexity 565.0965
-8000 train 6.521562 (lr=5.6809e-06) (hash(x)=161180944)
-8100 val loss 6.3356
-8100 val perplexity 564.3146
-8100 train 6.367569 (lr=5.4277e-06) (hash(x)=154107345)
-8200 val loss 6.3359
-8200 val perplexity 564.4596
-8200 train 6.350157 (lr=5.1858e-06) (hash(x)=152486517)
-8300 val loss 6.3262
-8300 val perplexity 559.0314
-8300 train 6.355000 (lr=4.9556e-06) (hash(x)=156167749)
-8400 val loss 6.3224
-8400 val perplexity 556.9216
-8400 train 6.399919 (lr=4.7372e-06) (hash(x)=149155006)
-8500 val loss 6.3170
-8500 val perplexity 553.9238
-8500 train 6.291015 (lr=4.5309e-06) (hash(x)=147844390)
-8600 val loss 6.3133
-8600 val perplexity 551.8853
-8600 train 6.573327 (lr=4.3369e-06) (hash(x)=165753320)
-8700 val loss 6.3135
-8700 val perplexity 551.9460
-8700 train 6.098110 (lr=4.1554e-06) (hash(x)=146079979)
-8800 val loss 6.3118
-8800 val perplexity 551.0446
-8800 train 6.534912 (lr=3.9866e-06) (hash(x)=172259509)
-8900 val loss 6.3120
-8900 val perplexity 551.1379
-8900 train 5.987774 (lr=3.8307e-06) (hash(x)=145148314)
-9000 val loss 6.3112
-9000 val perplexity 550.7257
-9000 train 6.234556 (lr=3.6877e-06) (hash(x)=144250633)
-9100 val loss 6.3057
-9100 val perplexity 547.6868
-9100 train 6.414195 (lr=3.5580e-06) (hash(x)=157219797)
-9200 val loss 6.2999
-9200 val perplexity 544.5214
-9200 train 6.288376 (lr=3.4415e-06) (hash(x)=142743778)
-9300 val loss 6.2994
-9300 val perplexity 544.2646
-9300 train 6.183533 (lr=3.3385e-06) (hash(x)=139669771)
-9400 val loss 6.2957
-9400 val perplexity 542.2340
-9400 train 6.265487 (lr=3.2490e-06) (hash(x)=145916843)
-9500 val loss 6.2935
-9500 val perplexity 541.0405
-9500 train 6.209871 (lr=3.1730e-06) (hash(x)=150196125)
-9600 val loss 6.2931
-9600 val perplexity 540.8432
-9600 train 6.577899 (lr=3.1108e-06) (hash(x)=160041419)
-9700 val loss 6.2934
-9700 val perplexity 540.9856
-9700 train 6.079773 (lr=3.0624e-06) (hash(x)=139931627)
-9800 val loss 6.2910
-9800 val perplexity 539.6734
-9800 train 6.253589 (lr=3.0277e-06) (hash(x)=150370792)
-9900 val loss 6.2893
-9900 val perplexity 538.8027
-9900 train 6.149877 (lr=3.0069e-06) (hash(x)=153014886)
-9999 val loss 6.2893
-9999 val perplexity 538.7606
+max_steps: 10000
+0 val loss 11.7640
+0 val perplexity 128545.8516
+0 val loss 11.7640
+0 val perplexity 128545.8516
+0 train 11.762399 (lr=2.5000e-07) (hash(x)=50671684)
+0 train 11.762399 (lr=3.5000e-07) (hash(x)=50671684)
+100 val loss 10.2612
+100 val perplexity 28599.8301
+100 val loss 10.0855
+100 val perplexity 23992.4160
+100 train 10.199640 (lr=2.5250e-05) (hash(x)=52740221)
+100 train 10.027347 (lr=3.5350e-05) (hash(x)=52740221)
+200 val loss 9.0641
+200 val perplexity 8639.7002
+200 train 9.082980 (lr=5.0000e-05) (hash(x)=49034180)
+200 val loss 8.3482
+200 val perplexity 4222.5566
+200 train 8.351851 (lr=7.0000e-05) (hash(x)=49034180)
+300 val loss 8.1940
+300 val perplexity 3619.2537
+300 train 8.292101 (lr=4.9988e-05) (hash(x)=63180688)
+300 val loss 7.7296
+300 val perplexity 2274.6714
+300 train 7.949653 (lr=6.9984e-05) (hash(x)=63180688)
+400 val loss 7.8544
+400 val perplexity 2577.1091
+400 train 7.816474 (lr=4.9954e-05) (hash(x)=50373500)
+400 val loss 7.6567
+400 val perplexity 2114.7312
+400 train 7.666688 (lr=6.9935e-05) (hash(x)=50373500)
+500 val loss 7.6951
+500 val perplexity 2197.5679
+500 train 7.606697 (lr=4.9896e-05) (hash(x)=44547422)
+500 val loss 7.6381
+500 val perplexity 2075.7231
+500 train 7.539029 (lr=6.9854e-05) (hash(x)=44547422)
+600 val loss 7.6593
+600 val perplexity 2120.3745
+600 train 7.554819 (lr=4.9815e-05) (hash(x)=47184699)
+600 val loss 7.6413
+600 val perplexity 2082.3872
+600 train 7.530231 (lr=6.9741e-05) (hash(x)=47184699)
+700 val loss 7.6521
+700 val perplexity 2104.9644
+700 train 7.565280 (lr=4.9712e-05) (hash(x)=51374582)
+700 val loss 7.6389
+700 val perplexity 2077.5461
+700 train 7.553903 (lr=6.9596e-05) (hash(x)=51374582)
+800 val loss 7.6369
+800 val perplexity 2073.2136
+800 train 7.378779 (lr=4.9585e-05) (hash(x)=46264805)
+800 val loss 7.6290
+800 val perplexity 2057.0747
+800 train 7.379150 (lr=6.9419e-05) (hash(x)=46264805)
+900 val loss 7.6547
+900 val perplexity 2110.6108
+900 train 7.937618 (lr=4.9436e-05) (hash(x)=61178712)
+900 val loss 7.6323
+900 val perplexity 2063.8235
+900 train 7.921933 (lr=6.9210e-05) (hash(x)=61178712)
+1000 val loss 7.6362
+1000 val perplexity 2071.7578
+1000 train 7.647681 (lr=4.9264e-05) (hash(x)=50886520)
+1000 val loss 7.6263
+1000 val perplexity 2051.3552
+1000 train 7.646755 (lr=6.8970e-05) (hash(x)=50886520)
+1100 val loss 7.6266
+1100 val perplexity 2052.0225
+1100 train 7.411226 (lr=4.9070e-05) (hash(x)=48600099)
+1100 val loss 7.6231
+1100 val perplexity 2044.9672
+1100 train 7.445827 (lr=6.8698e-05) (hash(x)=48600099)
+1200 val loss 7.6176
+1200 val perplexity 2033.6812
+1200 train 7.354834 (lr=4.8854e-05) (hash(x)=50146792)
+1200 val loss 7.6272
+1200 val perplexity 2053.3645
+1200 train 7.383700 (lr=6.8395e-05) (hash(x)=50146792)
+1300 val loss 7.6195
+1300 val perplexity 2037.5619
+1300 train 7.491829 (lr=4.8616e-05) (hash(x)=52617313)
+1300 val loss 7.6346
+1300 val perplexity 2068.6394
+1300 train 7.519730 (lr=6.8062e-05) (hash(x)=52617313)
+1400 val loss 7.6085
+1400 val perplexity 2015.1982
+1400 train 7.284451 (lr=4.8356e-05) (hash(x)=49794446)
+1400 val loss 7.6309
+1400 val perplexity 2060.8892
+1400 train 7.316226 (lr=6.7698e-05) (hash(x)=49794446)
+1500 val loss 7.6101
+1500 val perplexity 2018.5585
+1500 train 7.262372 (lr=4.8074e-05) (hash(x)=50766317)
+1500 val loss 7.6201
+1500 val perplexity 2038.8419
+1500 train 7.261213 (lr=6.7304e-05) (hash(x)=50766317)
+1600 val loss 7.6007
+1600 val perplexity 1999.5605
+1600 train 7.493066 (lr=4.7772e-05) (hash(x)=55551175)
+1600 val loss 7.5985
+1600 val perplexity 1995.1804
+1600 train 7.499768 (lr=6.6881e-05) (hash(x)=55551175)
+1700 val loss 7.5795
+1700 val perplexity 1957.7035
+1700 train 7.733157 (lr=4.7448e-05) (hash(x)=56717172)
+1700 val loss 7.5674
+1700 val perplexity 1934.0377
+1700 train 7.691600 (lr=6.6428e-05) (hash(x)=56717172)
+1800 val loss 7.5356
+1800 val perplexity 1873.6436
+1800 train 7.837944 (lr=4.7105e-05) (hash(x)=55376447)
+1800 val loss 7.5258
+1800 val perplexity 1855.3097
+1800 train 7.796223 (lr=6.5947e-05) (hash(x)=55376447)
+1900 val loss 7.5072
+1900 val perplexity 1821.0981
+1900 train 7.223547 (lr=4.6741e-05) (hash(x)=43810837)
+1900 val loss 7.5078
+1900 val perplexity 1822.2770
+1900 train 7.237650 (lr=6.5437e-05) (hash(x)=43810837)
+2000 val loss 7.4870
+2000 val perplexity 1784.7079
+2000 train 7.566131 (lr=4.6357e-05) (hash(x)=50881655)
+2000 val loss 7.4835
+2000 val perplexity 1778.4316
+2000 train 7.579944 (lr=6.4900e-05) (hash(x)=50881655)
+2100 val loss 7.4819
+2100 val perplexity 1775.6853
+2100 train 7.358100 (lr=4.5954e-05) (hash(x)=49386015)
+2100 val loss 7.4574
+2100 val perplexity 1732.5850
+2100 train 7.333408 (lr=6.4335e-05) (hash(x)=49386015)
+2200 val loss 7.4734
+2200 val perplexity 1760.6473
+2200 train 7.392198 (lr=4.5532e-05) (hash(x)=48572079)
+2200 val loss 7.4651
+2200 val perplexity 1745.9648
+2200 train 7.405998 (lr=6.3745e-05) (hash(x)=48572079)
+2300 val loss 7.4649
+2300 val perplexity 1745.6702
+2300 train 7.528904 (lr=4.5091e-05) (hash(x)=54950719)
+2300 val loss 7.4181
+2300 val perplexity 1665.8230
+2300 train 7.485063 (lr=6.3128e-05) (hash(x)=54950719)
+2400 val loss 7.4481
+2400 val perplexity 1716.5986
+2400 train 7.104608 (lr=4.4633e-05) (hash(x)=42190240)
+2400 val loss 7.3888
+2400 val perplexity 1617.7377
+2400 train 7.051908 (lr=6.2486e-05) (hash(x)=42190240)
+2500 val loss 7.4384
+2500 val perplexity 1700.0844
+2500 train 7.503536 (lr=4.4156e-05) (hash(x)=45223539)
+2500 val loss 7.3688
+2500 val perplexity 1585.7074
+2500 train 7.440190 (lr=6.1819e-05) (hash(x)=45223539)
+2600 val loss 7.4214
+2600 val perplexity 1671.3839
+2600 train 7.425591 (lr=4.3663e-05) (hash(x)=54037353)
+2600 val loss 7.3434
+2600 val perplexity 1546.0243
+2600 train 7.361808 (lr=6.1128e-05) (hash(x)=54037353)
+2700 val loss 7.3990
+2700 val perplexity 1634.3760
+2700 train 7.943274 (lr=4.3153e-05) (hash(x)=59131616)
+2700 val loss 7.3256
+2700 val perplexity 1518.7589
+2700 train 7.771473 (lr=6.0414e-05) (hash(x)=59131616)
+2800 val loss 7.3895
+2800 val perplexity 1618.9755
+2800 train 7.214322 (lr=4.2627e-05) (hash(x)=45882743)
+2800 val loss 7.2809
+2800 val perplexity 1452.2246
+2800 train 7.131896 (lr=5.9677e-05) (hash(x)=45882743)
+2900 val loss 7.3756
+2900 val perplexity 1596.5599
+2900 train 6.986534 (lr=4.2085e-05) (hash(x)=43758910)
+3000 val loss 7.4025
+3000 val perplexity 1640.0563
+3000 train 7.337453 (lr=4.1529e-05) (hash(x)=47965974)
+2900 val loss 7.2484
+2900 val perplexity 1405.8976
+2900 train 6.918225 (lr=5.8919e-05) (hash(x)=43758910)
+3100 val loss 7.3828
+3100 val perplexity 1608.0157
+3100 train 7.282964 (lr=4.0957e-05) (hash(x)=48205243)
+3000 val loss 7.2421
+3000 val perplexity 1397.0543
+3000 train 7.161748 (lr=5.8140e-05) (hash(x)=47965974)
+3200 val loss 7.3542
+3200 val perplexity 1562.7103
+3200 train 7.431534 (lr=4.0373e-05) (hash(x)=54511383)
+3100 val loss 7.2156
+3100 val perplexity 1360.4982
+3100 train 7.109374 (lr=5.7340e-05) (hash(x)=48205243)
+3300 val loss 7.3452
+3300 val perplexity 1548.7839
+3300 train 7.324124 (lr=3.9775e-05) (hash(x)=54428388)
+3200 val loss 7.2021
+3200 val perplexity 1342.2188
+3200 train 7.273560 (lr=5.6522e-05) (hash(x)=54511383)
+3400 val loss 7.3255
+3400 val perplexity 1518.5460
+3400 train 7.373497 (lr=3.9164e-05) (hash(x)=48115990)
+3300 val loss 7.1854
+3300 val perplexity 1320.0015
+3300 train 7.137845 (lr=5.5684e-05) (hash(x)=54428388)
+3500 val loss 7.3067
+3500 val perplexity 1490.2999
+3500 train 6.951129 (lr=3.8541e-05) (hash(x)=41137345)
+3400 val loss 7.1642
+3400 val perplexity 1292.2944
+3400 train 7.198205 (lr=5.4829e-05) (hash(x)=48115990)
+3600 val loss 7.3099
+3600 val perplexity 1495.0908
+3600 train 7.210977 (lr=3.7907e-05) (hash(x)=55186224)
+3500 val loss 7.1480
+3500 val perplexity 1271.5587
+3500 train 6.770042 (lr=5.3958e-05) (hash(x)=41137345)
+3700 val loss 7.2953
+3700 val perplexity 1473.4081
+3700 train 7.162465 (lr=3.7262e-05) (hash(x)=54990049)
+3600 val loss 7.1600
+3600 val perplexity 1286.9684
+3600 train 7.056310 (lr=5.3070e-05) (hash(x)=55186224)
+3800 val loss 7.2900
+3800 val perplexity 1465.5238
+3800 train 7.036756 (lr=3.6608e-05) (hash(x)=46288812)
+3700 val loss 7.1365
+3700 val perplexity 1257.0605
+3700 train 7.001248 (lr=5.2167e-05) (hash(x)=54990049)
+3900 val loss 7.2680
+3900 val perplexity 1433.7078
+3900 train 6.930634 (lr=3.5944e-05) (hash(x)=45829773)
+3800 val loss 7.1138
+3800 val perplexity 1228.7883
+3800 train 6.855778 (lr=5.1251e-05) (hash(x)=46288812)
+4000 val loss 7.2640
+4000 val perplexity 1427.9847
+4000 train 7.045815 (lr=3.5271e-05) (hash(x)=52499943)
+3900 val loss 7.1105
+3900 val perplexity 1224.7800
+3900 train 6.770267 (lr=5.0321e-05) (hash(x)=45829773)
+4100 val loss 7.2214
+4100 val perplexity 1368.4528
+4100 train 7.119732 (lr=3.4590e-05) (hash(x)=48563796)
+4000 val loss 7.0887
+4000 val perplexity 1198.3544
+4000 train 6.878484 (lr=4.9379e-05) (hash(x)=52499943)
+4200 val loss 7.1987
+4200 val perplexity 1337.7422
+4200 train 7.161106 (lr=3.3902e-05) (hash(x)=49165143)
+4100 val loss 7.0505
+4100 val perplexity 1153.4874
+4100 train 6.955088 (lr=4.8426e-05) (hash(x)=48563796)
+4300 val loss 7.1848
+4300 val perplexity 1319.2406
+4300 train 7.263635 (lr=3.3207e-05) (hash(x)=50973176)
+4200 val loss 6.9988
+4200 val perplexity 1095.3329
+4200 train 6.969129 (lr=4.7463e-05) (hash(x)=49165143)
+4400 val loss 7.1592
+4400 val perplexity 1285.8815
+4400 train 7.206085 (lr=3.2507e-05) (hash(x)=55275124)
+4300 val loss 6.9748
+4300 val perplexity 1069.3761
+4300 train 7.032444 (lr=4.6490e-05) (hash(x)=50973176)
+4500 val loss 7.1423
+4500 val perplexity 1264.3616
+4500 train 7.480568 (lr=3.1801e-05) (hash(x)=58646505)
+4400 val loss 6.9587
+4400 val perplexity 1052.2388
+4400 train 7.023558 (lr=4.5509e-05) (hash(x)=55275124)
+4600 val loss 7.1201
+4600 val perplexity 1236.5511
+4600 train 6.934882 (lr=3.1091e-05) (hash(x)=42554666)
+4500 val loss 6.9399
+4500 val perplexity 1032.6294
+4500 train 7.294427 (lr=4.4521e-05) (hash(x)=58646505)
+4700 val loss 7.1123
+4700 val perplexity 1226.9720
+4700 train 7.021177 (lr=3.0377e-05) (hash(x)=47846764)
+4600 val loss 6.9231
+4600 val perplexity 1015.4349
+4600 train 6.728717 (lr=4.3527e-05) (hash(x)=42554666)
+4800 val loss 7.1072
+4800 val perplexity 1220.7145
+4800 train 7.640239 (lr=2.9661e-05) (hash(x)=58239019)
+4700 val loss 6.9024
+4700 val perplexity 994.6642
+4700 train 6.794252 (lr=4.2528e-05) (hash(x)=47846764)
+4900 val loss 7.0912
+4900 val perplexity 1201.3014
+4900 train 7.122570 (lr=2.8942e-05) (hash(x)=50711220)
+4800 val loss 6.8864
+4800 val perplexity 978.8246
+4800 train 7.385535 (lr=4.1525e-05) (hash(x)=58239019)
+5000 val loss 7.0806
+5000 val perplexity 1188.6388
+5000 train 7.011305 (lr=2.8221e-05) (hash(x)=45994194)
+4900 val loss 6.8732
+4900 val perplexity 966.0390
+4900 train 6.911656 (lr=4.0518e-05) (hash(x)=50711220)
+5100 val loss 7.0847
+5100 val perplexity 1193.5157
+5100 train 6.948140 (lr=2.7500e-05) (hash(x)=48659050)
+5000 val loss 6.8794
+5000 val perplexity 972.0590
+5000 train 6.808244 (lr=3.9510e-05) (hash(x)=45994194)
+5200 val loss 7.0560
+5200 val perplexity 1159.7809
+5200 train 6.998636 (lr=2.6779e-05) (hash(x)=49369682)
+5100 val loss 6.8575
+5100 val perplexity 950.9432
+5100 train 6.727626 (lr=3.8500e-05) (hash(x)=48659050)
+5300 val loss 7.0524
+5300 val perplexity 1155.6340
+5300 train 7.432145 (lr=2.6058e-05) (hash(x)=57787700)
+5200 val loss 6.8367
+5200 val perplexity 931.4388
+5200 train 6.751755 (lr=3.7490e-05) (hash(x)=49369682)
+5400 val loss 7.0336
+5400 val perplexity 1134.0551
+5400 train 6.964849 (lr=2.5339e-05) (hash(x)=49365400)
+5300 val loss 6.8303
+5300 val perplexity 925.4502
+5300 train 7.234496 (lr=3.6482e-05) (hash(x)=57787700)
+5500 val loss 7.0214
+5500 val perplexity 1120.3920
+5500 train 6.973766 (lr=2.4623e-05) (hash(x)=48720412)
+5400 val loss 6.8203
+5400 val perplexity 916.2604
+5400 train 6.747429 (lr=3.5475e-05) (hash(x)=49365400)
+5600 val loss 7.0085
+5600 val perplexity 1106.0455
+5600 train 7.324727 (lr=2.3909e-05) (hash(x)=55784800)
+5500 val loss 6.8173
+5500 val perplexity 913.5103
+5500 train 6.778089 (lr=3.4472e-05) (hash(x)=48720412)
+5700 val loss 7.0040
+5700 val perplexity 1101.0533
+5700 train 6.863188 (lr=2.3199e-05) (hash(x)=50073634)
+5600 val loss 6.8012
+5600 val perplexity 898.9412
+5600 train 7.131571 (lr=3.3473e-05) (hash(x)=55784800)
+5800 val loss 6.9817
+5800 val perplexity 1076.7632
+5800 train 6.810428 (lr=2.2493e-05) (hash(x)=50170324)
+5700 val loss 6.7964
+5700 val perplexity 894.6154
+5700 train 6.646752 (lr=3.2479e-05) (hash(x)=50073634)
+5900 val loss 6.9721
+5900 val perplexity 1066.4613
+5900 train 6.721549 (lr=2.1793e-05) (hash(x)=48410268)
+5800 val loss 6.7835
+5800 val perplexity 883.1710
+5800 train 6.606258 (lr=3.1491e-05) (hash(x)=50170324)
+6000 val loss 6.9624
+6000 val perplexity 1056.1648
+6000 train 6.858176 (lr=2.1098e-05) (hash(x)=49527342)
+5900 val loss 6.7863
+5900 val perplexity 885.6093
+5900 train 6.532552 (lr=3.0510e-05) (hash(x)=48410268)
+6100 val loss 6.9515
+6100 val perplexity 1044.7220
+6100 train 6.763265 (lr=2.0410e-05) (hash(x)=49550294)
+6000 val loss 6.7822
+6000 val perplexity 881.9771
+6000 train 6.668611 (lr=2.9537e-05) (hash(x)=49527342)
+6200 val loss 6.9387
+6200 val perplexity 1031.4410
+6200 train 6.494091 (lr=1.9729e-05) (hash(x)=42126106)
+6100 val loss 6.7750
+6100 val perplexity 875.6949
+6100 train 6.587618 (lr=2.8574e-05) (hash(x)=49550294)
+6300 val loss 6.9254
+6300 val perplexity 1017.7720
+6300 train 6.696783 (lr=1.9056e-05) (hash(x)=49608772)
+6200 val loss 6.7762
+6200 val perplexity 876.6934
+6200 train 6.334079 (lr=2.7621e-05) (hash(x)=42126106)
+6400 val loss 6.9128
+6400 val perplexity 1005.0568
+6400 train 6.485902 (lr=1.8392e-05) (hash(x)=52324417)
+6300 val loss 6.7694
+6300 val perplexity 870.7931
+6300 train 6.520870 (lr=2.6679e-05) (hash(x)=49608772)
+6500 val loss 6.8815
+6500 val perplexity 974.0597
+6500 train 6.923303 (lr=1.7738e-05) (hash(x)=46207215)
+6400 val loss 6.7631
+6400 val perplexity 865.2852
+6400 train 6.338704 (lr=2.5749e-05) (hash(x)=52324417)
+6600 val loss 6.8604
+6600 val perplexity 953.7169
+6600 train 6.797771 (lr=1.7093e-05) (hash(x)=49027014)
+6500 val loss 6.7358
+6500 val perplexity 841.9837
+6500 train 6.789601 (lr=2.4833e-05) (hash(x)=46207215)
+6700 val loss 6.8459
+6700 val perplexity 939.9939
+6700 train 6.819520 (lr=1.6459e-05) (hash(x)=46232513)
+6800 val loss 6.8301
+6800 val perplexity 925.2843
+6800 train 6.766226 (lr=1.5836e-05) (hash(x)=47348403)
+6600 val loss 6.7171
+6600 val perplexity 826.3766
+6600 train 6.647032 (lr=2.3930e-05) (hash(x)=49027014)
+6900 val loss 6.8254
+6900 val perplexity 920.9736
+6900 train 6.799843 (lr=1.5225e-05) (hash(x)=49806647)
+6700 val loss 6.7033
+6700 val perplexity 815.0651
+6700 train 6.672481 (lr=2.3042e-05) (hash(x)=46232513)
+7000 val loss 6.8056
+7000 val perplexity 902.8452
+7000 train 6.843875 (lr=1.4627e-05) (hash(x)=50893018)
+6800 val loss 6.7007
+6800 val perplexity 812.9648
+6800 train 6.624757 (lr=2.2171e-05) (hash(x)=47348403)
+7100 val loss 6.7984
+7100 val perplexity 896.4370
+7100 train 6.822917 (lr=1.4043e-05) (hash(x)=49157639)
+6900 val loss 6.7069
+6900 val perplexity 817.9907
+6900 train 6.675504 (lr=2.1316e-05) (hash(x)=49806647)
+7200 val loss 6.7897
+7200 val perplexity 888.6813
+7200 train 6.761259 (lr=1.3471e-05) (hash(x)=47014759)
+7000 val loss 6.6812
+7000 val perplexity 797.3110
+7000 train 6.741058 (lr=2.0478e-05) (hash(x)=50893018)
+7300 val loss 6.7779
+7300 val perplexity 878.1854
+7300 train 6.779035 (lr=1.2915e-05) (hash(x)=47325591)
+7100 val loss 6.6752
+7100 val perplexity 792.4863
+7100 train 6.693850 (lr=1.9660e-05) (hash(x)=49157639)
+7400 val loss 6.7720
+7400 val perplexity 873.0865
+7400 train 6.654572 (lr=1.2373e-05) (hash(x)=49184604)
+7200 val loss 6.6680
+7200 val perplexity 786.8536
+7200 train 6.651248 (lr=1.8860e-05) (hash(x)=47014759)
+7500 val loss 6.7629
+7500 val perplexity 865.1734
+7500 train 7.052269 (lr=1.1847e-05) (hash(x)=55053584)
+7300 val loss 6.6610
+7300 val perplexity 781.3257
+7300 train 6.657533 (lr=1.8081e-05) (hash(x)=47325591)
+7600 val loss 6.7585
+7600 val perplexity 861.3661
+7600 train 6.715505 (lr=1.1337e-05) (hash(x)=48693923)
+7400 val loss 6.6557
+7400 val perplexity 777.1942
+7400 train 6.538068 (lr=1.7323e-05) (hash(x)=49184604)
+7700 val loss 6.7544
+7700 val perplexity 857.8153
+7700 train 6.298505 (lr=1.0844e-05) (hash(x)=40952882)
+7500 val loss 6.6487
+7500 val perplexity 771.8184
+7500 train 6.927235 (lr=1.6586e-05) (hash(x)=55053584)
+7800 val loss 6.7480
+7800 val perplexity 852.3216
+7800 train 6.836084 (lr=1.0367e-05) (hash(x)=52487845)
+7600 val loss 6.6504
+7600 val perplexity 773.0807
+7600 train 6.592214 (lr=1.5872e-05) (hash(x)=48693923)
+7900 val loss 6.7444
+7900 val perplexity 849.3254
+7900 train 6.821626 (lr=9.9088e-06) (hash(x)=50221547)
+7700 val loss 6.6482
+7700 val perplexity 771.4020
+7700 train 6.223924 (lr=1.5181e-05) (hash(x)=40952882)
+8000 val loss 6.7372
+8000 val perplexity 843.2168
+8000 train 7.036058 (lr=9.4682e-06) (hash(x)=62294204)
+7800 val loss 6.6412
+7800 val perplexity 765.9936
+7800 train 6.724183 (lr=1.4514e-05) (hash(x)=52487845)
+8100 val loss 6.7324
+8100 val perplexity 839.1255
+8100 train 6.425050 (lr=9.0461e-06) (hash(x)=44401967)
+7900 val loss 6.6360
+7900 val perplexity 762.0717
+7900 train 6.738249 (lr=1.3872e-05) (hash(x)=50221547)
+8200 val loss 6.7270
+8200 val perplexity 834.6295
+8200 train 6.687708 (lr=8.6430e-06) (hash(x)=52769095)
+8000 val loss 6.6358
+8000 val perplexity 761.8933
+8000 train 6.951126 (lr=1.3255e-05) (hash(x)=62294204)
+8300 val loss 6.7317
+8300 val perplexity 838.6075
+8300 train 6.673268 (lr=8.2593e-06) (hash(x)=56829883)
+8100 val loss 6.6309
+8100 val perplexity 758.1688
+8100 train 6.340541 (lr=1.2665e-05) (hash(x)=44401967)
+8400 val loss 6.7297
+8400 val perplexity 836.9218
+8400 train 6.691971 (lr=7.8953e-06) (hash(x)=52147375)
+8200 val loss 6.6290
+8200 val perplexity 756.7592
+8200 train 6.576173 (lr=1.2100e-05) (hash(x)=52769095)
+8500 val loss 6.7269
+8500 val perplexity 834.5331
+8500 train 6.963628 (lr=7.5515e-06) (hash(x)=60197820)
+8300 val loss 6.6358
+8300 val perplexity 761.8755
+8300 train 6.583430 (lr=1.1563e-05) (hash(x)=56829883)
+8600 val loss 6.7228
+8600 val perplexity 831.1801
+8600 train 6.488226 (lr=7.2282e-06) (hash(x)=49377068)
+8400 val loss 6.6368
+8400 val perplexity 762.6450
+8400 train 6.574768 (lr=1.1053e-05) (hash(x)=52147375)
+8700 val loss 6.7243
+8700 val perplexity 832.3748
+8700 train 6.683083 (lr=6.9257e-06) (hash(x)=51092724)
+8500 val loss 6.6341
+8500 val perplexity 760.6315
+8500 train 6.861284 (lr=1.0572e-05) (hash(x)=60197820)
+8800 val loss 6.7192
+8800 val perplexity 828.1299
+8800 train 6.728143 (lr=6.6444e-06) (hash(x)=48642928)
+8600 val loss 6.6271
+8600 val perplexity 755.2836
+8600 train 6.383428 (lr=1.0119e-05) (hash(x)=49377068)
+8900 val loss 6.7147
+8900 val perplexity 824.3984
+8900 train 6.926672 (lr=6.3845e-06) (hash(x)=55342246)
+8700 val loss 6.6293
+8700 val perplexity 756.9796
+8700 train 6.602247 (lr=9.6960e-06) (hash(x)=51092724)
+9000 val loss 6.6997
+9000 val perplexity 812.1678
+9000 train 6.671376 (lr=6.1462e-06) (hash(x)=48093368)
+8800 val loss 6.6263
+8800 val perplexity 754.6838
+8800 train 6.621269 (lr=9.3021e-06) (hash(x)=48642928)
+9100 val loss 6.6943
+9100 val perplexity 807.8050
+9100 train 6.789246 (lr=5.9300e-06) (hash(x)=48578183)
+8900 val loss 6.6234
+8900 val perplexity 752.5152
+8900 train 6.830116 (lr=8.9382e-06) (hash(x)=55342246)
+9200 val loss 6.6896
+9200 val perplexity 803.9971
+9200 train 6.888481 (lr=5.7359e-06) (hash(x)=50794720)
+9000 val loss 6.6079
+9000 val perplexity 740.9231
+9000 train 6.601912 (lr=8.6047e-06) (hash(x)=48093368)
+9300 val loss 6.6857
+9300 val perplexity 800.8321
+9300 train 6.454147 (lr=5.5641e-06) (hash(x)=46513190)
+9100 val loss 6.6025
+9100 val perplexity 736.9377
+9100 train 6.679001 (lr=8.3020e-06) (hash(x)=48578183)
+9400 val loss 6.6830
+9400 val perplexity 798.7258
+9400 train 6.323279 (lr=5.4149e-06) (hash(x)=43808238)
+9200 val loss 6.5976
+9200 val perplexity 733.3257
+9200 train 6.786611 (lr=8.0302e-06) (hash(x)=50794720)
+9500 val loss 6.6799
+9500 val perplexity 796.2480
+9500 train 6.484011 (lr=5.2884e-06) (hash(x)=45021888)
+9300 val loss 6.5947
+9300 val perplexity 731.1756
+9300 train 6.369024 (lr=7.7898e-06) (hash(x)=46513190)
+9600 val loss 6.6766
+9600 val perplexity 793.6359
+9600 train 6.773294 (lr=5.1847e-06) (hash(x)=56525570)
+9400 val loss 6.5938
+9400 val perplexity 730.5848
+9400 train 6.250168 (lr=7.5809e-06) (hash(x)=43808238)
+9700 val loss 6.6736
+9700 val perplexity 791.2244
+9700 train 6.871308 (lr=5.1040e-06) (hash(x)=52585913)
+9500 val loss 6.5891
+9500 val perplexity 727.1143
+9500 train 6.406715 (lr=7.4038e-06) (hash(x)=45021888)
+9800 val loss 6.6725
+9800 val perplexity 790.3431
+9800 train 6.852726 (lr=5.0462e-06) (hash(x)=52344698)
+9600 val loss 6.5886
+9600 val perplexity 726.7500
+9600 train 6.674443 (lr=7.2586e-06) (hash(x)=56525570)
+9900 val loss 6.6691
+9900 val perplexity 787.7227
+9900 train 6.682109 (lr=5.0116e-06) (hash(x)=51740945)
+9700 val loss 6.5857
+9700 val perplexity 724.6240
+9700 train 6.775486 (lr=7.1456e-06) (hash(x)=52585913)
+9999 val loss 6.6652
+9999 val perplexity 784.6176
+9800 val loss 6.5830
+9800 val perplexity 722.7222
+9800 train 6.780724 (lr=7.0647e-06) (hash(x)=52344698)
+9900 val loss 6.5810
+9900 val perplexity 721.2598
+9900 train 6.589230 (lr=7.0162e-06) (hash(x)=51740945)
+9999 val loss 6.5788
+9999 val perplexity 719.6898

attention_kindselective_n_heads2_seed1338/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bfe50e723e825bd0786c1a4596ac4cdaf796676df7fdc5da3f87bcf867afc971
+oid sha256:88a13d00e75e1cf66932d118b2f84bb24f00babd0ee75e7f5801725ef5c13418
 size 38587970

attention_kindselective_n_heads2_seed1338/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2e8ba950312264226829ac705df43b9cdffaf5bcc036d9b24aef5f3e832494c
+oid sha256:32d5afc7324a3d22f4c1218ec09c6c54149343ba625fcba9940acb396b796c11
 size 38587970

attention_kindselective_n_heads2_seed1338/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:720ce17fda15dca0483b5bee479b09521f324cbda2f55ee89297aeb3815e7127
+oid sha256:05a121807594d5ecb5feed74e34dfa1e69b69860960bb12751f372377bb006a6
 size 38587970

attention_kindselective_n_heads2_seed1338/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e1a2367f59732516e841fec0ec047482f7b40395d9a8c4e9bfd7434db667578
+oid sha256:40e0b112f277f6cbba421303357ec83f7697b417831c9e4ff43b473f4f3e1114
 size 38587970

attention_kindselective_n_heads2_seed1338/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8266da6cdaef57d288608149918e3d4cc507a0f73a368f4c248cc84d130fa87
+oid sha256:8d833216fc7bfe1147572ad064c3757e6d11dab6d14b15ed9dbc193d3fe0c652
 size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1cf4773a63f7bd4d9b9b266dfca8bb2631ec02374f0d2e94c402329e5b41680
+oid sha256:42125ee96db8d16feb270e483c050b0f0ad3baa06ce04fb49898faaa1eb18ca6
 size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28ffac77284f9c9c07e805be4e544c3bf52c59e2e9c317ae438e92592ac76c33
+oid sha256:b4c2b49578a7dc4e2283cd26e5e389f35e17e5b918115b0f054f8c487f39e3d0
 size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d24ecb4199ebaa5047064c533fe7647126557157f3557829d39923f2d8fba6fd
+oid sha256:42529f3655fc336117c939c1c8ec0fb125634901be3c730bed8da0466e4741ef
 size 70895430