Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_30720_4_1340", "n_embd": 256}

attention_kindselective_n_heads4_seed1340/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2714
-0 val perplexity 78546.7344
-0 train 11.276529 (lr=1.6250e-07) (hash(x)=164406924)
-100 val loss 9.9427
-100 val perplexity 20799.7773
-100 train 9.986905 (lr=1.6412e-05) (hash(x)=177407419)
-200 val loss 9.0545
-200 val perplexity 8556.6611
-200 train 9.022396 (lr=3.2500e-05) (hash(x)=144903932)
-300 val loss 7.9905
-300 val perplexity 2952.6941
-300 train 8.233212 (lr=3.2498e-05) (hash(x)=173839165)
-400 val loss 7.7050
-400 val perplexity 2219.5210
-400 train 7.775531 (lr=3.2490e-05) (hash(x)=167734596)
-500 val loss 7.5884
-500 val perplexity 1975.1251
-500 train 7.603640 (lr=3.2478e-05) (hash(x)=153224076)
-600 val loss 7.5186
-600 val perplexity 1842.0447
-600 train 7.431871 (lr=3.2461e-05) (hash(x)=149619098)
-700 val loss 7.4777
-700 val perplexity 1768.2009
-700 train 7.410966 (lr=3.2440e-05) (hash(x)=146539909)
-800 val loss 7.4248
-800 val perplexity 1677.1040
-800 train 7.388150 (lr=3.2413e-05) (hash(x)=153710890)
-900 val loss 7.3854
-900 val perplexity 1612.2769
-900 train 7.327601 (lr=3.2382e-05) (hash(x)=155873620)
-1000 val loss 7.3567
-1000 val perplexity 1566.6251
-1000 train 7.275003 (lr=3.2346e-05) (hash(x)=145450636)
-1100 val loss 7.3128
-1100 val perplexity 1499.4359
-1100 train 7.385886 (lr=3.2305e-05) (hash(x)=154123388)
-1200 val loss 7.2874
-1200 val perplexity 1461.7990
-1200 train 7.163799 (lr=3.2260e-05) (hash(x)=145249251)
-1300 val loss 7.2463
-1300 val perplexity 1402.8694
-1300 train 7.144474 (lr=3.2209e-05) (hash(x)=148937127)
-1400 val loss 7.1995
-1400 val perplexity 1338.7473
-1400 train 7.317604 (lr=3.2154e-05) (hash(x)=150475545)
-1500 val loss 7.1654
-1500 val perplexity 1293.8778
-1500 train 7.090920 (lr=3.2094e-05) (hash(x)=154653428)
-1600 val loss 7.1204
-1600 val perplexity 1236.9946
-1600 train 7.097936 (lr=3.2030e-05) (hash(x)=144483776)
-1700 val loss 7.0663
-1700 val perplexity 1171.8545
-1700 train 7.221328 (lr=3.1961e-05) (hash(x)=157395496)
-1800 val loss 7.0269
-1800 val perplexity 1126.5381
-1800 train 7.044158 (lr=3.1887e-05) (hash(x)=157916369)
-1900 val loss 6.9853
-1900 val perplexity 1080.6533
-1900 train 7.202100 (lr=3.1809e-05) (hash(x)=166073923)
-2000 val loss 6.9577
-2000 val perplexity 1051.2118
-2000 train 7.038825 (lr=3.1726e-05) (hash(x)=154856891)
-2100 val loss 6.9203
-2100 val perplexity 1012.6016
-2100 train 6.912439 (lr=3.1638e-05) (hash(x)=151925203)
-2200 val loss 6.8823
-2200 val perplexity 974.8445
-2200 train 6.584104 (lr=3.1546e-05) (hash(x)=136191502)
-2300 val loss 6.8423
-2300 val perplexity 936.6110
-2300 train 7.000276 (lr=3.1449e-05) (hash(x)=153273362)
-2400 val loss 6.8023
-2400 val perplexity 899.9409
-2400 train 6.751945 (lr=3.1348e-05) (hash(x)=148021541)
-2500 val loss 6.7656
-2500 val perplexity 867.4752
-2500 train 6.709182 (lr=3.1243e-05) (hash(x)=141356608)
-2600 val loss 6.7277
-2600 val perplexity 835.2031
-2600 train 6.673390 (lr=3.1133e-05) (hash(x)=146005217)
-2700 val loss 6.6951
-2700 val perplexity 808.4293
-2700 train 6.534774 (lr=3.1019e-05) (hash(x)=144511718)
-2800 val loss 6.6608
-2800 val perplexity 781.1756
-2800 train 6.540296 (lr=3.0900e-05) (hash(x)=146019502)
-2900 val loss 6.6303
-2900 val perplexity 757.6885
-2900 train 6.544481 (lr=3.0777e-05) (hash(x)=146496200)
-3000 val loss 6.5995
-3000 val perplexity 734.6974
-3000 train 6.548376 (lr=3.0650e-05) (hash(x)=150127281)
-3100 val loss 6.5634
-3100 val perplexity 708.6861
-3100 train 6.541520 (lr=3.0518e-05) (hash(x)=142022255)
-3200 val loss 6.5337
-3200 val perplexity 687.9238
-3200 train 6.587421 (lr=3.0383e-05) (hash(x)=154120875)
-3300 val loss 6.5133
-3300 val perplexity 674.0387
-3300 train 6.571418 (lr=3.0243e-05) (hash(x)=153999717)
-3400 val loss 6.4929
-3400 val perplexity 660.4374
-3400 train 6.258515 (lr=3.0099e-05) (hash(x)=139694097)
-3500 val loss 6.4649
-3500 val perplexity 642.1870
-3500 train 6.649519 (lr=2.9952e-05) (hash(x)=162992732)
-3600 val loss 6.4301
-3600 val perplexity 620.2365
-3600 train 6.405787 (lr=2.9800e-05) (hash(x)=147574101)
-3700 val loss 6.4131
-3700 val perplexity 609.7664
-3700 train 6.523832 (lr=2.9644e-05) (hash(x)=157763099)
-3800 val loss 6.3888
-3800 val perplexity 595.1349
-3800 train 6.553192 (lr=2.9485e-05) (hash(x)=170800034)
-3900 val loss 6.3732
-3900 val perplexity 585.9507
-3900 train 6.449968 (lr=2.9321e-05) (hash(x)=164984528)
-4000 val loss 6.3589
-4000 val perplexity 577.6016
-4000 train 6.219861 (lr=2.9154e-05) (hash(x)=141743323)
-4100 val loss 6.3496
-4100 val perplexity 572.2399
-4100 train 6.364213 (lr=2.8983e-05) (hash(x)=153392872)
-4200 val loss 6.3388
-4200 val perplexity 566.0950
-4200 train 6.200565 (lr=2.8808e-05) (hash(x)=149074933)
-4300 val loss 6.3134
-4300 val perplexity 551.9182
-4300 train 6.704398 (lr=2.8630e-05) (hash(x)=167823423)
-4400 val loss 6.2896
-4400 val perplexity 538.9551
-4400 train 6.058969 (lr=2.8448e-05) (hash(x)=141203114)
-4500 val loss 6.2706
-4500 val perplexity 528.7738
-4500 train 6.267660 (lr=2.8263e-05) (hash(x)=146284780)
-4600 val loss 6.2508
-4600 val perplexity 518.4062
-4600 train 6.053355 (lr=2.8075e-05) (hash(x)=141126464)
-4700 val loss 6.2356
-4700 val perplexity 510.5971
-4700 train 6.261318 (lr=2.7883e-05) (hash(x)=154751926)
-4800 val loss 6.2221
-4800 val perplexity 503.7436
-4800 train 6.305309 (lr=2.7687e-05) (hash(x)=154793198)
-4900 val loss 6.2138
-4900 val perplexity 499.5989
-4900 train 5.932302 (lr=2.7489e-05) (hash(x)=139406392)
-5000 val loss 6.2221
-5000 val perplexity 503.7762
-5000 train 5.932709 (lr=2.7287e-05) (hash(x)=153548741)
-5100 val loss 6.1919
-5100 val perplexity 488.7556
-5100 train 6.270887 (lr=2.7082e-05) (hash(x)=160488568)
-5200 val loss 6.1705
-5200 val perplexity 478.4274
-5200 train 6.178745 (lr=2.6874e-05) (hash(x)=149645053)
-5300 val loss 6.1507
-5300 val perplexity 469.0544
-5300 train 6.247884 (lr=2.6664e-05) (hash(x)=155820556)
-5400 val loss 6.1414
-5400 val perplexity 464.7147
-5400 train 6.085527 (lr=2.6450e-05) (hash(x)=147538134)
-5500 val loss 6.1284
-5500 val perplexity 458.7081
-5500 train 6.339289 (lr=2.6233e-05) (hash(x)=166889307)
-5600 val loss 6.1148
-5600 val perplexity 452.4975
-5600 train 5.816920 (lr=2.6014e-05) (hash(x)=139516699)
-5700 val loss 6.1153
-5700 val perplexity 452.7293
-5700 train 5.789756 (lr=2.5792e-05) (hash(x)=140453511)
-5800 val loss 6.1057
-5800 val perplexity 448.4195
-5800 train 6.046035 (lr=2.5567e-05) (hash(x)=162964847)
-5900 val loss 6.0923
-5900 val perplexity 442.4182
-5900 train 6.095953 (lr=2.5340e-05) (hash(x)=150606634)
-6000 val loss 6.0706
-6000 val perplexity 432.9581
-6000 train 6.155572 (lr=2.5111e-05) (hash(x)=149890857)
-6100 val loss 6.0665
-6100 val perplexity 431.1557
-6100 train 6.164382 (lr=2.4879e-05) (hash(x)=173884145)
-6200 val loss 6.0532
-6200 val perplexity 425.4883
-6200 train 6.114431 (lr=2.4644e-05) (hash(x)=151987098)
-6300 val loss 6.0394
-6300 val perplexity 419.6264
-6300 train 5.991861 (lr=2.4408e-05) (hash(x)=148853562)
-6400 val loss 6.0311
-6400 val perplexity 416.1762
-6400 train 5.778894 (lr=2.4169e-05) (hash(x)=141530101)
-6500 val loss 6.0240
-6500 val perplexity 413.2226
-6500 train 5.882457 (lr=2.3928e-05) (hash(x)=142297809)
-6600 val loss 6.0092
-6600 val perplexity 407.1544
-6600 train 5.900824 (lr=2.3686e-05) (hash(x)=142447782)
-6700 val loss 6.0062
-6700 val perplexity 405.9474
-6700 train 5.991254 (lr=2.3441e-05) (hash(x)=147004686)
-6800 val loss 5.9924
-6800 val perplexity 400.3843
-6800 train 5.738660 (lr=2.3195e-05) (hash(x)=133438702)
-6900 val loss 5.9919
-6900 val perplexity 400.1860
-6900 train 6.006359 (lr=2.2946e-05) (hash(x)=157085143)
-7000 val loss 5.9796
-7000 val perplexity 395.2977
-7000 train 5.865448 (lr=2.2696e-05) (hash(x)=139437666)
-7100 val loss 5.9743
-7100 val perplexity 393.1910
-7100 train 5.988501 (lr=2.2445e-05) (hash(x)=159792986)
-7200 val loss 5.9581
-7200 val perplexity 386.8573
-7200 train 5.942185 (lr=2.2192e-05) (hash(x)=144930687)
-7300 val loss 5.9490
-7300 val perplexity 383.3533
-7300 train 6.009021 (lr=2.1937e-05) (hash(x)=156242690)
-7400 val loss 5.9416
-7400 val perplexity 380.5278
-7400 train 5.777550 (lr=2.1682e-05) (hash(x)=148183719)
-7500 val loss 5.9370
-7500 val perplexity 378.7969
-7500 train 5.975434 (lr=2.1425e-05) (hash(x)=152494758)
-7600 val loss 5.9312
-7600 val perplexity 376.6238
-7600 train 5.619554 (lr=2.1166e-05) (hash(x)=142485027)
-7700 val loss 5.9281
-7700 val perplexity 375.4567
-7700 train 5.795169 (lr=2.0907e-05) (hash(x)=147512165)
-7800 val loss 5.9124
-7800 val perplexity 369.5941
-7800 train 5.783484 (lr=2.0647e-05) (hash(x)=160346994)
-7900 val loss 5.9094
-7900 val perplexity 368.4920
-7900 train 5.783132 (lr=2.0386e-05) (hash(x)=144488254)
-8000 val loss 5.8979
-8000 val perplexity 364.2743
-8000 train 5.709069 (lr=2.0123e-05) (hash(x)=147637019)
-8100 val loss 5.8961
-8100 val perplexity 363.6187
-8100 train 5.744081 (lr=1.9861e-05) (hash(x)=147340534)
-8200 val loss 5.8903
-8200 val perplexity 361.5074
-8200 train 5.946971 (lr=1.9597e-05) (hash(x)=151630665)
-8300 val loss 5.8814
-8300 val perplexity 358.3273
-8300 train 5.982193 (lr=1.9333e-05) (hash(x)=149747064)
-8400 val loss 5.8713
-8400 val perplexity 354.7009
-8400 train 6.068233 (lr=1.9069e-05) (hash(x)=154245770)
-8500 val loss 5.8662
-8500 val perplexity 352.9071
-8500 train 5.753820 (lr=1.8804e-05) (hash(x)=152559100)
-8600 val loss 5.8618
-8600 val perplexity 351.3425
-8600 train 6.275620 (lr=1.8539e-05) (hash(x)=181365926)
-8700 val loss 5.8589
-8700 val perplexity 350.3235
-8700 train 5.692950 (lr=1.8273e-05) (hash(x)=154405991)
-8800 val loss 5.8489
-8800 val perplexity 346.8562
-8800 train 5.792358 (lr=1.8008e-05) (hash(x)=153755904)
-8900 val loss 5.8465
-8900 val perplexity 346.0291
-8900 train 5.764337 (lr=1.7742e-05) (hash(x)=152120568)
-9000 val loss 5.8297
-9000 val perplexity 340.2657
-9000 train 5.617670 (lr=1.7477e-05) (hash(x)=142797279)
-9100 val loss 5.8270
-9100 val perplexity 339.3379
-9100 train 5.691728 (lr=1.7211e-05) (hash(x)=143037503)
-9200 val loss 5.8270
-9200 val perplexity 339.3499
-9200 train 5.694878 (lr=1.6946e-05) (hash(x)=113690273)
-9300 val loss 5.8200
-9300 val perplexity 336.9655
-9300 train 5.757028 (lr=1.6681e-05) (hash(x)=158025077)
-9400 val loss 5.8099
-9400 val perplexity 333.5787
-9400 train 5.883915 (lr=1.6417e-05) (hash(x)=158251718)
-9500 val loss 5.8112
-9500 val perplexity 334.0197
-9500 train 5.848474 (lr=1.6153e-05) (hash(x)=154752610)
-9600 val loss 5.8045
-9600 val perplexity 331.7947
-9600 train 5.681903 (lr=1.5889e-05) (hash(x)=146889093)
-9700 val loss 5.7924
-9700 val perplexity 327.8095
-9700 train 5.801524 (lr=1.5627e-05) (hash(x)=156906516)
-9800 val loss 5.7899
-9800 val perplexity 326.9742
-9800 train 5.621650 (lr=1.5364e-05) (hash(x)=153841927)
-9900 val loss 5.7852
-9900 val perplexity 325.4485
-9900 train 5.999937 (lr=1.5103e-05) (hash(x)=163514334)
-10000 val loss 5.7790
-10000 val perplexity 323.4331
-10000 train 5.769652 (lr=1.4843e-05) (hash(x)=144745561)
-10100 val loss 5.7745
-10100 val perplexity 321.9720
-10100 train 6.025921 (lr=1.4584e-05) (hash(x)=161651149)
-10200 val loss 5.7679
-10200 val perplexity 319.8694
-10200 train 5.710236 (lr=1.4325e-05) (hash(x)=143667037)
-10300 val loss 5.7619
-10300 val perplexity 317.9444
-10300 train 5.654992 (lr=1.4068e-05) (hash(x)=151687262)
-10400 val loss 5.7627
-10400 val perplexity 318.2142
-10400 train 5.627633 (lr=1.3813e-05) (hash(x)=153694955)
-10500 val loss 5.7588
-10500 val perplexity 316.9752
-10500 train 5.704593 (lr=1.3558e-05) (hash(x)=150218029)
-10600 val loss 5.7528
-10600 val perplexity 315.0698
-10600 train 5.851352 (lr=1.3305e-05) (hash(x)=155093716)
-10700 val loss 5.7477
-10700 val perplexity 313.4625
-10700 train 5.787298 (lr=1.3054e-05) (hash(x)=151631831)
-10800 val loss 5.7389
-10800 val perplexity 310.7081
-10800 train 5.586615 (lr=1.2804e-05) (hash(x)=136465175)
-10900 val loss 5.7369
-10900 val perplexity 310.0930
-10900 train 5.669595 (lr=1.2555e-05) (hash(x)=154461514)
-11000 val loss 5.7358
-11000 val perplexity 309.7675
-11000 train 5.728212 (lr=1.2309e-05) (hash(x)=141877432)
-11100 val loss 5.7325
-11100 val perplexity 308.7520
-11100 train 5.526520 (lr=1.2064e-05) (hash(x)=139572894)
-11200 val loss 5.7246
-11200 val perplexity 306.2994
-11200 train 5.771356 (lr=1.1822e-05) (hash(x)=155591443)
-11300 val loss 5.7183
-11300 val perplexity 304.3907
-11300 train 5.733082 (lr=1.1581e-05) (hash(x)=153349136)
-11400 val loss 5.7172
-11400 val perplexity 304.0620
-11400 train 5.671595 (lr=1.1342e-05) (hash(x)=148109395)
-11500 val loss 5.7130
-11500 val perplexity 302.7882
-11500 train 5.702454 (lr=1.1106e-05) (hash(x)=153276620)
-11600 val loss 5.7125
-11600 val perplexity 302.6309
-11600 train 5.571287 (lr=1.0871e-05) (hash(x)=151416388)
-11700 val loss 5.7101
-11700 val perplexity 301.9037
-11700 train 5.582951 (lr=1.0639e-05) (hash(x)=143420927)
-11800 val loss 5.7050
-11800 val perplexity 300.3752
-11800 train 5.772466 (lr=1.0410e-05) (hash(x)=165023996)
-11900 val loss 5.6986
-11900 val perplexity 298.4425
-11900 train 5.614121 (lr=1.0183e-05) (hash(x)=144465076)
-12000 val loss 5.6963
-12000 val perplexity 297.7500
-12000 train 5.707811 (lr=9.9580e-06) (hash(x)=155991892)
-12100 val loss 5.6944
-12100 val perplexity 297.1982
-12100 train 5.728391 (lr=9.7360e-06) (hash(x)=150748844)
-12200 val loss 5.6913
-12200 val perplexity 296.2873
-12200 train 5.653107 (lr=9.5167e-06) (hash(x)=149712638)
-12300 val loss 5.6921
-12300 val perplexity 296.5135
-12300 train 5.510970 (lr=9.3001e-06) (hash(x)=140428230)
-12400 val loss 5.6856
-12400 val perplexity 294.6078
-12400 train 5.866589 (lr=9.0864e-06) (hash(x)=163614308)
-12500 val loss 5.6804
-12500 val perplexity 293.0593
-12500 train 5.644887 (lr=8.8756e-06) (hash(x)=153366153)
-12600 val loss 5.6785
-12600 val perplexity 292.5041
-12600 train 5.570187 (lr=8.6677e-06) (hash(x)=146832066)
-12700 val loss 5.6759
-12700 val perplexity 291.7508
-12700 train 5.852587 (lr=8.4629e-06) (hash(x)=154387414)
-12800 val loss 5.6739
-12800 val perplexity 291.1724
-12800 train 5.470691 (lr=8.2612e-06) (hash(x)=139465199)
-12900 val loss 5.6761
-12900 val perplexity 291.7956
-12900 train 5.386293 (lr=8.0627e-06) (hash(x)=143326374)
-13000 val loss 5.6690
-13000 val perplexity 289.7305
-13000 train 5.826969 (lr=7.8674e-06) (hash(x)=158817835)
-13100 val loss 5.6663
-13100 val perplexity 288.9554
-13100 train 5.546062 (lr=7.6754e-06) (hash(x)=142027874)
-13200 val loss 5.6657
-13200 val perplexity 288.7859
-13200 train 5.648112 (lr=7.4867e-06) (hash(x)=148497730)
-13300 val loss 5.6641
-13300 val perplexity 288.3189
-13300 train 5.566710 (lr=7.3015e-06) (hash(x)=148094633)
-13400 val loss 5.6629
-13400 val perplexity 287.9900
-13400 train 5.538298 (lr=7.1198e-06) (hash(x)=150548214)
-13500 val loss 5.6550
-13500 val perplexity 285.7133
-13500 train 5.763052 (lr=6.9416e-06) (hash(x)=152789725)
-13600 val loss 5.6540
-13600 val perplexity 285.4225
-13600 train 5.692722 (lr=6.7670e-06) (hash(x)=153220230)
-13700 val loss 5.6499
-13700 val perplexity 284.2614
-13700 train 5.663307 (lr=6.5961e-06) (hash(x)=147805699)
-13800 val loss 5.6506
-13800 val perplexity 284.4547
-13800 train 5.465456 (lr=6.4289e-06) (hash(x)=146267341)
-13900 val loss 5.6502
-13900 val perplexity 284.3566
-13900 train 5.469760 (lr=6.2655e-06) (hash(x)=139467501)
-14000 val loss 5.6485
-14000 val perplexity 283.8530
-14000 train 5.660518 (lr=6.1059e-06) (hash(x)=152185436)
-14100 val loss 5.6417
-14100 val perplexity 281.9511
-14100 train 5.761265 (lr=5.9502e-06) (hash(x)=176479139)
-14200 val loss 5.6392
-14200 val perplexity 281.2393
-14200 train 5.739030 (lr=5.7984e-06) (hash(x)=151052040)
-14300 val loss 5.6375
-14300 val perplexity 280.7668
-14300 train 5.578081 (lr=5.6506e-06) (hash(x)=149668097)
-14400 val loss 5.6364
-14400 val perplexity 280.4461
-14400 train 5.648204 (lr=5.5068e-06) (hash(x)=151761941)
-14500 val loss 5.6360
-14500 val perplexity 280.3417
-14500 train 5.591083 (lr=5.3671e-06) (hash(x)=149412009)
-14600 val loss 5.6344
-14600 val perplexity 279.9049
-14600 train 5.499872 (lr=5.2316e-06) (hash(x)=145048020)
-14700 val loss 5.6318
-14700 val perplexity 279.1771
-14700 train 5.587432 (lr=5.1002e-06) (hash(x)=141771666)
-14800 val loss 5.6281
-14800 val perplexity 278.1451
-14800 train 5.560456 (lr=4.9730e-06) (hash(x)=148759740)
-14900 val loss 5.6269
-14900 val perplexity 277.7952
-14900 train 5.551805 (lr=4.8501e-06) (hash(x)=146287032)
-15000 val loss 5.6254
-15000 val perplexity 277.3827
-15000 train 5.572130 (lr=4.7314e-06) (hash(x)=143519156)
-15100 val loss 5.6268
-15100 val perplexity 277.7812
-15100 train 5.608938 (lr=4.6171e-06) (hash(x)=151330688)
-15200 val loss 5.6293
-15200 val perplexity 278.4634
-15200 train 5.424197 (lr=4.5072e-06) (hash(x)=140380090)
-15300 val loss 5.6243
-15300 val perplexity 277.0752
-15300 train 5.524822 (lr=4.4017e-06) (hash(x)=152598160)
-15400 val loss 5.6190
-15400 val perplexity 275.6197
-15400 train 5.879935 (lr=4.3006e-06) (hash(x)=164136966)
-15500 val loss 5.6199
-15500 val perplexity 275.8588
-15500 train 5.826211 (lr=4.2040e-06) (hash(x)=158733990)
-15600 val loss 5.6162
-15600 val perplexity 274.8502
-15600 train 5.653137 (lr=4.1119e-06) (hash(x)=149149390)
-15700 val loss 5.6159
-15700 val perplexity 274.7505
-15700 train 5.570819 (lr=4.0244e-06) (hash(x)=146746394)
-15800 val loss 5.6159
-15800 val perplexity 274.7523
-15800 train 5.637339 (lr=3.9414e-06) (hash(x)=159405219)
-15900 val loss 5.6143
-15900 val perplexity 274.3232
-15900 train 5.642344 (lr=3.8630e-06) (hash(x)=154034029)
-16000 val loss 5.6118
-16000 val perplexity 273.6403
-16000 train 5.622232 (lr=3.7892e-06) (hash(x)=153634210)
-16100 val loss 5.6090
-16100 val perplexity 272.8640
-16100 train 5.539718 (lr=3.7201e-06) (hash(x)=155646721)
-16200 val loss 5.6092
-16200 val perplexity 272.9309
-16200 train 5.523324 (lr=3.6556e-06) (hash(x)=142461643)
-16300 val loss 5.6083
-16300 val perplexity 272.6779
-16300 train 5.506813 (lr=3.5959e-06) (hash(x)=141080038)
-16400 val loss 5.6087
-16400 val perplexity 272.7780
-16400 train 5.624062 (lr=3.5408e-06) (hash(x)=153744166)
-16500 val loss 5.6120
-16500 val perplexity 273.6787
-16500 train 5.419729 (lr=3.4905e-06) (hash(x)=149783517)
-16600 val loss 5.6132
-16600 val perplexity 274.0175
-16600 train 5.444574 (lr=3.4449e-06) (hash(x)=142659127)
-16700 val loss 5.6057
-16700 val perplexity 271.9776
-16700 train 5.668278 (lr=3.4041e-06) (hash(x)=149725921)
-16800 val loss 5.6037
-16800 val perplexity 271.4249
-16800 train 5.455229 (lr=3.3680e-06) (hash(x)=140678087)
-16900 val loss 5.6021
-16900 val perplexity 270.9842
-16900 train 5.614109 (lr=3.3367e-06) (hash(x)=146935413)
-17000 val loss 5.6004
-17000 val perplexity 270.5381
-17000 train 5.614009 (lr=3.3102e-06) (hash(x)=144408437)
-17100 val loss 5.6000
-17100 val perplexity 270.4159
-17100 train 5.451910 (lr=3.2886e-06) (hash(x)=149229905)
-17200 val loss 5.6000
-17200 val perplexity 270.4264
-17200 train 5.549056 (lr=3.2717e-06) (hash(x)=150471865)
-17300 val loss 5.6026
-17300 val perplexity 271.1258
-17300 train 5.492773 (lr=3.2596e-06) (hash(x)=155784916)
-17400 val loss 5.6052
-17400 val perplexity 271.8368
-17400 train 5.381482 (lr=3.2524e-06) (hash(x)=148482413)
-17499 val loss 5.5979
-17499 val perplexity 269.8707
+0 val perplexity 78545.6094
+0 train 11.276529 (lr=1.7500e-07) (hash(x)=164406924)
+100 val loss 9.9266
+100 val perplexity 20467.4668
+100 train 9.971992 (lr=1.7675e-05) (hash(x)=177407419)
+200 val loss 8.9790
+200 val perplexity 7934.9438
+200 train 8.945872 (lr=3.5000e-05) (hash(x)=144903932)
+300 val loss 7.9072
+300 val perplexity 2716.8372
+300 train 8.167640 (lr=3.4997e-05) (hash(x)=173839165)
+400 val loss 7.6143
+400 val perplexity 2026.9836
+400 train 7.693684 (lr=3.4990e-05) (hash(x)=167734596)
+500 val loss 7.5204
+500 val perplexity 1845.3423
+500 train 7.531204 (lr=3.4977e-05) (hash(x)=153224076)
+600 val loss 7.4565
+600 val perplexity 1731.0846
+600 train 7.358781 (lr=3.4958e-05) (hash(x)=149619098)
+700 val loss 7.4099
+700 val perplexity 1652.3087
+700 train 7.336334 (lr=3.4935e-05) (hash(x)=146539909)
+800 val loss 7.3613
+800 val perplexity 1573.9480
+800 train 7.321278 (lr=3.4907e-05) (hash(x)=153710890)
+900 val loss 7.3258
+900 val perplexity 1519.0479
+900 train 7.265766 (lr=3.4873e-05) (hash(x)=155873620)
+1000 val loss 7.2840
+1000 val perplexity 1456.8236
+1000 train 7.201667 (lr=3.4834e-05) (hash(x)=145450636)
+1100 val loss 7.2368
+1100 val perplexity 1389.6337
+1100 train 7.307060 (lr=3.4790e-05) (hash(x)=154123388)
+1200 val loss 7.1929
+1200 val perplexity 1329.9440
+1200 train 7.065642 (lr=3.4741e-05) (hash(x)=145249251)
+1300 val loss 7.1508
+1300 val perplexity 1275.1411
+1300 train 7.040481 (lr=3.4687e-05) (hash(x)=148937127)
+1400 val loss 7.1070
+1400 val perplexity 1220.4199
+1400 train 7.223106 (lr=3.4628e-05) (hash(x)=150475545)
+1500 val loss 7.0655
+1500 val perplexity 1170.8129
+1500 train 6.988626 (lr=3.4563e-05) (hash(x)=154653428)
+1600 val loss 7.0237
+1600 val perplexity 1122.9769
+1600 train 7.005347 (lr=3.4494e-05) (hash(x)=144483776)
+1700 val loss 6.9803
+1700 val perplexity 1075.2378
+1700 train 7.135699 (lr=3.4419e-05) (hash(x)=157395496)
+1800 val loss 6.9478
+1800 val perplexity 1040.8788
+1800 train 6.965749 (lr=3.4340e-05) (hash(x)=157916369)
+1900 val loss 6.9103
+1900 val perplexity 1002.5468
+1900 train 7.111033 (lr=3.4255e-05) (hash(x)=166073923)
+2000 val loss 6.8849
+2000 val perplexity 977.3700
+2000 train 6.956383 (lr=3.4166e-05) (hash(x)=154856891)
+2100 val loss 6.8511
+2100 val perplexity 944.8918
+2100 train 6.848049 (lr=3.4072e-05) (hash(x)=151925203)
+2200 val loss 6.8133
+2200 val perplexity 909.8325
+2200 train 6.523357 (lr=3.3973e-05) (hash(x)=136191502)
+2300 val loss 6.7732
+2300 val perplexity 874.0762
+2300 train 6.939043 (lr=3.3869e-05) (hash(x)=153273362)
+2400 val loss 6.7307
+2400 val perplexity 837.6920
+2400 train 6.681937 (lr=3.3760e-05) (hash(x)=148021541)
+2500 val loss 6.7012
+2500 val perplexity 813.3631
+2500 train 6.649411 (lr=3.3646e-05) (hash(x)=141356608)
+2600 val loss 6.6608
+2600 val perplexity 781.1562
+2600 train 6.598667 (lr=3.3528e-05) (hash(x)=146005217)
+2700 val loss 6.6327
+2700 val perplexity 759.5594
+2700 train 6.472603 (lr=3.3405e-05) (hash(x)=144511718)
+2800 val loss 6.5913
+2800 val perplexity 728.7283
+2800 train 6.472763 (lr=3.3277e-05) (hash(x)=146019502)
+2900 val loss 6.5594
+2900 val perplexity 705.8724
+2900 train 6.474362 (lr=3.3144e-05) (hash(x)=146496200)
+3000 val loss 6.5411
+3000 val perplexity 693.0455
+3000 train 6.485529 (lr=3.3008e-05) (hash(x)=150127281)
+3100 val loss 6.5020
+3100 val perplexity 666.4407
+3100 train 6.481001 (lr=3.2866e-05) (hash(x)=142022255)
+3200 val loss 6.4705
+3200 val perplexity 645.7927
+3200 train 6.523926 (lr=3.2720e-05) (hash(x)=154120875)
+3300 val loss 6.4562
+3300 val perplexity 636.6580
+3300 train 6.525292 (lr=3.2570e-05) (hash(x)=153999717)
+3400 val loss 6.4321
+3400 val perplexity 621.4645
+3400 train 6.200135 (lr=3.2415e-05) (hash(x)=139694097)
+3500 val loss 6.4008
+3500 val perplexity 602.3418
+3500 train 6.595932 (lr=3.2256e-05) (hash(x)=162992732)
+3600 val loss 6.3711
+3600 val perplexity 584.6736
+3600 train 6.347825 (lr=3.2092e-05) (hash(x)=147574101)
+3700 val loss 6.3570
+3700 val perplexity 576.4910
+3700 train 6.464335 (lr=3.1924e-05) (hash(x)=157763099)
+3800 val loss 6.3282
+3800 val perplexity 560.1225
+3800 train 6.493472 (lr=3.1753e-05) (hash(x)=170800034)
+3900 val loss 6.3173
+3900 val perplexity 554.0912
+3900 train 6.395955 (lr=3.1577e-05) (hash(x)=164984528)
+4000 val loss 6.2971
+4000 val perplexity 542.9831
+4000 train 6.155862 (lr=3.1397e-05) (hash(x)=141743323)
+4100 val loss 6.2924
+4100 val perplexity 540.4485
+4100 train 6.310122 (lr=3.1212e-05) (hash(x)=153392872)
+4200 val loss 6.2781
+4200 val perplexity 532.7952
+4200 train 6.143710 (lr=3.1024e-05) (hash(x)=149074933)
+4300 val loss 6.2582
+4300 val perplexity 522.2766
+4300 train 6.652179 (lr=3.0833e-05) (hash(x)=167823423)
+4400 val loss 6.2320
+4400 val perplexity 508.7494
+4400 train 5.998601 (lr=3.0637e-05) (hash(x)=141203114)
+4500 val loss 6.2189
+4500 val perplexity 502.1626
+4500 train 6.219765 (lr=3.0437e-05) (hash(x)=146284780)
+4600 val loss 6.2007
+4600 val perplexity 493.1036
+4600 train 6.002645 (lr=3.0234e-05) (hash(x)=141126464)
+4700 val loss 6.1858
+4700 val perplexity 485.7832
+4700 train 6.216115 (lr=3.0027e-05) (hash(x)=154751926)
+4800 val loss 6.1708
+4800 val perplexity 478.5557
+4800 train 6.251617 (lr=2.9817e-05) (hash(x)=154793198)
+4900 val loss 6.1639
+4900 val perplexity 475.2658
+4900 train 5.881503 (lr=2.9603e-05) (hash(x)=139406392)
+5000 val loss 6.1723
+5000 val perplexity 479.2990
+5000 train 5.887846 (lr=2.9386e-05) (hash(x)=153548741)
+5100 val loss 6.1475
+5100 val perplexity 467.5700
+5100 train 6.233172 (lr=2.9166e-05) (hash(x)=160488568)
+5200 val loss 6.1319
+5200 val perplexity 460.3081
+5200 train 6.151352 (lr=2.8942e-05) (hash(x)=149645053)
+5300 val loss 6.1078
+5300 val perplexity 449.3437
+5300 train 6.208061 (lr=2.8715e-05) (hash(x)=155820556)
+5400 val loss 6.0966
+5400 val perplexity 444.3400
+5400 train 6.046060 (lr=2.8484e-05) (hash(x)=147538134)
+5500 val loss 6.0916
+5500 val perplexity 442.1204
+5500 train 6.301843 (lr=2.8251e-05) (hash(x)=166889307)
+5600 val loss 6.0740
+5600 val perplexity 434.4076
+5600 train 5.769093 (lr=2.8015e-05) (hash(x)=139516699)
+5700 val loss 6.0724
+5700 val perplexity 433.7223
+5700 train 5.755292 (lr=2.7776e-05) (hash(x)=140453511)
+5800 val loss 6.0626
+5800 val perplexity 429.4872
+5800 train 6.000001 (lr=2.7534e-05) (hash(x)=162964847)
+5900 val loss 6.0503
+5900 val perplexity 424.2379
+5900 train 6.053433 (lr=2.7290e-05) (hash(x)=150606634)
+6000 val loss 6.0302
+6000 val perplexity 415.8105
+6000 train 6.121263 (lr=2.7042e-05) (hash(x)=149890857)
+6100 val loss 6.0265
+6100 val perplexity 414.2556
+6100 train 6.125873 (lr=2.6792e-05) (hash(x)=173884145)
+6200 val loss 6.0170
+6200 val perplexity 410.3571
+6200 train 6.064613 (lr=2.6540e-05) (hash(x)=151987098)
+6300 val loss 6.0003
+6300 val perplexity 403.5415
+6300 train 5.954297 (lr=2.6285e-05) (hash(x)=148853562)
+6400 val loss 5.9930
+6400 val perplexity 400.5982
+6400 train 5.743896 (lr=2.6028e-05) (hash(x)=141530101)
+6500 val loss 5.9873
+6500 val perplexity 398.3281
+6500 train 5.854073 (lr=2.5769e-05) (hash(x)=142297809)
+6600 val loss 5.9702
+6600 val perplexity 391.5672
+6600 train 5.868578 (lr=2.5508e-05) (hash(x)=142447782)
+6700 val loss 5.9670
+6700 val perplexity 390.3195
+6700 train 5.964384 (lr=2.5244e-05) (hash(x)=147004686)
+6800 val loss 5.9555
+6800 val perplexity 385.8641
+6800 train 5.699266 (lr=2.4979e-05) (hash(x)=133438702)
+6900 val loss 5.9538
+6900 val perplexity 385.2148
+6900 train 5.983283 (lr=2.4711e-05) (hash(x)=157085143)
+7000 val loss 5.9415
+7000 val perplexity 380.5093
+7000 train 5.814411 (lr=2.4442e-05) (hash(x)=139437666)
+7100 val loss 5.9337
+7100 val perplexity 377.5412
+7100 train 5.945026 (lr=2.4171e-05) (hash(x)=159792986)
+7200 val loss 5.9151
+7200 val perplexity 370.6091
+7200 train 5.902672 (lr=2.3899e-05) (hash(x)=144930687)
+7300 val loss 5.9125
+7300 val perplexity 369.6437
+7300 train 5.960479 (lr=2.3625e-05) (hash(x)=156242690)
+7400 val loss 5.9014
+7400 val perplexity 365.5585
+7400 train 5.731031 (lr=2.3349e-05) (hash(x)=148183719)
+7500 val loss 5.8968
+7500 val perplexity 363.8860
+7500 train 5.943295 (lr=2.3073e-05) (hash(x)=152494758)
+7600 val loss 5.8931
+7600 val perplexity 362.5366
+7600 train 5.588400 (lr=2.2795e-05) (hash(x)=142485027)
+7700 val loss 5.8946
+7700 val perplexity 363.0880
+7700 train 5.764569 (lr=2.2515e-05) (hash(x)=147512165)
+7800 val loss 5.8700
+7800 val perplexity 354.2464
+7800 train 5.741778 (lr=2.2235e-05) (hash(x)=160346994)
+7900 val loss 5.8666
+7900 val perplexity 353.0454
+7900 train 5.738772 (lr=2.1954e-05) (hash(x)=144488254)
+8000 val loss 5.8550
+8000 val perplexity 348.9649
+8000 train 5.665582 (lr=2.1671e-05) (hash(x)=147637019)
+8100 val loss 5.8522
+8100 val perplexity 348.0111
+8100 train 5.713085 (lr=2.1388e-05) (hash(x)=147340534)
+8200 val loss 5.8435
+8200 val perplexity 344.9934
+8200 train 5.907599 (lr=2.1105e-05) (hash(x)=151630665)
+8300 val loss 5.8371
+8300 val perplexity 342.7936
+8300 train 5.941170 (lr=2.0820e-05) (hash(x)=149747064)
+8400 val loss 5.8255
+8400 val perplexity 338.8247
+8400 train 6.025146 (lr=2.0536e-05) (hash(x)=154245770)
+8500 val loss 5.8218
+8500 val perplexity 337.5670
+8500 train 5.710631 (lr=2.0250e-05) (hash(x)=152559100)
+8600 val loss 5.8137
+8600 val perplexity 334.8488
+8600 train 6.208880 (lr=1.9965e-05) (hash(x)=181365926)
+8700 val loss 5.8088
+8700 val perplexity 333.2325
+8700 train 5.632130 (lr=1.9679e-05) (hash(x)=154405991)
+8800 val loss 5.7990
+8800 val perplexity 329.9627
+8800 train 5.749090 (lr=1.9393e-05) (hash(x)=153755904)
+8900 val loss 5.7975
+8900 val perplexity 329.4634
+8900 train 5.708569 (lr=1.9107e-05) (hash(x)=152120568)
+9000 val loss 5.7785
+9000 val perplexity 323.2753
+9000 train 5.573270 (lr=1.8821e-05) (hash(x)=142797279)
+9100 val loss 5.7769
+9100 val perplexity 322.7535
+9100 train 5.646819 (lr=1.8535e-05) (hash(x)=143037503)
+9200 val loss 5.7753
+9200 val perplexity 322.2328
+9200 train 5.649633 (lr=1.8250e-05) (hash(x)=113690273)
+9300 val loss 5.7678
+9300 val perplexity 319.8191
+9300 train 5.708163 (lr=1.7964e-05) (hash(x)=158025077)
+9400 val loss 5.7595
+9400 val perplexity 317.1894
+9400 train 5.840967 (lr=1.7680e-05) (hash(x)=158251718)
+9500 val loss 5.7562
+9500 val perplexity 316.1416
+9500 train 5.786424 (lr=1.7395e-05) (hash(x)=154752610)
+9600 val loss 5.7477
+9600 val perplexity 313.4607
+9600 train 5.630883 (lr=1.7112e-05) (hash(x)=146889093)
+9700 val loss 5.7382
+9700 val perplexity 310.4960
+9700 train 5.741565 (lr=1.6829e-05) (hash(x)=156906516)
+9800 val loss 5.7365
+9800 val perplexity 309.9757
+9800 train 5.562198 (lr=1.6546e-05) (hash(x)=153841927)
+9900 val loss 5.7311
+9900 val perplexity 308.3038
+9900 train 5.949374 (lr=1.6265e-05) (hash(x)=163514334)
+10000 val loss 5.7238
+10000 val perplexity 306.0793
+10000 train 5.723799 (lr=1.5985e-05) (hash(x)=144745561)
+10100 val loss 5.7185
+10100 val perplexity 304.4456
+10100 train 5.935614 (lr=1.5705e-05) (hash(x)=161651149)
+10200 val loss 5.7124
+10200 val perplexity 302.5875
+10200 train 5.650381 (lr=1.5427e-05) (hash(x)=143667037)
+10300 val loss 5.7086
+10300 val perplexity 301.4551
+10300 train 5.606673 (lr=1.5151e-05) (hash(x)=151687262)
+10400 val loss 5.7048
+10400 val perplexity 300.2989
+10400 train 5.547319 (lr=1.4875e-05) (hash(x)=153694955)
+10500 val loss 5.7046
+10500 val perplexity 300.2592
+10500 train 5.655263 (lr=1.4601e-05) (hash(x)=150218029)
+10600 val loss 5.6985
+10600 val perplexity 298.4055
+10600 train 5.777823 (lr=1.4329e-05) (hash(x)=155093716)
+10700 val loss 5.6915
+10700 val perplexity 296.3455
+10700 train 5.739979 (lr=1.4058e-05) (hash(x)=151631831)
+10800 val loss 5.6844
+10800 val perplexity 294.2519
+10800 train 5.547766 (lr=1.3789e-05) (hash(x)=136465175)
+10900 val loss 5.6813
+10900 val perplexity 293.3297
+10900 train 5.622696 (lr=1.3521e-05) (hash(x)=154461514)
+11000 val loss 5.6797
+11000 val perplexity 292.8474
+11000 train 5.673146 (lr=1.3256e-05) (hash(x)=141877432)
+11100 val loss 5.6754
+11100 val perplexity 291.5960
+11100 train 5.466080 (lr=1.2992e-05) (hash(x)=139572894)
+11200 val loss 5.6679
+11200 val perplexity 289.4243
+11200 train 5.721173 (lr=1.2731e-05) (hash(x)=155591443)
+11300 val loss 5.6625
+11300 val perplexity 287.8601
+11300 train 5.691182 (lr=1.2472e-05) (hash(x)=153349136)
+11400 val loss 5.6604
+11400 val perplexity 287.2627
+11400 train 5.621408 (lr=1.2215e-05) (hash(x)=148109395)
+11500 val loss 5.6564
+11500 val perplexity 286.1266
+11500 train 5.653296 (lr=1.1960e-05) (hash(x)=153276620)
+11600 val loss 5.6545
+11600 val perplexity 285.5773
+11600 train 5.523118 (lr=1.1708e-05) (hash(x)=151416388)
+11700 val loss 5.6520
+11700 val perplexity 284.8488
+11700 train 5.519225 (lr=1.1458e-05) (hash(x)=143420927)
+11800 val loss 5.6470
+11800 val perplexity 283.4373
+11800 train 5.705198 (lr=1.1210e-05) (hash(x)=165023996)
+11900 val loss 5.6393
+11900 val perplexity 281.2567
+11900 train 5.543955 (lr=1.0966e-05) (hash(x)=144465076)
+12000 val loss 5.6378
+12000 val perplexity 280.8501
+12000 train 5.653812 (lr=1.0724e-05) (hash(x)=155991892)
+12100 val loss 5.6366
+12100 val perplexity 280.5099
+12100 train 5.683947 (lr=1.0485e-05) (hash(x)=150748844)
+12200 val loss 5.6320
+12200 val perplexity 279.2109
+12200 train 5.605118 (lr=1.0249e-05) (hash(x)=149712638)
+12300 val loss 5.6304
+12300 val perplexity 278.7651
+12300 train 5.462433 (lr=1.0016e-05) (hash(x)=140428230)
+12400 val loss 5.6273
+12400 val perplexity 277.9107
+12400 train 5.813768 (lr=9.7854e-06) (hash(x)=163614308)
+12500 val loss 5.6219
+12500 val perplexity 276.4029
+12500 train 5.587658 (lr=9.5583e-06) (hash(x)=153366153)
+12600 val loss 5.6200
+12600 val perplexity 275.8958
+12600 train 5.525670 (lr=9.3345e-06) (hash(x)=146832066)
+12700 val loss 5.6159
+12700 val perplexity 274.7627
+12700 train 5.789717 (lr=9.1139e-06) (hash(x)=154387414)
+12800 val loss 5.6145
+12800 val perplexity 274.3859
+12800 train 5.408749 (lr=8.8967e-06) (hash(x)=139465199)
+12900 val loss 5.6157
+12900 val perplexity 274.7184
+12900 train 5.331150 (lr=8.6829e-06) (hash(x)=143326374)
+13000 val loss 5.6096
+13000 val perplexity 273.0276
+13000 train 5.762860 (lr=8.4726e-06) (hash(x)=158817835)
+13100 val loss 5.6064
+13100 val perplexity 272.1617
+13100 train 5.488267 (lr=8.2658e-06) (hash(x)=142027874)
+13200 val loss 5.6051
+13200 val perplexity 271.8146
+13200 train 5.591190 (lr=8.0626e-06) (hash(x)=148497730)
+13300 val loss 5.6034
+13300 val perplexity 271.3353
+13300 train 5.507898 (lr=7.8632e-06) (hash(x)=148094633)
+13400 val loss 5.5998
+13400 val perplexity 270.3817
+13400 train 5.463295 (lr=7.6674e-06) (hash(x)=150548214)
+13500 val loss 5.5936
+13500 val perplexity 268.7015
+13500 train 5.709482 (lr=7.4756e-06) (hash(x)=152789725)
+13600 val loss 5.5917
+13600 val perplexity 268.1876
+13600 train 5.634488 (lr=7.2875e-06) (hash(x)=153220230)
+13700 val loss 5.5894
+13700 val perplexity 267.5828
+13700 train 5.616460 (lr=7.1035e-06) (hash(x)=147805699)
+13800 val loss 5.5893
+13800 val perplexity 267.5524
+13800 train 5.399392 (lr=6.9234e-06) (hash(x)=146267341)
+13900 val loss 5.5881
+13900 val perplexity 267.2294
+13900 train 5.423314 (lr=6.7474e-06) (hash(x)=139467501)
+14000 val loss 5.5856
+14000 val perplexity 266.5664
+14000 train 5.602534 (lr=6.5756e-06) (hash(x)=152185436)
+14100 val loss 5.5805
+14100 val perplexity 265.2093
+14100 train 5.709170 (lr=6.4079e-06) (hash(x)=176479139)
+14200 val loss 5.5764
+14200 val perplexity 264.1297
+14200 train 5.654222 (lr=6.2444e-06) (hash(x)=151052040)
+14300 val loss 5.5752
+14300 val perplexity 263.8138
+14300 train 5.522494 (lr=6.0852e-06) (hash(x)=149668097)
+14400 val loss 5.5749
+14400 val perplexity 263.7227
+14400 train 5.588047 (lr=5.9304e-06) (hash(x)=151761941)
+14500 val loss 5.5726
+14500 val perplexity 263.1170
+14500 train 5.530627 (lr=5.7800e-06) (hash(x)=149412009)
+14600 val loss 5.5700
+14600 val perplexity 262.4285
+14600 train 5.451919 (lr=5.6340e-06) (hash(x)=145048020)
+14700 val loss 5.5677
+14700 val perplexity 261.8238
+14700 train 5.520936 (lr=5.4925e-06) (hash(x)=141771666)
+14800 val loss 5.5643
+14800 val perplexity 260.9447
+14800 train 5.486890 (lr=5.3555e-06) (hash(x)=148759740)
+14900 val loss 5.5629
+14900 val perplexity 260.5715
+14900 train 5.491292 (lr=5.2231e-06) (hash(x)=146287032)
+15000 val loss 5.5606
+15000 val perplexity 259.9802
+15000 train 5.507795 (lr=5.0954e-06) (hash(x)=143519156)
+15100 val loss 5.5632
+15100 val perplexity 260.6473
+15100 train 5.548056 (lr=4.9723e-06) (hash(x)=151330688)
+15200 val loss 5.5655
+15200 val perplexity 261.2667
+15200 train 5.371873 (lr=4.8539e-06) (hash(x)=140380090)
+15300 val loss 5.5599
+15300 val perplexity 259.7881
+15300 train 5.470203 (lr=4.7403e-06) (hash(x)=152598160)
+15400 val loss 5.5553
+15400 val perplexity 258.5987
+15400 train 5.792345 (lr=4.6314e-06) (hash(x)=164136966)
+15500 val loss 5.5544
+15500 val perplexity 258.3689
+15500 train 5.751496 (lr=4.5274e-06) (hash(x)=158733990)
+15600 val loss 5.5519
+15600 val perplexity 257.7298
+15600 train 5.593985 (lr=4.4282e-06) (hash(x)=149149390)
+15700 val loss 5.5506
+15700 val perplexity 257.3911
+15700 train 5.507520 (lr=4.3339e-06) (hash(x)=146746394)
+15800 val loss 5.5500
+15800 val perplexity 257.2358
+15800 train 5.569361 (lr=4.2446e-06) (hash(x)=159405219)
+15900 val loss 5.5485
+15900 val perplexity 256.8521
+15900 train 5.591457 (lr=4.1601e-06) (hash(x)=154034029)
+16000 val loss 5.5467
+16000 val perplexity 256.3889
+16000 train 5.543062 (lr=4.0807e-06) (hash(x)=153634210)
+16100 val loss 5.5438
+16100 val perplexity 255.6543
+16100 train 5.479768 (lr=4.0063e-06) (hash(x)=155646721)
+16200 val loss 5.5435
+16200 val perplexity 255.5702
+16200 train 5.458189 (lr=3.9368e-06) (hash(x)=142461643)
+16300 val loss 5.5426
+16300 val perplexity 255.3481
+16300 train 5.459714 (lr=3.8725e-06) (hash(x)=141080038)
+16400 val loss 5.5416
+16400 val perplexity 255.0929
+16400 train 5.554860 (lr=3.8132e-06) (hash(x)=153744166)
+16500 val loss 5.5441
+16500 val perplexity 255.7334
+16500 train 5.367388 (lr=3.7590e-06) (hash(x)=149783517)
+16600 val loss 5.5463
+16600 val perplexity 256.2921
+16600 train 5.386395 (lr=3.7099e-06) (hash(x)=142659127)
+16700 val loss 5.5397
+16700 val perplexity 254.5953
+16700 train 5.616637 (lr=3.6659e-06) (hash(x)=149725921)
+16800 val loss 5.5367
+16800 val perplexity 253.8382
+16800 train 5.392825 (lr=3.6271e-06) (hash(x)=140678087)
+16900 val loss 5.5347
+16900 val perplexity 253.3222
+16900 train 5.555767 (lr=3.5934e-06) (hash(x)=146935413)
+17000 val loss 5.5331
+17000 val perplexity 252.9307
+17000 train 5.549460 (lr=3.5649e-06) (hash(x)=144408437)
+17100 val loss 5.5327
+17100 val perplexity 252.8303
+17100 train 5.365889 (lr=3.5415e-06) (hash(x)=149229905)
+17200 val loss 5.5338
+17200 val perplexity 253.1139
+17200 train 5.497445 (lr=3.5234e-06) (hash(x)=150471865)
+17300 val loss 5.5350
+17300 val perplexity 253.3969
+17300 train 5.428278 (lr=3.5104e-06) (hash(x)=155784916)
+17400 val loss 5.5377
+17400 val perplexity 254.0839
+17400 train 5.318845 (lr=3.5026e-06) (hash(x)=148482413)
+17499 val loss 5.5305
+17499 val perplexity 252.2734

attention_kindselective_n_heads4_seed1340/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a172398ba2e69bcf26655d3b124c6e88d83154a9dbaee1270ed90a1898458663
+oid sha256:4923e7ab702d1408072d4b06f276cf73b3ab667510dbbd6f90cda737d2a3559a
 size 92843394

attention_kindselective_n_heads4_seed1340/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db5e34eda7ce004231ee080ffa4fbd862d3bb53a9f408e9f8709da0053c1c277
+oid sha256:82254200685547f8a942c48ae6e428bb336a452a15fc717fa5d70f603e7ad765
 size 179406214