Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1342/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1342", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1342", "n_embd": 256}

attention_kindselective_n_heads4_seed1342/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2907
 0 val perplexity 80077.4453
-0 train 11.286213 (lr=1.6250e-07) (hash(x)=150332693)
-100 val loss 9.9210
-100 val perplexity 20352.7031
-100 train 9.943149 (lr=1.6412e-05) (hash(x)=148948662)
-200 val loss 8.8456
-200 val perplexity 6943.5645
-200 train 8.764943 (lr=3.2500e-05) (hash(x)=149277319)
-300 val loss 7.8158
-300 val perplexity 2479.5896
-300 train 7.699074 (lr=3.2498e-05) (hash(x)=146429920)
-400 val loss 7.5422
-400 val perplexity 1886.0404
-400 train 7.658668 (lr=3.2490e-05) (hash(x)=155934075)
-500 val loss 7.4469
-500 val perplexity 1714.5961
-500 train 7.449210 (lr=3.2478e-05) (hash(x)=157510366)
-600 val loss 7.3775
-600 val perplexity 1599.5120
-600 train 7.494113 (lr=3.2461e-05) (hash(x)=165402628)
-700 val loss 7.3098
-700 val perplexity 1494.8848
-700 train 7.372377 (lr=3.2440e-05) (hash(x)=166427433)
-800 val loss 7.2453
-800 val perplexity 1401.5107
-800 train 7.043656 (lr=3.2413e-05) (hash(x)=139168939)
-900 val loss 7.1972
-900 val perplexity 1335.7433
-900 train 7.097889 (lr=3.2382e-05) (hash(x)=144698667)
-1000 val loss 7.1481
-1000 val perplexity 1271.6613
-1000 train 7.133333 (lr=3.2346e-05) (hash(x)=147134338)
-1100 val loss 7.0959
-1100 val perplexity 1207.0054
-1100 train 7.058327 (lr=3.2305e-05) (hash(x)=147172640)
-1200 val loss 7.0486
-1200 val perplexity 1151.2061
-1200 train 7.053260 (lr=3.2260e-05) (hash(x)=159086459)
-1300 val loss 7.0029
-1300 val perplexity 1099.8055
-1300 train 6.757029 (lr=3.2209e-05) (hash(x)=141578940)
-1400 val loss 6.9632
-1400 val perplexity 1057.0172
-1400 train 7.025721 (lr=3.2154e-05) (hash(x)=152399099)
-1500 val loss 6.9023
-1500 val perplexity 994.5381
-1500 train 6.985551 (lr=3.2094e-05) (hash(x)=158569712)
-1600 val loss 6.8529
-1600 val perplexity 946.5969
-1600 train 6.953071 (lr=3.2030e-05) (hash(x)=156548843)
-1700 val loss 6.7977
-1700 val perplexity 895.8251
-1700 train 6.806039 (lr=3.1961e-05) (hash(x)=152608173)
-1800 val loss 6.7500
-1800 val perplexity 854.0690
-1800 train 6.826288 (lr=3.1887e-05) (hash(x)=157889911)
-1900 val loss 6.7062
-1900 val perplexity 817.4597
-1900 train 6.672256 (lr=3.1809e-05) (hash(x)=140839139)
-2000 val loss 6.6697
-2000 val perplexity 788.1506
-2000 train 6.555364 (lr=3.1726e-05) (hash(x)=142943707)
-2100 val loss 6.6246
-2100 val perplexity 753.4261
-2100 train 6.591334 (lr=3.1638e-05) (hash(x)=151025344)
-2200 val loss 6.5868
-2200 val perplexity 725.4727
-2200 train 6.594067 (lr=3.1546e-05) (hash(x)=148938689)
-2300 val loss 6.5577
-2300 val perplexity 704.6324
-2300 train 6.518873 (lr=3.1449e-05) (hash(x)=150321226)
-2400 val loss 6.5091
-2400 val perplexity 671.1996
-2400 train 6.753350 (lr=3.1348e-05) (hash(x)=161962903)
-2500 val loss 6.4833
-2500 val perplexity 654.1196
-2500 train 6.418224 (lr=3.1243e-05) (hash(x)=153085222)
-2600 val loss 6.4487
-2600 val perplexity 631.8550
-2600 train 6.508657 (lr=3.1133e-05) (hash(x)=145979235)
-2700 val loss 6.4258
-2700 val perplexity 617.5670
-2700 train 6.490528 (lr=3.1019e-05) (hash(x)=149091945)
-2800 val loss 6.4006
-2800 val perplexity 602.2289
-2800 train 6.484369 (lr=3.0900e-05) (hash(x)=159157507)
-2900 val loss 6.3882
-2900 val perplexity 594.7667
-2900 train 6.396844 (lr=3.0777e-05) (hash(x)=155377499)
-3000 val loss 6.3600
-3000 val perplexity 578.2189
-3000 train 6.319705 (lr=3.0650e-05) (hash(x)=145943130)
-3100 val loss 6.3322
-3100 val perplexity 562.4135
-3100 train 6.391010 (lr=3.0518e-05) (hash(x)=153264973)
-3200 val loss 6.3080
-3200 val perplexity 548.9350
-3200 train 6.146701 (lr=3.0383e-05) (hash(x)=143249173)
-3300 val loss 6.2994
-3300 val perplexity 544.2535
-3300 train 6.307387 (lr=3.0243e-05) (hash(x)=155558767)
-3400 val loss 6.2723
-3400 val perplexity 529.6979
-3400 train 6.214176 (lr=3.0099e-05) (hash(x)=144137265)
-3500 val loss 6.2528
-3500 val perplexity 519.4819
-3500 train 6.451112 (lr=2.9952e-05) (hash(x)=154379201)
-3600 val loss 6.2379
-3600 val perplexity 511.8052
-3600 train 6.370066 (lr=2.9800e-05) (hash(x)=158728918)
-3700 val loss 6.2239
-3700 val perplexity 504.6459
-3700 train 5.971830 (lr=2.9644e-05) (hash(x)=139902844)
-3800 val loss 6.2162
-3800 val perplexity 500.8030
-3800 train 6.299497 (lr=2.9485e-05) (hash(x)=160738928)
-3900 val loss 6.1909
-3900 val perplexity 488.2827
-3900 train 6.231839 (lr=2.9321e-05) (hash(x)=156122233)
-4000 val loss 6.1714
-4000 val perplexity 478.8499
-4000 train 6.007371 (lr=2.9154e-05) (hash(x)=136691174)
-4100 val loss 6.1697
-4100 val perplexity 478.0639
-4100 train 6.138481 (lr=2.8983e-05) (hash(x)=150135523)
-4200 val loss 6.1566
-4200 val perplexity 471.8432
-4200 train 6.149134 (lr=2.8808e-05) (hash(x)=154053307)
-4300 val loss 6.1458
-4300 val perplexity 466.7565
-4300 train 5.931820 (lr=2.8630e-05) (hash(x)=136076439)
-4400 val loss 6.1281
-4400 val perplexity 458.5824
-4400 train 6.025293 (lr=2.8448e-05) (hash(x)=147368042)
-4500 val loss 6.1132
-4500 val perplexity 451.7802
-4500 train 6.206073 (lr=2.8263e-05) (hash(x)=163726147)
-4600 val loss 6.0976
-4600 val perplexity 444.7792
-4600 train 6.113523 (lr=2.8075e-05) (hash(x)=151996388)
-4700 val loss 6.0855
-4700 val perplexity 439.4369
-4700 train 6.137928 (lr=2.7883e-05) (hash(x)=148952763)
-4800 val loss 6.0812
-4800 val perplexity 437.5425
-4800 train 6.004941 (lr=2.7687e-05) (hash(x)=139747769)
-4900 val loss 6.0717
-4900 val perplexity 433.4262
-4900 train 5.939377 (lr=2.7489e-05) (hash(x)=145011383)
-5000 val loss 6.0599
-5000 val perplexity 428.3344
-5000 train 6.082546 (lr=2.7287e-05) (hash(x)=153449428)
-5100 val loss 6.0476
-5100 val perplexity 423.1035
-5100 train 6.091887 (lr=2.7082e-05) (hash(x)=161397672)
-5200 val loss 6.0315
-5200 val perplexity 416.3513
-5200 train 6.013671 (lr=2.6874e-05) (hash(x)=164813156)
-5300 val loss 6.0209
-5300 val perplexity 411.9349
-5300 train 6.068652 (lr=2.6664e-05) (hash(x)=146155759)
-5400 val loss 6.0193
-5400 val perplexity 411.3075
-5400 train 6.090827 (lr=2.6450e-05) (hash(x)=152337652)
-5500 val loss 6.0072
-5500 val perplexity 406.3346
-5500 train 5.868421 (lr=2.6233e-05) (hash(x)=148346631)
-5600 val loss 5.9981
-5600 val perplexity 402.6518
-5600 train 5.910365 (lr=2.6014e-05) (hash(x)=146652463)
-5700 val loss 5.9894
-5700 val perplexity 399.1822
-5700 train 5.936297 (lr=2.5792e-05) (hash(x)=147553632)
-5800 val loss 5.9710
-5800 val perplexity 391.8972
-5800 train 6.067301 (lr=2.5567e-05) (hash(x)=153512314)
-5900 val loss 5.9650
-5900 val perplexity 389.5436
-5900 train 5.942406 (lr=2.5340e-05) (hash(x)=137351707)
-6000 val loss 5.9541
-6000 val perplexity 385.3337
-6000 train 5.976089 (lr=2.5111e-05) (hash(x)=143375055)
-6100 val loss 5.9517
-6100 val perplexity 384.4194
-6100 train 5.977683 (lr=2.4879e-05) (hash(x)=144781857)
-6200 val loss 5.9378
-6200 val perplexity 379.0844
-6200 train 6.171885 (lr=2.4644e-05) (hash(x)=167556812)
-6300 val loss 5.9337
-6300 val perplexity 377.5300
-6300 train 6.092905 (lr=2.4408e-05) (hash(x)=162290597)
-6400 val loss 5.9223
-6400 val perplexity 373.2822
-6400 train 5.821190 (lr=2.4169e-05) (hash(x)=144836361)
-6500 val loss 5.9054
-6500 val perplexity 367.0234
-6500 train 5.778796 (lr=2.3928e-05) (hash(x)=150949131)
-6600 val loss 5.9223
-6600 val perplexity 373.2838
-6600 train 5.985580 (lr=2.3686e-05) (hash(x)=151386312)
-6700 val loss 5.8876
-6700 val perplexity 360.5525
-6700 train 5.970166 (lr=2.3441e-05) (hash(x)=159840718)
-6800 val loss 5.8800
-6800 val perplexity 357.8123
-6800 train 5.971055 (lr=2.3195e-05) (hash(x)=158843242)
-6900 val loss 5.8727
-6900 val perplexity 355.2012
-6900 train 6.100095 (lr=2.2946e-05) (hash(x)=160544793)
-7000 val loss 5.8631
-7000 val perplexity 351.8195
-7000 train 5.713056 (lr=2.2696e-05) (hash(x)=139031423)
-7100 val loss 5.8606
-7100 val perplexity 350.9410
-7100 train 5.813060 (lr=2.2445e-05) (hash(x)=148444150)
-7200 val loss 5.8529
-7200 val perplexity 348.2327
-7200 train 6.190878 (lr=2.2192e-05) (hash(x)=162348298)
-7300 val loss 5.8435
-7300 val perplexity 345.0013
-7300 train 5.890631 (lr=2.1937e-05) (hash(x)=160070640)
-7400 val loss 5.8353
-7400 val perplexity 342.1762
-7400 train 5.759228 (lr=2.1682e-05) (hash(x)=142907242)
-7500 val loss 5.8300
-7500 val perplexity 340.3468
-7500 train 5.815858 (lr=2.1425e-05) (hash(x)=155567735)
-7600 val loss 5.8206
-7600 val perplexity 337.1619
-7600 train 5.675367 (lr=2.1166e-05) (hash(x)=149026190)
-7700 val loss 5.8122
-7700 val perplexity 334.3678
-7700 train 6.354319 (lr=2.0907e-05) (hash(x)=172973386)
-7800 val loss 5.8100
-7800 val perplexity 333.6221
-7800 train 5.794294 (lr=2.0647e-05) (hash(x)=148166384)
-7900 val loss 5.8004
-7900 val perplexity 330.4375
-7900 train 6.223359 (lr=2.0386e-05) (hash(x)=174580801)
-8000 val loss 5.7914
-8000 val perplexity 327.4816
-8000 train 5.803993 (lr=2.0123e-05) (hash(x)=153576391)
-8100 val loss 5.7841
-8100 val perplexity 325.0847
-8100 train 5.681134 (lr=1.9861e-05) (hash(x)=143483378)
-8200 val loss 5.7820
-8200 val perplexity 324.3921
-8200 train 6.249942 (lr=1.9597e-05) (hash(x)=125210829)
-8300 val loss 5.7756
-8300 val perplexity 322.3530
-8300 train 5.698894 (lr=1.9333e-05) (hash(x)=152432601)
-8400 val loss 5.7777
-8400 val perplexity 323.0283
-8400 train 5.651137 (lr=1.9069e-05) (hash(x)=148759990)
-8500 val loss 5.7614
-8500 val perplexity 317.8048
-8500 train 5.744183 (lr=1.8804e-05) (hash(x)=156974533)
-8600 val loss 5.7548
-8600 val perplexity 315.6933
-8600 train 5.871877 (lr=1.8539e-05) (hash(x)=155441439)
-8700 val loss 5.7501
-8700 val perplexity 314.2266
-8700 train 5.732445 (lr=1.8273e-05) (hash(x)=149001504)
-8800 val loss 5.7452
-8800 val perplexity 312.6789
-8800 train 5.619228 (lr=1.8008e-05) (hash(x)=144721722)
-8900 val loss 5.7395
-8900 val perplexity 310.9202
-8900 train 5.648099 (lr=1.7742e-05) (hash(x)=151789511)
-9000 val loss 5.7362
-9000 val perplexity 309.8998
-9000 train 5.668345 (lr=1.7477e-05) (hash(x)=151307797)
-9100 val loss 5.7284
-9100 val perplexity 307.4894
-9100 train 5.858921 (lr=1.7211e-05) (hash(x)=155534554)
-9200 val loss 5.7233
-9200 val perplexity 305.9167
-9200 train 5.622472 (lr=1.6946e-05) (hash(x)=138467356)
-9300 val loss 5.7209
-9300 val perplexity 305.1840
-9300 train 5.654151 (lr=1.6681e-05) (hash(x)=148622510)
-9400 val loss 5.7137
-9400 val perplexity 303.0007
-9400 train 5.844534 (lr=1.6417e-05) (hash(x)=154343672)
-9500 val loss 5.7113
-9500 val perplexity 302.2711
-9500 train 5.699077 (lr=1.6153e-05) (hash(x)=156831628)
-9600 val loss 5.7072
-9600 val perplexity 301.0198
-9600 train 5.483487 (lr=1.5889e-05) (hash(x)=142190812)
-9700 val loss 5.6971
-9700 val perplexity 298.0132
-9700 train 5.597317 (lr=1.5627e-05) (hash(x)=146487376)
-9800 val loss 5.6923
-9800 val perplexity 296.5642
-9800 train 5.762100 (lr=1.5364e-05) (hash(x)=164178945)
-9900 val loss 5.6894
-9900 val perplexity 295.7070
-9900 train 5.551723 (lr=1.5103e-05) (hash(x)=142224918)
-10000 val loss 5.6884
-10000 val perplexity 295.4115
-10000 train 5.614565 (lr=1.4843e-05) (hash(x)=148509740)
-10100 val loss 5.6823
-10100 val perplexity 293.6255
-10100 train 5.746253 (lr=1.4584e-05) (hash(x)=153536520)
-10200 val loss 5.6733
-10200 val perplexity 290.9957
-10200 train 5.608801 (lr=1.4325e-05) (hash(x)=153740168)
-10300 val loss 5.6889
-10300 val perplexity 295.5677
-10300 train 5.645096 (lr=1.4068e-05) (hash(x)=160410260)
-10400 val loss 5.6682
-10400 val perplexity 289.5168
-10400 train 5.824097 (lr=1.3813e-05) (hash(x)=136788294)
-10500 val loss 5.6622
-10500 val perplexity 287.7668
-10500 train 5.517078 (lr=1.3558e-05) (hash(x)=153988760)
-10600 val loss 5.6585
-10600 val perplexity 286.7173
-10600 train 5.546037 (lr=1.3305e-05) (hash(x)=141896522)
-10700 val loss 5.6583
-10700 val perplexity 286.6493
-10700 train 6.007439 (lr=1.3054e-05) (hash(x)=147057702)
-10800 val loss 5.6483
-10800 val perplexity 283.8185
-10800 train 5.720152 (lr=1.2804e-05) (hash(x)=157542287)
-10900 val loss 5.6443
-10900 val perplexity 282.6749
-10900 train 5.721572 (lr=1.2555e-05) (hash(x)=153860040)
-11000 val loss 5.6408
-11000 val perplexity 281.6780
-11000 train 5.501269 (lr=1.2309e-05) (hash(x)=142932555)
-11100 val loss 5.6365
-11100 val perplexity 280.4904
-11100 train 5.674755 (lr=1.2064e-05) (hash(x)=146260760)
-11200 val loss 5.6382
-11200 val perplexity 280.9466
-11200 train 5.741637 (lr=1.1822e-05) (hash(x)=148970533)
-11300 val loss 5.6323
-11300 val perplexity 279.3171
-11300 train 5.654080 (lr=1.1581e-05) (hash(x)=149421645)
-11400 val loss 5.6261
-11400 val perplexity 277.5911
-11400 train 5.560232 (lr=1.1342e-05) (hash(x)=158728429)
-11500 val loss 5.6225
-11500 val perplexity 276.5673
-11500 train 5.730070 (lr=1.1106e-05) (hash(x)=139524335)
-11600 val loss 5.6225
-11600 val perplexity 276.5689
-11600 train 5.659407 (lr=1.0871e-05) (hash(x)=152760174)
-11700 val loss 5.6180
-11700 val perplexity 275.3245
-11700 train 5.519762 (lr=1.0639e-05) (hash(x)=140740682)
-11800 val loss 5.6172
-11800 val perplexity 275.1151
-11800 train 5.465315 (lr=1.0410e-05) (hash(x)=153665407)
-11900 val loss 5.6156
-11900 val perplexity 274.6813
-11900 train 5.564779 (lr=1.0183e-05) (hash(x)=163068424)
-12000 val loss 5.6077
-12000 val perplexity 272.5207
-12000 train 5.665866 (lr=9.9580e-06) (hash(x)=145166541)
-12100 val loss 5.6057
-12100 val perplexity 271.9738
-12100 train 5.569032 (lr=9.7360e-06) (hash(x)=143228042)
-12200 val loss 5.6028
-12200 val perplexity 271.1841
-12200 train 5.703817 (lr=9.5167e-06) (hash(x)=159222680)
-12300 val loss 5.6012
-12300 val perplexity 270.7473
-12300 train 5.551047 (lr=9.3001e-06) (hash(x)=145196157)
-12400 val loss 5.5979
-12400 val perplexity 269.8643
-12400 train 5.523250 (lr=9.0864e-06) (hash(x)=143033742)
-12500 val loss 5.5977
-12500 val perplexity 269.8084
-12500 train 5.637140 (lr=8.8756e-06) (hash(x)=151133346)
-12600 val loss 5.5936
-12600 val perplexity 268.7136
-12600 train 5.616040 (lr=8.6677e-06) (hash(x)=153607184)
-12700 val loss 5.5890
-12700 val perplexity 267.4702
-12700 train 5.499484 (lr=8.4629e-06) (hash(x)=136931911)
-12800 val loss 5.5876
-12800 val perplexity 267.0939
-12800 train 5.697507 (lr=8.2612e-06) (hash(x)=155105402)
-12900 val loss 5.5868
-12900 val perplexity 266.8801
-12900 train 5.419711 (lr=8.0627e-06) (hash(x)=144141636)
-13000 val loss 5.5876
-13000 val perplexity 267.0950
-13000 train 5.509489 (lr=7.8674e-06) (hash(x)=139380218)
-13100 val loss 5.5797
-13100 val perplexity 264.9858
-13100 train 5.683111 (lr=7.6754e-06) (hash(x)=157695174)
-13200 val loss 5.5790
-13200 val perplexity 264.8096
-13200 train 5.524367 (lr=7.4867e-06) (hash(x)=143697560)
-13300 val loss 5.5759
-13300 val perplexity 263.9969
-13300 train 5.406800 (lr=7.3015e-06) (hash(x)=156314952)
-13400 val loss 5.5758
-13400 val perplexity 263.9568
-13400 train 5.547408 (lr=7.1198e-06) (hash(x)=136720279)
-13500 val loss 5.5748
-13500 val perplexity 263.7006
-13500 train 5.532155 (lr=6.9416e-06) (hash(x)=142579957)
-13600 val loss 5.5756
-13600 val perplexity 263.9139
-13600 train 5.289562 (lr=6.7670e-06) (hash(x)=140719440)
-13700 val loss 5.5755
-13700 val perplexity 263.8818
-13700 train 5.670376 (lr=6.5961e-06) (hash(x)=164396003)
-13800 val loss 5.5686
-13800 val perplexity 262.0556
-13800 train 5.625392 (lr=6.4289e-06) (hash(x)=152712862)
-13900 val loss 5.5644
-13900 val perplexity 260.9780
-13900 train 5.587784 (lr=6.2655e-06) (hash(x)=160985139)
-14000 val loss 5.5626
-14000 val perplexity 260.4993
-14000 train 5.565255 (lr=6.1059e-06) (hash(x)=147429379)
-14100 val loss 5.5613
-14100 val perplexity 260.1722
-14100 train 5.627103 (lr=5.9502e-06) (hash(x)=131830470)
-14200 val loss 5.5633
-14200 val perplexity 260.6945
-14200 train 5.305113 (lr=5.7984e-06) (hash(x)=146383028)
-14300 val loss 5.5595
-14300 val perplexity 259.6993
-14300 train 5.609846 (lr=5.6506e-06) (hash(x)=157158747)
-14400 val loss 5.5622
-14400 val perplexity 260.3876
-14400 train 5.420488 (lr=5.5068e-06) (hash(x)=143911520)
-14500 val loss 5.5625
-14500 val perplexity 260.4818
-14500 train 5.335642 (lr=5.3671e-06) (hash(x)=159555347)
-14600 val loss 5.5564
-14600 val perplexity 258.8873
-14600 train 5.560647 (lr=5.2316e-06) (hash(x)=152957062)
-14700 val loss 5.5522
-14700 val perplexity 257.7971
-14700 train 5.625054 (lr=5.1002e-06) (hash(x)=156001594)
-14800 val loss 5.5512
-14800 val perplexity 257.5373
-14800 train 5.657235 (lr=4.9730e-06) (hash(x)=145885458)
-14900 val loss 5.5487
-14900 val perplexity 256.9017
-14900 train 5.622381 (lr=4.8501e-06) (hash(x)=144544291)
-15000 val loss 5.5491
-15000 val perplexity 257.0116
-15000 train 5.556798 (lr=4.7314e-06) (hash(x)=154606709)
-15100 val loss 5.5484
-15100 val perplexity 256.8292
-15100 train 5.593152 (lr=4.6171e-06) (hash(x)=156005324)
-15200 val loss 5.5506
-15200 val perplexity 257.3899
-15200 train 5.648378 (lr=4.5072e-06) (hash(x)=158847373)
-15300 val loss 5.5508
-15300 val perplexity 257.4484
-15300 train 5.334248 (lr=4.4017e-06) (hash(x)=159609594)
-15400 val loss 5.5455
-15400 val perplexity 256.0708
-15400 train 5.643424 (lr=4.3006e-06) (hash(x)=152428492)
-15500 val loss 5.5420
-15500 val perplexity 255.1837
-15500 train 5.612495 (lr=4.2040e-06) (hash(x)=162251192)
-15600 val loss 5.5408
-15600 val perplexity 254.8778
-15600 train 5.792667 (lr=4.1119e-06) (hash(x)=155072002)
-15700 val loss 5.5408
-15700 val perplexity 254.8784
-15700 train 5.562519 (lr=4.0244e-06) (hash(x)=151203094)
-15800 val loss 5.5395
-15800 val perplexity 254.5390
-15800 train 5.467110 (lr=3.9414e-06) (hash(x)=146125932)
-15900 val loss 5.5390
-15900 val perplexity 254.4307
-15900 train 5.412131 (lr=3.8630e-06) (hash(x)=148060195)
-16000 val loss 5.5403
-16000 val perplexity 254.7613
-16000 train 5.244052 (lr=3.7892e-06) (hash(x)=137406416)
-16100 val loss 5.5406
-16100 val perplexity 254.8330
-16100 train 5.603872 (lr=3.7201e-06) (hash(x)=160149755)
-16200 val loss 5.5347
-16200 val perplexity 253.3356
-16200 train 5.715085 (lr=3.6556e-06) (hash(x)=151371894)
-16300 val loss 5.5332
-16300 val perplexity 252.9408
-16300 train 5.589443 (lr=3.5959e-06) (hash(x)=151810252)
-16400 val loss 5.5319
-16400 val perplexity 252.6117
-16400 train 5.517071 (lr=3.5408e-06) (hash(x)=153088303)
-16500 val loss 5.5314
-16500 val perplexity 252.5005
-16500 train 5.410724 (lr=3.4905e-06) (hash(x)=151706615)
-16600 val loss 5.5295
-16600 val perplexity 252.0252
-16600 train 5.427650 (lr=3.4449e-06) (hash(x)=142110626)
-16700 val loss 5.5305
-16700 val perplexity 252.2643
-16700 train 5.427150 (lr=3.4041e-06) (hash(x)=153222439)
-16800 val loss 5.5297
-16800 val perplexity 252.0710
-16800 train 5.448761 (lr=3.3680e-06) (hash(x)=154964902)
-16900 val loss 5.5264
-16900 val perplexity 251.2368
-16900 train 5.686378 (lr=3.3367e-06) (hash(x)=158893204)
-17000 val loss 5.5250
-17000 val perplexity 250.8958
-17000 train 5.454472 (lr=3.3102e-06) (hash(x)=148929955)
-17100 val loss 5.5259
-17100 val perplexity 251.1136
-17100 train 5.539378 (lr=3.2886e-06) (hash(x)=147818677)
-17200 val loss 5.5243
-17200 val perplexity 250.7198
-17200 train 5.456254 (lr=3.2717e-06) (hash(x)=147119446)
-17300 val loss 5.5240
-17300 val perplexity 250.6337
-17300 train 5.575593 (lr=3.2596e-06) (hash(x)=150494142)
-17400 val loss 5.5245
-17400 val perplexity 250.7691
-17400 train 5.557147 (lr=3.2524e-06) (hash(x)=155918058)
-17499 val loss 5.5225
-17499 val perplexity 250.2664
+0 train 11.286213 (lr=1.5000e-07) (hash(x)=150332693)
+100 val loss 9.9351
+100 val perplexity 20642.3594
+100 train 9.958242 (lr=1.5150e-05) (hash(x)=148948662)
+200 val loss 9.0369
+200 val perplexity 8407.2920
+200 train 8.958506 (lr=3.0000e-05) (hash(x)=149277319)
+300 val loss 7.9533
+300 val perplexity 2844.9207
+300 train 7.853283 (lr=2.9998e-05) (hash(x)=146429920)
+400 val loss 7.6313
+400 val perplexity 2061.7974
+400 train 7.745924 (lr=2.9991e-05) (hash(x)=155934075)
+500 val loss 7.5210
+500 val perplexity 1846.3237
+500 train 7.530861 (lr=2.9980e-05) (hash(x)=157510366)
+600 val loss 7.4547
+600 val perplexity 1727.9935
+600 train 7.567689 (lr=2.9964e-05) (hash(x)=165402628)
+700 val loss 7.3864
+700 val perplexity 1613.9584
+700 train 7.445437 (lr=2.9944e-05) (hash(x)=166427433)
+800 val loss 7.3378
+800 val perplexity 1537.3270
+800 train 7.138241 (lr=2.9920e-05) (hash(x)=139168939)
+900 val loss 7.2903
+900 val perplexity 1466.0508
+900 train 7.199311 (lr=2.9891e-05) (hash(x)=144698667)
+1000 val loss 7.2443
+1000 val perplexity 1400.1667
+1000 train 7.230426 (lr=2.9858e-05) (hash(x)=147134338)
+1100 val loss 7.2013
+1100 val perplexity 1341.2264
+1100 train 7.165863 (lr=2.9820e-05) (hash(x)=147172640)
+1200 val loss 7.1529
+1200 val perplexity 1277.8662
+1200 train 7.153451 (lr=2.9778e-05) (hash(x)=159086459)
+1300 val loss 7.1123
+1300 val perplexity 1226.9387
+1300 train 6.877395 (lr=2.9732e-05) (hash(x)=141578940)
+1400 val loss 7.0736
+1400 val perplexity 1180.4430
+1400 train 7.141044 (lr=2.9681e-05) (hash(x)=152399099)
+1500 val loss 7.0265
+1500 val perplexity 1126.0300
+1500 train 7.104778 (lr=2.9626e-05) (hash(x)=158569712)
+1600 val loss 6.9794
+1600 val perplexity 1074.2599
+1600 train 7.075292 (lr=2.9566e-05) (hash(x)=156548843)
+1700 val loss 6.9464
+1700 val perplexity 1039.4301
+1700 train 6.948898 (lr=2.9502e-05) (hash(x)=152608173)
+1800 val loss 6.9093
+1800 val perplexity 1001.5162
+1800 train 6.986165 (lr=2.9434e-05) (hash(x)=157889911)
+1900 val loss 6.8804
+1900 val perplexity 973.0444
+1900 train 6.833936 (lr=2.9362e-05) (hash(x)=140839139)
+2000 val loss 6.8449
+2000 val perplexity 939.1225
+2000 train 6.733738 (lr=2.9285e-05) (hash(x)=142943707)
+2100 val loss 6.8209
+2100 val perplexity 916.8447
+2100 train 6.793978 (lr=2.9204e-05) (hash(x)=151025344)
+2200 val loss 6.7804
+2200 val perplexity 880.4022
+2200 train 6.775433 (lr=2.9119e-05) (hash(x)=148938689)
+2300 val loss 6.7393
+2300 val perplexity 844.9785
+2300 train 6.695773 (lr=2.9030e-05) (hash(x)=150321226)
+2400 val loss 6.6971
+2400 val perplexity 810.0812
+2400 train 6.919641 (lr=2.8937e-05) (hash(x)=161962903)
+2500 val loss 6.6544
+2500 val perplexity 776.1724
+2500 train 6.579124 (lr=2.8840e-05) (hash(x)=153085222)
+2600 val loss 6.6196
+2600 val perplexity 749.6511
+2600 train 6.666867 (lr=2.8738e-05) (hash(x)=145979235)
+2700 val loss 6.5862
+2700 val perplexity 725.0187
+2700 train 6.645664 (lr=2.8633e-05) (hash(x)=149091945)
+2800 val loss 6.5506
+2800 val perplexity 699.6385
+2800 train 6.620130 (lr=2.8523e-05) (hash(x)=159157507)
+2900 val loss 6.5289
+2900 val perplexity 684.6458
+2900 train 6.537559 (lr=2.8410e-05) (hash(x)=155377499)
+3000 val loss 6.4989
+3000 val perplexity 664.4088
+3000 train 6.464838 (lr=2.8292e-05) (hash(x)=145943130)
+3100 val loss 6.4650
+3100 val perplexity 642.2914
+3100 train 6.520201 (lr=2.8171e-05) (hash(x)=153264973)
+3200 val loss 6.4402
+3200 val perplexity 626.5162
+3200 train 6.270744 (lr=2.8046e-05) (hash(x)=143249173)
+3300 val loss 6.4214
+3300 val perplexity 614.8851
+3300 train 6.430489 (lr=2.7917e-05) (hash(x)=155558767)
+3400 val loss 6.3902
+3400 val perplexity 596.0014
+3400 train 6.332700 (lr=2.7784e-05) (hash(x)=144137265)
+3500 val loss 6.3664
+3500 val perplexity 581.9711
+3500 train 6.553246 (lr=2.7648e-05) (hash(x)=154379201)
+3600 val loss 6.3509
+3600 val perplexity 572.9807
+3600 train 6.470469 (lr=2.7508e-05) (hash(x)=158728918)
+3700 val loss 6.3331
+3700 val perplexity 562.8910
+3700 train 6.070055 (lr=2.7364e-05) (hash(x)=139902844)
+3800 val loss 6.3185
+3800 val perplexity 554.7255
+3800 train 6.389813 (lr=2.7216e-05) (hash(x)=160738928)
+3900 val loss 6.2904
+3900 val perplexity 539.3649
+3900 train 6.331081 (lr=2.7066e-05) (hash(x)=156122233)
+4000 val loss 6.2721
+4000 val perplexity 529.5984
+4000 train 6.095728 (lr=2.6911e-05) (hash(x)=136691174)
+4100 val loss 6.2625
+4100 val perplexity 524.5030
+4100 train 6.228635 (lr=2.6754e-05) (hash(x)=150135523)
+4200 val loss 6.2536
+4200 val perplexity 519.8756
+4200 train 6.243220 (lr=2.6592e-05) (hash(x)=154053307)
+4300 val loss 6.2403
+4300 val perplexity 513.0081
+4300 train 6.015927 (lr=2.6428e-05) (hash(x)=136076439)
+4400 val loss 6.2177
+4400 val perplexity 501.5292
+4400 train 6.107508 (lr=2.6260e-05) (hash(x)=147368042)
+4500 val loss 6.2021
+4500 val perplexity 493.7754
+4500 train 6.297653 (lr=2.6089e-05) (hash(x)=163726147)
+4600 val loss 6.1854
+4600 val perplexity 485.5912
+4600 train 6.203592 (lr=2.5915e-05) (hash(x)=151996388)
+4700 val loss 6.1712
+4700 val perplexity 478.7798
+4700 train 6.213857 (lr=2.5738e-05) (hash(x)=148952763)
+4800 val loss 6.1600
+4800 val perplexity 473.4117
+4800 train 6.077888 (lr=2.5558e-05) (hash(x)=139747769)
+4900 val loss 6.1494
+4900 val perplexity 468.4167
+4900 train 6.010595 (lr=2.5374e-05) (hash(x)=145011383)
+5000 val loss 6.1393
+5000 val perplexity 463.7233
+5000 train 6.162326 (lr=2.5188e-05) (hash(x)=153449428)
+5100 val loss 6.1294
+5100 val perplexity 459.1557
+5100 train 6.172811 (lr=2.4999e-05) (hash(x)=161397672)
+5200 val loss 6.1093
+5200 val perplexity 450.0402
+5200 train 6.097622 (lr=2.4807e-05) (hash(x)=164813156)
+5300 val loss 6.0982
+5300 val perplexity 445.0724
+5300 train 6.152255 (lr=2.4613e-05) (hash(x)=146155759)
+5400 val loss 6.0879
+5400 val perplexity 440.5091
+5400 train 6.156091 (lr=2.4415e-05) (hash(x)=152337652)
+5500 val loss 6.0800
+5500 val perplexity 437.0487
+5500 train 5.936502 (lr=2.4215e-05) (hash(x)=148346631)
+5600 val loss 6.0698
+5600 val perplexity 432.5997
+5600 train 5.978254 (lr=2.4013e-05) (hash(x)=146652463)
+5700 val loss 6.0636
+5700 val perplexity 429.9003
+5700 train 6.000385 (lr=2.3808e-05) (hash(x)=147553632)
+5800 val loss 6.0496
+5800 val perplexity 423.9548
+5800 train 6.145557 (lr=2.3601e-05) (hash(x)=153512314)
+5900 val loss 6.0372
+5900 val perplexity 418.7395
+5900 train 6.004522 (lr=2.3391e-05) (hash(x)=137351707)
+6000 val loss 6.0272
+6000 val perplexity 414.5595
+6000 train 6.050917 (lr=2.3179e-05) (hash(x)=143375055)
+6100 val loss 6.0245
+6100 val perplexity 413.4346
+6100 train 6.047802 (lr=2.2965e-05) (hash(x)=144781857)
+6200 val loss 6.0119
+6200 val perplexity 408.2610
+6200 train 6.255000 (lr=2.2749e-05) (hash(x)=167556812)
+6300 val loss 6.0016
+6300 val perplexity 404.0685
+6300 train 6.156122 (lr=2.2530e-05) (hash(x)=162290597)
+6400 val loss 5.9908
+6400 val perplexity 399.7199
+6400 train 5.886322 (lr=2.2310e-05) (hash(x)=144836361)
+6500 val loss 5.9783
+6500 val perplexity 394.7825
+6500 train 5.851570 (lr=2.2088e-05) (hash(x)=150949131)
+6600 val loss 5.9938
+6600 val perplexity 400.9352
+6600 train 6.061947 (lr=2.1864e-05) (hash(x)=151386312)
+6700 val loss 5.9631
+6700 val perplexity 388.8249
+6700 train 6.041219 (lr=2.1638e-05) (hash(x)=159840718)
+6800 val loss 5.9529
+6800 val perplexity 384.8733
+6800 train 6.046275 (lr=2.1410e-05) (hash(x)=158843242)
+6900 val loss 5.9471
+6900 val perplexity 382.6416
+6900 train 6.188788 (lr=2.1181e-05) (hash(x)=160544793)
+7000 val loss 5.9385
+7000 val perplexity 379.3763
+7000 train 5.780041 (lr=2.0951e-05) (hash(x)=139031423)
+7100 val loss 5.9357
+7100 val perplexity 378.2969
+7100 train 5.874920 (lr=2.0718e-05) (hash(x)=148444150)
+7200 val loss 5.9244
+7200 val perplexity 374.0450
+7200 train 6.257408 (lr=2.0485e-05) (hash(x)=162348298)
+7300 val loss 5.9135
+7300 val perplexity 370.0092
+7300 train 5.969735 (lr=2.0250e-05) (hash(x)=160070640)
+7400 val loss 5.9077
+7400 val perplexity 367.8553
+7400 train 5.828381 (lr=2.0014e-05) (hash(x)=142907242)
+7500 val loss 5.9027
+7500 val perplexity 366.0135
+7500 train 5.909079 (lr=1.9777e-05) (hash(x)=155567735)
+7600 val loss 5.8932
+7600 val perplexity 362.5581
+7600 train 5.749373 (lr=1.9538e-05) (hash(x)=149026190)
+7700 val loss 5.8864
+7700 val perplexity 360.1118
+7700 train 6.437926 (lr=1.9299e-05) (hash(x)=172973386)
+7800 val loss 5.8815
+7800 val perplexity 358.3377
+7800 train 5.852612 (lr=1.9059e-05) (hash(x)=148166384)
+7900 val loss 5.8721
+7900 val perplexity 354.9937
+7900 train 6.311577 (lr=1.8817e-05) (hash(x)=174580801)
+8000 val loss 5.8638
+8000 val perplexity 352.0487
+8000 train 5.881599 (lr=1.8576e-05) (hash(x)=153576391)
+8100 val loss 5.8601
+8100 val perplexity 350.7613
+8100 train 5.759035 (lr=1.8333e-05) (hash(x)=143483378)
+8200 val loss 5.8529
+8200 val perplexity 348.2314
+8200 train 6.302994 (lr=1.8090e-05) (hash(x)=125210829)
+8300 val loss 5.8485
+8300 val perplexity 346.7147
+8300 train 5.778973 (lr=1.7846e-05) (hash(x)=152432601)
+8400 val loss 5.8504
+8400 val perplexity 347.3867
+8400 train 5.727871 (lr=1.7602e-05) (hash(x)=148759990)
+8500 val loss 5.8366
+8500 val perplexity 342.6263
+8500 train 5.820582 (lr=1.7357e-05) (hash(x)=156974533)
+8600 val loss 5.8261
+8600 val perplexity 339.0219
+8600 train 5.939592 (lr=1.7113e-05) (hash(x)=155441439)
+8700 val loss 5.8206
+8700 val perplexity 337.1772
+8700 train 5.801907 (lr=1.6868e-05) (hash(x)=149001504)
+8800 val loss 5.8160
+8800 val perplexity 335.6134
+8800 train 5.685404 (lr=1.6623e-05) (hash(x)=144721722)
+8900 val loss 5.8102
+8900 val perplexity 333.6932
+8900 train 5.717410 (lr=1.6377e-05) (hash(x)=151789511)
+9000 val loss 5.8033
+9000 val perplexity 331.3896
+9000 train 5.732379 (lr=1.6132e-05) (hash(x)=151307797)
+9100 val loss 5.7979
+9100 val perplexity 329.5970
+9100 train 5.928390 (lr=1.5887e-05) (hash(x)=155534554)
+9200 val loss 5.7888
+9200 val perplexity 326.6216
+9200 train 5.677572 (lr=1.5643e-05) (hash(x)=138467356)
+9300 val loss 5.7872
+9300 val perplexity 326.0889
+9300 train 5.718623 (lr=1.5398e-05) (hash(x)=148622510)
+9400 val loss 5.7821
+9400 val perplexity 324.4314
+9400 train 5.915169 (lr=1.5154e-05) (hash(x)=154343672)
+9500 val loss 5.7790
+9500 val perplexity 323.4408
+9500 train 5.763975 (lr=1.4910e-05) (hash(x)=156831628)
+9600 val loss 5.7742
+9600 val perplexity 321.8810
+9600 train 5.550517 (lr=1.4667e-05) (hash(x)=142190812)
+9700 val loss 5.7644
+9700 val perplexity 318.7634
+9700 train 5.670130 (lr=1.4424e-05) (hash(x)=146487376)
+9800 val loss 5.7629
+9800 val perplexity 318.2787
+9800 train 5.842342 (lr=1.4183e-05) (hash(x)=164178945)
+9900 val loss 5.7580
+9900 val perplexity 316.7204
+9900 train 5.622567 (lr=1.3941e-05) (hash(x)=142224918)
+10000 val loss 5.7564
+10000 val perplexity 316.2114
+10000 train 5.682513 (lr=1.3701e-05) (hash(x)=148509740)
+10100 val loss 5.7488
+10100 val perplexity 313.8015
+10100 train 5.813766 (lr=1.3462e-05) (hash(x)=153536520)
+10200 val loss 5.7386
+10200 val perplexity 310.6290
+10200 train 5.667438 (lr=1.3223e-05) (hash(x)=153740168)
+10300 val loss 5.7613
+10300 val perplexity 317.7690
+10300 train 5.717767 (lr=1.2986e-05) (hash(x)=160410260)
+10400 val loss 5.7330
+10400 val perplexity 308.9095
+10400 train 5.894064 (lr=1.2750e-05) (hash(x)=136788294)
+10500 val loss 5.7284
+10500 val perplexity 307.4727
+10500 train 5.576824 (lr=1.2515e-05) (hash(x)=153988760)
+10600 val loss 5.7258
+10600 val perplexity 306.6924
+10600 train 5.606709 (lr=1.2282e-05) (hash(x)=141896522)
+10700 val loss 5.7242
+10700 val perplexity 306.1778
+10700 train 6.063038 (lr=1.2049e-05) (hash(x)=147057702)
+10800 val loss 5.7162
+10800 val perplexity 303.7487
+10800 train 5.782267 (lr=1.1819e-05) (hash(x)=157542287)
+10900 val loss 5.7106
+10900 val perplexity 302.0526
+10900 train 5.783550 (lr=1.1590e-05) (hash(x)=153860040)
+11000 val loss 5.7061
+11000 val perplexity 300.7032
+11000 train 5.563945 (lr=1.1362e-05) (hash(x)=142932555)
+11100 val loss 5.7038
+11100 val perplexity 299.9987
+11100 train 5.739359 (lr=1.1136e-05) (hash(x)=146260760)
+11200 val loss 5.7056
+11200 val perplexity 300.5415
+11200 train 5.801177 (lr=1.0912e-05) (hash(x)=148970533)
+11300 val loss 5.6988
+11300 val perplexity 298.5046
+11300 train 5.713600 (lr=1.0690e-05) (hash(x)=149421645)
+11400 val loss 5.6921
+11400 val perplexity 296.5246
+11400 train 5.633717 (lr=1.0470e-05) (hash(x)=158728429)
+11500 val loss 5.6890
+11500 val perplexity 295.6100
+11500 train 5.784037 (lr=1.0251e-05) (hash(x)=139524335)
+11600 val loss 5.6881
+11600 val perplexity 295.3450
+11600 train 5.721812 (lr=1.0035e-05) (hash(x)=152760174)
+11700 val loss 5.6831
+11700 val perplexity 293.8480
+11700 train 5.586452 (lr=9.8209e-06) (hash(x)=140740682)
+11800 val loss 5.6822
+11800 val perplexity 293.5990
+11800 train 5.538492 (lr=9.6090e-06) (hash(x)=153665407)
+11900 val loss 5.6813
+11900 val perplexity 293.3422
+11900 train 5.654160 (lr=9.3993e-06) (hash(x)=163068424)
+12000 val loss 5.6737
+12000 val perplexity 291.1071
+12000 train 5.718170 (lr=9.1920e-06) (hash(x)=145166541)
+12100 val loss 5.6692
+12100 val perplexity 289.8016
+12100 train 5.626221 (lr=8.9871e-06) (hash(x)=143228042)
+12200 val loss 5.6683
+12200 val perplexity 289.5381
+12200 train 5.776905 (lr=8.7846e-06) (hash(x)=159222680)
+12300 val loss 5.6676
+12300 val perplexity 289.3356
+12300 train 5.611221 (lr=8.5847e-06) (hash(x)=145196157)
+12400 val loss 5.6638
+12400 val perplexity 288.2523
+12400 train 5.575829 (lr=8.3875e-06) (hash(x)=143033742)
+12500 val loss 5.6615
+12500 val perplexity 287.5922
+12500 train 5.694445 (lr=8.1929e-06) (hash(x)=151133346)
+12600 val loss 5.6591
+12600 val perplexity 286.8763
+12600 train 5.681115 (lr=8.0010e-06) (hash(x)=153607184)
+12700 val loss 5.6548
+12700 val perplexity 285.6623
+12700 train 5.569495 (lr=7.8119e-06) (hash(x)=136931911)
+12800 val loss 5.6528
+12800 val perplexity 285.0946
+12800 train 5.761200 (lr=7.6257e-06) (hash(x)=155105402)
+12900 val loss 5.6501
+12900 val perplexity 284.3116
+12900 train 5.479981 (lr=7.4425e-06) (hash(x)=144141636)
+13000 val loss 5.6512
+13000 val perplexity 284.6234
+13000 train 5.568130 (lr=7.2622e-06) (hash(x)=139380218)
+13100 val loss 5.6434
+13100 val perplexity 282.4235
+13100 train 5.737788 (lr=7.0850e-06) (hash(x)=157695174)
+13200 val loss 5.6419
+13200 val perplexity 281.9885
+13200 train 5.578032 (lr=6.9108e-06) (hash(x)=143697560)
+13300 val loss 5.6394
+13300 val perplexity 281.2981
+13300 train 5.478682 (lr=6.7398e-06) (hash(x)=156314952)
+13400 val loss 5.6395
+13400 val perplexity 281.3124
+13400 train 5.608853 (lr=6.5721e-06) (hash(x)=136720279)
+13500 val loss 5.6387
+13500 val perplexity 281.0854
+13500 train 5.592543 (lr=6.4076e-06) (hash(x)=142579957)
+13600 val loss 5.6389
+13600 val perplexity 281.1602
+13600 train 5.352937 (lr=6.2465e-06) (hash(x)=140719440)
+13700 val loss 5.6381
+13700 val perplexity 280.9380
+13700 train 5.736770 (lr=6.0887e-06) (hash(x)=164396003)
+13800 val loss 5.6320
+13800 val perplexity 279.2188
+13800 train 5.693521 (lr=5.9344e-06) (hash(x)=152712862)
+13900 val loss 5.6282
+13900 val perplexity 278.1716
+13900 train 5.652749 (lr=5.7835e-06) (hash(x)=160985139)
+14000 val loss 5.6259
+14000 val perplexity 277.5321
+14000 train 5.626481 (lr=5.6362e-06) (hash(x)=147429379)
+14100 val loss 5.6248
+14100 val perplexity 277.2046
+14100 train 5.689951 (lr=5.4925e-06) (hash(x)=131830470)
+14200 val loss 5.6258
+14200 val perplexity 277.4954
+14200 train 5.367467 (lr=5.3524e-06) (hash(x)=146383028)
+14300 val loss 5.6231
+14300 val perplexity 276.7373
+14300 train 5.671228 (lr=5.2159e-06) (hash(x)=157158747)
+14400 val loss 5.6251
+14400 val perplexity 277.3032
+14400 train 5.476326 (lr=5.0832e-06) (hash(x)=143911520)
+14500 val loss 5.6246
+14500 val perplexity 277.1737
+14500 train 5.399273 (lr=4.9543e-06) (hash(x)=159555347)
+14600 val loss 5.6189
+14600 val perplexity 275.5745
+14600 train 5.619324 (lr=4.8291e-06) (hash(x)=152957062)
+14700 val loss 5.6143
+14700 val perplexity 274.3109
+14700 train 5.683186 (lr=4.7079e-06) (hash(x)=156001594)
+14800 val loss 5.6143
+14800 val perplexity 274.3225
+14800 train 5.715827 (lr=4.5905e-06) (hash(x)=145885458)
+14900 val loss 5.6122
+14900 val perplexity 273.7392
+14900 train 5.676468 (lr=4.4770e-06) (hash(x)=144544291)
+15000 val loss 5.6122
+15000 val perplexity 273.7438
+15000 train 5.622100 (lr=4.3675e-06) (hash(x)=154606709)
+15100 val loss 5.6118
+15100 val perplexity 273.6255
+15100 train 5.653947 (lr=4.2620e-06) (hash(x)=156005324)
+15200 val loss 5.6125
+15200 val perplexity 273.8391
+15200 train 5.712396 (lr=4.1605e-06) (hash(x)=158847373)
+15300 val loss 5.6134
+15300 val perplexity 274.0727
+15300 train 5.397266 (lr=4.0631e-06) (hash(x)=159609594)
+15400 val loss 5.6084
+15400 val perplexity 272.7195
+15400 train 5.706025 (lr=3.9698e-06) (hash(x)=152428492)
+15500 val loss 5.6049
+15500 val perplexity 271.7659
+15500 train 5.666723 (lr=3.8806e-06) (hash(x)=162251192)
+15600 val loss 5.6044
+15600 val perplexity 271.6171
+15600 train 5.850659 (lr=3.7956e-06) (hash(x)=155072002)
+15700 val loss 5.6027
+15700 val perplexity 271.1621
+15700 train 5.619005 (lr=3.7148e-06) (hash(x)=151203094)
+15800 val loss 5.6023
+15800 val perplexity 271.0609
+15800 train 5.521658 (lr=3.6382e-06) (hash(x)=146125932)
+15900 val loss 5.6018
+15900 val perplexity 270.9266
+15900 train 5.488575 (lr=3.5658e-06) (hash(x)=148060195)
+16000 val loss 5.6027
+16000 val perplexity 271.1473
+16000 train 5.299024 (lr=3.4977e-06) (hash(x)=137406416)
+16100 val loss 5.6029
+16100 val perplexity 271.2044
+16100 train 5.664294 (lr=3.4339e-06) (hash(x)=160149755)
+16200 val loss 5.5976
+16200 val perplexity 269.7787
+16200 train 5.769825 (lr=3.3744e-06) (hash(x)=151371894)
+16300 val loss 5.5959
+16300 val perplexity 269.3203
+16300 train 5.638423 (lr=3.3193e-06) (hash(x)=151810252)
+16400 val loss 5.5946
+16400 val perplexity 268.9703
+16400 train 5.578471 (lr=3.2684e-06) (hash(x)=153088303)
+16500 val loss 5.5948
+16500 val perplexity 269.0238
+16500 train 5.474696 (lr=3.2220e-06) (hash(x)=151706615)
+16600 val loss 5.5926
+16600 val perplexity 268.4265
+16600 train 5.483826 (lr=3.1799e-06) (hash(x)=142110626)
+16700 val loss 5.5937
+16700 val perplexity 268.7342
+16700 train 5.486968 (lr=3.1422e-06) (hash(x)=153222439)
+16800 val loss 5.5921
+16800 val perplexity 268.2885
+16800 train 5.519623 (lr=3.1089e-06) (hash(x)=154964902)
+16900 val loss 5.5899
+16900 val perplexity 267.6995
+16900 train 5.766605 (lr=3.0801e-06) (hash(x)=158893204)
+17000 val loss 5.5877
+17000 val perplexity 267.1279
+17000 train 5.519545 (lr=3.0556e-06) (hash(x)=148929955)
+17100 val loss 5.5881
+17100 val perplexity 267.2181
+17100 train 5.599936 (lr=3.0356e-06) (hash(x)=147818677)
+17200 val loss 5.5868
+17200 val perplexity 266.8931
+17200 train 5.519412 (lr=3.0200e-06) (hash(x)=147119446)
+17300 val loss 5.5870
+17300 val perplexity 266.9319
+17300 train 5.630085 (lr=3.0089e-06) (hash(x)=150494142)
+17400 val loss 5.5871
+17400 val perplexity 266.9568
+17400 train 5.614751 (lr=3.0022e-06) (hash(x)=155918058)
+17499 val loss 5.5846
+17499 val perplexity 266.2846

attention_kindselective_n_heads4_seed1342/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3bfcb58747e9c1a758d5d5cc9c3751f077e21cf4336de84d9b33c35dcc8c3ccb
+oid sha256:48e71907b9a9c5cb915194076023f3234729f187f59bea2b256a18008ea6605b
 size 92843394

attention_kindselective_n_heads4_seed1342/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d724ae9cc4ad82d4bca42989b48a19daf300f0696210809e9b80107bbccec269
+oid sha256:751318665afe834c16da4ecbe1c1b7971a7477cea8c663e1ec88b6fc29605938
 size 179406214