Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_30720_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2663
 0 val perplexity 78147.0000
-0 train 11.263341 (lr=1.3750e-07) (hash(x)=150327452)
-100 val loss 9.9216
-100 val perplexity 20365.0898
-100 train 9.932128 (lr=1.3888e-05) (hash(x)=166441190)
-200 val loss 9.2026
-200 val perplexity 9923.1172
-200 train 9.311575 (lr=2.7500e-05) (hash(x)=166780046)
-300 val loss 8.1964
-300 val perplexity 3627.8376
-300 train 8.155242 (lr=2.7498e-05) (hash(x)=159835303)
-400 val loss 7.7685
-400 val perplexity 2365.0269
-400 train 7.570842 (lr=2.7492e-05) (hash(x)=155040610)
-500 val loss 7.6098
-500 val perplexity 2017.9415
-500 train 7.360526 (lr=2.7482e-05) (hash(x)=130190460)
-600 val loss 7.5112
-600 val perplexity 1828.4420
-600 train 7.535985 (lr=2.7467e-05) (hash(x)=155504036)
-700 val loss 7.4475
-700 val perplexity 1715.5283
-700 train 7.301879 (lr=2.7449e-05) (hash(x)=137347213)
-800 val loss 7.3965
-800 val perplexity 1630.2454
-800 train 7.251807 (lr=2.7427e-05) (hash(x)=143823248)
-900 val loss 7.3470
-900 val perplexity 1551.5825
-900 train 7.458832 (lr=2.7400e-05) (hash(x)=156260416)
-1000 val loss 7.3139
-1000 val perplexity 1501.0842
-1000 train 7.281650 (lr=2.7370e-05) (hash(x)=143734685)
-1100 val loss 7.2764
-1100 val perplexity 1445.7637
-1100 train 7.209399 (lr=2.7335e-05) (hash(x)=160013925)
-1200 val loss 7.2480
-1200 val perplexity 1405.2816
-1200 train 6.993395 (lr=2.7297e-05) (hash(x)=150678249)
-1300 val loss 7.2195
-1300 val perplexity 1365.8458
-1300 train 7.003422 (lr=2.7254e-05) (hash(x)=149073315)
-1400 val loss 7.1645
-1400 val perplexity 1292.6741
-1400 train 7.483765 (lr=2.7207e-05) (hash(x)=175802021)
-1500 val loss 7.1143
-1500 val perplexity 1229.3901
-1500 train 7.375745 (lr=2.7157e-05) (hash(x)=171034639)
-1600 val loss 7.0674
-1600 val perplexity 1173.0409
-1600 train 7.270476 (lr=2.7102e-05) (hash(x)=158681215)
-1700 val loss 7.0231
-1700 val perplexity 1122.2725
-1700 train 6.961975 (lr=2.7044e-05) (hash(x)=152116061)
-1800 val loss 6.9929
-1800 val perplexity 1088.8434
-1800 train 6.930839 (lr=2.6981e-05) (hash(x)=146108145)
-1900 val loss 6.9646
-1900 val perplexity 1058.4460
-1900 train 6.793094 (lr=2.6915e-05) (hash(x)=147598108)
-2000 val loss 6.9345
-2000 val perplexity 1027.1322
-2000 train 6.798426 (lr=2.6845e-05) (hash(x)=154996086)
-2100 val loss 6.9084
-2100 val perplexity 1000.6855
-2100 train 6.694521 (lr=2.6771e-05) (hash(x)=153396183)
-2200 val loss 6.8577
-2200 val perplexity 951.1731
-2200 train 6.844943 (lr=2.6693e-05) (hash(x)=153885445)
-2300 val loss 6.8186
-2300 val perplexity 914.6971
-2300 train 6.820184 (lr=2.6611e-05) (hash(x)=159666385)
-2400 val loss 6.7737
-2400 val perplexity 874.5115
-2400 train 6.748491 (lr=2.6525e-05) (hash(x)=142353087)
-2500 val loss 6.7371
-2500 val perplexity 843.0906
-2500 train 6.756777 (lr=2.6436e-05) (hash(x)=146491718)
-2600 val loss 6.7059
-2600 val perplexity 817.2048
-2600 train 6.631540 (lr=2.6343e-05) (hash(x)=150750353)
-2700 val loss 6.6686
-2700 val perplexity 787.2690
-2700 train 6.380807 (lr=2.6246e-05) (hash(x)=129849193)
-2800 val loss 6.6495
-2800 val perplexity 772.4222
-2800 train 6.494568 (lr=2.6146e-05) (hash(x)=152767913)
-2900 val loss 6.6275
-2900 val perplexity 755.5663
-2900 train 6.406610 (lr=2.6042e-05) (hash(x)=146531140)
-3000 val loss 6.5894
-3000 val perplexity 727.3140
-3000 train 6.665761 (lr=2.5934e-05) (hash(x)=151562048)
-3100 val loss 6.5494
-3100 val perplexity 698.8493
-3100 train 6.516375 (lr=2.5823e-05) (hash(x)=146001424)
-3200 val loss 6.5272
-3200 val perplexity 683.4663
-3200 train 6.600415 (lr=2.5709e-05) (hash(x)=166486165)
-3300 val loss 6.4946
-3300 val perplexity 661.5607
-3300 train 6.426333 (lr=2.5590e-05) (hash(x)=150866680)
-3400 val loss 6.4801
-3400 val perplexity 652.0275
-3400 train 6.374853 (lr=2.5469e-05) (hash(x)=143900419)
-3500 val loss 6.4553
-3500 val perplexity 636.0541
-3500 train 6.270348 (lr=2.5344e-05) (hash(x)=148845794)
-3600 val loss 6.4393
-3600 val perplexity 625.9676
-3600 train 6.269843 (lr=2.5215e-05) (hash(x)=145667796)
-3700 val loss 6.4257
-3700 val perplexity 617.5016
-3700 train 6.442654 (lr=2.5083e-05) (hash(x)=163563851)
-3800 val loss 6.4055
-3800 val perplexity 605.1692
-3800 train 6.348926 (lr=2.4948e-05) (hash(x)=147488689)
-3900 val loss 6.3750
-3900 val perplexity 586.9639
-3900 train 6.376982 (lr=2.4810e-05) (hash(x)=148186608)
-4000 val loss 6.3525
-4000 val perplexity 573.9432
-4000 train 6.228961 (lr=2.4669e-05) (hash(x)=142970187)
-4100 val loss 6.3316
-4100 val perplexity 562.0504
-4100 train 6.424266 (lr=2.4524e-05) (hash(x)=141584883)
-4200 val loss 6.3206
-4200 val perplexity 555.9155
-4200 train 6.121674 (lr=2.4376e-05) (hash(x)=145664585)
-4300 val loss 6.3036
-4300 val perplexity 546.5199
-4300 train 6.176545 (lr=2.4226e-05) (hash(x)=143736499)
-4400 val loss 6.3040
-4400 val perplexity 546.7621
-4400 train 6.126653 (lr=2.4072e-05) (hash(x)=151883322)
-4500 val loss 6.2887
-4500 val perplexity 538.4496
-4500 train 6.172879 (lr=2.3915e-05) (hash(x)=153904871)
-4600 val loss 6.2713
-4600 val perplexity 529.1572
-4600 train 6.360380 (lr=2.3755e-05) (hash(x)=154893521)
-4700 val loss 6.2472
-4700 val perplexity 516.5841
-4700 train 6.304053 (lr=2.3593e-05) (hash(x)=152323949)
-4800 val loss 6.2306
-4800 val perplexity 508.0774
-4800 train 6.176684 (lr=2.3428e-05) (hash(x)=154104619)
-4900 val loss 6.2180
-4900 val perplexity 501.6885
-4900 train 6.347137 (lr=2.3260e-05) (hash(x)=146311426)
-5000 val loss 6.2067
-5000 val perplexity 496.0761
-5000 train 6.195480 (lr=2.3089e-05) (hash(x)=156741847)
-5100 val loss 6.1953
-5100 val perplexity 490.4566
-5100 train 5.973955 (lr=2.2916e-05) (hash(x)=142086346)
-5200 val loss 6.1901
-5200 val perplexity 487.8952
-5200 train 5.958094 (lr=2.2740e-05) (hash(x)=150265428)
-5300 val loss 6.1848
-5300 val perplexity 485.3176
-5300 train 6.061728 (lr=2.2561e-05) (hash(x)=151339108)
-5400 val loss 6.1701
-5400 val perplexity 478.2155
-5400 train 6.146971 (lr=2.2381e-05) (hash(x)=154654372)
-5500 val loss 6.1531
-5500 val perplexity 470.1668
-5500 train 6.165415 (lr=2.2197e-05) (hash(x)=150575051)
-5600 val loss 6.1412
-5600 val perplexity 464.6232
-5600 train 6.048434 (lr=2.2012e-05) (hash(x)=140396423)
-5700 val loss 6.1312
-5700 val perplexity 459.9658
-5700 train 5.991282 (lr=2.1824e-05) (hash(x)=144678758)
-5800 val loss 6.1224
-5800 val perplexity 455.9356
-5800 train 6.170418 (lr=2.1634e-05) (hash(x)=151992743)
-5900 val loss 6.1131
-5900 val perplexity 451.7320
-5900 train 5.902652 (lr=2.1442e-05) (hash(x)=144396927)
-6000 val loss 6.1034
-6000 val perplexity 447.3857
-6000 train 6.065875 (lr=2.1248e-05) (hash(x)=165478625)
-6100 val loss 6.1096
-6100 val perplexity 450.1805
-6100 train 5.843530 (lr=2.1051e-05) (hash(x)=147088621)
-6200 val loss 6.0995
-6200 val perplexity 445.6127
-6200 train 5.866304 (lr=2.0853e-05) (hash(x)=140794994)
-6300 val loss 6.0785
-6300 val perplexity 436.3955
-6300 train 6.008928 (lr=2.0653e-05) (hash(x)=134780906)
-6400 val loss 6.0650
-6400 val perplexity 430.5202
-6400 train 6.043757 (lr=2.0451e-05) (hash(x)=149023655)
-6500 val loss 6.0570
-6500 val perplexity 427.0772
-6500 train 5.909537 (lr=2.0247e-05) (hash(x)=147497796)
-6600 val loss 6.0540
-6600 val perplexity 425.8057
-6600 train 5.906900 (lr=2.0042e-05) (hash(x)=152902689)
-6700 val loss 6.0523
-6700 val perplexity 425.0714
-6700 train 6.086189 (lr=1.9835e-05) (hash(x)=153846046)
-6800 val loss 6.0303
-6800 val perplexity 415.8329
-6800 train 6.160307 (lr=1.9626e-05) (hash(x)=158512738)
-6900 val loss 6.0207
-6900 val perplexity 411.8505
-6900 train 6.544151 (lr=1.9416e-05) (hash(x)=156849968)
-7000 val loss 6.0165
-7000 val perplexity 410.1386
-7000 train 5.866353 (lr=1.9205e-05) (hash(x)=142395855)
-7100 val loss 6.0088
-7100 val perplexity 406.9989
-7100 train 5.857893 (lr=1.8992e-05) (hash(x)=147114884)
-7200 val loss 5.9996
-7200 val perplexity 403.2617
-7200 train 6.032767 (lr=1.8778e-05) (hash(x)=156979839)
-7300 val loss 5.9959
-7300 val perplexity 401.7586
-7300 train 5.781147 (lr=1.8562e-05) (hash(x)=145584373)
-7400 val loss 5.9900
-7400 val perplexity 399.4259
-7400 train 5.729474 (lr=1.8346e-05) (hash(x)=141508204)
-7500 val loss 5.9780
-7500 val perplexity 394.6485
-7500 train 6.012234 (lr=1.8128e-05) (hash(x)=148803965)
-7600 val loss 5.9718
-7600 val perplexity 392.1917
-7600 train 6.031247 (lr=1.7910e-05) (hash(x)=151019676)
-7700 val loss 5.9586
-7700 val perplexity 387.0653
-7700 train 6.031113 (lr=1.7691e-05) (hash(x)=143155750)
-7800 val loss 5.9556
-7800 val perplexity 385.9111
-7800 train 5.981885 (lr=1.7470e-05) (hash(x)=152569653)
-7900 val loss 5.9519
-7900 val perplexity 384.4711
-7900 train 5.823013 (lr=1.7249e-05) (hash(x)=143519455)
-8000 val loss 5.9416
-8000 val perplexity 380.5505
-8000 train 6.052700 (lr=1.7028e-05) (hash(x)=161180944)
-8100 val loss 5.9420
-8100 val perplexity 380.7064
-8100 train 5.966490 (lr=1.6805e-05) (hash(x)=154107345)
-8200 val loss 5.9371
-8200 val perplexity 378.8401
-8200 train 5.968087 (lr=1.6582e-05) (hash(x)=152486517)
-8300 val loss 5.9261
-8300 val perplexity 374.6855
-8300 train 5.935761 (lr=1.6359e-05) (hash(x)=156167749)
-8400 val loss 5.9156
-8400 val perplexity 370.7589
-8400 train 6.000555 (lr=1.6135e-05) (hash(x)=149155006)
-8500 val loss 5.9067
-8500 val perplexity 367.4887
-8500 train 5.864130 (lr=1.5911e-05) (hash(x)=147844390)
-8600 val loss 5.9047
-8600 val perplexity 366.7408
-8600 train 6.157965 (lr=1.5687e-05) (hash(x)=165753320)
-8700 val loss 5.8990
-8700 val perplexity 364.6836
-8700 train 5.683146 (lr=1.5462e-05) (hash(x)=146079979)
-8800 val loss 5.8990
-8800 val perplexity 364.6555
-8800 train 6.090808 (lr=1.5237e-05) (hash(x)=172259509)
-8900 val loss 5.8907
-8900 val perplexity 361.6673
-8900 train 5.573002 (lr=1.5013e-05) (hash(x)=145148314)
-9000 val loss 5.8871
-9000 val perplexity 360.3432
-9000 train 5.803849 (lr=1.4788e-05) (hash(x)=144250633)
-9100 val loss 5.8804
-9100 val perplexity 357.9615
-9100 train 5.972807 (lr=1.4563e-05) (hash(x)=157219797)
-9200 val loss 5.8735
-9200 val perplexity 355.4825
-9200 train 5.910276 (lr=1.4339e-05) (hash(x)=142743778)
-9300 val loss 5.8653
-9300 val perplexity 352.5787
-9300 train 5.783849 (lr=1.4115e-05) (hash(x)=139669771)
-9400 val loss 5.8574
-9400 val perplexity 349.8001
-9400 train 5.884781 (lr=1.3891e-05) (hash(x)=145916843)
-9500 val loss 5.8574
-9500 val perplexity 349.8114
-9500 train 5.776641 (lr=1.3668e-05) (hash(x)=150196125)
-9600 val loss 5.8502
-9600 val perplexity 347.2911
-9600 train 6.161810 (lr=1.3445e-05) (hash(x)=160041419)
-9700 val loss 5.8489
-9700 val perplexity 346.8675
-9700 train 5.629546 (lr=1.3222e-05) (hash(x)=139931627)
-9800 val loss 5.8436
-9800 val perplexity 345.0097
-9800 train 5.823668 (lr=1.3001e-05) (hash(x)=150370792)
-9900 val loss 5.8368
-9900 val perplexity 342.6939
-9900 train 5.680959 (lr=1.2780e-05) (hash(x)=153014886)
-10000 val loss 5.8341
-10000 val perplexity 341.7490
-10000 train 5.737649 (lr=1.2559e-05) (hash(x)=141915265)
-10100 val loss 5.8270
-10100 val perplexity 339.3311
-10100 train 6.341434 (lr=1.2340e-05) (hash(x)=197770952)
-10200 val loss 5.8213
-10200 val perplexity 337.4204
-10200 train 5.808439 (lr=1.2122e-05) (hash(x)=156806005)
-10300 val loss 5.8179
-10300 val perplexity 336.2575
-10300 train 5.622517 (lr=1.1904e-05) (hash(x)=145378216)
-10400 val loss 5.8126
-10400 val perplexity 334.4817
-10400 train 5.716724 (lr=1.1688e-05) (hash(x)=145125407)
-10500 val loss 5.8088
-10500 val perplexity 333.2034
-10500 train 5.622625 (lr=1.1472e-05) (hash(x)=147553745)
-10600 val loss 5.8047
-10600 val perplexity 331.8625
-10600 train 5.440615 (lr=1.1258e-05) (hash(x)=134894889)
-10700 val loss 5.7991
-10700 val perplexity 330.0065
-10700 train 5.726994 (lr=1.1045e-05) (hash(x)=151041040)
-10800 val loss 5.7921
-10800 val perplexity 327.6916
-10800 train 5.724377 (lr=1.0834e-05) (hash(x)=145685829)
-10900 val loss 5.7874
-10900 val perplexity 326.1582
-10900 train 5.860260 (lr=1.0624e-05) (hash(x)=156000716)
-11000 val loss 5.7849
-11000 val perplexity 325.3531
-11000 train 5.782057 (lr=1.0415e-05) (hash(x)=149858163)
-11100 val loss 5.7831
-11100 val perplexity 324.7552
-11100 train 6.055070 (lr=1.0208e-05) (hash(x)=164873575)
-11200 val loss 5.7819
-11200 val perplexity 324.3619
-11200 train 5.665148 (lr=1.0003e-05) (hash(x)=156370069)
-11300 val loss 5.7809
-11300 val perplexity 324.0505
-11300 train 5.885826 (lr=9.7991e-06) (hash(x)=160356892)
-11400 val loss 5.7694
-11400 val perplexity 320.3412
-11400 train 5.853624 (lr=9.5972e-06) (hash(x)=158955430)
-11500 val loss 5.7655
-11500 val perplexity 319.0882
-11500 train 5.735012 (lr=9.3970e-06) (hash(x)=146598377)
-11600 val loss 5.7783
-11600 val perplexity 323.2089
-11600 train 5.920930 (lr=9.1988e-06) (hash(x)=156045795)
-11700 val loss 5.7633
-11700 val perplexity 318.4037
-11700 train 5.620872 (lr=9.0025e-06) (hash(x)=143963334)
-11800 val loss 5.7613
-11800 val perplexity 317.7613
-11800 train 5.593094 (lr=8.8082e-06) (hash(x)=145831579)
-11900 val loss 5.7564
-11900 val perplexity 316.1924
-11900 train 5.932192 (lr=8.6160e-06) (hash(x)=165154141)
-12000 val loss 5.7496
-12000 val perplexity 314.0803
-12000 train 5.828736 (lr=8.4260e-06) (hash(x)=160373794)
-12100 val loss 5.7464
-12100 val perplexity 313.0729
-12100 train 5.533817 (lr=8.2381e-06) (hash(x)=141476346)
-12200 val loss 5.7444
-12200 val perplexity 312.4352
-12200 train 6.004211 (lr=8.0526e-06) (hash(x)=164508058)
-12300 val loss 5.7431
-12300 val perplexity 312.0411
-12300 train 5.710019 (lr=7.8694e-06) (hash(x)=140407664)
-12400 val loss 5.7408
-12400 val perplexity 311.3231
-12400 train 5.591258 (lr=7.6885e-06) (hash(x)=149526189)
-12500 val loss 5.7348
-12500 val perplexity 309.4465
-12500 train 5.577487 (lr=7.5101e-06) (hash(x)=142060916)
-12600 val loss 5.7295
-12600 val perplexity 307.8142
-12600 train 5.919009 (lr=7.3342e-06) (hash(x)=168650848)
-12700 val loss 5.7286
-12700 val perplexity 307.5334
-12700 train 5.749731 (lr=7.1609e-06) (hash(x)=154353018)
-12800 val loss 5.7300
-12800 val perplexity 307.9663
-12800 train 5.888818 (lr=6.9903e-06) (hash(x)=156115265)
-12900 val loss 5.7250
-12900 val perplexity 306.4193
-12900 train 5.652087 (lr=6.8223e-06) (hash(x)=143378175)
-13000 val loss 5.7236
-13000 val perplexity 306.0145
-13000 train 5.839179 (lr=6.6570e-06) (hash(x)=166124864)
-13100 val loss 5.7219
-13100 val perplexity 305.4771
-13100 train 5.596782 (lr=6.4945e-06) (hash(x)=146615385)
-13200 val loss 5.7215
-13200 val perplexity 305.3489
-13200 train 5.621416 (lr=6.3349e-06) (hash(x)=149236047)
-13300 val loss 5.7149
-13300 val perplexity 303.3439
-13300 train 5.648214 (lr=6.1782e-06) (hash(x)=151346972)
-13400 val loss 5.7142
-13400 val perplexity 303.1342
-13400 train 5.801619 (lr=6.0244e-06) (hash(x)=159167093)
-13500 val loss 5.7126
-13500 val perplexity 302.6487
-13500 train 5.428577 (lr=5.8737e-06) (hash(x)=135054650)
-13600 val loss 5.7108
-13600 val perplexity 302.1104
-13600 train 5.596169 (lr=5.7259e-06) (hash(x)=151033927)
-13700 val loss 5.7104
-13700 val perplexity 301.9891
-13700 train 5.409321 (lr=5.5813e-06) (hash(x)=141119172)
-13800 val loss 5.7096
-13800 val perplexity 301.7523
-13800 train 5.430826 (lr=5.4398e-06) (hash(x)=144197766)
-13900 val loss 5.7053
-13900 val perplexity 300.4471
-13900 train 5.645763 (lr=5.3016e-06) (hash(x)=145148158)
-14000 val loss 5.7019
-14000 val perplexity 299.4242
-14000 train 6.197217 (lr=5.1665e-06) (hash(x)=179068751)
-14100 val loss 5.7002
-14100 val perplexity 298.9149
-14100 train 5.569189 (lr=5.0348e-06) (hash(x)=144988373)
-14200 val loss 5.6993
-14200 val perplexity 298.6475
-14200 train 5.623942 (lr=4.9063e-06) (hash(x)=133379696)
-14300 val loss 5.6992
-14300 val perplexity 298.6255
-14300 train 5.580496 (lr=4.7813e-06) (hash(x)=149883422)
-14400 val loss 5.6996
-14400 val perplexity 298.7478
-14400 train 5.567074 (lr=4.6596e-06) (hash(x)=145709198)
-14500 val loss 5.6944
-14500 val perplexity 297.2122
-14500 train 5.709787 (lr=4.5414e-06) (hash(x)=148016579)
-14600 val loss 5.6922
-14600 val perplexity 296.5331
-14600 train 5.658023 (lr=4.4267e-06) (hash(x)=150218222)
-14700 val loss 5.6919
-14700 val perplexity 296.4617
-14700 train 5.611262 (lr=4.3155e-06) (hash(x)=142258737)
-14800 val loss 5.6890
-14800 val perplexity 295.5974
-14800 train 5.584694 (lr=4.2079e-06) (hash(x)=147628003)
-14900 val loss 5.6895
-14900 val perplexity 295.7564
-14900 train 5.539468 (lr=4.1039e-06) (hash(x)=149144036)
-15000 val loss 5.6894
-15000 val perplexity 295.7050
-15000 train 5.623379 (lr=4.0035e-06) (hash(x)=145925409)
-15100 val loss 5.6857
-15100 val perplexity 294.6349
-15100 train 5.563715 (lr=3.9068e-06) (hash(x)=144555751)
-15200 val loss 5.6844
-15200 val perplexity 294.2293
-15200 train 5.501987 (lr=3.8138e-06) (hash(x)=143448852)
-15300 val loss 5.6836
-15300 val perplexity 294.0114
-15300 train 5.680522 (lr=3.7245e-06) (hash(x)=156347253)
-15400 val loss 5.6814
-15400 val perplexity 293.3521
-15400 train 5.635643 (lr=3.6390e-06) (hash(x)=153010179)
-15500 val loss 5.6814
-15500 val perplexity 293.3618
-15500 train 5.463892 (lr=3.5572e-06) (hash(x)=144066445)
-15600 val loss 5.6817
-15600 val perplexity 293.4398
-15600 train 5.423588 (lr=3.4793e-06) (hash(x)=146107722)
-15700 val loss 5.6776
-15700 val perplexity 292.2488
-15700 train 5.816041 (lr=3.4052e-06) (hash(x)=152533799)
-15800 val loss 5.6749
-15800 val perplexity 291.4473
-15800 train 5.726212 (lr=3.3350e-06) (hash(x)=151756366)
-15900 val loss 5.6743
-15900 val perplexity 291.2946
-15900 train 5.659422 (lr=3.2687e-06) (hash(x)=151735039)
-16000 val loss 5.6746
-16000 val perplexity 291.3792
-16000 train 5.703759 (lr=3.2063e-06) (hash(x)=154633226)
-16100 val loss 5.6751
-16100 val perplexity 291.5109
-16100 train 5.528335 (lr=3.1478e-06) (hash(x)=158094333)
-16200 val loss 5.6740
-16200 val perplexity 291.1947
-16200 train 5.437234 (lr=3.0932e-06) (hash(x)=142772927)
-16300 val loss 5.6733
-16300 val perplexity 290.9790
-16300 train 5.568473 (lr=3.0427e-06) (hash(x)=148042896)
-16400 val loss 5.6704
-16400 val perplexity 290.1549
-16400 train 6.154361 (lr=2.9961e-06) (hash(x)=170306399)
-16500 val loss 5.6695
-16500 val perplexity 289.8988
-16500 train 5.688489 (lr=2.9535e-06) (hash(x)=141049320)
-16600 val loss 5.6693
-16600 val perplexity 289.8406
-16600 train 5.527473 (lr=2.9149e-06) (hash(x)=150512136)
-16700 val loss 5.6678
-16700 val perplexity 289.3889
-16700 train 5.678548 (lr=2.8804e-06) (hash(x)=151843645)
-16800 val loss 5.6655
-16800 val perplexity 288.7390
-16800 train 5.724266 (lr=2.8498e-06) (hash(x)=140073659)
-16900 val loss 5.6635
-16900 val perplexity 288.1412
-16900 train 5.852492 (lr=2.8234e-06) (hash(x)=171157212)
-17000 val loss 5.6630
-17000 val perplexity 288.0170
-17000 train 5.594303 (lr=2.8010e-06) (hash(x)=144553276)
-17100 val loss 5.6631
-17100 val perplexity 288.0265
-17100 train 5.660722 (lr=2.7826e-06) (hash(x)=157717565)
-17200 val loss 5.6633
-17200 val perplexity 288.0857
-17200 train 5.528824 (lr=2.7684e-06) (hash(x)=149843860)
-17300 val loss 5.6609
-17300 val perplexity 287.3978
-17300 train 5.721745 (lr=2.7582e-06) (hash(x)=162171979)
-17400 val loss 5.6599
-17400 val perplexity 287.1299
-17400 train 5.633310 (lr=2.7520e-06) (hash(x)=148400702)
-17499 val loss 5.6588
-17499 val perplexity 286.7997
+0 train 11.263341 (lr=1.7500e-07) (hash(x)=150327452)
+100 val loss 9.8666
+100 val perplexity 19276.3164
+100 train 9.875949 (lr=1.7675e-05) (hash(x)=166441190)
+200 val loss 8.5172
+200 val perplexity 5000.0649
+200 train 8.645861 (lr=3.5000e-05) (hash(x)=166780046)
+300 val loss 7.6495
+300 val perplexity 2099.6785
+300 train 7.605138 (lr=3.4997e-05) (hash(x)=159835303)
+400 val loss 7.5080
+400 val perplexity 1822.5958
+400 train 7.283498 (lr=3.4990e-05) (hash(x)=155040610)
+500 val loss 7.4459
+500 val perplexity 1712.7689
+500 train 7.172822 (lr=3.4977e-05) (hash(x)=130190460)
+600 val loss 7.3656
+600 val perplexity 1580.6713
+600 train 7.384192 (lr=3.4958e-05) (hash(x)=155504036)
+700 val loss 7.2947
+700 val perplexity 1472.5343
+700 train 7.147750 (lr=3.4935e-05) (hash(x)=137347213)
+800 val loss 7.2237
+800 val perplexity 1371.5957
+800 train 7.065663 (lr=3.4907e-05) (hash(x)=143823248)
+900 val loss 7.1532
+900 val perplexity 1278.2330
+900 train 7.261928 (lr=3.4873e-05) (hash(x)=156260416)
+1000 val loss 7.0941
+1000 val perplexity 1204.8857
+1000 train 7.049677 (lr=3.4834e-05) (hash(x)=143734685)
+1100 val loss 7.0353
+1100 val perplexity 1136.0641
+1100 train 6.952003 (lr=3.4790e-05) (hash(x)=160013925)
+1200 val loss 6.9993
+1200 val perplexity 1095.8428
+1200 train 6.722279 (lr=3.4741e-05) (hash(x)=150678249)
+1300 val loss 6.9463
+1300 val perplexity 1039.2848
+1300 train 6.712331 (lr=3.4687e-05) (hash(x)=149073315)
+1400 val loss 6.8822
+1400 val perplexity 974.7924
+1400 train 7.177571 (lr=3.4628e-05) (hash(x)=175802021)
+1500 val loss 6.8204
+1500 val perplexity 916.3141
+1500 train 7.088338 (lr=3.4563e-05) (hash(x)=171034639)
+1600 val loss 6.7715
+1600 val perplexity 872.6337
+1600 train 6.987903 (lr=3.4494e-05) (hash(x)=158681215)
+1700 val loss 6.7281
+1700 val perplexity 835.5875
+1700 train 6.675895 (lr=3.4419e-05) (hash(x)=152116061)
+1800 val loss 6.6849
+1800 val perplexity 800.2023
+1800 train 6.617714 (lr=3.4340e-05) (hash(x)=146108145)
+1900 val loss 6.6472
+1900 val perplexity 770.6526
+1900 train 6.465954 (lr=3.4255e-05) (hash(x)=147598108)
+2000 val loss 6.6229
+2000 val perplexity 752.1500
+2000 train 6.451444 (lr=3.4166e-05) (hash(x)=154996086)
+2100 val loss 6.5878
+2100 val perplexity 726.2009
+2100 train 6.356413 (lr=3.4072e-05) (hash(x)=153396183)
+2200 val loss 6.5320
+2200 val perplexity 686.7681
+2200 train 6.529510 (lr=3.3973e-05) (hash(x)=153885445)
+2300 val loss 6.4981
+2300 val perplexity 663.9014
+2300 train 6.507496 (lr=3.3869e-05) (hash(x)=159666385)
+2400 val loss 6.4592
+2400 val perplexity 638.5427
+2400 train 6.455539 (lr=3.3760e-05) (hash(x)=142353087)
+2500 val loss 6.4283
+2500 val perplexity 619.1046
+2500 train 6.443000 (lr=3.3646e-05) (hash(x)=146491718)
+2600 val loss 6.4042
+2600 val perplexity 604.3793
+2600 train 6.326907 (lr=3.3528e-05) (hash(x)=150750353)
+2700 val loss 6.3784
+2700 val perplexity 588.9938
+2700 train 6.114101 (lr=3.3405e-05) (hash(x)=129849193)
+2800 val loss 6.3725
+2800 val perplexity 585.5407
+2800 train 6.196213 (lr=3.3277e-05) (hash(x)=152767913)
+2900 val loss 6.3522
+2900 val perplexity 573.7309
+2900 train 6.126045 (lr=3.3144e-05) (hash(x)=146531140)
+3000 val loss 6.3223
+3000 val perplexity 556.8661
+3000 train 6.413922 (lr=3.3008e-05) (hash(x)=151562048)
+3100 val loss 6.2940
+3100 val perplexity 541.3404
+3100 train 6.252115 (lr=3.2866e-05) (hash(x)=146001424)
+3200 val loss 6.2814
+3200 val perplexity 534.5200
+3200 train 6.352508 (lr=3.2720e-05) (hash(x)=166486165)
+3300 val loss 6.2526
+3300 val perplexity 519.3692
+3300 train 6.191504 (lr=3.2570e-05) (hash(x)=150866680)
+3400 val loss 6.2387
+3400 val perplexity 512.1734
+3400 train 6.131408 (lr=3.2415e-05) (hash(x)=143900419)
+3500 val loss 6.2234
+3500 val perplexity 504.4341
+3500 train 6.034927 (lr=3.2256e-05) (hash(x)=148845794)
+3600 val loss 6.2103
+3600 val perplexity 497.8275
+3600 train 6.038193 (lr=3.2092e-05) (hash(x)=145667796)
+3700 val loss 6.2000
+3700 val perplexity 492.7471
+3700 train 6.219586 (lr=3.1924e-05) (hash(x)=163563851)
+3800 val loss 6.1898
+3800 val perplexity 487.7324
+3800 train 6.139465 (lr=3.1753e-05) (hash(x)=147488689)
+3900 val loss 6.1621
+3900 val perplexity 474.4142
+3900 train 6.174405 (lr=3.1577e-05) (hash(x)=148186608)
+4000 val loss 6.1436
+4000 val perplexity 465.7114
+4000 train 6.032261 (lr=3.1397e-05) (hash(x)=142970187)
+4100 val loss 6.1282
+4100 val perplexity 458.5933
+4100 train 6.276835 (lr=3.1212e-05) (hash(x)=141584883)
+4200 val loss 6.1200
+4200 val perplexity 454.8668
+4200 train 5.921480 (lr=3.1024e-05) (hash(x)=145664585)
+4300 val loss 6.1068
+4300 val perplexity 448.8869
+4300 train 5.991696 (lr=3.0833e-05) (hash(x)=143736499)
+4400 val loss 6.1066
+4400 val perplexity 448.7936
+4400 train 5.929280 (lr=3.0637e-05) (hash(x)=151883322)
+4500 val loss 6.0970
+4500 val perplexity 444.5335
+4500 train 5.972202 (lr=3.0437e-05) (hash(x)=153904871)
+4600 val loss 6.0794
+4600 val perplexity 436.7792
+4600 train 6.184064 (lr=3.0234e-05) (hash(x)=154893521)
+4700 val loss 6.0611
+4700 val perplexity 428.8307
+4700 train 6.102214 (lr=3.0027e-05) (hash(x)=152323949)
+4800 val loss 6.0513
+4800 val perplexity 424.6822
+4800 train 5.999310 (lr=2.9817e-05) (hash(x)=154104619)
+4900 val loss 6.0326
+4900 val perplexity 416.8012
+4900 train 6.160482 (lr=2.9603e-05) (hash(x)=146311426)
+5000 val loss 6.0271
+5000 val perplexity 414.5144
+5000 train 5.998255 (lr=2.9386e-05) (hash(x)=156741847)
+5100 val loss 6.0212
+5100 val perplexity 412.0829
+5100 train 5.801046 (lr=2.9166e-05) (hash(x)=142086346)
+5200 val loss 6.0182
+5200 val perplexity 410.8245
+5200 train 5.785246 (lr=2.8942e-05) (hash(x)=150265428)
+5300 val loss 6.0053
+5300 val perplexity 405.5529
+5300 train 5.873408 (lr=2.8715e-05) (hash(x)=151339108)
+5400 val loss 5.9988
+5400 val perplexity 402.9337
+5400 train 5.956289 (lr=2.8484e-05) (hash(x)=154654372)
+5500 val loss 5.9793
+5500 val perplexity 395.1808
+5500 train 5.988954 (lr=2.8251e-05) (hash(x)=150575051)
+5600 val loss 5.9670
+5600 val perplexity 390.3333
+5600 train 5.884884 (lr=2.8015e-05) (hash(x)=140396423)
+5700 val loss 5.9546
+5700 val perplexity 385.5175
+5700 train 5.811714 (lr=2.7776e-05) (hash(x)=144678758)
+5800 val loss 5.9474
+5800 val perplexity 382.7544
+5800 train 6.006756 (lr=2.7534e-05) (hash(x)=151992743)
+5900 val loss 5.9399
+5900 val perplexity 379.9125
+5900 train 5.736372 (lr=2.7290e-05) (hash(x)=144396927)
+6000 val loss 5.9352
+6000 val perplexity 378.1267
+6000 train 5.888235 (lr=2.7042e-05) (hash(x)=165478625)
+6100 val loss 5.9382
+6100 val perplexity 379.2473
+6100 train 5.665012 (lr=2.6792e-05) (hash(x)=147088621)
+6200 val loss 5.9269
+6200 val perplexity 374.9813
+6200 train 5.690819 (lr=2.6540e-05) (hash(x)=140794994)
+6300 val loss 5.9086
+6300 val perplexity 368.1940
+6300 train 5.854196 (lr=2.6285e-05) (hash(x)=134780906)
+6400 val loss 5.8961
+6400 val perplexity 363.6166
+6400 train 5.876956 (lr=2.6028e-05) (hash(x)=149023655)
+6500 val loss 5.8875
+6500 val perplexity 360.5128
+6500 train 5.736791 (lr=2.5769e-05) (hash(x)=147497796)
+6600 val loss 5.8871
+6600 val perplexity 360.3513
+6600 train 5.739757 (lr=2.5508e-05) (hash(x)=152902689)
+6700 val loss 5.8851
+6700 val perplexity 359.6428
+6700 train 5.922760 (lr=2.5244e-05) (hash(x)=153846046)
+6800 val loss 5.8640
+6800 val perplexity 352.1464
+6800 train 5.991194 (lr=2.4979e-05) (hash(x)=158512738)
+6900 val loss 5.8555
+6900 val perplexity 349.1653
+6900 train 6.398537 (lr=2.4711e-05) (hash(x)=156849968)
+7000 val loss 5.8474
+7000 val perplexity 346.3384
+7000 train 5.704376 (lr=2.4442e-05) (hash(x)=142395855)
+7100 val loss 5.8435
+7100 val perplexity 344.9890
+7100 train 5.685997 (lr=2.4171e-05) (hash(x)=147114884)
+7200 val loss 5.8335
+7200 val perplexity 341.5437
+7200 train 5.867506 (lr=2.3899e-05) (hash(x)=156979839)
+7300 val loss 5.8267
+7300 val perplexity 339.2277
+7300 train 5.626989 (lr=2.3625e-05) (hash(x)=145584373)
+7400 val loss 5.8235
+7400 val perplexity 338.1602
+7400 train 5.572762 (lr=2.3349e-05) (hash(x)=141508204)
+7500 val loss 5.8126
+7500 val perplexity 334.4917
+7500 train 5.851742 (lr=2.3073e-05) (hash(x)=148803965)
+7600 val loss 5.8067
+7600 val perplexity 332.5083
+7600 train 5.863437 (lr=2.2795e-05) (hash(x)=151019676)
+7700 val loss 5.7939
+7700 val perplexity 328.2856
+7700 train 5.906473 (lr=2.2515e-05) (hash(x)=143155750)
+7800 val loss 5.7892
+7800 val perplexity 326.7418
+7800 train 5.820032 (lr=2.2235e-05) (hash(x)=152569653)
+7900 val loss 5.7830
+7900 val perplexity 324.7243
+7900 train 5.654320 (lr=2.1954e-05) (hash(x)=143519455)
+8000 val loss 5.7751
+8000 val perplexity 322.1766
+8000 train 5.862270 (lr=2.1671e-05) (hash(x)=161180944)
+8100 val loss 5.7742
+8100 val perplexity 321.8991
+8100 train 5.804288 (lr=2.1388e-05) (hash(x)=154107345)
+8200 val loss 5.7724
+8200 val perplexity 321.2982
+8200 train 5.814382 (lr=2.1105e-05) (hash(x)=152486517)
+8300 val loss 5.7592
+8300 val perplexity 317.1099
+8300 train 5.759204 (lr=2.0820e-05) (hash(x)=156167749)
+8400 val loss 5.7493
+8400 val perplexity 313.9706
+8400 train 5.835824 (lr=2.0536e-05) (hash(x)=149155006)
+8500 val loss 5.7415
+8500 val perplexity 311.5284
+8500 train 5.687334 (lr=2.0250e-05) (hash(x)=147844390)
+8600 val loss 5.7388
+8600 val perplexity 310.6940
+8600 train 5.997749 (lr=1.9965e-05) (hash(x)=165753320)
+8700 val loss 5.7355
+8700 val perplexity 309.6590
+8700 train 5.514201 (lr=1.9679e-05) (hash(x)=146079979)
+8800 val loss 5.7332
+8800 val perplexity 308.9646
+8800 train 5.915848 (lr=1.9393e-05) (hash(x)=172259509)
+8900 val loss 5.7253
+8900 val perplexity 306.5404
+8900 train 5.405478 (lr=1.9107e-05) (hash(x)=145148314)
+9000 val loss 5.7233
+9000 val perplexity 305.9171
+9000 train 5.642859 (lr=1.8821e-05) (hash(x)=144250633)
+9100 val loss 5.7168
+9100 val perplexity 303.9372
+9100 train 5.809632 (lr=1.8535e-05) (hash(x)=157219797)
+9200 val loss 5.7049
+9200 val perplexity 300.3360
+9200 train 5.752951 (lr=1.8250e-05) (hash(x)=142743778)
+9300 val loss 5.7006
+9300 val perplexity 299.0528
+9300 train 5.633637 (lr=1.7964e-05) (hash(x)=139669771)
+9400 val loss 5.6915
+9400 val perplexity 296.3345
+9400 train 5.733429 (lr=1.7680e-05) (hash(x)=145916843)
+9500 val loss 5.6955
+9500 val perplexity 297.5302
+9500 train 5.614336 (lr=1.7395e-05) (hash(x)=150196125)
+9600 val loss 5.6859
+9600 val perplexity 294.6796
+9600 train 6.000877 (lr=1.7112e-05) (hash(x)=160041419)
+9700 val loss 5.6857
+9700 val perplexity 294.6254
+9700 train 5.472782 (lr=1.6829e-05) (hash(x)=139931627)
+9800 val loss 5.6803
+9800 val perplexity 293.0335
+9800 train 5.666329 (lr=1.6546e-05) (hash(x)=150370792)
+9900 val loss 5.6722
+9900 val perplexity 290.6800
+9900 train 5.500357 (lr=1.6265e-05) (hash(x)=153014886)
+10000 val loss 5.6696
+10000 val perplexity 289.9293
+10000 train 5.582622 (lr=1.5985e-05) (hash(x)=141915265)
+10100 val loss 5.6658
+10100 val perplexity 288.8095
+10100 train 6.173605 (lr=1.5705e-05) (hash(x)=197770952)
+10200 val loss 5.6572
+10200 val perplexity 286.3348
+10200 train 5.623529 (lr=1.5427e-05) (hash(x)=156806005)
+10300 val loss 5.6543
+10300 val perplexity 285.5200
+10300 train 5.462598 (lr=1.5151e-05) (hash(x)=145378216)
+10400 val loss 5.6495
+10400 val perplexity 284.1413
+10400 train 5.563924 (lr=1.4875e-05) (hash(x)=145125407)
+10500 val loss 5.6463
+10500 val perplexity 283.2400
+10500 train 5.467100 (lr=1.4601e-05) (hash(x)=147553745)
+10600 val loss 5.6426
+10600 val perplexity 282.1878
+10600 train 5.293966 (lr=1.4329e-05) (hash(x)=134894889)
+10700 val loss 5.6358
+10700 val perplexity 280.2725
+10700 train 5.560827 (lr=1.4058e-05) (hash(x)=151041040)
+10800 val loss 5.6313
+10800 val perplexity 279.0345
+10800 train 5.559671 (lr=1.3789e-05) (hash(x)=145685829)
+10900 val loss 5.6262
+10900 val perplexity 277.6062
+10900 train 5.715626 (lr=1.3521e-05) (hash(x)=156000716)
+11000 val loss 5.6236
+11000 val perplexity 276.8808
+11000 train 5.636567 (lr=1.3256e-05) (hash(x)=149858163)
+11100 val loss 5.6234
+11100 val perplexity 276.8307
+11100 train 5.906702 (lr=1.2992e-05) (hash(x)=164873575)
+11200 val loss 5.6220
+11200 val perplexity 276.4303
+11200 train 5.503514 (lr=1.2731e-05) (hash(x)=156370069)
+11300 val loss 5.6210
+11300 val perplexity 276.1702
+11300 train 5.706107 (lr=1.2472e-05) (hash(x)=160356892)
+11400 val loss 5.6073
+11400 val perplexity 272.3991
+11400 train 5.678671 (lr=1.2215e-05) (hash(x)=158955430)
+11500 val loss 5.6066
+11500 val perplexity 272.2245
+11500 train 5.573491 (lr=1.1960e-05) (hash(x)=146598377)
+11600 val loss 5.6235
+11600 val perplexity 276.8658
+11600 train 5.762615 (lr=1.1708e-05) (hash(x)=156045795)
+11700 val loss 5.6025
+11700 val perplexity 271.1102
+11700 train 5.456719 (lr=1.1458e-05) (hash(x)=143963334)
+11800 val loss 5.6006
+11800 val perplexity 270.5925
+11800 train 5.440306 (lr=1.1210e-05) (hash(x)=145831579)
+11900 val loss 5.5957
+11900 val perplexity 269.2682
+11900 train 5.755351 (lr=1.0966e-05) (hash(x)=165154141)
+12000 val loss 5.5908
+12000 val perplexity 267.9550
+12000 train 5.674036 (lr=1.0724e-05) (hash(x)=160373794)
+12100 val loss 5.5874
+12100 val perplexity 267.0450
+12100 train 5.375375 (lr=1.0485e-05) (hash(x)=141476346)
+12200 val loss 5.5856
+12200 val perplexity 266.5506
+12200 train 5.835909 (lr=1.0249e-05) (hash(x)=164508058)
+12300 val loss 5.5835
+12300 val perplexity 265.9999
+12300 train 5.569179 (lr=1.0016e-05) (hash(x)=140407664)
+12400 val loss 5.5811
+12400 val perplexity 265.3701
+12400 train 5.437617 (lr=9.7854e-06) (hash(x)=149526189)
+12500 val loss 5.5757
+12500 val perplexity 263.9212
+12500 train 5.430196 (lr=9.5583e-06) (hash(x)=142060916)
+12600 val loss 5.5706
+12600 val perplexity 262.5867
+12600 train 5.744236 (lr=9.3345e-06) (hash(x)=168650848)
+12700 val loss 5.5701
+12700 val perplexity 262.4474
+12700 train 5.596745 (lr=9.1139e-06) (hash(x)=154353018)
+12800 val loss 5.5697
+12800 val perplexity 262.3454
+12800 train 5.737992 (lr=8.8967e-06) (hash(x)=156115265)
+12900 val loss 5.5665
+12900 val perplexity 261.5076
+12900 train 5.494372 (lr=8.6829e-06) (hash(x)=143378175)
+13000 val loss 5.5639
+13000 val perplexity 260.8358
+13000 train 5.656729 (lr=8.4726e-06) (hash(x)=166124864)
+13100 val loss 5.5626
+13100 val perplexity 260.4978
+13100 train 5.440381 (lr=8.2658e-06) (hash(x)=146615385)
+13200 val loss 5.5623
+13200 val perplexity 260.4130
+13200 train 5.458254 (lr=8.0626e-06) (hash(x)=149236047)
+13300 val loss 5.5570
+13300 val perplexity 259.0388
+13300 train 5.490791 (lr=7.8632e-06) (hash(x)=151346972)
+13400 val loss 5.5544
+13400 val perplexity 258.3721
+13400 train 5.652813 (lr=7.6674e-06) (hash(x)=159167093)
+13500 val loss 5.5534
+13500 val perplexity 258.1016
+13500 train 5.282621 (lr=7.4756e-06) (hash(x)=135054650)
+13600 val loss 5.5507
+13600 val perplexity 257.4178
+13600 train 5.435532 (lr=7.2875e-06) (hash(x)=151033927)
+13700 val loss 5.5507
+13700 val perplexity 257.4250
+13700 train 5.252520 (lr=7.1035e-06) (hash(x)=141119172)
+13800 val loss 5.5498
+13800 val perplexity 257.1873
+13800 train 5.263982 (lr=6.9234e-06) (hash(x)=144197766)
+13900 val loss 5.5452
+13900 val perplexity 256.0157
+13900 train 5.482715 (lr=6.7474e-06) (hash(x)=145148158)
+14000 val loss 5.5415
+14000 val perplexity 255.0668
+14000 train 6.022576 (lr=6.5756e-06) (hash(x)=179068751)
+14100 val loss 5.5413
+14100 val perplexity 255.0131
+14100 train 5.412605 (lr=6.4079e-06) (hash(x)=144988373)
+14200 val loss 5.5398
+14200 val perplexity 254.6327
+14200 train 5.489853 (lr=6.2444e-06) (hash(x)=133379696)
+14300 val loss 5.5397
+14300 val perplexity 254.6027
+14300 train 5.420756 (lr=6.0852e-06) (hash(x)=149883422)
+14400 val loss 5.5397
+14400 val perplexity 254.5917
+14400 train 5.406084 (lr=5.9304e-06) (hash(x)=145709198)
+14500 val loss 5.5349
+14500 val perplexity 253.3946
+14500 train 5.546863 (lr=5.7800e-06) (hash(x)=148016579)
+14600 val loss 5.5333
+14600 val perplexity 252.9894
+14600 train 5.513137 (lr=5.6340e-06) (hash(x)=150218222)
+14700 val loss 5.5329
+14700 val perplexity 252.8771
+14700 train 5.468463 (lr=5.4925e-06) (hash(x)=142258737)
+14800 val loss 5.5297
+14800 val perplexity 252.0703
+14800 train 5.427268 (lr=5.3555e-06) (hash(x)=147628003)
+14900 val loss 5.5314
+14900 val perplexity 252.5087
+14900 train 5.382735 (lr=5.2231e-06) (hash(x)=149144036)
+15000 val loss 5.5304
+15000 val perplexity 252.2545
+15000 train 5.463471 (lr=5.0954e-06) (hash(x)=145925409)
+15100 val loss 5.5261
+15100 val perplexity 251.1536
+15100 train 5.413812 (lr=4.9723e-06) (hash(x)=144555751)
+15200 val loss 5.5241
+15200 val perplexity 250.6498
+15200 train 5.348018 (lr=4.8539e-06) (hash(x)=143448852)
+15300 val loss 5.5229
+15300 val perplexity 250.3561
+15300 train 5.524452 (lr=4.7403e-06) (hash(x)=156347253)
+15400 val loss 5.5216
+15400 val perplexity 250.0448
+15400 train 5.474526 (lr=4.6314e-06) (hash(x)=153010179)
+15500 val loss 5.5222
+15500 val perplexity 250.1826
+15500 train 5.299636 (lr=4.5274e-06) (hash(x)=144066445)
+15600 val loss 5.5216
+15600 val perplexity 250.0413
+15600 train 5.255478 (lr=4.4282e-06) (hash(x)=146107722)
+15700 val loss 5.5178
+15700 val perplexity 249.0807
+15700 train 5.667055 (lr=4.3339e-06) (hash(x)=152533799)
+15800 val loss 5.5155
+15800 val perplexity 248.5203
+15800 train 5.569731 (lr=4.2446e-06) (hash(x)=151756366)
+15900 val loss 5.5149
+15900 val perplexity 248.3722
+15900 train 5.498937 (lr=4.1601e-06) (hash(x)=151735039)
+16000 val loss 5.5144
+16000 val perplexity 248.2384
+16000 train 5.541584 (lr=4.0807e-06) (hash(x)=154633226)
+16100 val loss 5.5148
+16100 val perplexity 248.3285
+16100 train 5.358697 (lr=4.0063e-06) (hash(x)=158094333)
+16200 val loss 5.5145
+16200 val perplexity 248.2621
+16200 train 5.270772 (lr=3.9368e-06) (hash(x)=142772927)
+16300 val loss 5.5136
+16300 val perplexity 248.0458
+16300 train 5.414096 (lr=3.8725e-06) (hash(x)=148042896)
+16400 val loss 5.5103
+16400 val perplexity 247.2169
+16400 train 5.995365 (lr=3.8132e-06) (hash(x)=170306399)
+16500 val loss 5.5097
+16500 val perplexity 247.0674
+16500 train 5.531100 (lr=3.7590e-06) (hash(x)=141049320)
+16600 val loss 5.5105
+16600 val perplexity 247.2748
+16600 train 5.366843 (lr=3.7099e-06) (hash(x)=150512136)
+16700 val loss 5.5079
+16700 val perplexity 246.6345
+16700 train 5.512495 (lr=3.6659e-06) (hash(x)=151843645)
+16800 val loss 5.5054
+16800 val perplexity 246.0190
+16800 train 5.574283 (lr=3.6271e-06) (hash(x)=140073659)
+16900 val loss 5.5040
+16900 val perplexity 245.6649
+16900 train 5.649993 (lr=3.5934e-06) (hash(x)=171157212)
+17000 val loss 5.5037
+17000 val perplexity 245.6022
+17000 train 5.439902 (lr=3.5649e-06) (hash(x)=144553276)
+17100 val loss 5.5033
+17100 val perplexity 245.5049
+17100 train 5.497724 (lr=3.5415e-06) (hash(x)=157717565)
+17200 val loss 5.5036
+17200 val perplexity 245.5721
+17200 train 5.361936 (lr=3.5234e-06) (hash(x)=149843860)
+17300 val loss 5.5012
+17300 val perplexity 244.9887
+17300 train 5.560598 (lr=3.5104e-06) (hash(x)=162171979)
+17400 val loss 5.5006
+17400 val perplexity 244.8329
+17400 train 5.472299 (lr=3.5026e-06) (hash(x)=148400702)
+17499 val loss 5.4991
+17499 val perplexity 244.4820

attention_kindselective_n_heads4_seed1338/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0c36683ee57463cfd86e7922a106b4b11abb68ae6f544a48c5187333df99ac6
+oid sha256:c9bccf2461b0c494a3687ecba37da35128d7decbe154d840de37be045dd40b15
 size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:749581d48e62690bb18401ec32aa78d78b9cf7705a7d783397a5bbdd00e78b06
+oid sha256:bbe1f41dc0665df8af9fb548e6cf38e4843c70c08313c5c880d130411c57a236
 size 179406214