Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2663
 0 val perplexity 78147.0000
-0 train 11.263341 (lr=1.6250e-07) (hash(x)=150327452)
-100 val loss 9.8875
-100 val perplexity 19683.5957
-100 train 9.898082 (lr=1.6412e-05) (hash(x)=166441190)
-200 val loss 8.9232
-200 val perplexity 7504.3047
-200 train 9.040984 (lr=3.2500e-05) (hash(x)=166780046)
-300 val loss 8.0895
-300 val perplexity 3259.9653
-300 train 8.044475 (lr=3.2498e-05) (hash(x)=159835303)
-400 val loss 7.7549
-400 val perplexity 2332.9883
-400 train 7.555726 (lr=3.2490e-05) (hash(x)=155040610)
-500 val loss 7.6166
-500 val perplexity 2031.6100
-500 train 7.373107 (lr=3.2478e-05) (hash(x)=130190460)
-600 val loss 7.5216
-600 val perplexity 1847.4545
-600 train 7.546905 (lr=3.2461e-05) (hash(x)=155504036)
-700 val loss 7.4629
-700 val perplexity 1742.1884
-700 train 7.319477 (lr=3.2440e-05) (hash(x)=137347213)
-800 val loss 7.4212
-800 val perplexity 1670.9910
-800 train 7.279414 (lr=3.2413e-05) (hash(x)=143823248)
-900 val loss 7.3721
-900 val perplexity 1590.9294
-900 train 7.489573 (lr=3.2382e-05) (hash(x)=156260416)
-1000 val loss 7.3399
-1000 val perplexity 1540.5684
-1000 train 7.312011 (lr=3.2346e-05) (hash(x)=143734685)
-1100 val loss 7.2992
-1100 val perplexity 1479.1761
-1100 train 7.239788 (lr=3.2305e-05) (hash(x)=160013925)
-1200 val loss 7.2686
-1200 val perplexity 1434.5974
-1200 train 7.015460 (lr=3.2260e-05) (hash(x)=150678249)
-1300 val loss 7.2366
-1300 val perplexity 1389.3865
-1300 train 7.022800 (lr=3.2209e-05) (hash(x)=149073315)
-1400 val loss 7.1896
-1400 val perplexity 1325.6152
-1400 train 7.533691 (lr=3.2154e-05) (hash(x)=175802021)
-1500 val loss 7.1516
-1500 val perplexity 1276.0918
-1500 train 7.414447 (lr=3.2094e-05) (hash(x)=171034639)
-1600 val loss 7.1099
-1600 val perplexity 1224.0310
-1600 train 7.326279 (lr=3.2030e-05) (hash(x)=158681215)
-1700 val loss 7.0736
-1700 val perplexity 1180.4250
-1700 train 7.009824 (lr=3.1961e-05) (hash(x)=152116061)
-1800 val loss 7.0371
-1800 val perplexity 1138.1099
-1800 train 6.974350 (lr=3.1887e-05) (hash(x)=146108145)
-1900 val loss 7.0033
-1900 val perplexity 1100.2341
-1900 train 6.837708 (lr=3.1809e-05) (hash(x)=147598108)
-2000 val loss 6.9814
-2000 val perplexity 1076.3971
-2000 train 6.851001 (lr=3.1726e-05) (hash(x)=154996086)
-2100 val loss 6.9515
-2100 val perplexity 1044.6981
-2100 train 6.739772 (lr=3.1638e-05) (hash(x)=153396183)
-2200 val loss 6.9003
-2200 val perplexity 992.5322
-2200 train 6.889448 (lr=3.1546e-05) (hash(x)=153885445)
-2300 val loss 6.8539
-2300 val perplexity 947.5439
-2300 train 6.854183 (lr=3.1449e-05) (hash(x)=159666385)
-2400 val loss 6.8063
-2400 val perplexity 903.5588
-2400 train 6.780859 (lr=3.1348e-05) (hash(x)=142353087)
-2500 val loss 6.7665
-2500 val perplexity 868.3074
-2500 train 6.773431 (lr=3.1243e-05) (hash(x)=146491718)
-2600 val loss 6.7330
-2600 val perplexity 839.6339
-2600 train 6.659631 (lr=3.1133e-05) (hash(x)=150750353)
-2700 val loss 6.6965
-2700 val perplexity 809.5696
-2700 train 6.414171 (lr=3.1019e-05) (hash(x)=129849193)
-2800 val loss 6.6659
-2800 val perplexity 785.1340
-2800 train 6.513141 (lr=3.0900e-05) (hash(x)=152767913)
-2900 val loss 6.6364
-2900 val perplexity 762.3094
-2900 train 6.420549 (lr=3.0777e-05) (hash(x)=146531140)
-3000 val loss 6.5941
-3000 val perplexity 730.7974
-3000 train 6.674080 (lr=3.0650e-05) (hash(x)=151562048)
-3100 val loss 6.5496
-3100 val perplexity 698.9839
-3100 train 6.522118 (lr=3.0518e-05) (hash(x)=146001424)
-3200 val loss 6.5231
-3200 val perplexity 680.6537
-3200 train 6.579503 (lr=3.0383e-05) (hash(x)=166486165)
-3300 val loss 6.4914
-3300 val perplexity 659.4257
-3300 train 6.422096 (lr=3.0243e-05) (hash(x)=150866680)
-3400 val loss 6.4730
-3400 val perplexity 647.4102
-3400 train 6.369732 (lr=3.0099e-05) (hash(x)=143900419)
-3500 val loss 6.4440
-3500 val perplexity 628.8882
-3500 train 6.262652 (lr=2.9952e-05) (hash(x)=148845794)
-3600 val loss 6.4252
-3600 val perplexity 617.2151
-3600 train 6.261324 (lr=2.9800e-05) (hash(x)=145667796)
-3700 val loss 6.4049
-3700 val perplexity 604.8305
-3700 train 6.426984 (lr=2.9644e-05) (hash(x)=163563851)
-3800 val loss 6.3892
-3800 val perplexity 595.3663
-3800 train 6.336480 (lr=2.9485e-05) (hash(x)=147488689)
-3900 val loss 6.3539
-3900 val perplexity 574.7558
-3900 train 6.357310 (lr=2.9321e-05) (hash(x)=148186608)
-4000 val loss 6.3303
-4000 val perplexity 561.3508
-4000 train 6.204438 (lr=2.9154e-05) (hash(x)=142970187)
-4100 val loss 6.3048
-4100 val perplexity 547.1708
-4100 train 6.399692 (lr=2.8983e-05) (hash(x)=141584883)
-4200 val loss 6.2908
-4200 val perplexity 539.5918
-4200 train 6.087615 (lr=2.8808e-05) (hash(x)=145664585)
-4300 val loss 6.2741
-4300 val perplexity 530.6376
-4300 train 6.144583 (lr=2.8630e-05) (hash(x)=143736499)
-4400 val loss 6.2689
-4400 val perplexity 527.8878
-4400 train 6.090772 (lr=2.8448e-05) (hash(x)=151883322)
-4500 val loss 6.2551
-4500 val perplexity 520.6404
-4500 train 6.139237 (lr=2.8263e-05) (hash(x)=153904871)
-4600 val loss 6.2374
-4600 val perplexity 511.5054
-4600 train 6.329777 (lr=2.8075e-05) (hash(x)=154893521)
-4700 val loss 6.2138
-4700 val perplexity 499.6078
-4700 train 6.267534 (lr=2.7883e-05) (hash(x)=152323949)
-4800 val loss 6.1972
-4800 val perplexity 491.3801
-4800 train 6.149027 (lr=2.7687e-05) (hash(x)=154104619)
-4900 val loss 6.1803
-4900 val perplexity 483.1391
-4900 train 6.309986 (lr=2.7489e-05) (hash(x)=146311426)
-5000 val loss 6.1733
-5000 val perplexity 479.7561
-5000 train 6.151302 (lr=2.7287e-05) (hash(x)=156741847)
-5100 val loss 6.1586
-5100 val perplexity 472.7873
-5100 train 5.937405 (lr=2.7082e-05) (hash(x)=142086346)
-5200 val loss 6.1561
-5200 val perplexity 471.5924
-5200 train 5.926026 (lr=2.6874e-05) (hash(x)=150265428)
-5300 val loss 6.1438
-5300 val perplexity 465.7993
-5300 train 6.022453 (lr=2.6664e-05) (hash(x)=151339108)
-5400 val loss 6.1304
-5400 val perplexity 459.6218
-5400 train 6.102451 (lr=2.6450e-05) (hash(x)=154654372)
-5500 val loss 6.1120
-5500 val perplexity 451.2368
-5500 train 6.118230 (lr=2.6233e-05) (hash(x)=150575051)
-5600 val loss 6.1000
-5600 val perplexity 445.8380
-5600 train 6.011166 (lr=2.6014e-05) (hash(x)=140396423)
-5700 val loss 6.0866
-5700 val perplexity 439.9342
-5700 train 5.939641 (lr=2.5792e-05) (hash(x)=144678758)
-5800 val loss 6.0762
-5800 val perplexity 435.3538
-5800 train 6.136963 (lr=2.5567e-05) (hash(x)=151992743)
-5900 val loss 6.0681
-5900 val perplexity 431.8787
-5900 train 5.861908 (lr=2.5340e-05) (hash(x)=144396927)
-6000 val loss 6.0589
-6000 val perplexity 427.8926
-6000 train 6.021150 (lr=2.5111e-05) (hash(x)=165478625)
-6100 val loss 6.0597
-6100 val perplexity 428.2602
-6100 train 5.792263 (lr=2.4879e-05) (hash(x)=147088621)
-6200 val loss 6.0481
-6200 val perplexity 423.3074
-6200 train 5.812227 (lr=2.4644e-05) (hash(x)=140794994)
-6300 val loss 6.0296
-6300 val perplexity 415.5572
-6300 train 5.968508 (lr=2.4408e-05) (hash(x)=134780906)
-6400 val loss 6.0144
-6400 val perplexity 409.2740
-6400 train 5.996778 (lr=2.4169e-05) (hash(x)=149023655)
-6500 val loss 6.0036
-6500 val perplexity 404.8894
-6500 train 5.856802 (lr=2.3928e-05) (hash(x)=147497796)
-6600 val loss 6.0013
-6600 val perplexity 403.9445
-6600 train 5.857316 (lr=2.3686e-05) (hash(x)=152902689)
-6700 val loss 6.0037
-6700 val perplexity 404.9313
-6700 train 6.043744 (lr=2.3441e-05) (hash(x)=153846046)
-6800 val loss 5.9799
-6800 val perplexity 395.4021
-6800 train 6.110096 (lr=2.3195e-05) (hash(x)=158512738)
-6900 val loss 5.9692
-6900 val perplexity 391.2107
-6900 train 6.498738 (lr=2.2946e-05) (hash(x)=156849968)
-7000 val loss 5.9604
-7000 val perplexity 387.7580
-7000 train 5.809161 (lr=2.2696e-05) (hash(x)=142395855)
-7100 val loss 5.9549
-7100 val perplexity 385.6508
-7100 train 5.797284 (lr=2.2445e-05) (hash(x)=147114884)
-7200 val loss 5.9443
-7200 val perplexity 381.5764
-7200 train 5.971154 (lr=2.2192e-05) (hash(x)=156979839)
-7300 val loss 5.9408
-7300 val perplexity 380.2552
-7300 train 5.738841 (lr=2.1937e-05) (hash(x)=145584373)
-7400 val loss 5.9331
-7400 val perplexity 377.3345
-7400 train 5.679769 (lr=2.1682e-05) (hash(x)=141508204)
-7500 val loss 5.9221
-7500 val perplexity 373.2067
-7500 train 5.954195 (lr=2.1425e-05) (hash(x)=148803965)
-7600 val loss 5.9160
-7600 val perplexity 370.9126
-7600 train 5.974942 (lr=2.1166e-05) (hash(x)=151019676)
-7700 val loss 5.9041
-7700 val perplexity 366.5331
-7700 train 5.988387 (lr=2.0907e-05) (hash(x)=143155750)
-7800 val loss 5.8963
-7800 val perplexity 363.6974
-7800 train 5.926476 (lr=2.0647e-05) (hash(x)=152569653)
-7900 val loss 5.8890
-7900 val perplexity 361.0439
-7900 train 5.763350 (lr=2.0386e-05) (hash(x)=143519455)
-8000 val loss 5.8823
-8000 val perplexity 358.6201
-8000 train 6.002029 (lr=2.0123e-05) (hash(x)=161180944)
-8100 val loss 5.8808
-8100 val perplexity 358.0812
-8100 train 5.909467 (lr=1.9861e-05) (hash(x)=154107345)
-8200 val loss 5.8773
-8200 val perplexity 356.8280
-8200 train 5.915339 (lr=1.9597e-05) (hash(x)=152486517)
-8300 val loss 5.8635
-8300 val perplexity 351.9369
-8300 train 5.868403 (lr=1.9333e-05) (hash(x)=156167749)
-8400 val loss 5.8580
-8400 val perplexity 350.0205
-8400 train 5.939692 (lr=1.9069e-05) (hash(x)=149155006)
-8500 val loss 5.8437
-8500 val perplexity 345.0668
-8500 train 5.792398 (lr=1.8804e-05) (hash(x)=147844390)
-8600 val loss 5.8417
-8600 val perplexity 344.3497
-8600 train 6.100732 (lr=1.8539e-05) (hash(x)=165753320)
-8700 val loss 5.8357
-8700 val perplexity 342.2876
-8700 train 5.617911 (lr=1.8273e-05) (hash(x)=146079979)
-8800 val loss 5.8368
-8800 val perplexity 342.6954
-8800 train 6.028148 (lr=1.8008e-05) (hash(x)=172259509)
-8900 val loss 5.8283
-8900 val perplexity 339.7885
-8900 train 5.512281 (lr=1.7742e-05) (hash(x)=145148314)
-9000 val loss 5.8204
-9000 val perplexity 337.1211
-9000 train 5.738827 (lr=1.7477e-05) (hash(x)=144250633)
-9100 val loss 5.8136
-9100 val perplexity 334.8091
-9100 train 5.910919 (lr=1.7211e-05) (hash(x)=157219797)
-9200 val loss 5.8091
-9200 val perplexity 333.3240
-9200 train 5.853679 (lr=1.6946e-05) (hash(x)=142743778)
-9300 val loss 5.7980
-9300 val perplexity 329.6350
-9300 train 5.722970 (lr=1.6681e-05) (hash(x)=139669771)
-9400 val loss 5.7902
-9400 val perplexity 327.0877
-9400 train 5.823825 (lr=1.6417e-05) (hash(x)=145916843)
-9500 val loss 5.7892
-9500 val perplexity 326.7543
-9500 train 5.710148 (lr=1.6153e-05) (hash(x)=150196125)
-9600 val loss 5.7812
-9600 val perplexity 324.1400
-9600 train 6.082446 (lr=1.5889e-05) (hash(x)=160041419)
-9700 val loss 5.7831
-9700 val perplexity 324.7499
-9700 train 5.570460 (lr=1.5627e-05) (hash(x)=139931627)
-9800 val loss 5.7745
-9800 val perplexity 321.9872
-9800 train 5.752125 (lr=1.5364e-05) (hash(x)=150370792)
-9900 val loss 5.7680
-9900 val perplexity 319.8929
-9900 train 5.602925 (lr=1.5103e-05) (hash(x)=153014886)
-10000 val loss 5.7651
-10000 val perplexity 318.9697
-10000 train 5.674670 (lr=1.4843e-05) (hash(x)=141915265)
-10100 val loss 5.7595
-10100 val perplexity 317.1992
-10100 train 6.257363 (lr=1.4584e-05) (hash(x)=197770952)
-10200 val loss 5.7518
-10200 val perplexity 314.7549
-10200 train 5.720849 (lr=1.4325e-05) (hash(x)=156806005)
-10300 val loss 5.7469
-10300 val perplexity 313.2228
-10300 train 5.550874 (lr=1.4068e-05) (hash(x)=145378216)
-10400 val loss 5.7420
-10400 val perplexity 311.6738
-10400 train 5.648454 (lr=1.3813e-05) (hash(x)=145125407)
-10500 val loss 5.7383
-10500 val perplexity 310.5409
-10500 train 5.556795 (lr=1.3558e-05) (hash(x)=147553745)
-10600 val loss 5.7375
-10600 val perplexity 310.2906
-10600 train 5.377072 (lr=1.3305e-05) (hash(x)=134894889)
-10700 val loss 5.7291
-10700 val perplexity 307.6816
-10700 train 5.653048 (lr=1.3054e-05) (hash(x)=151041040)
-10800 val loss 5.7224
-10800 val perplexity 305.6333
-10800 train 5.655978 (lr=1.2804e-05) (hash(x)=145685829)
-10900 val loss 5.7162
-10900 val perplexity 303.7596
-10900 train 5.794735 (lr=1.2555e-05) (hash(x)=156000716)
-11000 val loss 5.7139
-11000 val perplexity 303.0442
-11000 train 5.712698 (lr=1.2309e-05) (hash(x)=149858163)
-11100 val loss 5.7135
-11100 val perplexity 302.9354
-11100 train 5.992548 (lr=1.2064e-05) (hash(x)=164873575)
-11200 val loss 5.7104
-11200 val perplexity 301.9899
-11200 train 5.599405 (lr=1.1822e-05) (hash(x)=156370069)
-11300 val loss 5.7097
-11300 val perplexity 301.7715
-11300 train 5.798551 (lr=1.1581e-05) (hash(x)=160356892)
-11400 val loss 5.6994
-11400 val perplexity 298.6908
-11400 train 5.780049 (lr=1.1342e-05) (hash(x)=158955430)
-11500 val loss 5.6948
-11500 val perplexity 297.3225
-11500 train 5.655000 (lr=1.1106e-05) (hash(x)=146598377)
-11600 val loss 5.7098
-11600 val perplexity 301.8159
-11600 train 5.854637 (lr=1.0871e-05) (hash(x)=156045795)
-11700 val loss 5.6914
-11700 val perplexity 296.3024
-11700 train 5.548706 (lr=1.0639e-05) (hash(x)=143963334)
-11800 val loss 5.6887
-11800 val perplexity 295.4948
-11800 train 5.527222 (lr=1.0410e-05) (hash(x)=145831579)
-11900 val loss 5.6838
-11900 val perplexity 294.0689
-11900 train 5.845513 (lr=1.0183e-05) (hash(x)=165154141)
-12000 val loss 5.6766
-12000 val perplexity 291.9412
-12000 train 5.762044 (lr=9.9580e-06) (hash(x)=160373794)
-12100 val loss 5.6749
-12100 val perplexity 291.4481
-12100 train 5.463029 (lr=9.7360e-06) (hash(x)=141476346)
-12200 val loss 5.6715
-12200 val perplexity 290.4636
-12200 train 5.920372 (lr=9.5167e-06) (hash(x)=164508058)
-12300 val loss 5.6698
-12300 val perplexity 289.9705
-12300 train 5.644031 (lr=9.3001e-06) (hash(x)=140407664)
-12400 val loss 5.6671
-12400 val perplexity 289.1933
-12400 train 5.524893 (lr=9.0864e-06) (hash(x)=149526189)
-12500 val loss 5.6616
-12500 val perplexity 287.5968
-12500 train 5.514926 (lr=8.8756e-06) (hash(x)=142060916)
-12600 val loss 5.6573
-12600 val perplexity 286.3658
-12600 train 5.838615 (lr=8.6677e-06) (hash(x)=168650848)
-12700 val loss 5.6561
-12700 val perplexity 286.0426
-12700 train 5.684471 (lr=8.4629e-06) (hash(x)=154353018)
-12800 val loss 5.6554
-12800 val perplexity 285.8341
-12800 train 5.816707 (lr=8.2612e-06) (hash(x)=156115265)
-12900 val loss 5.6525
-12900 val perplexity 284.9962
-12900 train 5.576836 (lr=8.0627e-06) (hash(x)=143378175)
-13000 val loss 5.6485
-13000 val perplexity 283.8571
-13000 train 5.746207 (lr=7.8674e-06) (hash(x)=166124864)
-13100 val loss 5.6475
-13100 val perplexity 283.5952
-13100 train 5.527974 (lr=7.6754e-06) (hash(x)=146615385)
-13200 val loss 5.6470
-13200 val perplexity 283.4364
-13200 train 5.544142 (lr=7.4867e-06) (hash(x)=149236047)
-13300 val loss 5.6417
-13300 val perplexity 281.9557
-13300 train 5.580195 (lr=7.3015e-06) (hash(x)=151346972)
-13400 val loss 5.6390
-13400 val perplexity 281.1739
-13400 train 5.729071 (lr=7.1198e-06) (hash(x)=159167093)
-13500 val loss 5.6384
-13500 val perplexity 281.0078
-13500 train 5.365855 (lr=6.9416e-06) (hash(x)=135054650)
-13600 val loss 5.6369
-13600 val perplexity 280.6006
-13600 train 5.514214 (lr=6.7670e-06) (hash(x)=151033927)
-13700 val loss 5.6363
-13700 val perplexity 280.4331
-13700 train 5.341426 (lr=6.5961e-06) (hash(x)=141119172)
-13800 val loss 5.6337
-13800 val perplexity 279.6906
-13800 train 5.352831 (lr=6.4289e-06) (hash(x)=144197766)
-13900 val loss 5.6292
-13900 val perplexity 278.4267
-13900 train 5.567875 (lr=6.2655e-06) (hash(x)=145148158)
-14000 val loss 5.6262
-14000 val perplexity 277.5951
-14000 train 6.124477 (lr=6.1059e-06) (hash(x)=179068751)
-14100 val loss 5.6250
-14100 val perplexity 277.2643
-14100 train 5.494565 (lr=5.9502e-06) (hash(x)=144988373)
-14200 val loss 5.6235
-14200 val perplexity 276.8575
-14200 train 5.556700 (lr=5.7984e-06) (hash(x)=133379696)
-14300 val loss 5.6244
-14300 val perplexity 277.1109
-14300 train 5.506085 (lr=5.6506e-06) (hash(x)=149883422)
-14400 val loss 5.6243
-14400 val perplexity 277.0917
-14400 train 5.492159 (lr=5.5068e-06) (hash(x)=145709198)
-14500 val loss 5.6185
-14500 val perplexity 275.4824
-14500 train 5.636049 (lr=5.3671e-06) (hash(x)=148016579)
-14600 val loss 5.6168
-14600 val perplexity 274.9970
-14600 train 5.586943 (lr=5.2316e-06) (hash(x)=150218222)
-14700 val loss 5.6163
-14700 val perplexity 274.8758
-14700 train 5.543831 (lr=5.1002e-06) (hash(x)=142258737)
-14800 val loss 5.6128
-14800 val perplexity 273.9005
-14800 train 5.502282 (lr=4.9730e-06) (hash(x)=147628003)
-14900 val loss 5.6144
-14900 val perplexity 274.3428
-14900 train 5.465495 (lr=4.8501e-06) (hash(x)=149144036)
-15000 val loss 5.6134
-15000 val perplexity 274.0710
-15000 train 5.552163 (lr=4.7314e-06) (hash(x)=145925409)
-15100 val loss 5.6101
-15100 val perplexity 273.1843
-15100 train 5.497269 (lr=4.6171e-06) (hash(x)=144555751)
-15200 val loss 5.6077
-15200 val perplexity 272.5063
-15200 train 5.425412 (lr=4.5072e-06) (hash(x)=143448852)
-15300 val loss 5.6064
-15300 val perplexity 272.1697
-15300 train 5.610938 (lr=4.4017e-06) (hash(x)=156347253)
-15400 val loss 5.6046
-15400 val perplexity 271.6618
-15400 train 5.562627 (lr=4.3006e-06) (hash(x)=153010179)
-15500 val loss 5.6050
-15500 val perplexity 271.7854
-15500 train 5.388970 (lr=4.2040e-06) (hash(x)=144066445)
-15600 val loss 5.6043
-15600 val perplexity 271.5881
-15600 train 5.339193 (lr=4.1119e-06) (hash(x)=146107722)
-15700 val loss 5.5999
-15700 val perplexity 270.4047
-15700 train 5.747085 (lr=4.0244e-06) (hash(x)=152533799)
-15800 val loss 5.5981
-15800 val perplexity 269.9133
-15800 train 5.652522 (lr=3.9414e-06) (hash(x)=151756366)
-15900 val loss 5.5974
-15900 val perplexity 269.7231
-15900 train 5.583819 (lr=3.8630e-06) (hash(x)=151735039)
-16000 val loss 5.5969
-16000 val perplexity 269.5918
-16000 train 5.627756 (lr=3.7892e-06) (hash(x)=154633226)
-16100 val loss 5.5972
-16100 val perplexity 269.6637
-16100 train 5.441843 (lr=3.7201e-06) (hash(x)=158094333)
-16200 val loss 5.5964
-16200 val perplexity 269.4558
-16200 train 5.358812 (lr=3.6556e-06) (hash(x)=142772927)
-16300 val loss 5.5958
-16300 val perplexity 269.2827
-16300 train 5.492966 (lr=3.5959e-06) (hash(x)=148042896)
-16400 val loss 5.5922
-16400 val perplexity 268.3255
-16400 train 6.091084 (lr=3.5408e-06) (hash(x)=170306399)
-16500 val loss 5.5917
-16500 val perplexity 268.2008
-16500 train 5.594467 (lr=3.4905e-06) (hash(x)=141049320)
-16600 val loss 5.5916
-16600 val perplexity 268.1708
-16600 train 5.455967 (lr=3.4449e-06) (hash(x)=150512136)
-16700 val loss 5.5894
-16700 val perplexity 267.5735
-16700 train 5.592753 (lr=3.4041e-06) (hash(x)=151843645)
-16800 val loss 5.5878
-16800 val perplexity 267.1494
-16800 train 5.642298 (lr=3.3680e-06) (hash(x)=140073659)
-16900 val loss 5.5860
-16900 val perplexity 266.6786
-16900 train 5.755163 (lr=3.3367e-06) (hash(x)=171157212)
-17000 val loss 5.5844
-17000 val perplexity 266.2370
-17000 train 5.524979 (lr=3.3102e-06) (hash(x)=144553276)
-17100 val loss 5.5853
-17100 val perplexity 266.4875
-17100 train 5.585420 (lr=3.2886e-06) (hash(x)=157717565)
-17200 val loss 5.5848
-17200 val perplexity 266.3359
-17200 train 5.453957 (lr=3.2717e-06) (hash(x)=149843860)
-17300 val loss 5.5822
-17300 val perplexity 265.6631
-17300 train 5.640796 (lr=3.2596e-06) (hash(x)=162171979)
-17400 val loss 5.5809
-17400 val perplexity 265.3040
-17400 train 5.549406 (lr=3.2524e-06) (hash(x)=148400702)
-17499 val loss 5.5807
-17499 val perplexity 265.2500
+0 train 11.263341 (lr=1.3750e-07) (hash(x)=150327452)
+100 val loss 9.9216
+100 val perplexity 20365.0898
+100 train 9.932128 (lr=1.3888e-05) (hash(x)=166441190)
+200 val loss 9.2026
+200 val perplexity 9923.1172
+200 train 9.311575 (lr=2.7500e-05) (hash(x)=166780046)
+300 val loss 8.1964
+300 val perplexity 3627.8376
+300 train 8.155242 (lr=2.7498e-05) (hash(x)=159835303)
+400 val loss 7.7685
+400 val perplexity 2365.0269
+400 train 7.570842 (lr=2.7492e-05) (hash(x)=155040610)
+500 val loss 7.6098
+500 val perplexity 2017.9415
+500 train 7.360526 (lr=2.7482e-05) (hash(x)=130190460)
+600 val loss 7.5112
+600 val perplexity 1828.4420
+600 train 7.535985 (lr=2.7467e-05) (hash(x)=155504036)
+700 val loss 7.4475
+700 val perplexity 1715.5283
+700 train 7.301879 (lr=2.7449e-05) (hash(x)=137347213)
+800 val loss 7.3965
+800 val perplexity 1630.2454
+800 train 7.251807 (lr=2.7427e-05) (hash(x)=143823248)
+900 val loss 7.3470
+900 val perplexity 1551.5825
+900 train 7.458832 (lr=2.7400e-05) (hash(x)=156260416)
+1000 val loss 7.3139
+1000 val perplexity 1501.0842
+1000 train 7.281650 (lr=2.7370e-05) (hash(x)=143734685)
+1100 val loss 7.2764
+1100 val perplexity 1445.7637
+1100 train 7.209399 (lr=2.7335e-05) (hash(x)=160013925)
+1200 val loss 7.2480
+1200 val perplexity 1405.2816
+1200 train 6.993395 (lr=2.7297e-05) (hash(x)=150678249)
+1300 val loss 7.2195
+1300 val perplexity 1365.8458
+1300 train 7.003422 (lr=2.7254e-05) (hash(x)=149073315)
+1400 val loss 7.1645
+1400 val perplexity 1292.6741
+1400 train 7.483765 (lr=2.7207e-05) (hash(x)=175802021)
+1500 val loss 7.1143
+1500 val perplexity 1229.3901
+1500 train 7.375745 (lr=2.7157e-05) (hash(x)=171034639)
+1600 val loss 7.0674
+1600 val perplexity 1173.0409
+1600 train 7.270476 (lr=2.7102e-05) (hash(x)=158681215)
+1700 val loss 7.0231
+1700 val perplexity 1122.2725
+1700 train 6.961975 (lr=2.7044e-05) (hash(x)=152116061)
+1800 val loss 6.9929
+1800 val perplexity 1088.8434
+1800 train 6.930839 (lr=2.6981e-05) (hash(x)=146108145)
+1900 val loss 6.9646
+1900 val perplexity 1058.4460
+1900 train 6.793094 (lr=2.6915e-05) (hash(x)=147598108)
+2000 val loss 6.9345
+2000 val perplexity 1027.1322
+2000 train 6.798426 (lr=2.6845e-05) (hash(x)=154996086)
+2100 val loss 6.9084
+2100 val perplexity 1000.6855
+2100 train 6.694521 (lr=2.6771e-05) (hash(x)=153396183)
+2200 val loss 6.8577
+2200 val perplexity 951.1731
+2200 train 6.844943 (lr=2.6693e-05) (hash(x)=153885445)
+2300 val loss 6.8186
+2300 val perplexity 914.6971
+2300 train 6.820184 (lr=2.6611e-05) (hash(x)=159666385)
+2400 val loss 6.7737
+2400 val perplexity 874.5115
+2400 train 6.748491 (lr=2.6525e-05) (hash(x)=142353087)
+2500 val loss 6.7371
+2500 val perplexity 843.0906
+2500 train 6.756777 (lr=2.6436e-05) (hash(x)=146491718)
+2600 val loss 6.7059
+2600 val perplexity 817.2048
+2600 train 6.631540 (lr=2.6343e-05) (hash(x)=150750353)
+2700 val loss 6.6686
+2700 val perplexity 787.2690
+2700 train 6.380807 (lr=2.6246e-05) (hash(x)=129849193)
+2800 val loss 6.6495
+2800 val perplexity 772.4222
+2800 train 6.494568 (lr=2.6146e-05) (hash(x)=152767913)
+2900 val loss 6.6275
+2900 val perplexity 755.5663
+2900 train 6.406610 (lr=2.6042e-05) (hash(x)=146531140)
+3000 val loss 6.5894
+3000 val perplexity 727.3140
+3000 train 6.665761 (lr=2.5934e-05) (hash(x)=151562048)
+3100 val loss 6.5494
+3100 val perplexity 698.8493
+3100 train 6.516375 (lr=2.5823e-05) (hash(x)=146001424)
+3200 val loss 6.5272
+3200 val perplexity 683.4663
+3200 train 6.600415 (lr=2.5709e-05) (hash(x)=166486165)
+3300 val loss 6.4946
+3300 val perplexity 661.5607
+3300 train 6.426333 (lr=2.5590e-05) (hash(x)=150866680)
+3400 val loss 6.4801
+3400 val perplexity 652.0275
+3400 train 6.374853 (lr=2.5469e-05) (hash(x)=143900419)
+3500 val loss 6.4553
+3500 val perplexity 636.0541
+3500 train 6.270348 (lr=2.5344e-05) (hash(x)=148845794)
+3600 val loss 6.4393
+3600 val perplexity 625.9676
+3600 train 6.269843 (lr=2.5215e-05) (hash(x)=145667796)
+3700 val loss 6.4257
+3700 val perplexity 617.5016
+3700 train 6.442654 (lr=2.5083e-05) (hash(x)=163563851)
+3800 val loss 6.4055
+3800 val perplexity 605.1692
+3800 train 6.348926 (lr=2.4948e-05) (hash(x)=147488689)
+3900 val loss 6.3750
+3900 val perplexity 586.9639
+3900 train 6.376982 (lr=2.4810e-05) (hash(x)=148186608)
+4000 val loss 6.3525
+4000 val perplexity 573.9432
+4000 train 6.228961 (lr=2.4669e-05) (hash(x)=142970187)
+4100 val loss 6.3316
+4100 val perplexity 562.0504
+4100 train 6.424266 (lr=2.4524e-05) (hash(x)=141584883)
+4200 val loss 6.3206
+4200 val perplexity 555.9155
+4200 train 6.121674 (lr=2.4376e-05) (hash(x)=145664585)
+4300 val loss 6.3036
+4300 val perplexity 546.5199
+4300 train 6.176545 (lr=2.4226e-05) (hash(x)=143736499)
+4400 val loss 6.3040
+4400 val perplexity 546.7621
+4400 train 6.126653 (lr=2.4072e-05) (hash(x)=151883322)
+4500 val loss 6.2887
+4500 val perplexity 538.4496
+4500 train 6.172879 (lr=2.3915e-05) (hash(x)=153904871)
+4600 val loss 6.2713
+4600 val perplexity 529.1572
+4600 train 6.360380 (lr=2.3755e-05) (hash(x)=154893521)
+4700 val loss 6.2472
+4700 val perplexity 516.5841
+4700 train 6.304053 (lr=2.3593e-05) (hash(x)=152323949)
+4800 val loss 6.2306
+4800 val perplexity 508.0774
+4800 train 6.176684 (lr=2.3428e-05) (hash(x)=154104619)
+4900 val loss 6.2180
+4900 val perplexity 501.6885
+4900 train 6.347137 (lr=2.3260e-05) (hash(x)=146311426)
+5000 val loss 6.2067
+5000 val perplexity 496.0761
+5000 train 6.195480 (lr=2.3089e-05) (hash(x)=156741847)
+5100 val loss 6.1953
+5100 val perplexity 490.4566
+5100 train 5.973955 (lr=2.2916e-05) (hash(x)=142086346)
+5200 val loss 6.1901
+5200 val perplexity 487.8952
+5200 train 5.958094 (lr=2.2740e-05) (hash(x)=150265428)
+5300 val loss 6.1848
+5300 val perplexity 485.3176
+5300 train 6.061728 (lr=2.2561e-05) (hash(x)=151339108)
+5400 val loss 6.1701
+5400 val perplexity 478.2155
+5400 train 6.146971 (lr=2.2381e-05) (hash(x)=154654372)
+5500 val loss 6.1531
+5500 val perplexity 470.1668
+5500 train 6.165415 (lr=2.2197e-05) (hash(x)=150575051)
+5600 val loss 6.1412
+5600 val perplexity 464.6232
+5600 train 6.048434 (lr=2.2012e-05) (hash(x)=140396423)
+5700 val loss 6.1312
+5700 val perplexity 459.9658
+5700 train 5.991282 (lr=2.1824e-05) (hash(x)=144678758)
+5800 val loss 6.1224
+5800 val perplexity 455.9356
+5800 train 6.170418 (lr=2.1634e-05) (hash(x)=151992743)
+5900 val loss 6.1131
+5900 val perplexity 451.7320
+5900 train 5.902652 (lr=2.1442e-05) (hash(x)=144396927)
+6000 val loss 6.1034
+6000 val perplexity 447.3857
+6000 train 6.065875 (lr=2.1248e-05) (hash(x)=165478625)
+6100 val loss 6.1096
+6100 val perplexity 450.1805
+6100 train 5.843530 (lr=2.1051e-05) (hash(x)=147088621)
+6200 val loss 6.0995
+6200 val perplexity 445.6127
+6200 train 5.866304 (lr=2.0853e-05) (hash(x)=140794994)
+6300 val loss 6.0785
+6300 val perplexity 436.3955
+6300 train 6.008928 (lr=2.0653e-05) (hash(x)=134780906)
+6400 val loss 6.0650
+6400 val perplexity 430.5202
+6400 train 6.043757 (lr=2.0451e-05) (hash(x)=149023655)
+6500 val loss 6.0570
+6500 val perplexity 427.0772
+6500 train 5.909537 (lr=2.0247e-05) (hash(x)=147497796)
+6600 val loss 6.0540
+6600 val perplexity 425.8057
+6600 train 5.906900 (lr=2.0042e-05) (hash(x)=152902689)
+6700 val loss 6.0523
+6700 val perplexity 425.0714
+6700 train 6.086189 (lr=1.9835e-05) (hash(x)=153846046)
+6800 val loss 6.0303
+6800 val perplexity 415.8329
+6800 train 6.160307 (lr=1.9626e-05) (hash(x)=158512738)
+6900 val loss 6.0207
+6900 val perplexity 411.8505
+6900 train 6.544151 (lr=1.9416e-05) (hash(x)=156849968)
+7000 val loss 6.0165
+7000 val perplexity 410.1386
+7000 train 5.866353 (lr=1.9205e-05) (hash(x)=142395855)
+7100 val loss 6.0088
+7100 val perplexity 406.9989
+7100 train 5.857893 (lr=1.8992e-05) (hash(x)=147114884)
+7200 val loss 5.9996
+7200 val perplexity 403.2617
+7200 train 6.032767 (lr=1.8778e-05) (hash(x)=156979839)
+7300 val loss 5.9959
+7300 val perplexity 401.7586
+7300 train 5.781147 (lr=1.8562e-05) (hash(x)=145584373)
+7400 val loss 5.9900
+7400 val perplexity 399.4259
+7400 train 5.729474 (lr=1.8346e-05) (hash(x)=141508204)
+7500 val loss 5.9780
+7500 val perplexity 394.6485
+7500 train 6.012234 (lr=1.8128e-05) (hash(x)=148803965)
+7600 val loss 5.9718
+7600 val perplexity 392.1917
+7600 train 6.031247 (lr=1.7910e-05) (hash(x)=151019676)
+7700 val loss 5.9586
+7700 val perplexity 387.0653
+7700 train 6.031113 (lr=1.7691e-05) (hash(x)=143155750)
+7800 val loss 5.9556
+7800 val perplexity 385.9111
+7800 train 5.981885 (lr=1.7470e-05) (hash(x)=152569653)
+7900 val loss 5.9519
+7900 val perplexity 384.4711
+7900 train 5.823013 (lr=1.7249e-05) (hash(x)=143519455)
+8000 val loss 5.9416
+8000 val perplexity 380.5505
+8000 train 6.052700 (lr=1.7028e-05) (hash(x)=161180944)
+8100 val loss 5.9420
+8100 val perplexity 380.7064
+8100 train 5.966490 (lr=1.6805e-05) (hash(x)=154107345)
+8200 val loss 5.9371
+8200 val perplexity 378.8401
+8200 train 5.968087 (lr=1.6582e-05) (hash(x)=152486517)
+8300 val loss 5.9261
+8300 val perplexity 374.6855
+8300 train 5.935761 (lr=1.6359e-05) (hash(x)=156167749)
+8400 val loss 5.9156
+8400 val perplexity 370.7589
+8400 train 6.000555 (lr=1.6135e-05) (hash(x)=149155006)
+8500 val loss 5.9067
+8500 val perplexity 367.4887
+8500 train 5.864130 (lr=1.5911e-05) (hash(x)=147844390)
+8600 val loss 5.9047
+8600 val perplexity 366.7408
+8600 train 6.157965 (lr=1.5687e-05) (hash(x)=165753320)
+8700 val loss 5.8990
+8700 val perplexity 364.6836
+8700 train 5.683146 (lr=1.5462e-05) (hash(x)=146079979)
+8800 val loss 5.8990
+8800 val perplexity 364.6555
+8800 train 6.090808 (lr=1.5237e-05) (hash(x)=172259509)
+8900 val loss 5.8907
+8900 val perplexity 361.6673
+8900 train 5.573002 (lr=1.5013e-05) (hash(x)=145148314)
+9000 val loss 5.8871
+9000 val perplexity 360.3432
+9000 train 5.803849 (lr=1.4788e-05) (hash(x)=144250633)
+9100 val loss 5.8804
+9100 val perplexity 357.9615
+9100 train 5.972807 (lr=1.4563e-05) (hash(x)=157219797)
+9200 val loss 5.8735
+9200 val perplexity 355.4825
+9200 train 5.910276 (lr=1.4339e-05) (hash(x)=142743778)
+9300 val loss 5.8653
+9300 val perplexity 352.5787
+9300 train 5.783849 (lr=1.4115e-05) (hash(x)=139669771)
+9400 val loss 5.8574
+9400 val perplexity 349.8001
+9400 train 5.884781 (lr=1.3891e-05) (hash(x)=145916843)
+9500 val loss 5.8574
+9500 val perplexity 349.8114
+9500 train 5.776641 (lr=1.3668e-05) (hash(x)=150196125)
+9600 val loss 5.8502
+9600 val perplexity 347.2911
+9600 train 6.161810 (lr=1.3445e-05) (hash(x)=160041419)
+9700 val loss 5.8489
+9700 val perplexity 346.8675
+9700 train 5.629546 (lr=1.3222e-05) (hash(x)=139931627)
+9800 val loss 5.8436
+9800 val perplexity 345.0097
+9800 train 5.823668 (lr=1.3001e-05) (hash(x)=150370792)
+9900 val loss 5.8368
+9900 val perplexity 342.6939
+9900 train 5.680959 (lr=1.2780e-05) (hash(x)=153014886)
+10000 val loss 5.8341
+10000 val perplexity 341.7490
+10000 train 5.737649 (lr=1.2559e-05) (hash(x)=141915265)
+10100 val loss 5.8270
+10100 val perplexity 339.3311
+10100 train 6.341434 (lr=1.2340e-05) (hash(x)=197770952)
+10200 val loss 5.8213
+10200 val perplexity 337.4204
+10200 train 5.808439 (lr=1.2122e-05) (hash(x)=156806005)
+10300 val loss 5.8179
+10300 val perplexity 336.2575
+10300 train 5.622517 (lr=1.1904e-05) (hash(x)=145378216)
+10400 val loss 5.8126
+10400 val perplexity 334.4817
+10400 train 5.716724 (lr=1.1688e-05) (hash(x)=145125407)
+10500 val loss 5.8088
+10500 val perplexity 333.2034
+10500 train 5.622625 (lr=1.1472e-05) (hash(x)=147553745)
+10600 val loss 5.8047
+10600 val perplexity 331.8625
+10600 train 5.440615 (lr=1.1258e-05) (hash(x)=134894889)
+10700 val loss 5.7991
+10700 val perplexity 330.0065
+10700 train 5.726994 (lr=1.1045e-05) (hash(x)=151041040)
+10800 val loss 5.7921
+10800 val perplexity 327.6916
+10800 train 5.724377 (lr=1.0834e-05) (hash(x)=145685829)
+10900 val loss 5.7874
+10900 val perplexity 326.1582
+10900 train 5.860260 (lr=1.0624e-05) (hash(x)=156000716)
+11000 val loss 5.7849
+11000 val perplexity 325.3531
+11000 train 5.782057 (lr=1.0415e-05) (hash(x)=149858163)
+11100 val loss 5.7831
+11100 val perplexity 324.7552
+11100 train 6.055070 (lr=1.0208e-05) (hash(x)=164873575)
+11200 val loss 5.7819
+11200 val perplexity 324.3619
+11200 train 5.665148 (lr=1.0003e-05) (hash(x)=156370069)
+11300 val loss 5.7809
+11300 val perplexity 324.0505
+11300 train 5.885826 (lr=9.7991e-06) (hash(x)=160356892)
+11400 val loss 5.7694
+11400 val perplexity 320.3412
+11400 train 5.853624 (lr=9.5972e-06) (hash(x)=158955430)
+11500 val loss 5.7655
+11500 val perplexity 319.0882
+11500 train 5.735012 (lr=9.3970e-06) (hash(x)=146598377)
+11600 val loss 5.7783
+11600 val perplexity 323.2089
+11600 train 5.920930 (lr=9.1988e-06) (hash(x)=156045795)
+11700 val loss 5.7633
+11700 val perplexity 318.4037
+11700 train 5.620872 (lr=9.0025e-06) (hash(x)=143963334)
+11800 val loss 5.7613
+11800 val perplexity 317.7613
+11800 train 5.593094 (lr=8.8082e-06) (hash(x)=145831579)
+11900 val loss 5.7564
+11900 val perplexity 316.1924
+11900 train 5.932192 (lr=8.6160e-06) (hash(x)=165154141)
+12000 val loss 5.7496
+12000 val perplexity 314.0803
+12000 train 5.828736 (lr=8.4260e-06) (hash(x)=160373794)
+12100 val loss 5.7464
+12100 val perplexity 313.0729
+12100 train 5.533817 (lr=8.2381e-06) (hash(x)=141476346)
+12200 val loss 5.7444
+12200 val perplexity 312.4352
+12200 train 6.004211 (lr=8.0526e-06) (hash(x)=164508058)
+12300 val loss 5.7431
+12300 val perplexity 312.0411
+12300 train 5.710019 (lr=7.8694e-06) (hash(x)=140407664)
+12400 val loss 5.7408
+12400 val perplexity 311.3231
+12400 train 5.591258 (lr=7.6885e-06) (hash(x)=149526189)
+12500 val loss 5.7348
+12500 val perplexity 309.4465
+12500 train 5.577487 (lr=7.5101e-06) (hash(x)=142060916)
+12600 val loss 5.7295
+12600 val perplexity 307.8142
+12600 train 5.919009 (lr=7.3342e-06) (hash(x)=168650848)
+12700 val loss 5.7286
+12700 val perplexity 307.5334
+12700 train 5.749731 (lr=7.1609e-06) (hash(x)=154353018)
+12800 val loss 5.7300
+12800 val perplexity 307.9663
+12800 train 5.888818 (lr=6.9903e-06) (hash(x)=156115265)
+12900 val loss 5.7250
+12900 val perplexity 306.4193
+12900 train 5.652087 (lr=6.8223e-06) (hash(x)=143378175)
+13000 val loss 5.7236
+13000 val perplexity 306.0145
+13000 train 5.839179 (lr=6.6570e-06) (hash(x)=166124864)
+13100 val loss 5.7219
+13100 val perplexity 305.4771
+13100 train 5.596782 (lr=6.4945e-06) (hash(x)=146615385)
+13200 val loss 5.7215
+13200 val perplexity 305.3489
+13200 train 5.621416 (lr=6.3349e-06) (hash(x)=149236047)
+13300 val loss 5.7149
+13300 val perplexity 303.3439
+13300 train 5.648214 (lr=6.1782e-06) (hash(x)=151346972)
+13400 val loss 5.7142
+13400 val perplexity 303.1342
+13400 train 5.801619 (lr=6.0244e-06) (hash(x)=159167093)
+13500 val loss 5.7126
+13500 val perplexity 302.6487
+13500 train 5.428577 (lr=5.8737e-06) (hash(x)=135054650)
+13600 val loss 5.7108
+13600 val perplexity 302.1104
+13600 train 5.596169 (lr=5.7259e-06) (hash(x)=151033927)
+13700 val loss 5.7104
+13700 val perplexity 301.9891
+13700 train 5.409321 (lr=5.5813e-06) (hash(x)=141119172)
+13800 val loss 5.7096
+13800 val perplexity 301.7523
+13800 train 5.430826 (lr=5.4398e-06) (hash(x)=144197766)
+13900 val loss 5.7053
+13900 val perplexity 300.4471
+13900 train 5.645763 (lr=5.3016e-06) (hash(x)=145148158)
+14000 val loss 5.7019
+14000 val perplexity 299.4242
+14000 train 6.197217 (lr=5.1665e-06) (hash(x)=179068751)
+14100 val loss 5.7002
+14100 val perplexity 298.9149
+14100 train 5.569189 (lr=5.0348e-06) (hash(x)=144988373)
+14200 val loss 5.6993
+14200 val perplexity 298.6475
+14200 train 5.623942 (lr=4.9063e-06) (hash(x)=133379696)
+14300 val loss 5.6992
+14300 val perplexity 298.6255
+14300 train 5.580496 (lr=4.7813e-06) (hash(x)=149883422)
+14400 val loss 5.6996
+14400 val perplexity 298.7478
+14400 train 5.567074 (lr=4.6596e-06) (hash(x)=145709198)
+14500 val loss 5.6944
+14500 val perplexity 297.2122
+14500 train 5.709787 (lr=4.5414e-06) (hash(x)=148016579)
+14600 val loss 5.6922
+14600 val perplexity 296.5331
+14600 train 5.658023 (lr=4.4267e-06) (hash(x)=150218222)
+14700 val loss 5.6919
+14700 val perplexity 296.4617
+14700 train 5.611262 (lr=4.3155e-06) (hash(x)=142258737)
+14800 val loss 5.6890
+14800 val perplexity 295.5974
+14800 train 5.584694 (lr=4.2079e-06) (hash(x)=147628003)
+14900 val loss 5.6895
+14900 val perplexity 295.7564
+14900 train 5.539468 (lr=4.1039e-06) (hash(x)=149144036)
+15000 val loss 5.6894
+15000 val perplexity 295.7050
+15000 train 5.623379 (lr=4.0035e-06) (hash(x)=145925409)
+15100 val loss 5.6857
+15100 val perplexity 294.6349
+15100 train 5.563715 (lr=3.9068e-06) (hash(x)=144555751)
+15200 val loss 5.6844
+15200 val perplexity 294.2293
+15200 train 5.501987 (lr=3.8138e-06) (hash(x)=143448852)
+15300 val loss 5.6836
+15300 val perplexity 294.0114
+15300 train 5.680522 (lr=3.7245e-06) (hash(x)=156347253)
+15400 val loss 5.6814
+15400 val perplexity 293.3521
+15400 train 5.635643 (lr=3.6390e-06) (hash(x)=153010179)
+15500 val loss 5.6814
+15500 val perplexity 293.3618
+15500 train 5.463892 (lr=3.5572e-06) (hash(x)=144066445)
+15600 val loss 5.6817
+15600 val perplexity 293.4398
+15600 train 5.423588 (lr=3.4793e-06) (hash(x)=146107722)
+15700 val loss 5.6776
+15700 val perplexity 292.2488
+15700 train 5.816041 (lr=3.4052e-06) (hash(x)=152533799)
+15800 val loss 5.6749
+15800 val perplexity 291.4473
+15800 train 5.726212 (lr=3.3350e-06) (hash(x)=151756366)
+15900 val loss 5.6743
+15900 val perplexity 291.2946
+15900 train 5.659422 (lr=3.2687e-06) (hash(x)=151735039)
+16000 val loss 5.6746
+16000 val perplexity 291.3792
+16000 train 5.703759 (lr=3.2063e-06) (hash(x)=154633226)
+16100 val loss 5.6751
+16100 val perplexity 291.5109
+16100 train 5.528335 (lr=3.1478e-06) (hash(x)=158094333)
+16200 val loss 5.6740
+16200 val perplexity 291.1947
+16200 train 5.437234 (lr=3.0932e-06) (hash(x)=142772927)
+16300 val loss 5.6733
+16300 val perplexity 290.9790
+16300 train 5.568473 (lr=3.0427e-06) (hash(x)=148042896)
+16400 val loss 5.6704
+16400 val perplexity 290.1549
+16400 train 6.154361 (lr=2.9961e-06) (hash(x)=170306399)
+16500 val loss 5.6695
+16500 val perplexity 289.8988
+16500 train 5.688489 (lr=2.9535e-06) (hash(x)=141049320)
+16600 val loss 5.6693
+16600 val perplexity 289.8406
+16600 train 5.527473 (lr=2.9149e-06) (hash(x)=150512136)
+16700 val loss 5.6678
+16700 val perplexity 289.3889
+16700 train 5.678548 (lr=2.8804e-06) (hash(x)=151843645)
+16800 val loss 5.6655
+16800 val perplexity 288.7390
+16800 train 5.724266 (lr=2.8498e-06) (hash(x)=140073659)
+16900 val loss 5.6635
+16900 val perplexity 288.1412
+16900 train 5.852492 (lr=2.8234e-06) (hash(x)=171157212)
+17000 val loss 5.6630
+17000 val perplexity 288.0170
+17000 train 5.594303 (lr=2.8010e-06) (hash(x)=144553276)
+17100 val loss 5.6631
+17100 val perplexity 288.0265
+17100 train 5.660722 (lr=2.7826e-06) (hash(x)=157717565)
+17200 val loss 5.6633
+17200 val perplexity 288.0857
+17200 train 5.528824 (lr=2.7684e-06) (hash(x)=149843860)
+17300 val loss 5.6609
+17300 val perplexity 287.3978
+17300 train 5.721745 (lr=2.7582e-06) (hash(x)=162171979)
+17400 val loss 5.6599
+17400 val perplexity 287.1299
+17400 train 5.633310 (lr=2.7520e-06) (hash(x)=148400702)
+17499 val loss 5.6588
+17499 val perplexity 286.7997

attention_kindselective_n_heads4_seed1338/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:369ca0e325a307ae0fe4b31859c5e3039654368ff28c0b4fed2d96d8e4d73b93
+oid sha256:e0c36683ee57463cfd86e7922a106b4b11abb68ae6f544a48c5187333df99ac6
 size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b139874be23990c8467336f94877c955b85aec61bb9887d4c0ff395e2fd58926
+oid sha256:749581d48e62690bb18401ec32aa78d78b9cf7705a7d783397a5bbdd00e78b06
 size 179406214