Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1344/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1344", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1344", "n_embd": 256}

attention_kindselective_n_heads4_seed1344/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2848
 0 val perplexity 79602.4766
-0 train 11.283560 (lr=1.6250e-07) (hash(x)=145830960)
-100 val loss 9.9599
-100 val perplexity 21160.5156
-100 train 9.980744 (lr=1.6412e-05) (hash(x)=156335112)
-200 val loss 9.0912
-200 val perplexity 8876.4326
-200 train 9.151538 (lr=3.2500e-05) (hash(x)=144157453)
-300 val loss 8.1860
-300 val perplexity 3590.2598
-300 train 8.190341 (lr=3.2498e-05) (hash(x)=149245414)
-400 val loss 7.8564
-400 val perplexity 2582.1475
-400 train 7.900827 (lr=3.2490e-05) (hash(x)=146764602)
-500 val loss 7.6707
-500 val perplexity 2144.5127
-500 train 7.630148 (lr=3.2478e-05) (hash(x)=152540261)
-600 val loss 7.5701
-600 val perplexity 1939.3589
-600 train 7.349800 (lr=3.2461e-05) (hash(x)=140559124)
-700 val loss 7.4952
-700 val perplexity 1799.3420
-700 train 7.542197 (lr=3.2440e-05) (hash(x)=156003556)
-800 val loss 7.4500
-800 val perplexity 1719.9104
-800 train 7.963747 (lr=3.2413e-05) (hash(x)=166024176)
-900 val loss 7.4207
-900 val perplexity 1670.2000
-900 train 7.463220 (lr=3.2382e-05) (hash(x)=156196976)
-1000 val loss 7.3957
-1000 val perplexity 1628.8840
-1000 train 7.477394 (lr=3.2346e-05) (hash(x)=161040668)
-1100 val loss 7.3753
-1100 val perplexity 1596.0294
-1100 train 7.463724 (lr=3.2305e-05) (hash(x)=175899671)
-1200 val loss 7.3571
-1200 val perplexity 1567.2632
-1200 train 7.373929 (lr=3.2260e-05) (hash(x)=148878990)
-1300 val loss 7.3335
-1300 val perplexity 1530.7035
-1300 train 7.324839 (lr=3.2209e-05) (hash(x)=147253797)
-1400 val loss 7.3084
-1400 val perplexity 1492.7358
-1400 train 7.329314 (lr=3.2154e-05) (hash(x)=150420695)
-1500 val loss 7.2696
-1500 val perplexity 1435.9690
-1500 train 7.223187 (lr=3.2094e-05) (hash(x)=146933283)
-1600 val loss 7.2424
-1600 val perplexity 1397.3754
-1600 train 7.211983 (lr=3.2030e-05) (hash(x)=151995229)
-1700 val loss 7.2171
-1700 val perplexity 1362.4900
-1700 train 7.459997 (lr=3.1961e-05) (hash(x)=154468626)
-1800 val loss 7.2010
-1800 val perplexity 1340.7111
-1800 train 6.917479 (lr=3.1887e-05) (hash(x)=139853932)
-1900 val loss 7.1688
-1900 val perplexity 1298.2341
-1900 train 6.966431 (lr=3.1809e-05) (hash(x)=142492588)
-2000 val loss 7.1404
-2000 val perplexity 1261.8771
-2000 train 7.475838 (lr=3.1726e-05) (hash(x)=174104207)
-2100 val loss 7.1090
-2100 val perplexity 1222.9773
-2100 train 7.088420 (lr=3.1638e-05) (hash(x)=151046997)
-2200 val loss 7.0901
-2200 val perplexity 1200.0785
-2200 train 6.951235 (lr=3.1546e-05) (hash(x)=146275038)
-2300 val loss 7.0700
-2300 val perplexity 1176.1006
-2300 train 6.987461 (lr=3.1449e-05) (hash(x)=147779512)
-2400 val loss 7.0425
-2400 val perplexity 1144.2594
-2400 train 6.903728 (lr=3.1348e-05) (hash(x)=141403655)
-2500 val loss 7.0185
-2500 val perplexity 1117.1313
-2500 train 6.996836 (lr=3.1243e-05) (hash(x)=160143436)
-2600 val loss 6.9944
-2600 val perplexity 1090.4697
-2600 train 7.262850 (lr=3.1133e-05) (hash(x)=173675040)
-2700 val loss 6.9657
-2700 val perplexity 1059.6196
-2700 train 6.981391 (lr=3.1019e-05) (hash(x)=150124772)
-2800 val loss 6.9322
-2800 val perplexity 1024.7571
-2800 train 7.053523 (lr=3.0900e-05) (hash(x)=162475906)
-2900 val loss 6.9104
-2900 val perplexity 1002.6725
-2900 train 6.838933 (lr=3.0777e-05) (hash(x)=152789231)
-3000 val loss 6.8856
-3000 val perplexity 978.1113
-3000 train 6.811411 (lr=3.0650e-05) (hash(x)=152785612)
-3100 val loss 6.8575
-3100 val perplexity 950.9545
-3100 train 6.902193 (lr=3.0518e-05) (hash(x)=156140235)
-3200 val loss 6.8196
-3200 val perplexity 915.5900
-3200 train 6.835784 (lr=3.0383e-05) (hash(x)=151987383)
-3300 val loss 6.7851
-3300 val perplexity 884.5807
-3300 train 7.016383 (lr=3.0243e-05) (hash(x)=176259299)
-3400 val loss 6.7537
-3400 val perplexity 857.2028
-3400 train 6.583331 (lr=3.0099e-05) (hash(x)=144540187)
-3500 val loss 6.7176
-3500 val perplexity 826.8558
-3500 train 6.904707 (lr=2.9952e-05) (hash(x)=156773596)
-3600 val loss 6.6901
-3600 val perplexity 804.3963
-3600 train 6.913805 (lr=2.9800e-05) (hash(x)=160934110)
-3700 val loss 6.6640
-3700 val perplexity 783.6652
-3700 train 6.768859 (lr=2.9644e-05) (hash(x)=156100357)
-3800 val loss 6.6411
-3800 val perplexity 765.9611
-3800 train 6.768399 (lr=2.9485e-05) (hash(x)=158048867)
-3900 val loss 6.6292
-3900 val perplexity 756.8519
-3900 train 6.455472 (lr=2.9321e-05) (hash(x)=148195960)
-4000 val loss 6.5926
-4000 val perplexity 729.6400
-4000 train 6.582702 (lr=2.9154e-05) (hash(x)=151976963)
-4100 val loss 6.5624
-4100 val perplexity 707.9741
-4100 train 6.539586 (lr=2.8983e-05) (hash(x)=157598041)
-4200 val loss 6.5374
-4200 val perplexity 690.4905
-4200 train 6.390263 (lr=2.8808e-05) (hash(x)=139121993)
-4300 val loss 6.5188
-4300 val perplexity 677.7638
-4300 train 6.616024 (lr=2.8630e-05) (hash(x)=154546719)
-4400 val loss 6.4994
-4400 val perplexity 664.7399
-4400 train 6.433190 (lr=2.8448e-05) (hash(x)=148310119)
-4500 val loss 6.4796
-4500 val perplexity 651.6804
-4500 train 6.363911 (lr=2.8263e-05) (hash(x)=147070971)
-4600 val loss 6.4720
-4600 val perplexity 646.7939
-4600 train 7.847555 (lr=2.8075e-05) (hash(x)=232298831)
-4700 val loss 6.4593
-4700 val perplexity 638.5972
-4700 train 6.411119 (lr=2.7883e-05) (hash(x)=149549680)
-4800 val loss 6.4289
-4800 val perplexity 619.5186
-4800 train 6.571551 (lr=2.7687e-05) (hash(x)=155703970)
-4900 val loss 6.4157
-4900 val perplexity 611.3965
-4900 train 6.413851 (lr=2.7489e-05) (hash(x)=158725785)
-5000 val loss 6.3958
-5000 val perplexity 599.3246
-5000 train 6.331246 (lr=2.7287e-05) (hash(x)=146983088)
-5100 val loss 6.3799
-5100 val perplexity 589.8960
-5100 train 6.302820 (lr=2.7082e-05) (hash(x)=144882522)
-5200 val loss 6.3708
-5200 val perplexity 584.5114
-5200 train 6.247755 (lr=2.6874e-05) (hash(x)=145880132)
-5300 val loss 6.3553
-5300 val perplexity 575.5067
-5300 train 6.157270 (lr=2.6664e-05) (hash(x)=140482543)
-5400 val loss 6.3421
-5400 val perplexity 567.9766
-5400 train 6.185915 (lr=2.6450e-05) (hash(x)=148902645)
-5500 val loss 6.3351
-5500 val perplexity 564.0076
-5500 train 6.093838 (lr=2.6233e-05) (hash(x)=149411471)
-5600 val loss 6.3083
-5600 val perplexity 549.0921
-5600 train 6.244935 (lr=2.6014e-05) (hash(x)=148673383)
-5700 val loss 6.2897
-5700 val perplexity 538.9991
-5700 train 6.356031 (lr=2.5792e-05) (hash(x)=157502794)
-5800 val loss 6.2787
-5800 val perplexity 533.0735
-5800 train 6.129450 (lr=2.5567e-05) (hash(x)=148057218)
-5900 val loss 6.2668
-5900 val perplexity 526.7706
-5900 train 6.101112 (lr=2.5340e-05) (hash(x)=141239696)
-6000 val loss 6.2553
-6000 val perplexity 520.7495
-6000 train 6.325564 (lr=2.5111e-05) (hash(x)=152338728)
-6100 val loss 6.2476
-6100 val perplexity 516.7849
-6100 train 5.917364 (lr=2.4879e-05) (hash(x)=143749713)
-6200 val loss 6.2363
-6200 val perplexity 510.9547
-6200 train 6.289093 (lr=2.4644e-05) (hash(x)=152128518)
-6300 val loss 6.2393
-6300 val perplexity 512.4878
-6300 train 6.130268 (lr=2.4408e-05) (hash(x)=152992195)
-6400 val loss 6.2214
-6400 val perplexity 503.4134
-6400 train 6.114389 (lr=2.4169e-05) (hash(x)=143799591)
-6500 val loss 6.1948
-6500 val perplexity 490.1777
-6500 train 6.273989 (lr=2.3928e-05) (hash(x)=144266888)
-6600 val loss 6.1813
-6600 val perplexity 483.6074
-6600 train 6.451832 (lr=2.3686e-05) (hash(x)=167374028)
-6700 val loss 6.1793
-6700 val perplexity 482.6491
-6700 train 6.151701 (lr=2.3441e-05) (hash(x)=151301787)
-6800 val loss 6.1644
-6800 val perplexity 475.5119
-6800 train 5.995218 (lr=2.3195e-05) (hash(x)=143204898)
-6900 val loss 6.1564
-6900 val perplexity 471.7352
-6900 train 6.231202 (lr=2.2946e-05) (hash(x)=148058660)
-7000 val loss 6.1538
-7000 val perplexity 470.5068
-7000 train 6.126843 (lr=2.2696e-05) (hash(x)=157509461)
-7100 val loss 6.1411
-7100 val perplexity 464.5543
-7100 train 6.477844 (lr=2.2445e-05) (hash(x)=171579342)
-7200 val loss 6.1258
-7200 val perplexity 457.5132
-7200 train 6.116988 (lr=2.2192e-05) (hash(x)=158180787)
-7300 val loss 6.1153
-7300 val perplexity 452.7334
-7300 train 5.979165 (lr=2.1937e-05) (hash(x)=150122338)
-7400 val loss 6.1110
-7400 val perplexity 450.7910
-7400 train 6.041814 (lr=2.1682e-05) (hash(x)=147561852)
-7500 val loss 6.1040
-7500 val perplexity 447.6480
-7500 train 6.087430 (lr=2.1425e-05) (hash(x)=149353924)
-7600 val loss 6.0976
-7600 val perplexity 444.7703
-7600 train 6.114073 (lr=2.1166e-05) (hash(x)=144545768)
-7700 val loss 6.0831
-7700 val perplexity 438.3668
-7700 train 6.091139 (lr=2.0907e-05) (hash(x)=142691770)
-7800 val loss 6.0767
-7800 val perplexity 435.5718
-7800 train 6.184749 (lr=2.0647e-05) (hash(x)=158378295)
-7900 val loss 6.0706
-7900 val perplexity 432.9561
-7900 train 5.934933 (lr=2.0386e-05) (hash(x)=150691918)
-8000 val loss 6.0554
-8000 val perplexity 426.4263
-8000 train 5.936950 (lr=2.0123e-05) (hash(x)=147837896)
-8100 val loss 6.0555
-8100 val perplexity 426.4411
-8100 train 6.014183 (lr=1.9861e-05) (hash(x)=144751226)
-8200 val loss 6.0455
-8200 val perplexity 422.1995
-8200 train 5.816062 (lr=1.9597e-05) (hash(x)=139168726)
-8300 val loss 6.0436
-8300 val perplexity 421.4195
-8300 train 5.974075 (lr=1.9333e-05) (hash(x)=154279434)
-8400 val loss 6.0303
-8400 val perplexity 415.8287
-8400 train 5.884507 (lr=1.9069e-05) (hash(x)=140111678)
-8500 val loss 6.0229
-8500 val perplexity 412.7881
-8500 train 6.001132 (lr=1.8804e-05) (hash(x)=152124088)
-8600 val loss 6.0159
-8600 val perplexity 409.8954
-8600 train 5.797539 (lr=1.8539e-05) (hash(x)=142869347)
-8700 val loss 6.0099
-8700 val perplexity 407.4459
-8700 train 5.903512 (lr=1.8273e-05) (hash(x)=145697606)
-8800 val loss 6.0079
-8800 val perplexity 406.6162
-8800 train 5.987753 (lr=1.8008e-05) (hash(x)=157314231)
-8900 val loss 5.9946
-8900 val perplexity 401.2674
-8900 train 5.984644 (lr=1.7742e-05) (hash(x)=149357509)
-9000 val loss 5.9892
-9000 val perplexity 399.1118
-9000 train 5.898522 (lr=1.7477e-05) (hash(x)=140434750)
-9100 val loss 5.9785
-9100 val perplexity 394.8529
-9100 train 5.972139 (lr=1.7211e-05) (hash(x)=144954543)
-9200 val loss 5.9733
-9200 val perplexity 392.7979
-9200 train 5.902909 (lr=1.6946e-05) (hash(x)=156329199)
-9300 val loss 5.9674
-9300 val perplexity 390.4979
-9300 train 5.757013 (lr=1.6681e-05) (hash(x)=137925654)
-9400 val loss 5.9667
-9400 val perplexity 390.2034
-9400 train 5.850411 (lr=1.6417e-05) (hash(x)=146385919)
-9500 val loss 5.9551
-9500 val perplexity 385.7179
-9500 train 5.880592 (lr=1.6153e-05) (hash(x)=144400019)
-9600 val loss 5.9563
-9600 val perplexity 386.1725
-9600 train 5.890789 (lr=1.5889e-05) (hash(x)=130562204)
-9700 val loss 5.9439
-9700 val perplexity 381.4185
-9700 train 5.884833 (lr=1.5627e-05) (hash(x)=152732805)
-9800 val loss 5.9368
-9800 val perplexity 378.7395
-9800 train 6.004889 (lr=1.5364e-05) (hash(x)=151860383)
-9900 val loss 5.9330
-9900 val perplexity 377.3029
-9900 train 6.009057 (lr=1.5103e-05) (hash(x)=156732259)
-10000 val loss 5.9277
-10000 val perplexity 375.2834
-10000 train 5.854128 (lr=1.4843e-05) (hash(x)=139943350)
-10100 val loss 5.9212
-10100 val perplexity 372.8618
-10100 train 5.743155 (lr=1.4584e-05) (hash(x)=145593924)
-10200 val loss 5.9188
-10200 val perplexity 371.9498
-10200 train 5.780659 (lr=1.4325e-05) (hash(x)=153996667)
-10300 val loss 5.9219
-10300 val perplexity 373.1076
-10300 train 5.832670 (lr=1.4068e-05) (hash(x)=154200987)
-10400 val loss 5.9178
-10400 val perplexity 371.5873
-10400 train 5.739534 (lr=1.3813e-05) (hash(x)=156872686)
-10500 val loss 5.9038
-10500 val perplexity 366.4414
-10500 train 5.989748 (lr=1.3558e-05) (hash(x)=155008723)
-10600 val loss 5.8966
-10600 val perplexity 363.7883
-10600 train 5.831800 (lr=1.3305e-05) (hash(x)=152715441)
-10700 val loss 5.8915
-10700 val perplexity 361.9330
-10700 train 6.048138 (lr=1.3054e-05) (hash(x)=166668700)
-10800 val loss 5.8887
-10800 val perplexity 360.9201
-10800 train 5.889436 (lr=1.2804e-05) (hash(x)=150698847)
-10900 val loss 5.8845
-10900 val perplexity 359.4253
-10900 train 5.701465 (lr=1.2555e-05) (hash(x)=137094574)
-11000 val loss 5.8794
-11000 val perplexity 357.5838
-11000 train 5.731179 (lr=1.2309e-05) (hash(x)=144225378)
-11100 val loss 5.8834
-11100 val perplexity 359.0289
-11100 train 5.797372 (lr=1.2064e-05) (hash(x)=151666772)
-11200 val loss 5.8800
-11200 val perplexity 357.8239
-11200 train 5.896419 (lr=1.1822e-05) (hash(x)=157843445)
-11300 val loss 5.8663
-11300 val perplexity 352.9310
-11300 train 5.771248 (lr=1.1581e-05) (hash(x)=140305735)
-11400 val loss 5.8649
-11400 val perplexity 352.4369
-11400 train 5.781058 (lr=1.1342e-05) (hash(x)=143435929)
-11500 val loss 5.8583
-11500 val perplexity 350.1213
-11500 train 5.890773 (lr=1.1106e-05) (hash(x)=150858606)
-11600 val loss 5.8555
-11600 val perplexity 349.1357
-11600 train 5.671831 (lr=1.0871e-05) (hash(x)=141643887)
-11700 val loss 5.8525
-11700 val perplexity 348.1206
-11700 train 5.714521 (lr=1.0639e-05) (hash(x)=151097313)
-11800 val loss 5.8506
-11800 val perplexity 347.4531
-11800 train 5.658457 (lr=1.0410e-05) (hash(x)=150639616)
-11900 val loss 5.8537
-11900 val perplexity 348.5312
-11900 train 5.559586 (lr=1.0183e-05) (hash(x)=154557688)
-12000 val loss 5.8478
-12000 val perplexity 346.4668
-12000 train 5.983258 (lr=9.9580e-06) (hash(x)=159324159)
-12100 val loss 5.8373
-12100 val perplexity 342.8491
-12100 train 5.883004 (lr=9.7360e-06) (hash(x)=151185357)
-12200 val loss 5.8367
-12200 val perplexity 342.6588
-12200 train 5.716962 (lr=9.5167e-06) (hash(x)=137457899)
-12300 val loss 5.8313
-12300 val perplexity 340.8114
-12300 train 5.890007 (lr=9.3001e-06) (hash(x)=146370056)
-12400 val loss 5.8278
-12400 val perplexity 339.5993
-12400 train 5.673313 (lr=9.0864e-06) (hash(x)=148352898)
-12500 val loss 5.8252
-12500 val perplexity 338.7279
-12500 train 5.817701 (lr=8.8756e-06) (hash(x)=161514681)
-12600 val loss 5.8246
-12600 val perplexity 338.5426
-12600 train 5.698085 (lr=8.6677e-06) (hash(x)=146207280)
-12700 val loss 5.8231
-12700 val perplexity 338.0049
-12700 train 5.654124 (lr=8.4629e-06) (hash(x)=142615619)
-12800 val loss 5.8199
-12800 val perplexity 336.9413
-12800 train 5.843594 (lr=8.2612e-06) (hash(x)=144604802)
-12900 val loss 5.8149
-12900 val perplexity 335.2452
-12900 train 5.804498 (lr=8.0627e-06) (hash(x)=143212941)
-13000 val loss 5.8070
-13000 val perplexity 332.6330
-13000 train 5.882431 (lr=7.8674e-06) (hash(x)=155987866)
-13100 val loss 5.8062
-13100 val perplexity 332.3383
-13100 train 6.088308 (lr=7.6754e-06) (hash(x)=155920713)
-13200 val loss 5.8028
-13200 val perplexity 331.2296
-13200 train 5.985959 (lr=7.4867e-06) (hash(x)=152145878)
-13300 val loss 5.8005
-13300 val perplexity 330.4608
-13300 train 5.606394 (lr=7.3015e-06) (hash(x)=134141709)
-13400 val loss 5.8019
-13400 val perplexity 330.9319
-13400 train 5.717030 (lr=7.1198e-06) (hash(x)=146096299)
-13500 val loss 5.8000
-13500 val perplexity 330.3149
-13500 train 5.557479 (lr=6.9416e-06) (hash(x)=144011431)
-13600 val loss 5.7930
-13600 val perplexity 328.0098
-13600 train 6.033184 (lr=6.7670e-06) (hash(x)=163950932)
-13700 val loss 5.7892
-13700 val perplexity 326.7588
-13700 train 5.933305 (lr=6.5961e-06) (hash(x)=156652406)
-13800 val loss 5.7860
-13800 val perplexity 325.7170
-13800 train 5.694222 (lr=6.4289e-06) (hash(x)=145753166)
-13900 val loss 5.7844
-13900 val perplexity 325.1796
-13900 train 5.654222 (lr=6.2655e-06) (hash(x)=147065551)
-14000 val loss 5.7817
-14000 val perplexity 324.3047
-14000 train 6.244597 (lr=6.1059e-06) (hash(x)=179077542)
-14100 val loss 5.7802
-14100 val perplexity 323.8351
-14100 train 5.867098 (lr=5.9502e-06) (hash(x)=171962802)
-14200 val loss 5.7802
-14200 val perplexity 323.8290
-14200 train 5.847116 (lr=5.7984e-06) (hash(x)=153299913)
-14300 val loss 5.7801
-14300 val perplexity 323.8044
-14300 train 5.706236 (lr=5.6506e-06) (hash(x)=154750565)
-14400 val loss 5.7769
-14400 val perplexity 322.7437
-14400 train 6.051046 (lr=5.5068e-06) (hash(x)=154536694)
-14500 val loss 5.7720
-14500 val perplexity 321.1904
-14500 train 5.821687 (lr=5.3671e-06) (hash(x)=151236383)
-14600 val loss 5.7703
-14600 val perplexity 320.6415
-14600 train 5.790136 (lr=5.2316e-06) (hash(x)=152584272)
-14700 val loss 5.7687
-14700 val perplexity 320.1265
-14700 train 5.789655 (lr=5.1002e-06) (hash(x)=175301826)
-14800 val loss 5.7669
-14800 val perplexity 319.5508
-14800 train 5.700191 (lr=4.9730e-06) (hash(x)=148729408)
-14900 val loss 5.7655
-14900 val perplexity 319.1001
-14900 train 5.943477 (lr=4.8501e-06) (hash(x)=150978283)
-15000 val loss 5.7656
-15000 val perplexity 319.1203
-15000 train 5.704052 (lr=4.7314e-06) (hash(x)=153621389)
-15100 val loss 5.7652
-15100 val perplexity 319.0132
-15100 train 5.685680 (lr=4.6171e-06) (hash(x)=150653793)
-15200 val loss 5.7633
-15200 val perplexity 318.4034
-15200 train 5.915984 (lr=4.5072e-06) (hash(x)=140802934)
-15300 val loss 5.7578
-15300 val perplexity 316.6652
-15300 train 5.736080 (lr=4.4017e-06) (hash(x)=152007556)
-15400 val loss 5.7567
-15400 val perplexity 316.2986
-15400 train 5.709007 (lr=4.3006e-06) (hash(x)=149057446)
-15500 val loss 5.7558
-15500 val perplexity 316.0076
-15500 train 5.668916 (lr=4.2040e-06) (hash(x)=147521774)
-15600 val loss 5.7535
-15600 val perplexity 315.2928
-15600 train 5.722573 (lr=4.1119e-06) (hash(x)=148944405)
-15700 val loss 5.7537
-15700 val perplexity 315.3688
-15700 train 5.707011 (lr=4.0244e-06) (hash(x)=148472532)
-15800 val loss 5.7524
-15800 val perplexity 314.9471
-15800 train 5.531003 (lr=3.9414e-06) (hash(x)=149048780)
-15900 val loss 5.7525
-15900 val perplexity 314.9665
-15900 train 5.719010 (lr=3.8630e-06) (hash(x)=150175778)
-16000 val loss 5.7528
-16000 val perplexity 315.0829
-16000 train 5.550600 (lr=3.7892e-06) (hash(x)=145943831)
-16100 val loss 5.7465
-16100 val perplexity 313.0834
-16100 train 5.913585 (lr=3.7201e-06) (hash(x)=134314842)
-16200 val loss 5.7448
-16200 val perplexity 312.5483
-16200 train 5.767522 (lr=3.6556e-06) (hash(x)=153755135)
-16300 val loss 5.7441
-16300 val perplexity 312.3481
-16300 train 5.476809 (lr=3.5959e-06) (hash(x)=140070802)
-16400 val loss 5.7436
-16400 val perplexity 312.1762
-16400 train 5.759050 (lr=3.5408e-06) (hash(x)=151132329)
-16500 val loss 5.7412
-16500 val perplexity 311.4370
-16500 train 5.777473 (lr=3.4905e-06) (hash(x)=151585550)
-16600 val loss 5.7391
-16600 val perplexity 310.7890
-16600 train 5.895504 (lr=3.4449e-06) (hash(x)=164389325)
-16700 val loss 5.7378
-16700 val perplexity 310.3712
-16700 train 5.643941 (lr=3.4041e-06) (hash(x)=146016298)
-16800 val loss 5.7378
-16800 val perplexity 310.3949
-16800 train 5.648126 (lr=3.3680e-06) (hash(x)=147668192)
-16900 val loss 5.7370
-16900 val perplexity 310.1412
-16900 train 5.715910 (lr=3.3367e-06) (hash(x)=155692530)
-17000 val loss 5.7375
-17000 val perplexity 310.2915
-17000 train 5.663022 (lr=3.3102e-06) (hash(x)=156598351)
-17100 val loss 5.7380
-17100 val perplexity 310.4377
-17100 train 5.648726 (lr=3.2886e-06) (hash(x)=157344236)
-17200 val loss 5.7349
-17200 val perplexity 309.4906
-17200 train 5.538077 (lr=3.2717e-06) (hash(x)=126288656)
-17300 val loss 5.7323
-17300 val perplexity 308.6758
-17300 train 5.773144 (lr=3.2596e-06) (hash(x)=146365959)
-17400 val loss 5.7304
-17400 val perplexity 308.0987
-17400 train 5.714909 (lr=3.2524e-06) (hash(x)=152729388)
-17499 val loss 5.7284
-17499 val perplexity 307.4802
+0 train 11.283560 (lr=1.3750e-07) (hash(x)=145830960)
+100 val loss 9.9832
+100 val perplexity 21659.9883
+100 train 10.001309 (lr=1.3888e-05) (hash(x)=156335112)
+200 val loss 9.2596
+200 val perplexity 10504.6865
+200 train 9.321742 (lr=2.7500e-05) (hash(x)=144157453)
+300 val loss 8.2356
+300 val perplexity 3772.9011
+300 train 8.243886 (lr=2.7498e-05) (hash(x)=149245414)
+400 val loss 7.7924
+400 val perplexity 2422.0842
+400 train 7.838326 (lr=2.7492e-05) (hash(x)=146764602)
+500 val loss 7.6107
+500 val perplexity 2019.6174
+500 train 7.567578 (lr=2.7482e-05) (hash(x)=152540261)
+600 val loss 7.5276
+600 val perplexity 1858.6896
+600 train 7.308209 (lr=2.7467e-05) (hash(x)=140559124)
+700 val loss 7.4688
+700 val perplexity 1752.4941
+700 train 7.520562 (lr=2.7449e-05) (hash(x)=156003556)
+800 val loss 7.4240
+800 val perplexity 1675.7202
+800 train 7.942647 (lr=2.7427e-05) (hash(x)=166024176)
+900 val loss 7.3916
+900 val perplexity 1622.2867
+900 train 7.435807 (lr=2.7400e-05) (hash(x)=156196976)
+1000 val loss 7.3601
+1000 val perplexity 1571.9177
+1000 train 7.434725 (lr=2.7370e-05) (hash(x)=161040668)
+1100 val loss 7.3324
+1100 val perplexity 1529.0920
+1100 train 7.411966 (lr=2.7335e-05) (hash(x)=175899671)
+1200 val loss 7.3054
+1200 val perplexity 1488.2810
+1200 train 7.321064 (lr=2.7297e-05) (hash(x)=148878990)
+1300 val loss 7.2732
+1300 val perplexity 1441.1879
+1300 train 7.264565 (lr=2.7254e-05) (hash(x)=147253797)
+1400 val loss 7.2508
+1400 val perplexity 1409.1924
+1400 train 7.270267 (lr=2.7207e-05) (hash(x)=150420695)
+1500 val loss 7.2080
+1500 val perplexity 1350.1888
+1500 train 7.153911 (lr=2.7157e-05) (hash(x)=146933283)
+1600 val loss 7.1696
+1600 val perplexity 1299.3105
+1600 train 7.137910 (lr=2.7102e-05) (hash(x)=151995229)
+1700 val loss 7.1428
+1700 val perplexity 1265.0243
+1700 train 7.378381 (lr=2.7044e-05) (hash(x)=154468626)
+1800 val loss 7.1122
+1800 val perplexity 1226.8884
+1800 train 6.814721 (lr=2.6981e-05) (hash(x)=139853932)
+1900 val loss 7.0736
+1900 val perplexity 1180.3677
+1900 train 6.873425 (lr=2.6915e-05) (hash(x)=142492588)
+2000 val loss 7.0344
+2000 val perplexity 1135.0413
+2000 train 7.389661 (lr=2.6845e-05) (hash(x)=174104207)
+2100 val loss 7.0015
+2100 val perplexity 1098.2297
+2100 train 6.985461 (lr=2.6771e-05) (hash(x)=151046997)
+2200 val loss 6.9589
+2200 val perplexity 1052.4565
+2200 train 6.823278 (lr=2.6693e-05) (hash(x)=146275038)
+2300 val loss 6.9287
+2300 val perplexity 1021.2055
+2300 train 6.849849 (lr=2.6611e-05) (hash(x)=147779512)
+2400 val loss 6.8994
+2400 val perplexity 991.6996
+2400 train 6.756907 (lr=2.6525e-05) (hash(x)=141403655)
+2500 val loss 6.8715
+2500 val perplexity 964.3752
+2500 train 6.857760 (lr=2.6436e-05) (hash(x)=160143436)
+2600 val loss 6.8308
+2600 val perplexity 925.9529
+2600 train 7.097726 (lr=2.6343e-05) (hash(x)=173675040)
+2700 val loss 6.7961
+2700 val perplexity 894.3893
+2700 train 6.822687 (lr=2.6246e-05) (hash(x)=150124772)
+2800 val loss 6.7634
+2800 val perplexity 865.5720
+2800 train 6.884556 (lr=2.6146e-05) (hash(x)=162475906)
+2900 val loss 6.7419
+2900 val perplexity 847.1724
+2900 train 6.668697 (lr=2.6042e-05) (hash(x)=152789231)
+3000 val loss 6.7228
+3000 val perplexity 831.1171
+3000 train 6.648977 (lr=2.5934e-05) (hash(x)=152785612)
+3100 val loss 6.6874
+3100 val perplexity 802.2723
+3100 train 6.718865 (lr=2.5823e-05) (hash(x)=156140235)
+3200 val loss 6.6601
+3200 val perplexity 780.6371
+3200 train 6.678284 (lr=2.5709e-05) (hash(x)=151987383)
+3300 val loss 6.6303
+3300 val perplexity 757.7012
+3300 train 6.861026 (lr=2.5590e-05) (hash(x)=176259299)
+3400 val loss 6.6014
+3400 val perplexity 736.1032
+3400 train 6.427187 (lr=2.5469e-05) (hash(x)=144540187)
+3500 val loss 6.5812
+3500 val perplexity 721.3984
+3500 train 6.781282 (lr=2.5344e-05) (hash(x)=156773596)
+3600 val loss 6.5557
+3600 val perplexity 703.2559
+3600 train 6.781649 (lr=2.5215e-05) (hash(x)=160934110)
+3700 val loss 6.5336
+3700 val perplexity 687.8378
+3700 train 6.647962 (lr=2.5083e-05) (hash(x)=156100357)
+3800 val loss 6.5107
+3800 val perplexity 672.3105
+3800 train 6.634124 (lr=2.4948e-05) (hash(x)=158048867)
+3900 val loss 6.5010
+3900 val perplexity 665.8036
+3900 train 6.329262 (lr=2.4810e-05) (hash(x)=148195960)
+4000 val loss 6.4667
+4000 val perplexity 643.3888
+4000 train 6.454521 (lr=2.4669e-05) (hash(x)=151976963)
+4100 val loss 6.4453
+4100 val perplexity 629.7110
+4100 train 6.413161 (lr=2.4524e-05) (hash(x)=157598041)
+4200 val loss 6.4264
+4200 val perplexity 617.9375
+4200 train 6.282009 (lr=2.4376e-05) (hash(x)=139121993)
+4300 val loss 6.4095
+4300 val perplexity 607.5652
+4300 train 6.509076 (lr=2.4226e-05) (hash(x)=154546719)
+4400 val loss 6.3961
+4400 val perplexity 599.5090
+4400 train 6.335598 (lr=2.4072e-05) (hash(x)=148310119)
+4500 val loss 6.3747
+4500 val perplexity 586.8069
+4500 train 6.261201 (lr=2.3915e-05) (hash(x)=147070971)
+4600 val loss 6.3689
+4600 val perplexity 583.3914
+4600 train 7.790611 (lr=2.3755e-05) (hash(x)=232298831)
+4700 val loss 6.3547
+4700 val perplexity 575.1835
+4700 train 6.308467 (lr=2.3593e-05) (hash(x)=149549680)
+4800 val loss 6.3372
+4800 val perplexity 565.2316
+4800 train 6.495251 (lr=2.3428e-05) (hash(x)=155703970)
+4900 val loss 6.3163
+4900 val perplexity 553.5396
+4900 train 6.312998 (lr=2.3260e-05) (hash(x)=158725785)
+5000 val loss 6.3036
+5000 val perplexity 546.5110
+5000 train 6.237771 (lr=2.3089e-05) (hash(x)=146983088)
+5100 val loss 6.2881
+5100 val perplexity 538.1198
+5100 train 6.219689 (lr=2.2916e-05) (hash(x)=144882522)
+5200 val loss 6.2803
+5200 val perplexity 533.9525
+5200 train 6.160968 (lr=2.2740e-05) (hash(x)=145880132)
+5300 val loss 6.2636
+5300 val perplexity 525.0846
+5300 train 6.071547 (lr=2.2561e-05) (hash(x)=140482543)
+5400 val loss 6.2555
+5400 val perplexity 520.8649
+5400 train 6.101057 (lr=2.2381e-05) (hash(x)=148902645)
+5500 val loss 6.2465
+5500 val perplexity 516.1917
+5500 train 6.002904 (lr=2.2197e-05) (hash(x)=149411471)
+5600 val loss 6.2280
+5600 val perplexity 506.7292
+5600 train 6.174450 (lr=2.2012e-05) (hash(x)=148673383)
+5700 val loss 6.2122
+5700 val perplexity 498.8032
+5700 train 6.277672 (lr=2.1824e-05) (hash(x)=157502794)
+5800 val loss 6.2017
+5800 val perplexity 493.5743
+5800 train 6.055883 (lr=2.1634e-05) (hash(x)=148057218)
+5900 val loss 6.1857
+5900 val perplexity 485.7591
+5900 train 6.024046 (lr=2.1442e-05) (hash(x)=141239696)
+6000 val loss 6.1754
+6000 val perplexity 480.7598
+6000 train 6.262658 (lr=2.1248e-05) (hash(x)=152338728)
+6100 val loss 6.1686
+6100 val perplexity 477.4951
+6100 train 5.851945 (lr=2.1051e-05) (hash(x)=143749713)
+6200 val loss 6.1634
+6200 val perplexity 475.0218
+6200 train 6.232953 (lr=2.0853e-05) (hash(x)=152128518)
+6300 val loss 6.1582
+6300 val perplexity 472.5694
+6300 train 6.046199 (lr=2.0653e-05) (hash(x)=152992195)
+6400 val loss 6.1448
+6400 val perplexity 466.2785
+6400 train 6.042677 (lr=2.0451e-05) (hash(x)=143799591)
+6500 val loss 6.1241
+6500 val perplexity 456.7129
+6500 train 6.219780 (lr=2.0247e-05) (hash(x)=144266888)
+6600 val loss 6.1138
+6600 val perplexity 452.0628
+6600 train 6.387853 (lr=2.0042e-05) (hash(x)=167374028)
+6700 val loss 6.1144
+6700 val perplexity 452.3122
+6700 train 6.088706 (lr=1.9835e-05) (hash(x)=151301787)
+6800 val loss 6.1000
+6800 val perplexity 445.8788
+6800 train 5.931244 (lr=1.9626e-05) (hash(x)=143204898)
+6900 val loss 6.0944
+6900 val perplexity 443.3757
+6900 train 6.175731 (lr=1.9416e-05) (hash(x)=148058660)
+7000 val loss 6.0901
+7000 val perplexity 441.4510
+7000 train 6.061460 (lr=1.9205e-05) (hash(x)=157509461)
+7100 val loss 6.0809
+7100 val perplexity 437.4422
+7100 train 6.422208 (lr=1.8992e-05) (hash(x)=171579342)
+7200 val loss 6.0689
+7200 val perplexity 432.2213
+7200 train 6.070595 (lr=1.8778e-05) (hash(x)=158180787)
+7300 val loss 6.0610
+7300 val perplexity 428.8191
+7300 train 5.927678 (lr=1.8562e-05) (hash(x)=150122338)
+7400 val loss 6.0578
+7400 val perplexity 427.4535
+7400 train 5.988640 (lr=1.8346e-05) (hash(x)=147561852)
+7500 val loss 6.0510
+7500 val perplexity 424.5388
+7500 train 6.033875 (lr=1.8128e-05) (hash(x)=149353924)
+7600 val loss 6.0436
+7600 val perplexity 421.4078
+7600 train 6.076441 (lr=1.7910e-05) (hash(x)=144545768)
+7700 val loss 6.0334
+7700 val perplexity 417.1183
+7700 train 6.038865 (lr=1.7691e-05) (hash(x)=142691770)
+7800 val loss 6.0264
+7800 val perplexity 414.2370
+7800 train 6.139307 (lr=1.7470e-05) (hash(x)=158378295)
+7900 val loss 6.0266
+7900 val perplexity 414.3207
+7900 train 5.887660 (lr=1.7249e-05) (hash(x)=150691918)
+8000 val loss 6.0117
+8000 val perplexity 408.1709
+8000 train 5.894296 (lr=1.7028e-05) (hash(x)=147837896)
+8100 val loss 6.0090
+8100 val perplexity 407.0768
+8100 train 5.975699 (lr=1.6805e-05) (hash(x)=144751226)
+8200 val loss 6.0030
+8200 val perplexity 404.6476
+8200 train 5.773884 (lr=1.6582e-05) (hash(x)=139168726)
+8300 val loss 5.9937
+8300 val perplexity 400.9057
+8300 train 5.921565 (lr=1.6359e-05) (hash(x)=154279434)
+8400 val loss 5.9857
+8400 val perplexity 397.6878
+8400 train 5.840742 (lr=1.6135e-05) (hash(x)=140111678)
+8500 val loss 5.9783
+8500 val perplexity 394.7612
+8500 train 5.952585 (lr=1.5911e-05) (hash(x)=152124088)
+8600 val loss 5.9724
+8600 val perplexity 392.4519
+8600 train 5.750079 (lr=1.5687e-05) (hash(x)=142869347)
+8700 val loss 5.9666
+8700 val perplexity 390.1960
+8700 train 5.865661 (lr=1.5462e-05) (hash(x)=145697606)
+8800 val loss 5.9649
+8800 val perplexity 389.5243
+8800 train 5.930559 (lr=1.5237e-05) (hash(x)=157314231)
+8900 val loss 5.9548
+8900 val perplexity 385.5879
+8900 train 5.938760 (lr=1.5013e-05) (hash(x)=149357509)
+9000 val loss 5.9482
+9000 val perplexity 383.0534
+9000 train 5.865352 (lr=1.4788e-05) (hash(x)=140434750)
+9100 val loss 5.9425
+9100 val perplexity 380.8907
+9100 train 5.936190 (lr=1.4563e-05) (hash(x)=144954543)
+9200 val loss 5.9361
+9200 val perplexity 378.4528
+9200 train 5.861468 (lr=1.4339e-05) (hash(x)=156329199)
+9300 val loss 5.9322
+9300 val perplexity 376.9731
+9300 train 5.719663 (lr=1.4115e-05) (hash(x)=137925654)
+9400 val loss 5.9339
+9400 val perplexity 377.6240
+9400 train 5.817344 (lr=1.3891e-05) (hash(x)=146385919)
+9500 val loss 5.9215
+9500 val perplexity 372.9842
+9500 train 5.854023 (lr=1.3668e-05) (hash(x)=144400019)
+9600 val loss 5.9250
+9600 val perplexity 374.2687
+9600 train 5.854084 (lr=1.3445e-05) (hash(x)=130562204)
+9700 val loss 5.9090
+9700 val perplexity 368.3387
+9700 train 5.847058 (lr=1.3222e-05) (hash(x)=152732805)
+9800 val loss 5.9033
+9800 val perplexity 366.2617
+9800 train 5.970324 (lr=1.3001e-05) (hash(x)=151860383)
+9900 val loss 5.9004
+9900 val perplexity 365.1756
+9900 train 5.977099 (lr=1.2780e-05) (hash(x)=156732259)
+10000 val loss 5.8938
+10000 val perplexity 362.7967
+10000 train 5.818756 (lr=1.2559e-05) (hash(x)=139943350)
+10100 val loss 5.8912
+10100 val perplexity 361.8531
+10100 train 5.707589 (lr=1.2340e-05) (hash(x)=145593924)
+10200 val loss 5.8867
+10200 val perplexity 360.2231
+10200 train 5.756473 (lr=1.2122e-05) (hash(x)=153996667)
+10300 val loss 5.8922
+10300 val perplexity 362.1841
+10300 train 5.796888 (lr=1.1904e-05) (hash(x)=154200987)
+10400 val loss 5.8867
+10400 val perplexity 360.2222
+10400 train 5.700641 (lr=1.1688e-05) (hash(x)=156872686)
+10500 val loss 5.8742
+10500 val perplexity 355.7265
+10500 train 5.959209 (lr=1.1472e-05) (hash(x)=155008723)
+10600 val loss 5.8645
+10600 val perplexity 352.3051
+10600 train 5.802585 (lr=1.1258e-05) (hash(x)=152715441)
+10700 val loss 5.8600
+10700 val perplexity 350.7289
+10700 train 6.017454 (lr=1.1045e-05) (hash(x)=166668700)
+10800 val loss 5.8566
+10800 val perplexity 349.5440
+10800 train 5.859518 (lr=1.0834e-05) (hash(x)=150698847)
+10900 val loss 5.8535
+10900 val perplexity 348.4593
+10900 train 5.662449 (lr=1.0624e-05) (hash(x)=137094574)
+11000 val loss 5.8504
+11000 val perplexity 347.3706
+11000 train 5.693173 (lr=1.0415e-05) (hash(x)=144225378)
+11100 val loss 5.8551
+11100 val perplexity 349.0094
+11100 train 5.766787 (lr=1.0208e-05) (hash(x)=151666772)
+11200 val loss 5.8512
+11200 val perplexity 347.6467
+11200 train 5.861551 (lr=1.0003e-05) (hash(x)=157843445)
+11300 val loss 5.8398
+11300 val perplexity 343.7015
+11300 train 5.745340 (lr=9.7991e-06) (hash(x)=140305735)
+11400 val loss 5.8349
+11400 val perplexity 342.0187
+11400 train 5.751746 (lr=9.5972e-06) (hash(x)=143435929)
+11500 val loss 5.8314
+11500 val perplexity 340.8513
+11500 train 5.855046 (lr=9.3970e-06) (hash(x)=150858606)
+11600 val loss 5.8269
+11600 val perplexity 339.3110
+11600 train 5.643163 (lr=9.1988e-06) (hash(x)=141643887)
+11700 val loss 5.8253
+11700 val perplexity 338.7520
+11700 train 5.690266 (lr=9.0025e-06) (hash(x)=151097313)
+11800 val loss 5.8228
+11800 val perplexity 337.9084
+11800 train 5.633714 (lr=8.8082e-06) (hash(x)=150639616)
+11900 val loss 5.8246
+11900 val perplexity 338.5177
+11900 train 5.521413 (lr=8.6160e-06) (hash(x)=154557688)
+12000 val loss 5.8197
+12000 val perplexity 336.8878
+12000 train 5.955412 (lr=8.4260e-06) (hash(x)=159324159)
+12100 val loss 5.8112
+12100 val perplexity 334.0166
+12100 train 5.856526 (lr=8.2381e-06) (hash(x)=151185357)
+12200 val loss 5.8070
+12200 val perplexity 332.6033
+12200 train 5.682366 (lr=8.0526e-06) (hash(x)=137457899)
+12300 val loss 5.8042
+12300 val perplexity 331.6975
+12300 train 5.861910 (lr=7.8694e-06) (hash(x)=146370056)
+12400 val loss 5.8006
+12400 val perplexity 330.5125
+12400 train 5.651220 (lr=7.6885e-06) (hash(x)=148352898)
+12500 val loss 5.7993
+12500 val perplexity 330.0701
+12500 train 5.785968 (lr=7.5101e-06) (hash(x)=161514681)
+12600 val loss 5.7992
+12600 val perplexity 330.0287
+12600 train 5.666851 (lr=7.3342e-06) (hash(x)=146207280)
+12700 val loss 5.7996
+12700 val perplexity 330.1529
+12700 train 5.631870 (lr=7.1609e-06) (hash(x)=142615619)
+12800 val loss 5.7938
+12800 val perplexity 328.2441
+12800 train 5.813536 (lr=6.9903e-06) (hash(x)=144604802)
+12900 val loss 5.7887
+12900 val perplexity 326.5765
+12900 train 5.767271 (lr=6.8223e-06) (hash(x)=143212941)
+13000 val loss 5.7821
+13000 val perplexity 324.4371
+13000 train 5.861295 (lr=6.6570e-06) (hash(x)=155987866)
+13100 val loss 5.7809
+13100 val perplexity 324.0486
+13100 train 6.063510 (lr=6.4945e-06) (hash(x)=155920713)
+13200 val loss 5.7777
+13200 val perplexity 323.0061
+13200 train 5.963966 (lr=6.3349e-06) (hash(x)=152145878)
+13300 val loss 5.7759
+13300 val perplexity 322.4263
+13300 train 5.584373 (lr=6.1782e-06) (hash(x)=134141709)
+13400 val loss 5.7780
+13400 val perplexity 323.1029
+13400 train 5.699019 (lr=6.0244e-06) (hash(x)=146096299)
+13500 val loss 5.7766
+13500 val perplexity 322.6459
+13500 train 5.536699 (lr=5.8737e-06) (hash(x)=144011431)
+13600 val loss 5.7686
+13600 val perplexity 320.1033
+13600 train 6.021538 (lr=5.7259e-06) (hash(x)=163950932)
+13700 val loss 5.7658
+13700 val perplexity 319.2066
+13700 train 5.906261 (lr=5.5813e-06) (hash(x)=156652406)
+13800 val loss 5.7618
+13800 val perplexity 317.9240
+13800 train 5.673179 (lr=5.4398e-06) (hash(x)=145753166)
+13900 val loss 5.7605
+13900 val perplexity 317.4920
+13900 train 5.634323 (lr=5.3016e-06) (hash(x)=147065551)
+14000 val loss 5.7587
+14000 val perplexity 316.9253
+14000 train 6.219856 (lr=5.1665e-06) (hash(x)=179077542)
+14100 val loss 5.7573
+14100 val perplexity 316.5054
+14100 train 5.854602 (lr=5.0348e-06) (hash(x)=171962802)
+14200 val loss 5.7582
+14200 val perplexity 316.7881
+14200 train 5.827171 (lr=4.9063e-06) (hash(x)=153299913)
+14300 val loss 5.7577
+14300 val perplexity 316.6262
+14300 train 5.685266 (lr=4.7813e-06) (hash(x)=154750565)
+14400 val loss 5.7549
+14400 val perplexity 315.7469
+14400 train 6.028841 (lr=4.6596e-06) (hash(x)=154536694)
+14500 val loss 5.7505
+14500 val perplexity 314.3393
+14500 train 5.796143 (lr=4.5414e-06) (hash(x)=151236383)
+14600 val loss 5.7472
+14600 val perplexity 313.3142
+14600 train 5.765508 (lr=4.4267e-06) (hash(x)=152584272)
+14700 val loss 5.7450
+14700 val perplexity 312.6319
+14700 train 5.759209 (lr=4.3155e-06) (hash(x)=175301826)
+14800 val loss 5.7445
+14800 val perplexity 312.4728
+14800 train 5.673807 (lr=4.2079e-06) (hash(x)=148729408)
+14900 val loss 5.7430
+14900 val perplexity 312.0077
+14900 train 5.921260 (lr=4.1039e-06) (hash(x)=150978283)
+15000 val loss 5.7445
+15000 val perplexity 312.4808
+15000 train 5.690734 (lr=4.0035e-06) (hash(x)=153621389)
+15100 val loss 5.7436
+15100 val perplexity 312.1883
+15100 train 5.670405 (lr=3.9068e-06) (hash(x)=150653793)
+15200 val loss 5.7412
+15200 val perplexity 311.4480
+15200 train 5.897426 (lr=3.8138e-06) (hash(x)=140802934)
+15300 val loss 5.7364
+15300 val perplexity 309.9620
+15300 train 5.711238 (lr=3.7245e-06) (hash(x)=152007556)
+15400 val loss 5.7352
+15400 val perplexity 309.5629
+15400 train 5.682077 (lr=3.6390e-06) (hash(x)=149057446)
+15500 val loss 5.7342
+15500 val perplexity 309.2806
+15500 train 5.655005 (lr=3.5572e-06) (hash(x)=147521774)
+15600 val loss 5.7330
+15600 val perplexity 308.8904
+15600 train 5.700313 (lr=3.4793e-06) (hash(x)=148944405)
+15700 val loss 5.7316
+15700 val perplexity 308.4541
+15700 train 5.686961 (lr=3.4052e-06) (hash(x)=148472532)
+15800 val loss 5.7311
+15800 val perplexity 308.3131
+15800 train 5.509686 (lr=3.3350e-06) (hash(x)=149048780)
+15900 val loss 5.7322
+15900 val perplexity 308.6458
+15900 train 5.696492 (lr=3.2687e-06) (hash(x)=150175778)
+16000 val loss 5.7336
+16000 val perplexity 309.0713
+16000 train 5.528355 (lr=3.2063e-06) (hash(x)=145943831)
+16100 val loss 5.7262
+16100 val perplexity 306.7892
+16100 train 5.899349 (lr=3.1478e-06) (hash(x)=134314842)
+16200 val loss 5.7247
+16200 val perplexity 306.3504
+16200 train 5.751829 (lr=3.0932e-06) (hash(x)=153755135)
+16300 val loss 5.7249
+16300 val perplexity 306.4005
+16300 train 5.447047 (lr=3.0427e-06) (hash(x)=140070802)
+16400 val loss 5.7231
+16400 val perplexity 305.8436
+16400 train 5.745318 (lr=2.9961e-06) (hash(x)=151132329)
+16500 val loss 5.7205
+16500 val perplexity 305.0458
+16500 train 5.751915 (lr=2.9535e-06) (hash(x)=151585550)
+16600 val loss 5.7193
+16600 val perplexity 304.7025
+16600 train 5.882590 (lr=2.9149e-06) (hash(x)=164389325)
+16700 val loss 5.7181
+16700 val perplexity 304.3339
+16700 train 5.623256 (lr=2.8804e-06) (hash(x)=146016298)
+16800 val loss 5.7181
+16800 val perplexity 304.3256
+16800 train 5.625247 (lr=2.8498e-06) (hash(x)=147668192)
+16900 val loss 5.7173
+16900 val perplexity 304.0757
+16900 train 5.694380 (lr=2.8234e-06) (hash(x)=155692530)
+17000 val loss 5.7186
+17000 val perplexity 304.4734
+17000 train 5.640749 (lr=2.8010e-06) (hash(x)=156598351)
+17100 val loss 5.7186
+17100 val perplexity 304.4753
+17100 train 5.630686 (lr=2.7826e-06) (hash(x)=157344236)
+17200 val loss 5.7154
+17200 val perplexity 303.5097
+17200 train 5.522198 (lr=2.7684e-06) (hash(x)=126288656)
+17300 val loss 5.7130
+17300 val perplexity 302.7687
+17300 train 5.754097 (lr=2.7582e-06) (hash(x)=146365959)
+17400 val loss 5.7112
+17400 val perplexity 302.2420
+17400 train 5.698453 (lr=2.7520e-06) (hash(x)=152729388)
+17499 val loss 5.7100
+17499 val perplexity 301.8719

attention_kindselective_n_heads4_seed1344/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1b343ac4384c6e6dce1c9b65490a03b915bcd842d344c17e51207b80f2a41e8
+oid sha256:0e756f7d9491da10e8b323fb0adc65f8e710af9bae581acc742915d1250d7106
 size 92843394

attention_kindselective_n_heads4_seed1344/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:82f52a9e837518df1666833e3cb6f8e76aa3c06bb81e76837bee3913c88adac0
+oid sha256:ff2b75f5d4306450c1c59182852607172124dd504534b1a18fffba7ed3e8c308
 size 179406214