Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1340", "n_embd": 256}

attention_kindselective_n_heads4_seed1340/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2714
-0 val perplexity 78545.6094
-0 train 11.276529 (lr=1.5000e-07) (hash(x)=164406924)
-100 val loss 9.9565
-100 val perplexity 21088.7754
-100 train 10.003725 (lr=1.5150e-05) (hash(x)=177407419)
-200 val loss 9.1915
-200 val perplexity 9813.6758
-200 train 9.161508 (lr=3.0000e-05) (hash(x)=144903932)
-300 val loss 8.1490
-300 val perplexity 3459.9509
-300 train 8.363692 (lr=2.9998e-05) (hash(x)=173839165)
-400 val loss 7.7785
-400 val perplexity 2388.6050
-400 train 7.844526 (lr=2.9991e-05) (hash(x)=167734596)
-500 val loss 7.6193
-500 val perplexity 2037.1665
-500 train 7.635940 (lr=2.9980e-05) (hash(x)=153224076)
-600 val loss 7.5358
-600 val perplexity 1873.8973
-600 train 7.448798 (lr=2.9964e-05) (hash(x)=149619098)
-700 val loss 7.4718
-700 val perplexity 1757.8380
-700 train 7.403675 (lr=2.9944e-05) (hash(x)=146539909)
-800 val loss 7.4150
-800 val perplexity 1660.6387
-800 train 7.379327 (lr=2.9920e-05) (hash(x)=153710890)
-900 val loss 7.3763
-900 val perplexity 1597.6978
-900 train 7.319605 (lr=2.9891e-05) (hash(x)=155873620)
-1000 val loss 7.3407
-1000 val perplexity 1541.7302
-1000 train 7.259896 (lr=2.9858e-05) (hash(x)=145450636)
-1100 val loss 7.3055
-1100 val perplexity 1488.5031
-1100 train 7.378976 (lr=2.9820e-05) (hash(x)=154123388)
-1200 val loss 7.2695
-1200 val perplexity 1435.8259
-1200 train 7.137440 (lr=2.9778e-05) (hash(x)=145249251)
-1300 val loss 7.2262
-1300 val perplexity 1375.0448
-1300 train 7.121584 (lr=2.9732e-05) (hash(x)=148937127)
-1400 val loss 7.1918
-1400 val perplexity 1328.4684
-1400 train 7.305116 (lr=2.9681e-05) (hash(x)=150475545)
-1500 val loss 7.1589
-1500 val perplexity 1285.4811
-1500 train 7.085552 (lr=2.9626e-05) (hash(x)=154653428)
-1600 val loss 7.1252
-1600 val perplexity 1242.9048
-1600 train 7.103401 (lr=2.9566e-05) (hash(x)=144483776)
-1700 val loss 7.0894
-1700 val perplexity 1199.1776
-1700 train 7.245973 (lr=2.9502e-05) (hash(x)=157395496)
-1800 val loss 7.0573
-1800 val perplexity 1161.3492
-1800 train 7.074311 (lr=2.9434e-05) (hash(x)=157916369)
-1900 val loss 7.0313
-1900 val perplexity 1131.4462
-1900 train 7.244211 (lr=2.9362e-05) (hash(x)=166073923)
-2000 val loss 7.0124
-2000 val perplexity 1110.2701
-2000 train 7.089108 (lr=2.9285e-05) (hash(x)=154856891)
-2100 val loss 6.9878
-2100 val perplexity 1083.3145
-2100 train 6.974560 (lr=2.9204e-05) (hash(x)=151925203)
-2200 val loss 6.9606
-2200 val perplexity 1054.2246
-2200 train 6.682307 (lr=2.9119e-05) (hash(x)=136191502)
-2300 val loss 6.9316
-2300 val perplexity 1024.1567
-2300 train 7.086887 (lr=2.9030e-05) (hash(x)=153273362)
-2400 val loss 6.8998
-2400 val perplexity 992.0283
-2400 train 6.840942 (lr=2.8937e-05) (hash(x)=148021541)
-2500 val loss 6.8799
-2500 val perplexity 972.5198
-2500 train 6.809747 (lr=2.8840e-05) (hash(x)=141356608)
-2600 val loss 6.8514
-2600 val perplexity 945.1887
-2600 train 6.799051 (lr=2.8738e-05) (hash(x)=146005217)
-2700 val loss 6.8314
-2700 val perplexity 926.4821
-2700 train 6.665824 (lr=2.8633e-05) (hash(x)=144511718)
-2800 val loss 6.8044
-2800 val perplexity 901.8392
-2800 train 6.691870 (lr=2.8523e-05) (hash(x)=146019502)
-2900 val loss 6.7775
-2900 val perplexity 877.8367
-2900 train 6.692867 (lr=2.8410e-05) (hash(x)=146496200)
-3000 val loss 6.7542
-3000 val perplexity 857.6766
-3000 train 6.705166 (lr=2.8292e-05) (hash(x)=150127281)
-3100 val loss 6.7226
-3100 val perplexity 830.9784
-3100 train 6.695710 (lr=2.8171e-05) (hash(x)=142022255)
-3200 val loss 6.6930
-3200 val perplexity 806.7307
-3200 train 6.755077 (lr=2.8046e-05) (hash(x)=154120875)
-3300 val loss 6.6781
-3300 val perplexity 794.7838
-3300 train 6.725983 (lr=2.7917e-05) (hash(x)=153999717)
-3400 val loss 6.6518
-3400 val perplexity 774.1745
-3400 train 6.428479 (lr=2.7784e-05) (hash(x)=139694097)
-3500 val loss 6.6172
-3500 val perplexity 747.8224
-3500 train 6.814740 (lr=2.7648e-05) (hash(x)=162992732)
-3600 val loss 6.5832
-3600 val perplexity 722.8391
-3600 train 6.559596 (lr=2.7508e-05) (hash(x)=147574101)
-3700 val loss 6.5598
-3700 val perplexity 706.1619
-3700 train 6.671773 (lr=2.7364e-05) (hash(x)=157763099)
-3800 val loss 6.5318
-3800 val perplexity 686.6064
-3800 train 6.697332 (lr=2.7216e-05) (hash(x)=170800034)
-3900 val loss 6.5078
-3900 val perplexity 670.3498
-3900 train 6.578204 (lr=2.7066e-05) (hash(x)=164984528)
-4000 val loss 6.4896
-4000 val perplexity 658.2488
-4000 train 6.356799 (lr=2.6911e-05) (hash(x)=141743323)
-4100 val loss 6.4820
-4100 val perplexity 653.2877
-4100 train 6.494662 (lr=2.6754e-05) (hash(x)=153392872)
-4200 val loss 6.4724
-4200 val perplexity 647.0207
-4200 train 6.330730 (lr=2.6592e-05) (hash(x)=149074933)
-4300 val loss 6.4442
-4300 val perplexity 629.0538
-4300 train 6.854745 (lr=2.6428e-05) (hash(x)=167823423)
-4400 val loss 6.4200
-4400 val perplexity 614.0254
-4400 train 6.180253 (lr=2.6260e-05) (hash(x)=141203114)
-4500 val loss 6.3970
-4500 val perplexity 600.0132
-4500 train 6.389085 (lr=2.6089e-05) (hash(x)=146284780)
-4600 val loss 6.3813
-4600 val perplexity 590.6880
-4600 train 6.176318 (lr=2.5915e-05) (hash(x)=141126464)
-4700 val loss 6.3699
-4700 val perplexity 584.0157
-4700 train 6.391843 (lr=2.5738e-05) (hash(x)=154751926)
-4800 val loss 6.3497
-4800 val perplexity 572.3212
-4800 train 6.431994 (lr=2.5558e-05) (hash(x)=154793198)
-4900 val loss 6.3433
-4900 val perplexity 568.6953
-4900 train 6.067785 (lr=2.5374e-05) (hash(x)=139406392)
-5000 val loss 6.3467
-5000 val perplexity 570.6268
-5000 train 6.084831 (lr=2.5188e-05) (hash(x)=153548741)
-5100 val loss 6.3233
-5100 val perplexity 557.4250
-5100 train 6.399115 (lr=2.4999e-05) (hash(x)=160488568)
-5200 val loss 6.3002
-5200 val perplexity 544.6785
-5200 train 6.312397 (lr=2.4807e-05) (hash(x)=149645053)
-5300 val loss 6.2777
-5300 val perplexity 532.5682
-5300 train 6.371747 (lr=2.4613e-05) (hash(x)=155820556)
-5400 val loss 6.2651
-5400 val perplexity 525.9007
-5400 train 6.201801 (lr=2.4415e-05) (hash(x)=147538134)
-5500 val loss 6.2555
-5500 val perplexity 520.8438
-5500 train 6.460732 (lr=2.4215e-05) (hash(x)=166889307)
-5600 val loss 6.2463
-5600 val perplexity 516.0897
-5600 train 5.952559 (lr=2.4013e-05) (hash(x)=139516699)
-5700 val loss 6.2444
-5700 val perplexity 515.0945
-5700 train 5.911396 (lr=2.3808e-05) (hash(x)=140453511)
-5800 val loss 6.2360
-5800 val perplexity 510.8066
-5800 train 6.175674 (lr=2.3601e-05) (hash(x)=162964847)
-5900 val loss 6.2180
-5900 val perplexity 501.7165
-5900 train 6.221095 (lr=2.3391e-05) (hash(x)=150606634)
-6000 val loss 6.1979
-6000 val perplexity 491.7024
-6000 train 6.294567 (lr=2.3179e-05) (hash(x)=149890857)
-6100 val loss 6.1909
-6100 val perplexity 488.2983
-6100 train 6.293276 (lr=2.2965e-05) (hash(x)=173884145)
-6200 val loss 6.1782
-6200 val perplexity 482.1198
-6200 train 6.220095 (lr=2.2749e-05) (hash(x)=151987098)
-6300 val loss 6.1676
-6300 val perplexity 477.0301
-6300 train 6.116311 (lr=2.2530e-05) (hash(x)=148853562)
-6400 val loss 6.1559
-6400 val perplexity 471.4874
-6400 train 5.908284 (lr=2.2310e-05) (hash(x)=141530101)
-6500 val loss 6.1510
-6500 val perplexity 469.1747
-6500 train 6.006312 (lr=2.2088e-05) (hash(x)=142297809)
-6600 val loss 6.1356
-6600 val perplexity 462.0191
-6600 train 6.018552 (lr=2.1864e-05) (hash(x)=142447782)
-6700 val loss 6.1270
-6700 val perplexity 458.0419
-6700 train 6.093075 (lr=2.1638e-05) (hash(x)=147004686)
-6800 val loss 6.1177
-6800 val perplexity 453.8031
-6800 train 5.860326 (lr=2.1410e-05) (hash(x)=133438702)
-6900 val loss 6.1114
-6900 val perplexity 450.9759
-6900 train 6.136273 (lr=2.1181e-05) (hash(x)=157085143)
-7000 val loss 6.1088
-7000 val perplexity 449.8181
-7000 train 5.980198 (lr=2.0951e-05) (hash(x)=139437666)
-7100 val loss 6.0960
-7100 val perplexity 444.0860
-7100 train 6.120476 (lr=2.0718e-05) (hash(x)=159792986)
-7200 val loss 6.0802
-7200 val perplexity 437.0959
-7200 train 6.049283 (lr=2.0485e-05) (hash(x)=144930687)
-7300 val loss 6.0733
-7300 val perplexity 434.0986
-7300 train 6.140574 (lr=2.0250e-05) (hash(x)=156242690)
-7400 val loss 6.0667
-7400 val perplexity 431.2542
-7400 train 5.906787 (lr=2.0014e-05) (hash(x)=148183719)
-7500 val loss 6.0595
-7500 val perplexity 428.1465
-7500 train 6.092733 (lr=1.9777e-05) (hash(x)=152494758)
-7600 val loss 6.0554
-7600 val perplexity 426.3988
-7600 train 5.742143 (lr=1.9538e-05) (hash(x)=142485027)
-7700 val loss 6.0586
-7700 val perplexity 427.7714
-7700 train 5.919030 (lr=1.9299e-05) (hash(x)=147512165)
-7800 val loss 6.0394
-7800 val perplexity 419.6538
-7800 train 5.934808 (lr=1.9059e-05) (hash(x)=160346994)
-7900 val loss 6.0336
-7900 val perplexity 417.1983
-7900 train 5.895798 (lr=1.8817e-05) (hash(x)=144488254)
-8000 val loss 6.0241
-8000 val perplexity 413.2754
-8000 train 5.831774 (lr=1.8576e-05) (hash(x)=147637019)
-8100 val loss 6.0206
-8100 val perplexity 411.8124
-8100 train 5.868452 (lr=1.8333e-05) (hash(x)=147340534)
-8200 val loss 6.0152
-8200 val perplexity 409.6074
-8200 train 6.065320 (lr=1.8090e-05) (hash(x)=151630665)
-8300 val loss 6.0044
-8300 val perplexity 405.1965
-8300 train 6.097698 (lr=1.7846e-05) (hash(x)=149747064)
-8400 val loss 5.9969
-8400 val perplexity 402.1890
-8400 train 6.182949 (lr=1.7602e-05) (hash(x)=154245770)
-8500 val loss 5.9934
-8500 val perplexity 400.7683
-8500 train 5.872526 (lr=1.7357e-05) (hash(x)=152559100)
-8600 val loss 5.9838
-8600 val perplexity 396.9453
-8600 train 6.427598 (lr=1.7113e-05) (hash(x)=181365926)
-8700 val loss 5.9828
-8700 val perplexity 396.5321
-8700 train 5.817455 (lr=1.6868e-05) (hash(x)=154405991)
-8800 val loss 5.9769
-8800 val perplexity 394.2023
-8800 train 5.927083 (lr=1.6623e-05) (hash(x)=153755904)
-8900 val loss 5.9756
-8900 val perplexity 393.7231
-8900 train 5.897131 (lr=1.6377e-05) (hash(x)=152120568)
-9000 val loss 5.9572
-9000 val perplexity 386.5221
-9000 train 5.747784 (lr=1.6132e-05) (hash(x)=142797279)
-9100 val loss 5.9533
-9100 val perplexity 385.0257
-9100 train 5.806417 (lr=1.5887e-05) (hash(x)=143037503)
-9200 val loss 5.9542
-9200 val perplexity 385.3618
-9200 train 5.788256 (lr=1.5643e-05) (hash(x)=113690273)
-9300 val loss 5.9434
-9300 val perplexity 381.2329
-9300 train 5.889099 (lr=1.5398e-05) (hash(x)=158025077)
-9400 val loss 5.9370
-9400 val perplexity 378.8150
-9400 train 6.008280 (lr=1.5154e-05) (hash(x)=158251718)
-9500 val loss 5.9340
-9500 val perplexity 377.6508
-9500 train 5.972120 (lr=1.4910e-05) (hash(x)=154752610)
-9600 val loss 5.9324
-9600 val perplexity 377.0439
-9600 train 5.803311 (lr=1.4667e-05) (hash(x)=146889093)
-9700 val loss 5.9203
-9700 val perplexity 372.5305
-9700 train 5.924708 (lr=1.4424e-05) (hash(x)=156906516)
-9800 val loss 5.9181
-9800 val perplexity 371.7113
-9800 train 5.752685 (lr=1.4183e-05) (hash(x)=153841927)
-9900 val loss 5.9140
-9900 val perplexity 370.2012
-9900 train 6.127919 (lr=1.3941e-05) (hash(x)=163514334)
-10000 val loss 5.9059
-10000 val perplexity 367.1806
-10000 train 5.881135 (lr=1.3701e-05) (hash(x)=144745561)
-10100 val loss 5.9042
-10100 val perplexity 366.5601
-10100 train 6.180727 (lr=1.3462e-05) (hash(x)=161651149)
-10200 val loss 5.8971
-10200 val perplexity 363.9953
-10200 train 5.840960 (lr=1.3223e-05) (hash(x)=143667037)
-10300 val loss 5.8897
-10300 val perplexity 361.2955
-10300 train 5.783295 (lr=1.2986e-05) (hash(x)=151687262)
-10400 val loss 5.8871
-10400 val perplexity 360.3506
-10400 train 5.761328 (lr=1.2750e-05) (hash(x)=153694955)
-10500 val loss 5.8859
-10500 val perplexity 359.9370
-10500 train 5.823026 (lr=1.2515e-05) (hash(x)=150218029)
-10600 val loss 5.8790
-10600 val perplexity 357.4660
-10600 train 5.968370 (lr=1.2282e-05) (hash(x)=155093716)
-10700 val loss 5.8750
-10700 val perplexity 356.0114
-10700 train 5.911896 (lr=1.2049e-05) (hash(x)=151631831)
-10800 val loss 5.8672
-10800 val perplexity 353.2490
-10800 train 5.706692 (lr=1.1819e-05) (hash(x)=136465175)
-10900 val loss 5.8654
-10900 val perplexity 352.6364
-10900 train 5.803391 (lr=1.1590e-05) (hash(x)=154461514)
-11000 val loss 5.8660
-11000 val perplexity 352.8288
-11000 train 5.848099 (lr=1.1362e-05) (hash(x)=141877432)
-11100 val loss 5.8617
-11100 val perplexity 351.3274
-11100 train 5.658975 (lr=1.1136e-05) (hash(x)=139572894)
-11200 val loss 5.8546
-11200 val perplexity 348.8446
-11200 train 5.902207 (lr=1.0912e-05) (hash(x)=155591443)
-11300 val loss 5.8496
-11300 val perplexity 347.0927
-11300 train 5.856249 (lr=1.0690e-05) (hash(x)=153349136)
-11400 val loss 5.8450
-11400 val perplexity 345.5066
-11400 train 5.795953 (lr=1.0470e-05) (hash(x)=148109395)
-11500 val loss 5.8418
-11500 val perplexity 344.3930
-11500 train 5.824965 (lr=1.0251e-05) (hash(x)=153276620)
-11600 val loss 5.8417
-11600 val perplexity 344.3531
-11600 train 5.706350 (lr=1.0035e-05) (hash(x)=151416388)
-11700 val loss 5.8373
-11700 val perplexity 342.8495
-11700 train 5.715032 (lr=9.8209e-06) (hash(x)=143420927)
-11800 val loss 5.8348
-11800 val perplexity 342.0114
-11800 train 5.907176 (lr=9.6090e-06) (hash(x)=165023996)
-11900 val loss 5.8281
-11900 val perplexity 339.7224
-11900 train 5.743924 (lr=9.3993e-06) (hash(x)=144465076)
-12000 val loss 5.8271
-12000 val perplexity 339.3660
-12000 train 5.830256 (lr=9.1920e-06) (hash(x)=155991892)
-12100 val loss 5.8243
-12100 val perplexity 338.4118
-12100 train 5.848801 (lr=8.9871e-06) (hash(x)=150748844)
-12200 val loss 5.8223
-12200 val perplexity 337.7464
-12200 train 5.776937 (lr=8.7846e-06) (hash(x)=149712638)
-12300 val loss 5.8187
-12300 val perplexity 336.5458
-12300 train 5.631779 (lr=8.5847e-06) (hash(x)=140428230)
-12400 val loss 5.8181
-12400 val perplexity 336.3350
-12400 train 5.993639 (lr=8.3875e-06) (hash(x)=163614308)
-12500 val loss 5.8106
-12500 val perplexity 333.8238
-12500 train 5.770899 (lr=8.1929e-06) (hash(x)=153366153)
-12600 val loss 5.8090
-12600 val perplexity 333.3011
-12600 train 5.696667 (lr=8.0010e-06) (hash(x)=146832066)
-12700 val loss 5.8051
-12700 val perplexity 331.9938
-12700 train 5.979981 (lr=7.8119e-06) (hash(x)=154387414)
-12800 val loss 5.8036
-12800 val perplexity 331.4978
-12800 train 5.592911 (lr=7.6257e-06) (hash(x)=139465199)
-12900 val loss 5.8055
-12900 val perplexity 332.1220
-12900 train 5.516826 (lr=7.4425e-06) (hash(x)=143326374)
-13000 val loss 5.7991
-13000 val perplexity 329.9940
-13000 train 5.947831 (lr=7.2622e-06) (hash(x)=158817835)
-13100 val loss 5.7966
-13100 val perplexity 329.1757
-13100 train 5.674695 (lr=7.0850e-06) (hash(x)=142027874)
-13200 val loss 5.7976
-13200 val perplexity 329.5131
-13200 train 5.767667 (lr=6.9108e-06) (hash(x)=148497730)
-13300 val loss 5.7948
-13300 val perplexity 328.5707
-13300 train 5.695848 (lr=6.7398e-06) (hash(x)=148094633)
-13400 val loss 5.7902
-13400 val perplexity 327.0631
-13400 train 5.679173 (lr=6.5721e-06) (hash(x)=150548214)
-13500 val loss 5.7841
-13500 val perplexity 325.0811
-13500 train 5.893899 (lr=6.4076e-06) (hash(x)=152789725)
-13600 val loss 5.7834
-13600 val perplexity 324.8510
-13600 train 5.825988 (lr=6.2465e-06) (hash(x)=153220230)
-13700 val loss 5.7819
-13700 val perplexity 324.3703
-13700 train 5.790273 (lr=6.0887e-06) (hash(x)=147805699)
-13800 val loss 5.7813
-13800 val perplexity 324.1805
-13800 train 5.608267 (lr=5.9344e-06) (hash(x)=146267341)
-13900 val loss 5.7810
-13900 val perplexity 324.0743
-13900 train 5.592768 (lr=5.7835e-06) (hash(x)=139467501)
-14000 val loss 5.7778
-14000 val perplexity 323.0539
-14000 train 5.790003 (lr=5.6362e-06) (hash(x)=152185436)
-14100 val loss 5.7726
-14100 val perplexity 321.3810
-14100 train 5.895826 (lr=5.4925e-06) (hash(x)=176479139)
-14200 val loss 5.7686
-14200 val perplexity 320.0971
-14200 train 5.864861 (lr=5.3524e-06) (hash(x)=151052040)
-14300 val loss 5.7689
-14300 val perplexity 320.1862
-14300 train 5.703229 (lr=5.2159e-06) (hash(x)=149668097)
-14400 val loss 5.7663
-14400 val perplexity 319.3461
-14400 train 5.767529 (lr=5.0832e-06) (hash(x)=151761941)
-14500 val loss 5.7653
-14500 val perplexity 319.0283
-14500 train 5.714600 (lr=4.9543e-06) (hash(x)=149412009)
-14600 val loss 5.7643
-14600 val perplexity 318.7158
-14600 train 5.635766 (lr=4.8291e-06) (hash(x)=145048020)
-14700 val loss 5.7612
-14700 val perplexity 317.7242
-14700 train 5.712110 (lr=4.7079e-06) (hash(x)=141771666)
-14800 val loss 5.7578
-14800 val perplexity 316.6632
-14800 train 5.687315 (lr=4.5905e-06) (hash(x)=148759740)
-14900 val loss 5.7564
-14900 val perplexity 316.2151
-14900 train 5.687220 (lr=4.4770e-06) (hash(x)=146287032)
-15000 val loss 5.7551
-15000 val perplexity 315.8114
-15000 train 5.692285 (lr=4.3675e-06) (hash(x)=143519156)
-15100 val loss 5.7571
-15100 val perplexity 316.4424
-15100 train 5.728598 (lr=4.2620e-06) (hash(x)=151330688)
-15200 val loss 5.7589
-15200 val perplexity 316.9836
-15200 train 5.546316 (lr=4.1605e-06) (hash(x)=140380090)
-15300 val loss 5.7522
-15300 val perplexity 314.8869
-15300 train 5.660977 (lr=4.0631e-06) (hash(x)=152598160)
-15400 val loss 5.7480
-15400 val perplexity 313.5758
-15400 train 6.015850 (lr=3.9698e-06) (hash(x)=164136966)
-15500 val loss 5.7492
-15500 val perplexity 313.9401
-15500 train 5.947266 (lr=3.8806e-06) (hash(x)=158733990)
-15600 val loss 5.7463
-15600 val perplexity 313.0248
-15600 train 5.781134 (lr=3.7956e-06) (hash(x)=149149390)
-15700 val loss 5.7450
-15700 val perplexity 312.6340
-15700 train 5.706141 (lr=3.7148e-06) (hash(x)=146746394)
-15800 val loss 5.7451
-15800 val perplexity 312.6615
-15800 train 5.772112 (lr=3.6382e-06) (hash(x)=159405219)
-15900 val loss 5.7436
-15900 val perplexity 312.1776
-15900 train 5.769944 (lr=3.5658e-06) (hash(x)=154034029)
-16000 val loss 5.7419
-16000 val perplexity 311.6655
-16000 train 5.749253 (lr=3.4977e-06) (hash(x)=153634210)
-16100 val loss 5.7394
-16100 val perplexity 310.8738
-16100 train 5.670649 (lr=3.4339e-06) (hash(x)=155646721)
-16200 val loss 5.7383
-16200 val perplexity 310.5243
-16200 train 5.639257 (lr=3.3744e-06) (hash(x)=142461643)
-16300 val loss 5.7382
-16300 val perplexity 310.4923
-16300 train 5.627505 (lr=3.3193e-06) (hash(x)=141080038)
-16400 val loss 5.7378
-16400 val perplexity 310.3942
-16400 train 5.757590 (lr=3.2684e-06) (hash(x)=153744166)
-16500 val loss 5.7407
-16500 val perplexity 311.2946
-16500 train 5.545078 (lr=3.2220e-06) (hash(x)=149783517)
-16600 val loss 5.7420
-16600 val perplexity 311.6945
-16600 train 5.575537 (lr=3.1799e-06) (hash(x)=142659127)
-16700 val loss 5.7353
-16700 val perplexity 309.6186
-16700 train 5.796297 (lr=3.1422e-06) (hash(x)=149725921)
-16800 val loss 5.7324
-16800 val perplexity 308.6945
-16800 train 5.569037 (lr=3.1089e-06) (hash(x)=140678087)
-16900 val loss 5.7321
-16900 val perplexity 308.6020
-16900 train 5.739532 (lr=3.0801e-06) (hash(x)=146935413)
-17000 val loss 5.7298
-17000 val perplexity 307.9060
-17000 train 5.740740 (lr=3.0556e-06) (hash(x)=144408437)
-17100 val loss 5.7296
-17100 val perplexity 307.8506
-17100 train 5.590549 (lr=3.0356e-06) (hash(x)=149229905)
-17200 val loss 5.7297
-17200 val perplexity 307.8794
-17200 train 5.667690 (lr=3.0200e-06) (hash(x)=150471865)
-17300 val loss 5.7316
-17300 val perplexity 308.4719
-17300 train 5.632981 (lr=3.0089e-06) (hash(x)=155784916)
-17400 val loss 5.7341
-17400 val perplexity 309.2283
-17400 train 5.511448 (lr=3.0022e-06) (hash(x)=148482413)
-17499 val loss 5.7268
-17499 val perplexity 306.9891
+0 val perplexity 78546.7344
+0 train 11.276529 (lr=1.6250e-07) (hash(x)=164406924)
+100 val loss 9.9427
+100 val perplexity 20799.7773
+100 train 9.986905 (lr=1.6412e-05) (hash(x)=177407419)
+200 val loss 9.0545
+200 val perplexity 8556.6611
+200 train 9.022396 (lr=3.2500e-05) (hash(x)=144903932)
+300 val loss 7.9905
+300 val perplexity 2952.6941
+300 train 8.233212 (lr=3.2498e-05) (hash(x)=173839165)
+400 val loss 7.7050
+400 val perplexity 2219.5210
+400 train 7.775531 (lr=3.2490e-05) (hash(x)=167734596)
+500 val loss 7.5884
+500 val perplexity 1975.1251
+500 train 7.603640 (lr=3.2478e-05) (hash(x)=153224076)
+600 val loss 7.5186
+600 val perplexity 1842.0447
+600 train 7.431871 (lr=3.2461e-05) (hash(x)=149619098)
+700 val loss 7.4777
+700 val perplexity 1768.2009
+700 train 7.410966 (lr=3.2440e-05) (hash(x)=146539909)
+800 val loss 7.4248
+800 val perplexity 1677.1040
+800 train 7.388150 (lr=3.2413e-05) (hash(x)=153710890)
+900 val loss 7.3854
+900 val perplexity 1612.2769
+900 train 7.327601 (lr=3.2382e-05) (hash(x)=155873620)
+1000 val loss 7.3567
+1000 val perplexity 1566.6251
+1000 train 7.275003 (lr=3.2346e-05) (hash(x)=145450636)
+1100 val loss 7.3128
+1100 val perplexity 1499.4359
+1100 train 7.385886 (lr=3.2305e-05) (hash(x)=154123388)
+1200 val loss 7.2874
+1200 val perplexity 1461.7990
+1200 train 7.163799 (lr=3.2260e-05) (hash(x)=145249251)
+1300 val loss 7.2463
+1300 val perplexity 1402.8694
+1300 train 7.144474 (lr=3.2209e-05) (hash(x)=148937127)
+1400 val loss 7.1995
+1400 val perplexity 1338.7473
+1400 train 7.317604 (lr=3.2154e-05) (hash(x)=150475545)
+1500 val loss 7.1654
+1500 val perplexity 1293.8778
+1500 train 7.090920 (lr=3.2094e-05) (hash(x)=154653428)
+1600 val loss 7.1204
+1600 val perplexity 1236.9946
+1600 train 7.097936 (lr=3.2030e-05) (hash(x)=144483776)
+1700 val loss 7.0663
+1700 val perplexity 1171.8545
+1700 train 7.221328 (lr=3.1961e-05) (hash(x)=157395496)
+1800 val loss 7.0269
+1800 val perplexity 1126.5381
+1800 train 7.044158 (lr=3.1887e-05) (hash(x)=157916369)
+1900 val loss 6.9853
+1900 val perplexity 1080.6533
+1900 train 7.202100 (lr=3.1809e-05) (hash(x)=166073923)
+2000 val loss 6.9577
+2000 val perplexity 1051.2118
+2000 train 7.038825 (lr=3.1726e-05) (hash(x)=154856891)
+2100 val loss 6.9203
+2100 val perplexity 1012.6016
+2100 train 6.912439 (lr=3.1638e-05) (hash(x)=151925203)
+2200 val loss 6.8823
+2200 val perplexity 974.8445
+2200 train 6.584104 (lr=3.1546e-05) (hash(x)=136191502)
+2300 val loss 6.8423
+2300 val perplexity 936.6110
+2300 train 7.000276 (lr=3.1449e-05) (hash(x)=153273362)
+2400 val loss 6.8023
+2400 val perplexity 899.9409
+2400 train 6.751945 (lr=3.1348e-05) (hash(x)=148021541)
+2500 val loss 6.7656
+2500 val perplexity 867.4752
+2500 train 6.709182 (lr=3.1243e-05) (hash(x)=141356608)
+2600 val loss 6.7277
+2600 val perplexity 835.2031
+2600 train 6.673390 (lr=3.1133e-05) (hash(x)=146005217)
+2700 val loss 6.6951
+2700 val perplexity 808.4293
+2700 train 6.534774 (lr=3.1019e-05) (hash(x)=144511718)
+2800 val loss 6.6608
+2800 val perplexity 781.1756
+2800 train 6.540296 (lr=3.0900e-05) (hash(x)=146019502)
+2900 val loss 6.6303
+2900 val perplexity 757.6885
+2900 train 6.544481 (lr=3.0777e-05) (hash(x)=146496200)
+3000 val loss 6.5995
+3000 val perplexity 734.6974
+3000 train 6.548376 (lr=3.0650e-05) (hash(x)=150127281)
+3100 val loss 6.5634
+3100 val perplexity 708.6861
+3100 train 6.541520 (lr=3.0518e-05) (hash(x)=142022255)
+3200 val loss 6.5337
+3200 val perplexity 687.9238
+3200 train 6.587421 (lr=3.0383e-05) (hash(x)=154120875)
+3300 val loss 6.5133
+3300 val perplexity 674.0387
+3300 train 6.571418 (lr=3.0243e-05) (hash(x)=153999717)
+3400 val loss 6.4929
+3400 val perplexity 660.4374
+3400 train 6.258515 (lr=3.0099e-05) (hash(x)=139694097)
+3500 val loss 6.4649
+3500 val perplexity 642.1870
+3500 train 6.649519 (lr=2.9952e-05) (hash(x)=162992732)
+3600 val loss 6.4301
+3600 val perplexity 620.2365
+3600 train 6.405787 (lr=2.9800e-05) (hash(x)=147574101)
+3700 val loss 6.4131
+3700 val perplexity 609.7664
+3700 train 6.523832 (lr=2.9644e-05) (hash(x)=157763099)
+3800 val loss 6.3888
+3800 val perplexity 595.1349
+3800 train 6.553192 (lr=2.9485e-05) (hash(x)=170800034)
+3900 val loss 6.3732
+3900 val perplexity 585.9507
+3900 train 6.449968 (lr=2.9321e-05) (hash(x)=164984528)
+4000 val loss 6.3589
+4000 val perplexity 577.6016
+4000 train 6.219861 (lr=2.9154e-05) (hash(x)=141743323)
+4100 val loss 6.3496
+4100 val perplexity 572.2399
+4100 train 6.364213 (lr=2.8983e-05) (hash(x)=153392872)
+4200 val loss 6.3388
+4200 val perplexity 566.0950
+4200 train 6.200565 (lr=2.8808e-05) (hash(x)=149074933)
+4300 val loss 6.3134
+4300 val perplexity 551.9182
+4300 train 6.704398 (lr=2.8630e-05) (hash(x)=167823423)
+4400 val loss 6.2896
+4400 val perplexity 538.9551
+4400 train 6.058969 (lr=2.8448e-05) (hash(x)=141203114)
+4500 val loss 6.2706
+4500 val perplexity 528.7738
+4500 train 6.267660 (lr=2.8263e-05) (hash(x)=146284780)
+4600 val loss 6.2508
+4600 val perplexity 518.4062
+4600 train 6.053355 (lr=2.8075e-05) (hash(x)=141126464)
+4700 val loss 6.2356
+4700 val perplexity 510.5971
+4700 train 6.261318 (lr=2.7883e-05) (hash(x)=154751926)
+4800 val loss 6.2221
+4800 val perplexity 503.7436
+4800 train 6.305309 (lr=2.7687e-05) (hash(x)=154793198)
+4900 val loss 6.2138
+4900 val perplexity 499.5989
+4900 train 5.932302 (lr=2.7489e-05) (hash(x)=139406392)
+5000 val loss 6.2221
+5000 val perplexity 503.7762
+5000 train 5.932709 (lr=2.7287e-05) (hash(x)=153548741)
+5100 val loss 6.1919
+5100 val perplexity 488.7556
+5100 train 6.270887 (lr=2.7082e-05) (hash(x)=160488568)
+5200 val loss 6.1705
+5200 val perplexity 478.4274
+5200 train 6.178745 (lr=2.6874e-05) (hash(x)=149645053)
+5300 val loss 6.1507
+5300 val perplexity 469.0544
+5300 train 6.247884 (lr=2.6664e-05) (hash(x)=155820556)
+5400 val loss 6.1414
+5400 val perplexity 464.7147
+5400 train 6.085527 (lr=2.6450e-05) (hash(x)=147538134)
+5500 val loss 6.1284
+5500 val perplexity 458.7081
+5500 train 6.339289 (lr=2.6233e-05) (hash(x)=166889307)
+5600 val loss 6.1148
+5600 val perplexity 452.4975
+5600 train 5.816920 (lr=2.6014e-05) (hash(x)=139516699)
+5700 val loss 6.1153
+5700 val perplexity 452.7293
+5700 train 5.789756 (lr=2.5792e-05) (hash(x)=140453511)
+5800 val loss 6.1057
+5800 val perplexity 448.4195
+5800 train 6.046035 (lr=2.5567e-05) (hash(x)=162964847)
+5900 val loss 6.0923
+5900 val perplexity 442.4182
+5900 train 6.095953 (lr=2.5340e-05) (hash(x)=150606634)
+6000 val loss 6.0706
+6000 val perplexity 432.9581
+6000 train 6.155572 (lr=2.5111e-05) (hash(x)=149890857)
+6100 val loss 6.0665
+6100 val perplexity 431.1557
+6100 train 6.164382 (lr=2.4879e-05) (hash(x)=173884145)
+6200 val loss 6.0532
+6200 val perplexity 425.4883
+6200 train 6.114431 (lr=2.4644e-05) (hash(x)=151987098)
+6300 val loss 6.0394
+6300 val perplexity 419.6264
+6300 train 5.991861 (lr=2.4408e-05) (hash(x)=148853562)
+6400 val loss 6.0311
+6400 val perplexity 416.1762
+6400 train 5.778894 (lr=2.4169e-05) (hash(x)=141530101)
+6500 val loss 6.0240
+6500 val perplexity 413.2226
+6500 train 5.882457 (lr=2.3928e-05) (hash(x)=142297809)
+6600 val loss 6.0092
+6600 val perplexity 407.1544
+6600 train 5.900824 (lr=2.3686e-05) (hash(x)=142447782)
+6700 val loss 6.0062
+6700 val perplexity 405.9474
+6700 train 5.991254 (lr=2.3441e-05) (hash(x)=147004686)
+6800 val loss 5.9924
+6800 val perplexity 400.3843
+6800 train 5.738660 (lr=2.3195e-05) (hash(x)=133438702)
+6900 val loss 5.9919
+6900 val perplexity 400.1860
+6900 train 6.006359 (lr=2.2946e-05) (hash(x)=157085143)
+7000 val loss 5.9796
+7000 val perplexity 395.2977
+7000 train 5.865448 (lr=2.2696e-05) (hash(x)=139437666)
+7100 val loss 5.9743
+7100 val perplexity 393.1910
+7100 train 5.988501 (lr=2.2445e-05) (hash(x)=159792986)
+7200 val loss 5.9581
+7200 val perplexity 386.8573
+7200 train 5.942185 (lr=2.2192e-05) (hash(x)=144930687)
+7300 val loss 5.9490
+7300 val perplexity 383.3533
+7300 train 6.009021 (lr=2.1937e-05) (hash(x)=156242690)
+7400 val loss 5.9416
+7400 val perplexity 380.5278
+7400 train 5.777550 (lr=2.1682e-05) (hash(x)=148183719)
+7500 val loss 5.9370
+7500 val perplexity 378.7969
+7500 train 5.975434 (lr=2.1425e-05) (hash(x)=152494758)
+7600 val loss 5.9312
+7600 val perplexity 376.6238
+7600 train 5.619554 (lr=2.1166e-05) (hash(x)=142485027)
+7700 val loss 5.9281
+7700 val perplexity 375.4567
+7700 train 5.795169 (lr=2.0907e-05) (hash(x)=147512165)
+7800 val loss 5.9124
+7800 val perplexity 369.5941
+7800 train 5.783484 (lr=2.0647e-05) (hash(x)=160346994)
+7900 val loss 5.9094
+7900 val perplexity 368.4920
+7900 train 5.783132 (lr=2.0386e-05) (hash(x)=144488254)
+8000 val loss 5.8979
+8000 val perplexity 364.2743
+8000 train 5.709069 (lr=2.0123e-05) (hash(x)=147637019)
+8100 val loss 5.8961
+8100 val perplexity 363.6187
+8100 train 5.744081 (lr=1.9861e-05) (hash(x)=147340534)
+8200 val loss 5.8903
+8200 val perplexity 361.5074
+8200 train 5.946971 (lr=1.9597e-05) (hash(x)=151630665)
+8300 val loss 5.8814
+8300 val perplexity 358.3273
+8300 train 5.982193 (lr=1.9333e-05) (hash(x)=149747064)
+8400 val loss 5.8713
+8400 val perplexity 354.7009
+8400 train 6.068233 (lr=1.9069e-05) (hash(x)=154245770)
+8500 val loss 5.8662
+8500 val perplexity 352.9071
+8500 train 5.753820 (lr=1.8804e-05) (hash(x)=152559100)
+8600 val loss 5.8618
+8600 val perplexity 351.3425
+8600 train 6.275620 (lr=1.8539e-05) (hash(x)=181365926)
+8700 val loss 5.8589
+8700 val perplexity 350.3235
+8700 train 5.692950 (lr=1.8273e-05) (hash(x)=154405991)
+8800 val loss 5.8489
+8800 val perplexity 346.8562
+8800 train 5.792358 (lr=1.8008e-05) (hash(x)=153755904)
+8900 val loss 5.8465
+8900 val perplexity 346.0291
+8900 train 5.764337 (lr=1.7742e-05) (hash(x)=152120568)
+9000 val loss 5.8297
+9000 val perplexity 340.2657
+9000 train 5.617670 (lr=1.7477e-05) (hash(x)=142797279)
+9100 val loss 5.8270
+9100 val perplexity 339.3379
+9100 train 5.691728 (lr=1.7211e-05) (hash(x)=143037503)
+9200 val loss 5.8270
+9200 val perplexity 339.3499
+9200 train 5.694878 (lr=1.6946e-05) (hash(x)=113690273)
+9300 val loss 5.8200
+9300 val perplexity 336.9655
+9300 train 5.757028 (lr=1.6681e-05) (hash(x)=158025077)
+9400 val loss 5.8099
+9400 val perplexity 333.5787
+9400 train 5.883915 (lr=1.6417e-05) (hash(x)=158251718)
+9500 val loss 5.8112
+9500 val perplexity 334.0197
+9500 train 5.848474 (lr=1.6153e-05) (hash(x)=154752610)
+9600 val loss 5.8045
+9600 val perplexity 331.7947
+9600 train 5.681903 (lr=1.5889e-05) (hash(x)=146889093)
+9700 val loss 5.7924
+9700 val perplexity 327.8095
+9700 train 5.801524 (lr=1.5627e-05) (hash(x)=156906516)
+9800 val loss 5.7899
+9800 val perplexity 326.9742
+9800 train 5.621650 (lr=1.5364e-05) (hash(x)=153841927)
+9900 val loss 5.7852
+9900 val perplexity 325.4485
+9900 train 5.999937 (lr=1.5103e-05) (hash(x)=163514334)
+10000 val loss 5.7790
+10000 val perplexity 323.4331
+10000 train 5.769652 (lr=1.4843e-05) (hash(x)=144745561)
+10100 val loss 5.7745
+10100 val perplexity 321.9720
+10100 train 6.025921 (lr=1.4584e-05) (hash(x)=161651149)
+10200 val loss 5.7679
+10200 val perplexity 319.8694
+10200 train 5.710236 (lr=1.4325e-05) (hash(x)=143667037)
+10300 val loss 5.7619
+10300 val perplexity 317.9444
+10300 train 5.654992 (lr=1.4068e-05) (hash(x)=151687262)
+10400 val loss 5.7627
+10400 val perplexity 318.2142
+10400 train 5.627633 (lr=1.3813e-05) (hash(x)=153694955)
+10500 val loss 5.7588
+10500 val perplexity 316.9752
+10500 train 5.704593 (lr=1.3558e-05) (hash(x)=150218029)
+10600 val loss 5.7528
+10600 val perplexity 315.0698
+10600 train 5.851352 (lr=1.3305e-05) (hash(x)=155093716)
+10700 val loss 5.7477
+10700 val perplexity 313.4625
+10700 train 5.787298 (lr=1.3054e-05) (hash(x)=151631831)
+10800 val loss 5.7389
+10800 val perplexity 310.7081
+10800 train 5.586615 (lr=1.2804e-05) (hash(x)=136465175)
+10900 val loss 5.7369
+10900 val perplexity 310.0930
+10900 train 5.669595 (lr=1.2555e-05) (hash(x)=154461514)
+11000 val loss 5.7358
+11000 val perplexity 309.7675
+11000 train 5.728212 (lr=1.2309e-05) (hash(x)=141877432)
+11100 val loss 5.7325
+11100 val perplexity 308.7520
+11100 train 5.526520 (lr=1.2064e-05) (hash(x)=139572894)
+11200 val loss 5.7246
+11200 val perplexity 306.2994
+11200 train 5.771356 (lr=1.1822e-05) (hash(x)=155591443)
+11300 val loss 5.7183
+11300 val perplexity 304.3907
+11300 train 5.733082 (lr=1.1581e-05) (hash(x)=153349136)
+11400 val loss 5.7172
+11400 val perplexity 304.0620
+11400 train 5.671595 (lr=1.1342e-05) (hash(x)=148109395)
+11500 val loss 5.7130
+11500 val perplexity 302.7882
+11500 train 5.702454 (lr=1.1106e-05) (hash(x)=153276620)
+11600 val loss 5.7125
+11600 val perplexity 302.6309
+11600 train 5.571287 (lr=1.0871e-05) (hash(x)=151416388)
+11700 val loss 5.7101
+11700 val perplexity 301.9037
+11700 train 5.582951 (lr=1.0639e-05) (hash(x)=143420927)
+11800 val loss 5.7050
+11800 val perplexity 300.3752
+11800 train 5.772466 (lr=1.0410e-05) (hash(x)=165023996)
+11900 val loss 5.6986
+11900 val perplexity 298.4425
+11900 train 5.614121 (lr=1.0183e-05) (hash(x)=144465076)
+12000 val loss 5.6963
+12000 val perplexity 297.7500
+12000 train 5.707811 (lr=9.9580e-06) (hash(x)=155991892)
+12100 val loss 5.6944
+12100 val perplexity 297.1982
+12100 train 5.728391 (lr=9.7360e-06) (hash(x)=150748844)
+12200 val loss 5.6913
+12200 val perplexity 296.2873
+12200 train 5.653107 (lr=9.5167e-06) (hash(x)=149712638)
+12300 val loss 5.6921
+12300 val perplexity 296.5135
+12300 train 5.510970 (lr=9.3001e-06) (hash(x)=140428230)
+12400 val loss 5.6856
+12400 val perplexity 294.6078
+12400 train 5.866589 (lr=9.0864e-06) (hash(x)=163614308)
+12500 val loss 5.6804
+12500 val perplexity 293.0593
+12500 train 5.644887 (lr=8.8756e-06) (hash(x)=153366153)
+12600 val loss 5.6785
+12600 val perplexity 292.5041
+12600 train 5.570187 (lr=8.6677e-06) (hash(x)=146832066)
+12700 val loss 5.6759
+12700 val perplexity 291.7508
+12700 train 5.852587 (lr=8.4629e-06) (hash(x)=154387414)
+12800 val loss 5.6739
+12800 val perplexity 291.1724
+12800 train 5.470691 (lr=8.2612e-06) (hash(x)=139465199)
+12900 val loss 5.6761
+12900 val perplexity 291.7956
+12900 train 5.386293 (lr=8.0627e-06) (hash(x)=143326374)
+13000 val loss 5.6690
+13000 val perplexity 289.7305
+13000 train 5.826969 (lr=7.8674e-06) (hash(x)=158817835)
+13100 val loss 5.6663
+13100 val perplexity 288.9554
+13100 train 5.546062 (lr=7.6754e-06) (hash(x)=142027874)
+13200 val loss 5.6657
+13200 val perplexity 288.7859
+13200 train 5.648112 (lr=7.4867e-06) (hash(x)=148497730)
+13300 val loss 5.6641
+13300 val perplexity 288.3189
+13300 train 5.566710 (lr=7.3015e-06) (hash(x)=148094633)
+13400 val loss 5.6629
+13400 val perplexity 287.9900
+13400 train 5.538298 (lr=7.1198e-06) (hash(x)=150548214)
+13500 val loss 5.6550
+13500 val perplexity 285.7133
+13500 train 5.763052 (lr=6.9416e-06) (hash(x)=152789725)
+13600 val loss 5.6540
+13600 val perplexity 285.4225
+13600 train 5.692722 (lr=6.7670e-06) (hash(x)=153220230)
+13700 val loss 5.6499
+13700 val perplexity 284.2614
+13700 train 5.663307 (lr=6.5961e-06) (hash(x)=147805699)
+13800 val loss 5.6506
+13800 val perplexity 284.4547
+13800 train 5.465456 (lr=6.4289e-06) (hash(x)=146267341)
+13900 val loss 5.6502
+13900 val perplexity 284.3566
+13900 train 5.469760 (lr=6.2655e-06) (hash(x)=139467501)
+14000 val loss 5.6485
+14000 val perplexity 283.8530
+14000 train 5.660518 (lr=6.1059e-06) (hash(x)=152185436)
+14100 val loss 5.6417
+14100 val perplexity 281.9511
+14100 train 5.761265 (lr=5.9502e-06) (hash(x)=176479139)
+14200 val loss 5.6392
+14200 val perplexity 281.2393
+14200 train 5.739030 (lr=5.7984e-06) (hash(x)=151052040)
+14300 val loss 5.6375
+14300 val perplexity 280.7668
+14300 train 5.578081 (lr=5.6506e-06) (hash(x)=149668097)
+14400 val loss 5.6364
+14400 val perplexity 280.4461
+14400 train 5.648204 (lr=5.5068e-06) (hash(x)=151761941)
+14500 val loss 5.6360
+14500 val perplexity 280.3417
+14500 train 5.591083 (lr=5.3671e-06) (hash(x)=149412009)
+14600 val loss 5.6344
+14600 val perplexity 279.9049
+14600 train 5.499872 (lr=5.2316e-06) (hash(x)=145048020)
+14700 val loss 5.6318
+14700 val perplexity 279.1771
+14700 train 5.587432 (lr=5.1002e-06) (hash(x)=141771666)
+14800 val loss 5.6281
+14800 val perplexity 278.1451
+14800 train 5.560456 (lr=4.9730e-06) (hash(x)=148759740)
+14900 val loss 5.6269
+14900 val perplexity 277.7952
+14900 train 5.551805 (lr=4.8501e-06) (hash(x)=146287032)
+15000 val loss 5.6254
+15000 val perplexity 277.3827
+15000 train 5.572130 (lr=4.7314e-06) (hash(x)=143519156)
+15100 val loss 5.6268
+15100 val perplexity 277.7812
+15100 train 5.608938 (lr=4.6171e-06) (hash(x)=151330688)
+15200 val loss 5.6293
+15200 val perplexity 278.4634
+15200 train 5.424197 (lr=4.5072e-06) (hash(x)=140380090)
+15300 val loss 5.6243
+15300 val perplexity 277.0752
+15300 train 5.524822 (lr=4.4017e-06) (hash(x)=152598160)
+15400 val loss 5.6190
+15400 val perplexity 275.6197
+15400 train 5.879935 (lr=4.3006e-06) (hash(x)=164136966)
+15500 val loss 5.6199
+15500 val perplexity 275.8588
+15500 train 5.826211 (lr=4.2040e-06) (hash(x)=158733990)
+15600 val loss 5.6162
+15600 val perplexity 274.8502
+15600 train 5.653137 (lr=4.1119e-06) (hash(x)=149149390)
+15700 val loss 5.6159
+15700 val perplexity 274.7505
+15700 train 5.570819 (lr=4.0244e-06) (hash(x)=146746394)
+15800 val loss 5.6159
+15800 val perplexity 274.7523
+15800 train 5.637339 (lr=3.9414e-06) (hash(x)=159405219)
+15900 val loss 5.6143
+15900 val perplexity 274.3232
+15900 train 5.642344 (lr=3.8630e-06) (hash(x)=154034029)
+16000 val loss 5.6118
+16000 val perplexity 273.6403
+16000 train 5.622232 (lr=3.7892e-06) (hash(x)=153634210)
+16100 val loss 5.6090
+16100 val perplexity 272.8640
+16100 train 5.539718 (lr=3.7201e-06) (hash(x)=155646721)
+16200 val loss 5.6092
+16200 val perplexity 272.9309
+16200 train 5.523324 (lr=3.6556e-06) (hash(x)=142461643)
+16300 val loss 5.6083
+16300 val perplexity 272.6779
+16300 train 5.506813 (lr=3.5959e-06) (hash(x)=141080038)
+16400 val loss 5.6087
+16400 val perplexity 272.7780
+16400 train 5.624062 (lr=3.5408e-06) (hash(x)=153744166)
+16500 val loss 5.6120
+16500 val perplexity 273.6787
+16500 train 5.419729 (lr=3.4905e-06) (hash(x)=149783517)
+16600 val loss 5.6132
+16600 val perplexity 274.0175
+16600 train 5.444574 (lr=3.4449e-06) (hash(x)=142659127)
+16700 val loss 5.6057
+16700 val perplexity 271.9776
+16700 train 5.668278 (lr=3.4041e-06) (hash(x)=149725921)
+16800 val loss 5.6037
+16800 val perplexity 271.4249
+16800 train 5.455229 (lr=3.3680e-06) (hash(x)=140678087)
+16900 val loss 5.6021
+16900 val perplexity 270.9842
+16900 train 5.614109 (lr=3.3367e-06) (hash(x)=146935413)
+17000 val loss 5.6004
+17000 val perplexity 270.5381
+17000 train 5.614009 (lr=3.3102e-06) (hash(x)=144408437)
+17100 val loss 5.6000
+17100 val perplexity 270.4159
+17100 train 5.451910 (lr=3.2886e-06) (hash(x)=149229905)
+17200 val loss 5.6000
+17200 val perplexity 270.4264
+17200 train 5.549056 (lr=3.2717e-06) (hash(x)=150471865)
+17300 val loss 5.6026
+17300 val perplexity 271.1258
+17300 train 5.492773 (lr=3.2596e-06) (hash(x)=155784916)
+17400 val loss 5.6052
+17400 val perplexity 271.8368
+17400 train 5.381482 (lr=3.2524e-06) (hash(x)=148482413)
+17499 val loss 5.5979
+17499 val perplexity 269.8707

attention_kindselective_n_heads4_seed1340/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ad465790d12e81945fcd6204b4bdc039b76e45eaafcabb5731f4107a3f4fecf
+oid sha256:a172398ba2e69bcf26655d3b124c6e88d83154a9dbaee1270ed90a1898458663
 size 92843394

attention_kindselective_n_heads4_seed1340/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:af63791d2b571558242572efd3e76828c1fbeefe6a2aea1f5eb569e9f5e097c3
+oid sha256:db5e34eda7ce004231ee080ffa4fbd862d3bb53a9f408e9f8709da0053c1c277
 size 179406214