Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,528 +1,528 @@
 max_steps: 17500
 0 val loss 11.2663
 0 val perplexity 78147.0000
-0 train 11.263341 (lr=1.5000e-07) (hash(x)=150327452)
-100 val loss 9.9014
-100 val perplexity 19958.0000
-100 train 9.912344 (lr=1.5150e-05) (hash(x)=166441190)
-200 val loss 8.8366
-200 val perplexity 6881.3281
-200 train 8.955378 (lr=3.0000e-05) (hash(x)=166780046)
-300 val loss 7.8593
-300 val perplexity 2589.7876
-300 train 7.803794 (lr=2.9998e-05) (hash(x)=159835303)
-400 val loss 7.5790
-400 val perplexity 1956.7469
-400 train 7.361750 (lr=2.9991e-05) (hash(x)=155040610)
-500 val loss 7.4912
-500 val perplexity 1792.2339
-500 train 7.228088 (lr=2.9980e-05) (hash(x)=130190460)
-600 val loss 7.4113
-600 val perplexity 1654.6433
-600 train 7.433746 (lr=2.9964e-05) (hash(x)=155504036)
-700 val loss 7.3336
-700 val perplexity 1530.9218
-700 train 7.190465 (lr=2.9944e-05) (hash(x)=137347213)
-800 val loss 7.2757
-800 val perplexity 1444.7686
-800 train 7.128286 (lr=2.9920e-05) (hash(x)=143823248)
-900 val loss 7.2234
-900 val perplexity 1371.0818
-900 train 7.338418 (lr=2.9891e-05) (hash(x)=156260416)
-1000 val loss 7.1780
-1000 val perplexity 1310.3419
-1000 train 7.143191 (lr=2.9858e-05) (hash(x)=143734685)
-1100 val loss 7.1197
-1100 val perplexity 1236.0919
-1100 train 7.042824 (lr=2.9820e-05) (hash(x)=160013925)
-1200 val loss 7.0802
-1200 val perplexity 1188.1973
-1200 train 6.814084 (lr=2.9778e-05) (hash(x)=150678249)
-1300 val loss 7.0305
-1300 val perplexity 1130.6367
-1300 train 6.803394 (lr=2.9732e-05) (hash(x)=149073315)
-1400 val loss 6.9647
-1400 val perplexity 1058.5557
-1400 train 7.277344 (lr=2.9681e-05) (hash(x)=175802021)
-1500 val loss 6.9098
-1500 val perplexity 1002.0621
-1500 train 7.176047 (lr=2.9626e-05) (hash(x)=171034639)
-1600 val loss 6.8487
-1600 val perplexity 942.6979
-1600 train 7.057546 (lr=2.9566e-05) (hash(x)=158681215)
-1700 val loss 6.8005
-1700 val perplexity 898.2830
-1700 train 6.742936 (lr=2.9502e-05) (hash(x)=152116061)
-1800 val loss 6.7597
-1800 val perplexity 862.3545
-1800 train 6.696222 (lr=2.9434e-05) (hash(x)=146108145)
-1900 val loss 6.7213
-1900 val perplexity 829.9302
-1900 train 6.552172 (lr=2.9362e-05) (hash(x)=147598108)
-2000 val loss 6.6885
-2000 val perplexity 803.0875
-2000 train 6.533982 (lr=2.9285e-05) (hash(x)=154996086)
-2100 val loss 6.6541
-2100 val perplexity 775.9614
-2100 train 6.430016 (lr=2.9204e-05) (hash(x)=153396183)
-2200 val loss 6.5975
-2200 val perplexity 733.2267
-2200 train 6.589885 (lr=2.9119e-05) (hash(x)=153885445)
-2300 val loss 6.5654
-2300 val perplexity 710.1285
-2300 train 6.570909 (lr=2.9030e-05) (hash(x)=159666385)
-2400 val loss 6.5211
-2400 val perplexity 679.3126
-2400 train 6.510075 (lr=2.8937e-05) (hash(x)=142353087)
-2500 val loss 6.4907
-2500 val perplexity 658.9900
-2500 train 6.512077 (lr=2.8840e-05) (hash(x)=146491718)
-2600 val loss 6.4671
-2600 val perplexity 643.6051
-2600 train 6.386323 (lr=2.8738e-05) (hash(x)=150750353)
-2700 val loss 6.4439
-2700 val perplexity 628.8747
-2700 train 6.166628 (lr=2.8633e-05) (hash(x)=129849193)
-2800 val loss 6.4297
-2800 val perplexity 619.9781
-2800 train 6.259184 (lr=2.8523e-05) (hash(x)=152767913)
-2900 val loss 6.4164
-2900 val perplexity 611.7715
-2900 train 6.186160 (lr=2.8410e-05) (hash(x)=146531140)
-3000 val loss 6.3808
-3000 val perplexity 590.4099
-3000 train 6.471205 (lr=2.8292e-05) (hash(x)=151562048)
-3100 val loss 6.3554
-3100 val perplexity 575.6011
-3100 train 6.312563 (lr=2.8171e-05) (hash(x)=146001424)
-3200 val loss 6.3392
-3200 val perplexity 566.3698
-3200 train 6.420431 (lr=2.8046e-05) (hash(x)=166486165)
-3300 val loss 6.3088
-3300 val perplexity 549.3705
-3300 train 6.239015 (lr=2.7917e-05) (hash(x)=150866680)
-3400 val loss 6.3019
-3400 val perplexity 545.5984
-3400 train 6.192265 (lr=2.7784e-05) (hash(x)=143900419)
-3500 val loss 6.2890
-3500 val perplexity 538.6371
-3500 train 6.110422 (lr=2.7648e-05) (hash(x)=148845794)
-3600 val loss 6.2716
-3600 val perplexity 529.3439
-3600 train 6.100612 (lr=2.7508e-05) (hash(x)=145667796)
-3700 val loss 6.2611
-3700 val perplexity 523.8117
-3700 train 6.278327 (lr=2.7364e-05) (hash(x)=163563851)
-3800 val loss 6.2502
-3800 val perplexity 518.0978
-3800 train 6.200144 (lr=2.7216e-05) (hash(x)=147488689)
-3900 val loss 6.2217
-3900 val perplexity 503.5543
-3900 train 6.232891 (lr=2.7066e-05) (hash(x)=148186608)
-4000 val loss 6.2090
-4000 val perplexity 497.1790
-4000 train 6.097112 (lr=2.6911e-05) (hash(x)=142970187)
-4100 val loss 6.1900
-4100 val perplexity 487.8564
-4100 train 6.318220 (lr=2.6754e-05) (hash(x)=141584883)
-4200 val loss 6.1778
-4200 val perplexity 481.9508
-4200 train 5.972452 (lr=2.6592e-05) (hash(x)=145664585)
-4300 val loss 6.1629
-4300 val perplexity 474.8204
-4300 train 6.035709 (lr=2.6428e-05) (hash(x)=143736499)
-4400 val loss 6.1599
-4400 val perplexity 473.3930
-4400 train 5.981257 (lr=2.6260e-05) (hash(x)=151883322)
-4500 val loss 6.1525
-4500 val perplexity 469.8898
-4500 train 6.029921 (lr=2.6089e-05) (hash(x)=153904871)
-4600 val loss 6.1340
-4600 val perplexity 461.2766
-4600 train 6.236507 (lr=2.5915e-05) (hash(x)=154893521)
-4700 val loss 6.1143
-4700 val perplexity 452.2870
-4700 train 6.159433 (lr=2.5738e-05) (hash(x)=152323949)
-4800 val loss 6.0991
-4800 val perplexity 445.4421
-4800 train 6.051072 (lr=2.5558e-05) (hash(x)=154104619)
-4900 val loss 6.0881
-4900 val perplexity 440.5904
-4900 train 6.224781 (lr=2.5374e-05) (hash(x)=146311426)
-5000 val loss 6.0798
-5000 val perplexity 436.9629
-5000 train 6.060164 (lr=2.5188e-05) (hash(x)=156741847)
-5100 val loss 6.0701
-5100 val perplexity 432.7412
-5100 train 5.850953 (lr=2.4999e-05) (hash(x)=142086346)
-5200 val loss 6.0631
-5200 val perplexity 429.6880
-5200 train 5.839406 (lr=2.4807e-05) (hash(x)=150265428)
-5300 val loss 6.0595
-5300 val perplexity 428.1700
-5300 train 5.935390 (lr=2.4613e-05) (hash(x)=151339108)
-5400 val loss 6.0461
-5400 val perplexity 422.4570
-5400 train 6.015918 (lr=2.4415e-05) (hash(x)=154654372)
-5500 val loss 6.0314
-5500 val perplexity 416.3066
-5500 train 6.041653 (lr=2.4215e-05) (hash(x)=150575051)
-5600 val loss 6.0226
-5600 val perplexity 412.6557
-5600 train 5.942938 (lr=2.4013e-05) (hash(x)=140396423)
-5700 val loss 6.0114
-5700 val perplexity 408.0675
-5700 train 5.867752 (lr=2.3808e-05) (hash(x)=144678758)
-5800 val loss 6.0055
-5800 val perplexity 405.6359
-5800 train 6.061187 (lr=2.3601e-05) (hash(x)=151992743)
-5900 val loss 5.9950
-5900 val perplexity 401.4122
-5900 train 5.792812 (lr=2.3391e-05) (hash(x)=144396927)
-6000 val loss 5.9863
-6000 val perplexity 397.9262
-6000 train 5.942162 (lr=2.3179e-05) (hash(x)=165478625)
-6100 val loss 5.9891
-6100 val perplexity 399.0456
-6100 train 5.721336 (lr=2.2965e-05) (hash(x)=147088621)
-6200 val loss 5.9814
-6200 val perplexity 395.9857
-6200 train 5.744236 (lr=2.2749e-05) (hash(x)=140794994)
-6300 val loss 5.9617
-6300 val perplexity 388.2704
-6300 train 5.908691 (lr=2.2530e-05) (hash(x)=134780906)
-6400 val loss 5.9512
-6400 val perplexity 384.2211
-6400 train 5.939652 (lr=2.2310e-05) (hash(x)=149023655)
-6500 val loss 5.9371
-6500 val perplexity 378.8390
-6500 train 5.787305 (lr=2.2088e-05) (hash(x)=147497796)
-6600 val loss 5.9364
-6600 val perplexity 378.5837
-6600 train 5.789012 (lr=2.1864e-05) (hash(x)=152902689)
-6700 val loss 5.9391
-6700 val perplexity 379.5893
-6700 train 5.982219 (lr=2.1638e-05) (hash(x)=153846046)
-6800 val loss 5.9154
-6800 val perplexity 370.7135
-6800 train 6.043589 (lr=2.1410e-05) (hash(x)=158512738)
-6900 val loss 5.9053
-6900 val perplexity 366.9849
-6900 train 6.442078 (lr=2.1181e-05) (hash(x)=156849968)
-7000 val loss 5.8981
-7000 val perplexity 364.3483
-7000 train 5.747692 (lr=2.0951e-05) (hash(x)=142395855)
-7100 val loss 5.8983
-7100 val perplexity 364.4187
-7100 train 5.745164 (lr=2.0718e-05) (hash(x)=147114884)
-7200 val loss 5.8833
-7200 val perplexity 358.9946
-7200 train 5.913739 (lr=2.0485e-05) (hash(x)=156979839)
-7300 val loss 5.8794
-7300 val perplexity 357.5988
-7300 train 5.672119 (lr=2.0250e-05) (hash(x)=145584373)
-7400 val loss 5.8717
-7400 val perplexity 354.8532
-7400 train 5.615767 (lr=2.0014e-05) (hash(x)=141508204)
-7500 val loss 5.8635
-7500 val perplexity 351.9554
-7500 train 5.898952 (lr=1.9777e-05) (hash(x)=148803965)
-7600 val loss 5.8569
-7600 val perplexity 349.6490
-7600 train 5.917202 (lr=1.9538e-05) (hash(x)=151019676)
-7700 val loss 5.8453
-7700 val perplexity 345.6068
-7700 train 5.947480 (lr=1.9299e-05) (hash(x)=143155750)
-7800 val loss 5.8386
-7800 val perplexity 343.2999
-7800 train 5.866199 (lr=1.9059e-05) (hash(x)=152569653)
-7900 val loss 5.8312
-7900 val perplexity 340.7828
-7900 train 5.705254 (lr=1.8817e-05) (hash(x)=143519455)
-8000 val loss 5.8251
-8000 val perplexity 338.6843
-8000 train 5.929979 (lr=1.8576e-05) (hash(x)=161180944)
-8100 val loss 5.8268
-8100 val perplexity 339.2856
-8100 train 5.857555 (lr=1.8333e-05) (hash(x)=154107345)
-8200 val loss 5.8214
-8200 val perplexity 337.4545
-8200 train 5.856858 (lr=1.8090e-05) (hash(x)=152486517)
-8300 val loss 5.8096
-8300 val perplexity 333.4917
-8300 train 5.808879 (lr=1.7846e-05) (hash(x)=156167749)
-8400 val loss 5.8043
-8400 val perplexity 331.7203
-8400 train 5.892135 (lr=1.7602e-05) (hash(x)=149155006)
-8500 val loss 5.7925
-8500 val perplexity 327.8286
-8500 train 5.742015 (lr=1.7357e-05) (hash(x)=147844390)
-8600 val loss 5.7897
-8600 val perplexity 326.9220
-8600 train 6.050057 (lr=1.7113e-05) (hash(x)=165753320)
-8700 val loss 5.7839
-8700 val perplexity 325.0404
-8700 train 5.560269 (lr=1.6868e-05) (hash(x)=146079979)
-8800 val loss 5.7823
-8800 val perplexity 324.5202
-8800 train 5.968102 (lr=1.6623e-05) (hash(x)=172259509)
-8900 val loss 5.7766
-8900 val perplexity 322.6667
-8900 train 5.455958 (lr=1.6377e-05) (hash(x)=145148314)
-9000 val loss 5.7730
-9000 val perplexity 321.4933
-9000 train 5.693444 (lr=1.6132e-05) (hash(x)=144250633)
-9100 val loss 5.7631
-9100 val perplexity 318.3478
-9100 train 5.865222 (lr=1.5887e-05) (hash(x)=157219797)
-9200 val loss 5.7563
-9200 val perplexity 316.1644
-9200 train 5.801858 (lr=1.5643e-05) (hash(x)=142743778)
-9300 val loss 5.7519
-9300 val perplexity 314.7833
-9300 train 5.675515 (lr=1.5398e-05) (hash(x)=139669771)
-9400 val loss 5.7446
-9400 val perplexity 312.5020
-9400 train 5.790522 (lr=1.5154e-05) (hash(x)=145916843)
-9500 val loss 5.7433
-9500 val perplexity 312.0899
-9500 train 5.667969 (lr=1.4910e-05) (hash(x)=150196125)
-9600 val loss 5.7371
-9600 val perplexity 310.1722
-9600 train 6.052320 (lr=1.4667e-05) (hash(x)=160041419)
-9700 val loss 5.7378
-9700 val perplexity 310.3841
-9700 train 5.515315 (lr=1.4424e-05) (hash(x)=139931627)
-9800 val loss 5.7303
-9800 val perplexity 308.0637
-9800 train 5.713668 (lr=1.4183e-05) (hash(x)=150370792)
-9900 val loss 5.7241
-9900 val perplexity 306.1474
-9900 train 5.560071 (lr=1.3941e-05) (hash(x)=153014886)
-10000 val loss 5.7211
-10000 val perplexity 305.2354
-10000 train 5.627098 (lr=1.3701e-05) (hash(x)=141915265)
-10100 val loss 5.7186
-10100 val perplexity 304.4876
-10100 train 6.229914 (lr=1.3462e-05) (hash(x)=197770952)
-10200 val loss 5.7086
-10200 val perplexity 301.4522
-10200 train 5.686818 (lr=1.3223e-05) (hash(x)=156806005)
-10300 val loss 5.7046
-10300 val perplexity 300.2360
-10300 train 5.515134 (lr=1.2986e-05) (hash(x)=145378216)
-10400 val loss 5.7010
-10400 val perplexity 299.1595
-10400 train 5.611872 (lr=1.2750e-05) (hash(x)=145125407)
-10500 val loss 5.6972
-10500 val perplexity 298.0323
-10500 train 5.516371 (lr=1.2515e-05) (hash(x)=147553745)
-10600 val loss 5.6944
-10600 val perplexity 297.2043
-10600 train 5.339860 (lr=1.2282e-05) (hash(x)=134894889)
-10700 val loss 5.6894
-10700 val perplexity 295.7046
-10700 train 5.614913 (lr=1.2049e-05) (hash(x)=151041040)
-10800 val loss 5.6831
-10800 val perplexity 293.8572
-10800 train 5.614328 (lr=1.1819e-05) (hash(x)=145685829)
-10900 val loss 5.6789
-10900 val perplexity 292.6139
-10900 train 5.759324 (lr=1.1590e-05) (hash(x)=156000716)
-11000 val loss 5.6758
-11000 val perplexity 291.7285
-11000 train 5.686039 (lr=1.1362e-05) (hash(x)=149858163)
-11100 val loss 5.6741
-11100 val perplexity 291.2157
-11100 train 5.953506 (lr=1.1136e-05) (hash(x)=164873575)
-11200 val loss 5.6732
-11200 val perplexity 290.9572
-11200 train 5.551642 (lr=1.0912e-05) (hash(x)=156370069)
-11300 val loss 5.6717
-11300 val perplexity 290.5277
-11300 train 5.766811 (lr=1.0690e-05) (hash(x)=160356892)
-11400 val loss 5.6607
-11400 val perplexity 287.3395
-11400 train 5.736993 (lr=1.0470e-05) (hash(x)=158955430)
-11500 val loss 5.6569
-11500 val perplexity 286.2544
-11500 train 5.627035 (lr=1.0251e-05) (hash(x)=146598377)
-11600 val loss 5.6774
-11600 val perplexity 292.1937
-11600 train 5.819471 (lr=1.0035e-05) (hash(x)=156045795)
-11700 val loss 5.6552
-11700 val perplexity 285.7665
-11700 train 5.507897 (lr=9.8209e-06) (hash(x)=143963334)
-11800 val loss 5.6532
-11800 val perplexity 285.1887
-11800 train 5.492560 (lr=9.6090e-06) (hash(x)=145831579)
-11900 val loss 5.6478
-11900 val perplexity 283.6758
-11900 train 5.812625 (lr=9.3993e-06) (hash(x)=165154141)
-12000 val loss 5.6428
-12000 val perplexity 282.2388
-12000 train 5.726239 (lr=9.1920e-06) (hash(x)=160373794)
-12100 val loss 5.6393
-12100 val perplexity 281.2672
-12100 train 5.430665 (lr=8.9871e-06) (hash(x)=141476346)
-12200 val loss 5.6392
-12200 val perplexity 281.2291
-12200 train 5.895460 (lr=8.7846e-06) (hash(x)=164508058)
-12300 val loss 5.6363
-12300 val perplexity 280.4270
-12300 train 5.623287 (lr=8.5847e-06) (hash(x)=140407664)
-12400 val loss 5.6351
-12400 val perplexity 280.0981
-12400 train 5.487605 (lr=8.3875e-06) (hash(x)=149526189)
-12500 val loss 5.6286
-12500 val perplexity 278.2783
-12500 train 5.479519 (lr=8.1929e-06) (hash(x)=142060916)
-12600 val loss 5.6244
-12600 val perplexity 277.1057
-12600 train 5.806100 (lr=8.0010e-06) (hash(x)=168650848)
-12700 val loss 5.6219
-12700 val perplexity 276.4150
-12700 train 5.641911 (lr=7.8119e-06) (hash(x)=154353018)
-12800 val loss 5.6236
-12800 val perplexity 276.8924
-12800 train 5.778793 (lr=7.6257e-06) (hash(x)=156115265)
-12900 val loss 5.6207
-12900 val perplexity 276.0897
-12900 train 5.549165 (lr=7.4425e-06) (hash(x)=143378175)
-13000 val loss 5.6184
-13000 val perplexity 275.4428
-13000 train 5.725389 (lr=7.2622e-06) (hash(x)=166124864)
-13100 val loss 5.6173
-13100 val perplexity 275.1555
-13100 train 5.492951 (lr=7.0850e-06) (hash(x)=146615385)
-13200 val loss 5.6163
-13200 val perplexity 274.8670
-13200 train 5.517329 (lr=6.9108e-06) (hash(x)=149236047)
-13300 val loss 5.6108
-13300 val perplexity 273.3650
-13300 train 5.535242 (lr=6.7398e-06) (hash(x)=151346972)
-13400 val loss 5.6094
-13400 val perplexity 272.9731
-13400 train 5.699604 (lr=6.5721e-06) (hash(x)=159167093)
-13500 val loss 5.6083
-13500 val perplexity 272.6931
-13500 train 5.333128 (lr=6.4076e-06) (hash(x)=135054650)
-13600 val loss 5.6057
-13600 val perplexity 271.9643
-13600 train 5.492490 (lr=6.2465e-06) (hash(x)=151033927)
-13700 val loss 5.6058
-13700 val perplexity 271.9900
-13700 train 5.300904 (lr=6.0887e-06) (hash(x)=141119172)
-13800 val loss 5.6043
-13800 val perplexity 271.5937
-13800 train 5.319691 (lr=5.9344e-06) (hash(x)=144197766)
-13900 val loss 5.6004
-13900 val perplexity 270.5355
-13900 train 5.540504 (lr=5.7835e-06) (hash(x)=145148158)
-14000 val loss 5.5963
-14000 val perplexity 269.4165
-14000 train 6.091290 (lr=5.6362e-06) (hash(x)=179068751)
-14100 val loss 5.5959
-14100 val perplexity 269.3181
-14100 train 5.462370 (lr=5.4925e-06) (hash(x)=144988373)
-14200 val loss 5.5943
-14200 val perplexity 268.8805
-14200 train 5.537189 (lr=5.3524e-06) (hash(x)=133379696)
-14300 val loss 5.5946
-14300 val perplexity 268.9815
-14300 train 5.475476 (lr=5.2159e-06) (hash(x)=149883422)
-14400 val loss 5.5950
-14400 val perplexity 269.0731
-14400 train 5.464013 (lr=5.0832e-06) (hash(x)=145709198)
-14500 val loss 5.5900
-14500 val perplexity 267.7359
-14500 train 5.611162 (lr=4.9543e-06) (hash(x)=148016579)
-14600 val loss 5.5875
-14600 val perplexity 267.0565
-14600 train 5.556258 (lr=4.8291e-06) (hash(x)=150218222)
-14700 val loss 5.5881
-14700 val perplexity 267.2294
-14700 train 5.514695 (lr=4.7079e-06) (hash(x)=142258737)
-14800 val loss 5.5846
-14800 val perplexity 266.2829
-14800 train 5.476633 (lr=4.5905e-06) (hash(x)=147628003)
-14900 val loss 5.5862
-14900 val perplexity 266.7187
-14900 train 5.434494 (lr=4.4770e-06) (hash(x)=149144036)
-15000 val loss 5.5857
-15000 val perplexity 266.5893
-15000 train 5.518596 (lr=4.3675e-06) (hash(x)=145925409)
-15100 val loss 5.5817
-15100 val perplexity 265.5213
-15100 train 5.467402 (lr=4.2620e-06) (hash(x)=144555751)
-15200 val loss 5.5800
-15200 val perplexity 265.0629
-15200 train 5.394983 (lr=4.1605e-06) (hash(x)=143448852)
-15300 val loss 5.5793
-15300 val perplexity 264.8942
-15300 train 5.581673 (lr=4.0631e-06) (hash(x)=156347253)
-15400 val loss 5.5771
-15400 val perplexity 264.2980
-15400 train 5.532283 (lr=3.9698e-06) (hash(x)=153010179)
-15500 val loss 5.5772
-15500 val perplexity 264.3268
-15500 train 5.355952 (lr=3.8806e-06) (hash(x)=144066445)
-15600 val loss 5.5775
-15600 val perplexity 264.4116
-15600 train 5.308009 (lr=3.7956e-06) (hash(x)=146107722)
-15700 val loss 5.5734
-15700 val perplexity 263.3213
-15700 train 5.720090 (lr=3.7148e-06) (hash(x)=152533799)
-15800 val loss 5.5723
-15800 val perplexity 263.0380
-15800 train 5.625428 (lr=3.6382e-06) (hash(x)=151756366)
-15900 val loss 5.5707
-15900 val perplexity 262.6128
-15900 train 5.554205 (lr=3.5658e-06) (hash(x)=151735039)
-16000 val loss 5.5702
-16000 val perplexity 262.4980
-16000 train 5.604529 (lr=3.4977e-06) (hash(x)=154633226)
-16100 val loss 5.5719
-16100 val perplexity 262.9393
-16100 train 5.413339 (lr=3.4339e-06) (hash(x)=158094333)
-16200 val loss 5.5713
-16200 val perplexity 262.7858
-16200 train 5.327019 (lr=3.3744e-06) (hash(x)=142772927)
-16300 val loss 5.5702
-16300 val perplexity 262.4890
-16300 train 5.470050 (lr=3.3193e-06) (hash(x)=148042896)
-16400 val loss 5.5667
-16400 val perplexity 261.5669
-16400 train 6.053113 (lr=3.2684e-06) (hash(x)=170306399)
-16500 val loss 5.5664
-16500 val perplexity 261.4984
-16500 train 5.586644 (lr=3.2220e-06) (hash(x)=141049320)
-16600 val loss 5.5668
-16600 val perplexity 261.6037
-16600 train 5.419602 (lr=3.1799e-06) (hash(x)=150512136)
-16700 val loss 5.5640
-16700 val perplexity 260.8613
-16700 train 5.573275 (lr=3.1422e-06) (hash(x)=151843645)
-16800 val loss 5.5622
-16800 val perplexity 260.3896
-16800 train 5.632764 (lr=3.1089e-06) (hash(x)=140073659)
-16900 val loss 5.5605
-16900 val perplexity 259.9638
-16900 train 5.725865 (lr=3.0801e-06) (hash(x)=171157212)
-17000 val loss 5.5600
-17000 val perplexity 259.8117
-17000 train 5.495543 (lr=3.0556e-06) (hash(x)=144553276)
-17100 val loss 5.5607
-17100 val perplexity 260.0120
-17100 train 5.550316 (lr=3.0356e-06) (hash(x)=157717565)
-17200 val loss 5.5605
-17200 val perplexity 259.9615
-17200 train 5.423789 (lr=3.0200e-06) (hash(x)=149843860)
-17300 val loss 5.5578
-17300 val perplexity 259.2461
-17300 train 5.617573 (lr=3.0089e-06) (hash(x)=162171979)
-17400 val loss 5.5566
-17400 val perplexity 258.9522
-17400 train 5.530491 (lr=3.0022e-06) (hash(x)=148400702)
-17499 val loss 5.5565
-17499 val perplexity 258.9096
+0 train 11.263341 (lr=1.6250e-07) (hash(x)=150327452)
+100 val loss 9.8875
+100 val perplexity 19683.5957
+100 train 9.898082 (lr=1.6412e-05) (hash(x)=166441190)
+200 val loss 8.9232
+200 val perplexity 7504.3047
+200 train 9.040984 (lr=3.2500e-05) (hash(x)=166780046)
+300 val loss 8.0895
+300 val perplexity 3259.9653
+300 train 8.044475 (lr=3.2498e-05) (hash(x)=159835303)
+400 val loss 7.7549
+400 val perplexity 2332.9883
+400 train 7.555726 (lr=3.2490e-05) (hash(x)=155040610)
+500 val loss 7.6166
+500 val perplexity 2031.6100
+500 train 7.373107 (lr=3.2478e-05) (hash(x)=130190460)
+600 val loss 7.5216
+600 val perplexity 1847.4545
+600 train 7.546905 (lr=3.2461e-05) (hash(x)=155504036)
+700 val loss 7.4629
+700 val perplexity 1742.1884
+700 train 7.319477 (lr=3.2440e-05) (hash(x)=137347213)
+800 val loss 7.4212
+800 val perplexity 1670.9910
+800 train 7.279414 (lr=3.2413e-05) (hash(x)=143823248)
+900 val loss 7.3721
+900 val perplexity 1590.9294
+900 train 7.489573 (lr=3.2382e-05) (hash(x)=156260416)
+1000 val loss 7.3399
+1000 val perplexity 1540.5684
+1000 train 7.312011 (lr=3.2346e-05) (hash(x)=143734685)
+1100 val loss 7.2992
+1100 val perplexity 1479.1761
+1100 train 7.239788 (lr=3.2305e-05) (hash(x)=160013925)
+1200 val loss 7.2686
+1200 val perplexity 1434.5974
+1200 train 7.015460 (lr=3.2260e-05) (hash(x)=150678249)
+1300 val loss 7.2366
+1300 val perplexity 1389.3865
+1300 train 7.022800 (lr=3.2209e-05) (hash(x)=149073315)
+1400 val loss 7.1896
+1400 val perplexity 1325.6152
+1400 train 7.533691 (lr=3.2154e-05) (hash(x)=175802021)
+1500 val loss 7.1516
+1500 val perplexity 1276.0918
+1500 train 7.414447 (lr=3.2094e-05) (hash(x)=171034639)
+1600 val loss 7.1099
+1600 val perplexity 1224.0310
+1600 train 7.326279 (lr=3.2030e-05) (hash(x)=158681215)
+1700 val loss 7.0736
+1700 val perplexity 1180.4250
+1700 train 7.009824 (lr=3.1961e-05) (hash(x)=152116061)
+1800 val loss 7.0371
+1800 val perplexity 1138.1099
+1800 train 6.974350 (lr=3.1887e-05) (hash(x)=146108145)
+1900 val loss 7.0033
+1900 val perplexity 1100.2341
+1900 train 6.837708 (lr=3.1809e-05) (hash(x)=147598108)
+2000 val loss 6.9814
+2000 val perplexity 1076.3971
+2000 train 6.851001 (lr=3.1726e-05) (hash(x)=154996086)
+2100 val loss 6.9515
+2100 val perplexity 1044.6981
+2100 train 6.739772 (lr=3.1638e-05) (hash(x)=153396183)
+2200 val loss 6.9003
+2200 val perplexity 992.5322
+2200 train 6.889448 (lr=3.1546e-05) (hash(x)=153885445)
+2300 val loss 6.8539
+2300 val perplexity 947.5439
+2300 train 6.854183 (lr=3.1449e-05) (hash(x)=159666385)
+2400 val loss 6.8063
+2400 val perplexity 903.5588
+2400 train 6.780859 (lr=3.1348e-05) (hash(x)=142353087)
+2500 val loss 6.7665
+2500 val perplexity 868.3074
+2500 train 6.773431 (lr=3.1243e-05) (hash(x)=146491718)
+2600 val loss 6.7330
+2600 val perplexity 839.6339
+2600 train 6.659631 (lr=3.1133e-05) (hash(x)=150750353)
+2700 val loss 6.6965
+2700 val perplexity 809.5696
+2700 train 6.414171 (lr=3.1019e-05) (hash(x)=129849193)
+2800 val loss 6.6659
+2800 val perplexity 785.1340
+2800 train 6.513141 (lr=3.0900e-05) (hash(x)=152767913)
+2900 val loss 6.6364
+2900 val perplexity 762.3094
+2900 train 6.420549 (lr=3.0777e-05) (hash(x)=146531140)
+3000 val loss 6.5941
+3000 val perplexity 730.7974
+3000 train 6.674080 (lr=3.0650e-05) (hash(x)=151562048)
+3100 val loss 6.5496
+3100 val perplexity 698.9839
+3100 train 6.522118 (lr=3.0518e-05) (hash(x)=146001424)
+3200 val loss 6.5231
+3200 val perplexity 680.6537
+3200 train 6.579503 (lr=3.0383e-05) (hash(x)=166486165)
+3300 val loss 6.4914
+3300 val perplexity 659.4257
+3300 train 6.422096 (lr=3.0243e-05) (hash(x)=150866680)
+3400 val loss 6.4730
+3400 val perplexity 647.4102
+3400 train 6.369732 (lr=3.0099e-05) (hash(x)=143900419)
+3500 val loss 6.4440
+3500 val perplexity 628.8882
+3500 train 6.262652 (lr=2.9952e-05) (hash(x)=148845794)
+3600 val loss 6.4252
+3600 val perplexity 617.2151
+3600 train 6.261324 (lr=2.9800e-05) (hash(x)=145667796)
+3700 val loss 6.4049
+3700 val perplexity 604.8305
+3700 train 6.426984 (lr=2.9644e-05) (hash(x)=163563851)
+3800 val loss 6.3892
+3800 val perplexity 595.3663
+3800 train 6.336480 (lr=2.9485e-05) (hash(x)=147488689)
+3900 val loss 6.3539
+3900 val perplexity 574.7558
+3900 train 6.357310 (lr=2.9321e-05) (hash(x)=148186608)
+4000 val loss 6.3303
+4000 val perplexity 561.3508
+4000 train 6.204438 (lr=2.9154e-05) (hash(x)=142970187)
+4100 val loss 6.3048
+4100 val perplexity 547.1708
+4100 train 6.399692 (lr=2.8983e-05) (hash(x)=141584883)
+4200 val loss 6.2908
+4200 val perplexity 539.5918
+4200 train 6.087615 (lr=2.8808e-05) (hash(x)=145664585)
+4300 val loss 6.2741
+4300 val perplexity 530.6376
+4300 train 6.144583 (lr=2.8630e-05) (hash(x)=143736499)
+4400 val loss 6.2689
+4400 val perplexity 527.8878
+4400 train 6.090772 (lr=2.8448e-05) (hash(x)=151883322)
+4500 val loss 6.2551
+4500 val perplexity 520.6404
+4500 train 6.139237 (lr=2.8263e-05) (hash(x)=153904871)
+4600 val loss 6.2374
+4600 val perplexity 511.5054
+4600 train 6.329777 (lr=2.8075e-05) (hash(x)=154893521)
+4700 val loss 6.2138
+4700 val perplexity 499.6078
+4700 train 6.267534 (lr=2.7883e-05) (hash(x)=152323949)
+4800 val loss 6.1972
+4800 val perplexity 491.3801
+4800 train 6.149027 (lr=2.7687e-05) (hash(x)=154104619)
+4900 val loss 6.1803
+4900 val perplexity 483.1391
+4900 train 6.309986 (lr=2.7489e-05) (hash(x)=146311426)
+5000 val loss 6.1733
+5000 val perplexity 479.7561
+5000 train 6.151302 (lr=2.7287e-05) (hash(x)=156741847)
+5100 val loss 6.1586
+5100 val perplexity 472.7873
+5100 train 5.937405 (lr=2.7082e-05) (hash(x)=142086346)
+5200 val loss 6.1561
+5200 val perplexity 471.5924
+5200 train 5.926026 (lr=2.6874e-05) (hash(x)=150265428)
+5300 val loss 6.1438
+5300 val perplexity 465.7993
+5300 train 6.022453 (lr=2.6664e-05) (hash(x)=151339108)
+5400 val loss 6.1304
+5400 val perplexity 459.6218
+5400 train 6.102451 (lr=2.6450e-05) (hash(x)=154654372)
+5500 val loss 6.1120
+5500 val perplexity 451.2368
+5500 train 6.118230 (lr=2.6233e-05) (hash(x)=150575051)
+5600 val loss 6.1000
+5600 val perplexity 445.8380
+5600 train 6.011166 (lr=2.6014e-05) (hash(x)=140396423)
+5700 val loss 6.0866
+5700 val perplexity 439.9342
+5700 train 5.939641 (lr=2.5792e-05) (hash(x)=144678758)
+5800 val loss 6.0762
+5800 val perplexity 435.3538
+5800 train 6.136963 (lr=2.5567e-05) (hash(x)=151992743)
+5900 val loss 6.0681
+5900 val perplexity 431.8787
+5900 train 5.861908 (lr=2.5340e-05) (hash(x)=144396927)
+6000 val loss 6.0589
+6000 val perplexity 427.8926
+6000 train 6.021150 (lr=2.5111e-05) (hash(x)=165478625)
+6100 val loss 6.0597
+6100 val perplexity 428.2602
+6100 train 5.792263 (lr=2.4879e-05) (hash(x)=147088621)
+6200 val loss 6.0481
+6200 val perplexity 423.3074
+6200 train 5.812227 (lr=2.4644e-05) (hash(x)=140794994)
+6300 val loss 6.0296
+6300 val perplexity 415.5572
+6300 train 5.968508 (lr=2.4408e-05) (hash(x)=134780906)
+6400 val loss 6.0144
+6400 val perplexity 409.2740
+6400 train 5.996778 (lr=2.4169e-05) (hash(x)=149023655)
+6500 val loss 6.0036
+6500 val perplexity 404.8894
+6500 train 5.856802 (lr=2.3928e-05) (hash(x)=147497796)
+6600 val loss 6.0013
+6600 val perplexity 403.9445
+6600 train 5.857316 (lr=2.3686e-05) (hash(x)=152902689)
+6700 val loss 6.0037
+6700 val perplexity 404.9313
+6700 train 6.043744 (lr=2.3441e-05) (hash(x)=153846046)
+6800 val loss 5.9799
+6800 val perplexity 395.4021
+6800 train 6.110096 (lr=2.3195e-05) (hash(x)=158512738)
+6900 val loss 5.9692
+6900 val perplexity 391.2107
+6900 train 6.498738 (lr=2.2946e-05) (hash(x)=156849968)
+7000 val loss 5.9604
+7000 val perplexity 387.7580
+7000 train 5.809161 (lr=2.2696e-05) (hash(x)=142395855)
+7100 val loss 5.9549
+7100 val perplexity 385.6508
+7100 train 5.797284 (lr=2.2445e-05) (hash(x)=147114884)
+7200 val loss 5.9443
+7200 val perplexity 381.5764
+7200 train 5.971154 (lr=2.2192e-05) (hash(x)=156979839)
+7300 val loss 5.9408
+7300 val perplexity 380.2552
+7300 train 5.738841 (lr=2.1937e-05) (hash(x)=145584373)
+7400 val loss 5.9331
+7400 val perplexity 377.3345
+7400 train 5.679769 (lr=2.1682e-05) (hash(x)=141508204)
+7500 val loss 5.9221
+7500 val perplexity 373.2067
+7500 train 5.954195 (lr=2.1425e-05) (hash(x)=148803965)
+7600 val loss 5.9160
+7600 val perplexity 370.9126
+7600 train 5.974942 (lr=2.1166e-05) (hash(x)=151019676)
+7700 val loss 5.9041
+7700 val perplexity 366.5331
+7700 train 5.988387 (lr=2.0907e-05) (hash(x)=143155750)
+7800 val loss 5.8963
+7800 val perplexity 363.6974
+7800 train 5.926476 (lr=2.0647e-05) (hash(x)=152569653)
+7900 val loss 5.8890
+7900 val perplexity 361.0439
+7900 train 5.763350 (lr=2.0386e-05) (hash(x)=143519455)
+8000 val loss 5.8823
+8000 val perplexity 358.6201
+8000 train 6.002029 (lr=2.0123e-05) (hash(x)=161180944)
+8100 val loss 5.8808
+8100 val perplexity 358.0812
+8100 train 5.909467 (lr=1.9861e-05) (hash(x)=154107345)
+8200 val loss 5.8773
+8200 val perplexity 356.8280
+8200 train 5.915339 (lr=1.9597e-05) (hash(x)=152486517)
+8300 val loss 5.8635
+8300 val perplexity 351.9369
+8300 train 5.868403 (lr=1.9333e-05) (hash(x)=156167749)
+8400 val loss 5.8580
+8400 val perplexity 350.0205
+8400 train 5.939692 (lr=1.9069e-05) (hash(x)=149155006)
+8500 val loss 5.8437
+8500 val perplexity 345.0668
+8500 train 5.792398 (lr=1.8804e-05) (hash(x)=147844390)
+8600 val loss 5.8417
+8600 val perplexity 344.3497
+8600 train 6.100732 (lr=1.8539e-05) (hash(x)=165753320)
+8700 val loss 5.8357
+8700 val perplexity 342.2876
+8700 train 5.617911 (lr=1.8273e-05) (hash(x)=146079979)
+8800 val loss 5.8368
+8800 val perplexity 342.6954
+8800 train 6.028148 (lr=1.8008e-05) (hash(x)=172259509)
+8900 val loss 5.8283
+8900 val perplexity 339.7885
+8900 train 5.512281 (lr=1.7742e-05) (hash(x)=145148314)
+9000 val loss 5.8204
+9000 val perplexity 337.1211
+9000 train 5.738827 (lr=1.7477e-05) (hash(x)=144250633)
+9100 val loss 5.8136
+9100 val perplexity 334.8091
+9100 train 5.910919 (lr=1.7211e-05) (hash(x)=157219797)
+9200 val loss 5.8091
+9200 val perplexity 333.3240
+9200 train 5.853679 (lr=1.6946e-05) (hash(x)=142743778)
+9300 val loss 5.7980
+9300 val perplexity 329.6350
+9300 train 5.722970 (lr=1.6681e-05) (hash(x)=139669771)
+9400 val loss 5.7902
+9400 val perplexity 327.0877
+9400 train 5.823825 (lr=1.6417e-05) (hash(x)=145916843)
+9500 val loss 5.7892
+9500 val perplexity 326.7543
+9500 train 5.710148 (lr=1.6153e-05) (hash(x)=150196125)
+9600 val loss 5.7812
+9600 val perplexity 324.1400
+9600 train 6.082446 (lr=1.5889e-05) (hash(x)=160041419)
+9700 val loss 5.7831
+9700 val perplexity 324.7499
+9700 train 5.570460 (lr=1.5627e-05) (hash(x)=139931627)
+9800 val loss 5.7745
+9800 val perplexity 321.9872
+9800 train 5.752125 (lr=1.5364e-05) (hash(x)=150370792)
+9900 val loss 5.7680
+9900 val perplexity 319.8929
+9900 train 5.602925 (lr=1.5103e-05) (hash(x)=153014886)
+10000 val loss 5.7651
+10000 val perplexity 318.9697
+10000 train 5.674670 (lr=1.4843e-05) (hash(x)=141915265)
+10100 val loss 5.7595
+10100 val perplexity 317.1992
+10100 train 6.257363 (lr=1.4584e-05) (hash(x)=197770952)
+10200 val loss 5.7518
+10200 val perplexity 314.7549
+10200 train 5.720849 (lr=1.4325e-05) (hash(x)=156806005)
+10300 val loss 5.7469
+10300 val perplexity 313.2228
+10300 train 5.550874 (lr=1.4068e-05) (hash(x)=145378216)
+10400 val loss 5.7420
+10400 val perplexity 311.6738
+10400 train 5.648454 (lr=1.3813e-05) (hash(x)=145125407)
+10500 val loss 5.7383
+10500 val perplexity 310.5409
+10500 train 5.556795 (lr=1.3558e-05) (hash(x)=147553745)
+10600 val loss 5.7375
+10600 val perplexity 310.2906
+10600 train 5.377072 (lr=1.3305e-05) (hash(x)=134894889)
+10700 val loss 5.7291
+10700 val perplexity 307.6816
+10700 train 5.653048 (lr=1.3054e-05) (hash(x)=151041040)
+10800 val loss 5.7224
+10800 val perplexity 305.6333
+10800 train 5.655978 (lr=1.2804e-05) (hash(x)=145685829)
+10900 val loss 5.7162
+10900 val perplexity 303.7596
+10900 train 5.794735 (lr=1.2555e-05) (hash(x)=156000716)
+11000 val loss 5.7139
+11000 val perplexity 303.0442
+11000 train 5.712698 (lr=1.2309e-05) (hash(x)=149858163)
+11100 val loss 5.7135
+11100 val perplexity 302.9354
+11100 train 5.992548 (lr=1.2064e-05) (hash(x)=164873575)
+11200 val loss 5.7104
+11200 val perplexity 301.9899
+11200 train 5.599405 (lr=1.1822e-05) (hash(x)=156370069)
+11300 val loss 5.7097
+11300 val perplexity 301.7715
+11300 train 5.798551 (lr=1.1581e-05) (hash(x)=160356892)
+11400 val loss 5.6994
+11400 val perplexity 298.6908
+11400 train 5.780049 (lr=1.1342e-05) (hash(x)=158955430)
+11500 val loss 5.6948
+11500 val perplexity 297.3225
+11500 train 5.655000 (lr=1.1106e-05) (hash(x)=146598377)
+11600 val loss 5.7098
+11600 val perplexity 301.8159
+11600 train 5.854637 (lr=1.0871e-05) (hash(x)=156045795)
+11700 val loss 5.6914
+11700 val perplexity 296.3024
+11700 train 5.548706 (lr=1.0639e-05) (hash(x)=143963334)
+11800 val loss 5.6887
+11800 val perplexity 295.4948
+11800 train 5.527222 (lr=1.0410e-05) (hash(x)=145831579)
+11900 val loss 5.6838
+11900 val perplexity 294.0689
+11900 train 5.845513 (lr=1.0183e-05) (hash(x)=165154141)
+12000 val loss 5.6766
+12000 val perplexity 291.9412
+12000 train 5.762044 (lr=9.9580e-06) (hash(x)=160373794)
+12100 val loss 5.6749
+12100 val perplexity 291.4481
+12100 train 5.463029 (lr=9.7360e-06) (hash(x)=141476346)
+12200 val loss 5.6715
+12200 val perplexity 290.4636
+12200 train 5.920372 (lr=9.5167e-06) (hash(x)=164508058)
+12300 val loss 5.6698
+12300 val perplexity 289.9705
+12300 train 5.644031 (lr=9.3001e-06) (hash(x)=140407664)
+12400 val loss 5.6671
+12400 val perplexity 289.1933
+12400 train 5.524893 (lr=9.0864e-06) (hash(x)=149526189)
+12500 val loss 5.6616
+12500 val perplexity 287.5968
+12500 train 5.514926 (lr=8.8756e-06) (hash(x)=142060916)
+12600 val loss 5.6573
+12600 val perplexity 286.3658
+12600 train 5.838615 (lr=8.6677e-06) (hash(x)=168650848)
+12700 val loss 5.6561
+12700 val perplexity 286.0426
+12700 train 5.684471 (lr=8.4629e-06) (hash(x)=154353018)
+12800 val loss 5.6554
+12800 val perplexity 285.8341
+12800 train 5.816707 (lr=8.2612e-06) (hash(x)=156115265)
+12900 val loss 5.6525
+12900 val perplexity 284.9962
+12900 train 5.576836 (lr=8.0627e-06) (hash(x)=143378175)
+13000 val loss 5.6485
+13000 val perplexity 283.8571
+13000 train 5.746207 (lr=7.8674e-06) (hash(x)=166124864)
+13100 val loss 5.6475
+13100 val perplexity 283.5952
+13100 train 5.527974 (lr=7.6754e-06) (hash(x)=146615385)
+13200 val loss 5.6470
+13200 val perplexity 283.4364
+13200 train 5.544142 (lr=7.4867e-06) (hash(x)=149236047)
+13300 val loss 5.6417
+13300 val perplexity 281.9557
+13300 train 5.580195 (lr=7.3015e-06) (hash(x)=151346972)
+13400 val loss 5.6390
+13400 val perplexity 281.1739
+13400 train 5.729071 (lr=7.1198e-06) (hash(x)=159167093)
+13500 val loss 5.6384
+13500 val perplexity 281.0078
+13500 train 5.365855 (lr=6.9416e-06) (hash(x)=135054650)
+13600 val loss 5.6369
+13600 val perplexity 280.6006
+13600 train 5.514214 (lr=6.7670e-06) (hash(x)=151033927)
+13700 val loss 5.6363
+13700 val perplexity 280.4331
+13700 train 5.341426 (lr=6.5961e-06) (hash(x)=141119172)
+13800 val loss 5.6337
+13800 val perplexity 279.6906
+13800 train 5.352831 (lr=6.4289e-06) (hash(x)=144197766)
+13900 val loss 5.6292
+13900 val perplexity 278.4267
+13900 train 5.567875 (lr=6.2655e-06) (hash(x)=145148158)
+14000 val loss 5.6262
+14000 val perplexity 277.5951
+14000 train 6.124477 (lr=6.1059e-06) (hash(x)=179068751)
+14100 val loss 5.6250
+14100 val perplexity 277.2643
+14100 train 5.494565 (lr=5.9502e-06) (hash(x)=144988373)
+14200 val loss 5.6235
+14200 val perplexity 276.8575
+14200 train 5.556700 (lr=5.7984e-06) (hash(x)=133379696)
+14300 val loss 5.6244
+14300 val perplexity 277.1109
+14300 train 5.506085 (lr=5.6506e-06) (hash(x)=149883422)
+14400 val loss 5.6243
+14400 val perplexity 277.0917
+14400 train 5.492159 (lr=5.5068e-06) (hash(x)=145709198)
+14500 val loss 5.6185
+14500 val perplexity 275.4824
+14500 train 5.636049 (lr=5.3671e-06) (hash(x)=148016579)
+14600 val loss 5.6168
+14600 val perplexity 274.9970
+14600 train 5.586943 (lr=5.2316e-06) (hash(x)=150218222)
+14700 val loss 5.6163
+14700 val perplexity 274.8758
+14700 train 5.543831 (lr=5.1002e-06) (hash(x)=142258737)
+14800 val loss 5.6128
+14800 val perplexity 273.9005
+14800 train 5.502282 (lr=4.9730e-06) (hash(x)=147628003)
+14900 val loss 5.6144
+14900 val perplexity 274.3428
+14900 train 5.465495 (lr=4.8501e-06) (hash(x)=149144036)
+15000 val loss 5.6134
+15000 val perplexity 274.0710
+15000 train 5.552163 (lr=4.7314e-06) (hash(x)=145925409)
+15100 val loss 5.6101
+15100 val perplexity 273.1843
+15100 train 5.497269 (lr=4.6171e-06) (hash(x)=144555751)
+15200 val loss 5.6077
+15200 val perplexity 272.5063
+15200 train 5.425412 (lr=4.5072e-06) (hash(x)=143448852)
+15300 val loss 5.6064
+15300 val perplexity 272.1697
+15300 train 5.610938 (lr=4.4017e-06) (hash(x)=156347253)
+15400 val loss 5.6046
+15400 val perplexity 271.6618
+15400 train 5.562627 (lr=4.3006e-06) (hash(x)=153010179)
+15500 val loss 5.6050
+15500 val perplexity 271.7854
+15500 train 5.388970 (lr=4.2040e-06) (hash(x)=144066445)
+15600 val loss 5.6043
+15600 val perplexity 271.5881
+15600 train 5.339193 (lr=4.1119e-06) (hash(x)=146107722)
+15700 val loss 5.5999
+15700 val perplexity 270.4047
+15700 train 5.747085 (lr=4.0244e-06) (hash(x)=152533799)
+15800 val loss 5.5981
+15800 val perplexity 269.9133
+15800 train 5.652522 (lr=3.9414e-06) (hash(x)=151756366)
+15900 val loss 5.5974
+15900 val perplexity 269.7231
+15900 train 5.583819 (lr=3.8630e-06) (hash(x)=151735039)
+16000 val loss 5.5969
+16000 val perplexity 269.5918
+16000 train 5.627756 (lr=3.7892e-06) (hash(x)=154633226)
+16100 val loss 5.5972
+16100 val perplexity 269.6637
+16100 train 5.441843 (lr=3.7201e-06) (hash(x)=158094333)
+16200 val loss 5.5964
+16200 val perplexity 269.4558
+16200 train 5.358812 (lr=3.6556e-06) (hash(x)=142772927)
+16300 val loss 5.5958
+16300 val perplexity 269.2827
+16300 train 5.492966 (lr=3.5959e-06) (hash(x)=148042896)
+16400 val loss 5.5922
+16400 val perplexity 268.3255
+16400 train 6.091084 (lr=3.5408e-06) (hash(x)=170306399)
+16500 val loss 5.5917
+16500 val perplexity 268.2008
+16500 train 5.594467 (lr=3.4905e-06) (hash(x)=141049320)
+16600 val loss 5.5916
+16600 val perplexity 268.1708
+16600 train 5.455967 (lr=3.4449e-06) (hash(x)=150512136)
+16700 val loss 5.5894
+16700 val perplexity 267.5735
+16700 train 5.592753 (lr=3.4041e-06) (hash(x)=151843645)
+16800 val loss 5.5878
+16800 val perplexity 267.1494
+16800 train 5.642298 (lr=3.3680e-06) (hash(x)=140073659)
+16900 val loss 5.5860
+16900 val perplexity 266.6786
+16900 train 5.755163 (lr=3.3367e-06) (hash(x)=171157212)
+17000 val loss 5.5844
+17000 val perplexity 266.2370
+17000 train 5.524979 (lr=3.3102e-06) (hash(x)=144553276)
+17100 val loss 5.5853
+17100 val perplexity 266.4875
+17100 train 5.585420 (lr=3.2886e-06) (hash(x)=157717565)
+17200 val loss 5.5848
+17200 val perplexity 266.3359
+17200 train 5.453957 (lr=3.2717e-06) (hash(x)=149843860)
+17300 val loss 5.5822
+17300 val perplexity 265.6631
+17300 train 5.640796 (lr=3.2596e-06) (hash(x)=162171979)
+17400 val loss 5.5809
+17400 val perplexity 265.3040
+17400 train 5.549406 (lr=3.2524e-06) (hash(x)=148400702)
+17499 val loss 5.5807
+17499 val perplexity 265.2500

attention_kindselective_n_heads4_seed1338/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f5db00d51e9a60a9ec51cc45865e30b30d959eba2e4420d76385259452d3336c
+oid sha256:369ca0e325a307ae0fe4b31859c5e3039654368ff28c0b4fed2d96d8e4d73b93
 size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:131f98c60e821509e44529eec0de5685d6511c9b5b4badf98603ac9f496120d2
+oid sha256:b139874be23990c8467336f94877c955b85aec61bb9887d4c0ff395e2fd58926
 size 179406214