Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_61440_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,408 +1,533 @@
 max_steps: 8750
-2000 val loss 6.4370
-2000 val perplexity 624.5455
-2000 train 6.371128 (lr=3.7143e-05) (hash(x)=158245023)
-0 val loss 11.2647
-0 val perplexity 78014.9766
-2100 val loss 6.3948
-2100 val perplexity 598.6997
-2100 train 6.215977 (lr=3.6761e-05) (hash(x)=157204896)
-0 train 11.267745 (lr=1.0000e-07) (hash(x)=150327452)
-2200 val loss 6.3725
-2200 val perplexity 585.5281
-2200 train 6.217300 (lr=3.6358e-05) (hash(x)=137541932)
-100 val loss 9.9617
-100 val perplexity 21199.5801
-100 train 10.014555 (lr=1.0100e-05) (hash(x)=166780046)
-2300 val loss 6.3442
-2300 val perplexity 569.1939
-2300 train 6.298882 (lr=3.5935e-05) (hash(x)=150149692)
-200 val loss 9.2487
-200 val perplexity 10391.1982
-200 train 9.198502 (lr=2.0100e-05) (hash(x)=155040610)
-2400 val loss 6.3008
-2400 val perplexity 545.0193
-2400 train 6.289979 (lr=3.5491e-05) (hash(x)=151730720)
-300 val loss 8.1647
-300 val perplexity 3514.8396
-300 train 8.169972 (lr=3.0100e-05) (hash(x)=155504036)
-2500 val loss 6.2774
-2500 val perplexity 532.3767
-2500 train 6.057900 (lr=3.5027e-05) (hash(x)=143406752)
-400 val loss 7.5999
-400 val perplexity 1998.0118
-400 train 7.465411 (lr=4.0100e-05) (hash(x)=143823248)
-2600 val loss 6.2626
-2600 val perplexity 524.5986
-2600 train 6.088369 (lr=3.4545e-05) (hash(x)=157272496)
-500 val loss 7.4464
-500 val perplexity 1713.7411
-500 train 7.412394 (lr=5.0000e-05) (hash(x)=143734685)
-2700 val loss 6.2352
-2700 val perplexity 510.3958
-2700 train 6.233461 (lr=3.4044e-05) (hash(x)=155342327)
-600 val loss 7.3773
-600 val perplexity 1599.2358
-600 train 7.154145 (lr=4.9984e-05) (hash(x)=150678249)
-2800 val loss 6.2037
-2800 val perplexity 494.5931
-2800 train 6.083380 (lr=3.3526e-05) (hash(x)=140626679)
-700 val loss 7.2967
-700 val perplexity 1475.4744
-700 train 7.442945 (lr=4.9935e-05) (hash(x)=175802021)
-2900 val loss 6.1842
-2900 val perplexity 485.0013
-2900 train 6.041373 (lr=3.2992e-05) (hash(x)=144953350)
-800 val loss 7.1922
-800 val perplexity 1329.0032
-800 train 7.206111 (lr=4.9853e-05) (hash(x)=158681215)
-3000 val loss 6.1690
-3000 val perplexity 477.6920
-3000 train 6.072640 (lr=3.2441e-05) (hash(x)=172449837)
-900 val loss 7.1072
-900 val perplexity 1220.7802
-900 train 6.997451 (lr=4.9739e-05) (hash(x)=146108145)
-3100 val loss 6.1653
-3100 val perplexity 475.9668
-3100 train 5.957810 (lr=3.1875e-05) (hash(x)=141710086)
-1000 val loss 7.0298
-1000 val perplexity 1129.8363
-1000 train 6.867131 (lr=4.9593e-05) (hash(x)=154996086)
-3200 val loss 6.1250
-3200 val perplexity 457.1336
-3200 train 6.054896 (lr=3.1295e-05) (hash(x)=151299772)
-1100 val loss 6.9379
-1100 val perplexity 1030.5953
-1100 train 6.917721 (lr=4.9415e-05) (hash(x)=153885445)
-3300 val loss 6.1114
-3300 val perplexity 450.9624
-3300 train 5.977787 (lr=3.0702e-05) (hash(x)=146473110)
-1200 val loss 6.8587
-1200 val perplexity 952.1497
-1200 train 6.820927 (lr=4.9205e-05) (hash(x)=142353087)
-3400 val loss 6.0891
-3400 val perplexity 441.0146
-3400 train 6.166475 (lr=3.0095e-05) (hash(x)=153954157)
-1300 val loss 6.7949
-1300 val perplexity 893.2373
-1300 train 6.699144 (lr=4.8964e-05) (hash(x)=150750353)
-3500 val loss 6.0733
-3500 val perplexity 434.1111
-3500 train 6.005799 (lr=2.9477e-05) (hash(x)=153717336)
-1400 val loss 6.7528
-1400 val perplexity 856.4910
-1400 train 6.573970 (lr=4.8691e-05) (hash(x)=152767913)
-3600 val loss 6.0607
-3600 val perplexity 428.6727
-3600 train 5.809951 (lr=2.8849e-05) (hash(x)=144965161)
-1500 val loss 6.6831
-1500 val perplexity 798.7867
-1500 train 6.727711 (lr=4.8388e-05) (hash(x)=151562048)
-3700 val loss 6.0523
-3700 val perplexity 425.0963
-3700 train 5.825770 (lr=2.8210e-05) (hash(x)=125969741)
-1600 val loss 6.6191
-1600 val perplexity 749.2616
-1600 train 6.620916 (lr=4.8055e-05) (hash(x)=166486165)
-3800 val loss 6.0290
-3800 val perplexity 415.2866
-3800 train 5.888848 (lr=2.7562e-05) (hash(x)=155070487)
-1700 val loss 6.5918
-1700 val perplexity 729.0835
-1700 train 6.235582 (lr=4.7691e-05) (hash(x)=130835396)
-3900 val loss 6.0102
-3900 val perplexity 407.5825
-3900 train 5.883321 (lr=2.6907e-05) (hash(x)=149444644)
-1800 val loss 6.5444
-1800 val perplexity 695.3355
-1800 train 6.591407 (lr=4.7299e-05) (hash(x)=158851816)
-4000 val loss 6.0051
-4000 val perplexity 405.4889
-4000 train 5.854551 (lr=2.6244e-05) (hash(x)=151663033)
-1900 val loss 6.5057
-1900 val perplexity 668.9247
-1900 train 6.534745 (lr=4.6878e-05) (hash(x)=153313879)
-4100 val loss 6.0012
-4100 val perplexity 403.9044
-4100 train 5.955617 (lr=2.5575e-05) (hash(x)=143688282)
-2000 val loss 6.4488
-2000 val perplexity 631.9535
-2000 train 6.381792 (lr=4.6428e-05) (hash(x)=158245023)
-4200 val loss 5.9756
-4200 val perplexity 393.6931
-4200 train 5.975471 (lr=2.4900e-05) (hash(x)=163361651)
-2100 val loss 6.4032
-2100 val perplexity 603.7603
-2100 train 6.227605 (lr=4.5951e-05) (hash(x)=157204896)
-4300 val loss 5.9607
-4300 val perplexity 387.8712
-4300 train 6.028554 (lr=2.4222e-05) (hash(x)=153619361)
-2200 val loss 6.3820
-2200 val perplexity 591.1212
-4400 val loss 5.9584
-4400 val perplexity 386.9796
-2200 train 6.223777 (lr=4.5448e-05) (hash(x)=137541932)
-4400 train 6.271734 (lr=2.3540e-05) (hash(x)=168527064)
-4500 val loss 5.9460
-4500 val perplexity 382.2105
-4500 train 5.752192 (lr=2.2856e-05) (hash(x)=125588037)
-2300 val loss 6.3599
-2300 val perplexity 578.1849
-2300 train 6.312685 (lr=4.4918e-05) (hash(x)=150149692)
-4600 val loss 5.9269
-4600 val perplexity 374.9906
-4600 train 5.890114 (lr=2.2171e-05) (hash(x)=143710941)
-2400 val loss 6.3222
-2400 val perplexity 556.8002
-2400 train 6.310997 (lr=4.4363e-05) (hash(x)=151730720)
-4700 val loss 5.9159
-4700 val perplexity 370.8951
-4700 train 5.746759 (lr=2.1486e-05) (hash(x)=150952742)
-2500 val loss 6.2898
-2500 val perplexity 539.0579
-2500 train 6.072419 (lr=4.3784e-05) (hash(x)=143406752)
-4800 val loss 5.9073
-4800 val perplexity 367.7109
-4800 train 5.773638 (lr=2.0801e-05) (hash(x)=145323659)
-2600 val loss 6.2791
-2600 val perplexity 533.3013
-2600 train 6.115969 (lr=4.3181e-05) (hash(x)=157272496)
-4900 val loss 5.8995
-4900 val perplexity 364.8397
-4900 train 5.918860 (lr=2.0118e-05) (hash(x)=153151397)
-2700 val loss 6.2556
-2700 val perplexity 520.9327
-2700 train 6.248377 (lr=4.2555e-05) (hash(x)=155342327)
-5000 val loss 5.8912
-5000 val perplexity 361.8272
-5000 train 5.792883 (lr=1.9438e-05) (hash(x)=143182059)
-2800 val loss 6.2198
-2800 val perplexity 502.6056
-2800 train 6.090466 (lr=4.1908e-05) (hash(x)=140626679)
-5100 val loss 5.8823
-5100 val perplexity 358.6324
-5100 train 5.928619 (lr=1.8762e-05) (hash(x)=170083586)
-2900 val loss 6.1933
-2900 val perplexity 489.4789
-2900 train 6.050964 (lr=4.1240e-05) (hash(x)=144953350)
-5200 val loss 5.8692
-5200 val perplexity 353.9545
-5200 train 5.759492 (lr=1.8090e-05) (hash(x)=149363919)
-3000 val loss 6.1754
-3000 val perplexity 480.7940
-3000 train 6.073356 (lr=4.0551e-05) (hash(x)=172449837)
-5300 val loss 5.8625
-5300 val perplexity 351.5884
-5300 train 5.756340 (lr=1.7424e-05) (hash(x)=152033784)
-3100 val loss 6.1695
-3100 val perplexity 477.9614
-3100 train 5.960622 (lr=3.9844e-05) (hash(x)=141710086)
-5400 val loss 5.8494
-5400 val perplexity 347.0114
-5400 train 5.957279 (lr=1.6765e-05) (hash(x)=154614289)
-3200 val loss 6.1283
-3200 val perplexity 458.6766
-3200 train 6.060437 (lr=3.9119e-05) (hash(x)=151299772)
-5500 val loss 5.8406
-5500 val perplexity 343.9975
-5500 train 5.933627 (lr=1.6113e-05) (hash(x)=157745174)
-3300 val loss 6.1232
-3300 val perplexity 456.3084
-3300 train 5.992179 (lr=3.8377e-05) (hash(x)=146473110)
-5600 val loss 5.8365
-5600 val perplexity 342.5832
-5600 train 5.729990 (lr=1.5469e-05) (hash(x)=147693222)
-3400 val loss 6.0933
-3400 val perplexity 442.8724
-3400 train 6.177316 (lr=3.7619e-05) (hash(x)=153954157)
-5700 val loss 5.8250
-5700 val perplexity 338.6544
-5700 train 5.762623 (lr=1.4836e-05) (hash(x)=149784627)
-3500 val loss 6.0865
-3500 val perplexity 439.8985
-3500 train 6.025422 (lr=3.6847e-05) (hash(x)=153717336)
-5800 val loss 5.8244
-5800 val perplexity 338.4496
-5800 train 5.736485 (lr=1.4212e-05) (hash(x)=158620729)
-3600 val loss 6.0678
-3600 val perplexity 431.7099
-3600 train 5.823427 (lr=3.6061e-05) (hash(x)=144965161)
-5900 val loss 5.8146
-5900 val perplexity 335.1426
-5900 train 5.760200 (lr=1.3600e-05) (hash(x)=159763910)
-3700 val loss 6.0644
-3700 val perplexity 430.2674
-3700 train 5.841285 (lr=3.5263e-05) (hash(x)=125969741)
-6000 val loss 5.8055
-6000 val perplexity 332.1231
-6000 train 5.739980 (lr=1.3000e-05) (hash(x)=147640561)
-3800 val loss 6.0351
-3800 val perplexity 417.8310
-3800 train 5.895473 (lr=3.4453e-05) (hash(x)=155070487)
-6100 val loss 5.7986
-6100 val perplexity 329.8218
-6100 train 5.780902 (lr=1.2413e-05) (hash(x)=156613394)
-3900 val loss 6.0204
-3900 val perplexity 411.7543
-3900 train 5.894702 (lr=3.3633e-05) (hash(x)=149444644)
-6200 val loss 5.7929
-6200 val perplexity 327.9590
-6200 train 5.741335 (lr=1.1840e-05) (hash(x)=186221290)
-4000 val loss 6.0050
-4000 val perplexity 405.4474
-4000 train 5.864678 (lr=3.2805e-05) (hash(x)=151663033)
-6300 val loss 5.7831
-6300 val perplexity 324.7564
-6300 train 5.659614 (lr=1.1282e-05) (hash(x)=152081419)
-4100 val loss 5.9989
-4100 val perplexity 403.0023
-4100 train 5.955342 (lr=3.1968e-05) (hash(x)=143688282)
-6400 val loss 5.7833
-6400 val perplexity 324.8156
-6400 train 5.735166 (lr=1.0739e-05) (hash(x)=154808349)
-4200 val loss 5.9733
-4200 val perplexity 392.7840
-4200 train 5.975454 (lr=3.1126e-05) (hash(x)=163361651)
-6500 val loss 5.7760
-6500 val perplexity 322.4663
-6500 train 5.745718 (lr=1.0213e-05) (hash(x)=159437208)
-4300 val loss 5.9635
-4300 val perplexity 388.9864
-4300 train 6.007787 (lr=3.0277e-05) (hash(x)=153619361)
-6600 val loss 5.7731
-6600 val perplexity 321.5303
-6600 train 5.587155 (lr=9.7032e-06) (hash(x)=157933074)
-4400 val loss 5.9564
-4400 val perplexity 386.2238
-4400 train 6.267161 (lr=2.9425e-05) (hash(x)=168527064)
-6700 val loss 5.7666
-6700 val perplexity 319.4617
-6700 train 5.745788 (lr=9.2116e-06) (hash(x)=161560240)
-4500 val loss 5.9458
-4500 val perplexity 382.1341
-4500 train 5.755313 (lr=2.8571e-05) (hash(x)=125588037)
-6800 val loss 5.7636
-6800 val perplexity 318.4973
-6800 train 5.744992 (lr=8.7387e-06) (hash(x)=155424292)
-4600 val loss 5.9250
-4600 val perplexity 374.2863
-4600 train 5.879909 (lr=2.7714e-05) (hash(x)=143710941)
-6900 val loss 5.7626
-6900 val perplexity 318.1587
-6900 train 5.756405 (lr=8.2849e-06) (hash(x)=148561470)
-4700 val loss 5.9121
-4700 val perplexity 369.4801
-4700 train 5.743450 (lr=2.6857e-05) (hash(x)=150952742)
-7000 val loss 5.7545
-7000 val perplexity 315.6169
-7000 train 5.646148 (lr=7.8510e-06) (hash(x)=141527450)
-4800 val loss 5.9022
-4800 val perplexity 365.8246
-4800 train 5.772085 (lr=2.6002e-05) (hash(x)=145323659)
-7100 val loss 5.7503
-7100 val perplexity 314.2946
-7100 train 5.706478 (lr=7.4377e-06) (hash(x)=151066339)
-4900 val loss 5.8952
-4900 val perplexity 363.2955
-4900 train 5.914246 (lr=2.5148e-05) (hash(x)=153151397)
-7200 val loss 5.7511
-7200 val perplexity 314.5356
-7200 train 5.548562 (lr=7.0455e-06) (hash(x)=155231264)
-5000 val loss 5.8868
-5000 val perplexity 360.2437
-5000 train 5.788630 (lr=2.4298e-05) (hash(x)=143182059)
-7300 val loss 5.7421
-7300 val perplexity 311.7135
-7300 train 5.759835 (lr=6.6749e-06) (hash(x)=150281149)
-5100 val loss 5.8739
-5100 val perplexity 355.6327
-5100 train 5.913543 (lr=2.3452e-05) (hash(x)=170083586)
-7400 val loss 5.7399
-7400 val perplexity 311.0442
-7400 train 5.622359 (lr=6.3266e-06) (hash(x)=148421717)
-5200 val loss 5.8581
-5200 val perplexity 350.0584
-5200 train 5.746659 (lr=2.2613e-05) (hash(x)=149363919)
-7500 val loss 5.7394
-7500 val perplexity 310.8757
-7500 train 5.503174 (lr=6.0010e-06) (hash(x)=146921118)
-5300 val loss 5.8530
-5300 val perplexity 348.2945
-5300 train 5.744291 (lr=2.1780e-05) (hash(x)=152033784)
-7600 val loss 5.7331
-7600 val perplexity 308.9312
-7600 train 5.784346 (lr=5.6986e-06) (hash(x)=150660048)
-5400 val loss 5.8368
-5400 val perplexity 342.6903
-5400 train 5.944860 (lr=2.0956e-05) (hash(x)=154614289)
-7700 val loss 5.7314
-7700 val perplexity 308.3879
-7700 train 5.606787 (lr=5.4198e-06) (hash(x)=148059852)
-5500 val loss 5.8313
-5500 val perplexity 340.8127
-5500 train 5.921842 (lr=2.0141e-05) (hash(x)=157745174)
-7800 val loss 5.7305
-7800 val perplexity 308.1266
-7800 train 5.426733 (lr=5.1650e-06) (hash(x)=148331002)
-5600 val loss 5.8229
-5600 val perplexity 337.9419
-5600 train 5.719602 (lr=1.9337e-05) (hash(x)=147693222)
-7900 val loss 5.7251
-7900 val perplexity 306.4575
-7900 train 5.675220 (lr=4.9347e-06) (hash(x)=164923883)
-5700 val loss 5.8136
-5700 val perplexity 334.8164
-5700 train 5.748807 (lr=1.8545e-05) (hash(x)=149784627)
-8000 val loss 5.7236
-8000 val perplexity 306.0140
-8000 train 5.605746 (lr=4.7291e-06) (hash(x)=143545384)
-5800 val loss 5.8120
-5800 val perplexity 334.2779
-5800 train 5.732249 (lr=1.7765e-05) (hash(x)=158620729)
-8100 val loss 5.7227
-8100 val perplexity 305.7214
-8100 train 5.539697 (lr=4.5486e-06) (hash(x)=160686959)
-5900 val loss 5.8007
-5900 val perplexity 330.5159
-5900 train 5.749244 (lr=1.7000e-05) (hash(x)=159763910)
-8200 val loss 5.7206
-8200 val perplexity 305.1013
-8200 train 5.676816 (lr=4.3933e-06) (hash(x)=156501889)
-6000 val loss 5.7906
-6000 val perplexity 327.2256
-6000 train 5.725507 (lr=1.6250e-05) (hash(x)=147640561)
-8300 val loss 5.7185
-8300 val perplexity 304.4521
-8300 train 5.560667 (lr=4.2636e-06) (hash(x)=142716875)
-6100 val loss 5.7819
-6100 val perplexity 324.3689
-6100 train 5.764189 (lr=1.5516e-05) (hash(x)=156613394)
-8400 val loss 5.7141
-8400 val perplexity 303.1260
-8400 train 5.627058 (lr=4.1596e-06) (hash(x)=154436684)
-6200 val loss 5.7755
-6200 val perplexity 322.3004
-6200 train 5.712209 (lr=1.4800e-05) (hash(x)=186221290)
-8500 val loss 5.7127
-8500 val perplexity 302.6869
-8500 train 5.795162 (lr=4.0815e-06) (hash(x)=147965839)
-6300 val loss 5.7650
-6300 val perplexity 318.9550
-6300 train 5.640378 (lr=1.4102e-05) (hash(x)=152081419)
-8600 val loss 5.7114
-8600 val perplexity 302.3018
-8600 train 5.512054 (lr=4.0294e-06) (hash(x)=145228097)
-6400 val loss 5.7646
-6400 val perplexity 318.8096
-6400 train 5.714537 (lr=1.3424e-05) (hash(x)=154808349)
-8700 val loss 5.7076
-8700 val perplexity 301.1422
-8700 train 5.898399 (lr=4.0033e-06) (hash(x)=152910357)
-8749 val loss 5.7076
-8749 val perplexity 301.1334
-6500 val loss 5.7571
-6500 val perplexity 316.4409
-6500 train 5.723630 (lr=1.2766e-05) (hash(x)=159437208)
-6600 val loss 5.7538
-6600 val perplexity 315.3894
-6600 train 5.563268 (lr=1.2129e-05) (hash(x)=157933074)
+0 val loss 11.2646
+0 val perplexity 78014.3047
+0 val loss 11.2646
+0 val perplexity 78014.3047
+0 train 11.267765 (lr=6.0000e-08) (hash(x)=150327452)
+0 train 11.267765 (lr=7.0000e-08) (hash(x)=150327452)
+100 val loss 10.0785
+100 val perplexity 23826.3750
+100 val loss 10.1480
+100 val perplexity 25539.9902
+100 train 10.130300 (lr=7.0700e-06) (hash(x)=166780046)
+100 train 10.197817 (lr=6.0600e-06) (hash(x)=166780046)
+200 val loss 9.5828
+200 val perplexity 14512.7627
+200 train 9.549955 (lr=1.4070e-05) (hash(x)=155040610)
+200 val loss 9.6799
+200 val perplexity 15992.7021
+200 train 9.651791 (lr=1.2060e-05) (hash(x)=155040610)
+300 val loss 8.7068
+300 val perplexity 6043.8276
+300 train 8.732620 (lr=2.1070e-05) (hash(x)=155504036)
+300 val loss 8.8988
+300 val perplexity 7323.0049
+300 train 8.930824 (lr=1.8060e-05) (hash(x)=155504036)
+400 val loss 7.9358
+400 val perplexity 2795.6145
+400 train 7.833813 (lr=2.8070e-05) (hash(x)=143823248)
+400 val loss 8.0672
+400 val perplexity 3188.0217
+400 train 7.977346 (lr=2.4060e-05) (hash(x)=143823248)
+500 val loss 7.6097
+500 val perplexity 2017.7271
+500 train 7.582412 (lr=3.5000e-05) (hash(x)=143734685)
+500 val loss 7.6500
+500 val perplexity 2100.6929
+500 train 7.625039 (lr=3.0000e-05) (hash(x)=143734685)
+600 val loss 7.4992
+600 val perplexity 1806.5713
+600 train 7.291920 (lr=3.4989e-05) (hash(x)=150678249)
+600 val loss 7.5211
+600 val perplexity 1846.5060
+600 train 7.316416 (lr=2.9990e-05) (hash(x)=150678249)
+700 val loss 7.4154
+700 val perplexity 1661.3982
+700 train 7.564420 (lr=3.4954e-05) (hash(x)=175802021)
+700 val loss 7.4274
+700 val perplexity 1681.4119
+700 train 7.571382 (lr=2.9961e-05) (hash(x)=175802021)
+800 val loss 7.3271
+800 val perplexity 1520.9635
+800 train 7.331609 (lr=3.4897e-05) (hash(x)=158681215)
+800 val loss 7.3369
+800 val perplexity 1535.9385
+800 train 7.345709 (lr=2.9912e-05) (hash(x)=158681215)
+900 val loss 7.2689
+900 val perplexity 1434.9026
+900 train 7.165131 (lr=3.4818e-05) (hash(x)=146108145)
+900 val loss 7.2753
+900 val perplexity 1444.1128
+900 train 7.172338 (lr=2.9844e-05) (hash(x)=146108145)
+1000 val loss 7.2140
+1000 val perplexity 1358.2812
+1000 train 7.060000 (lr=3.4715e-05) (hash(x)=154996086)
+1000 val loss 7.2104
+1000 val perplexity 1353.4595
+1000 train 7.056768 (lr=2.9756e-05) (hash(x)=154996086)
+1100 val loss 7.1419
+1100 val perplexity 1263.8793
+1100 train 7.124806 (lr=3.4591e-05) (hash(x)=153885445)
+1100 val loss 7.1242
+1100 val perplexity 1241.7123
+1100 train 7.107671 (lr=2.9649e-05) (hash(x)=153885445)
+1200 val loss 7.0668
+1200 val perplexity 1172.3687
+1200 train 7.020203 (lr=3.4444e-05) (hash(x)=142353087)
+1200 val loss 7.0451
+1200 val perplexity 1147.1823
+1200 train 6.998048 (lr=2.9523e-05) (hash(x)=142353087)
+1300 val loss 7.0154
+1300 val perplexity 1113.6349
+1300 train 6.930557 (lr=3.4275e-05) (hash(x)=150750353)
+1300 val loss 6.9831
+1300 val perplexity 1078.2115
+1300 train 6.896412 (lr=2.9378e-05) (hash(x)=150750353)
+1400 val loss 6.9752
+1400 val perplexity 1069.7871
+1400 train 6.799040 (lr=3.4084e-05) (hash(x)=152767913)
+1400 val loss 6.9322
+1400 val perplexity 1024.7649
+1400 train 6.756323 (lr=2.9215e-05) (hash(x)=152767913)
+1500 val loss 6.9097
+1500 val perplexity 1001.9031
+1500 train 6.945250 (lr=3.3872e-05) (hash(x)=151562048)
+1500 val loss 6.8590
+1500 val perplexity 952.3980
+1500 train 6.893571 (lr=2.9033e-05) (hash(x)=151562048)
+1600 val loss 6.8390
+1600 val perplexity 933.5735
+1600 train 6.832361 (lr=3.3638e-05) (hash(x)=166486165)
+1600 val loss 6.7795
+1600 val perplexity 879.6003
+1600 train 6.775093 (lr=2.8833e-05) (hash(x)=166486165)
+1700 val loss 6.7954
+1700 val perplexity 893.7272
+1700 train 6.446530 (lr=3.3384e-05) (hash(x)=130835396)
+1700 val loss 6.7200
+1700 val perplexity 828.8434
+1700 train 6.368818 (lr=2.8615e-05) (hash(x)=130835396)
+1800 val loss 6.7493
+1800 val perplexity 853.4929
+1800 train 6.801020 (lr=3.3109e-05) (hash(x)=158851816)
+1800 val loss 6.6636
+1800 val perplexity 783.3477
+1800 train 6.714691 (lr=2.8379e-05) (hash(x)=158851816)
+1900 val loss 6.7007
+1900 val perplexity 812.9660
+1900 train 6.740700 (lr=3.2814e-05) (hash(x)=153313879)
+1900 val loss 6.6132
+1900 val perplexity 744.8441
+1900 train 6.650075 (lr=2.8127e-05) (hash(x)=153313879)
+2000 val loss 6.6387
+2000 val perplexity 764.0649
+2000 train 6.572113 (lr=3.2500e-05) (hash(x)=158245023)
+2000 val loss 6.5498
+2000 val perplexity 699.1063
+2000 train 6.483957 (lr=2.7857e-05) (hash(x)=158245023)
+2100 val loss 6.5899
+2100 val perplexity 727.7206
+2100 train 6.421700 (lr=3.2166e-05) (hash(x)=157204896)
+2100 val loss 6.5033
+2100 val perplexity 667.3728
+2100 train 6.327139 (lr=2.7571e-05) (hash(x)=157204896)
+2200 val loss 6.5629
+2200 val perplexity 708.3510
+2200 train 6.407813 (lr=3.1813e-05) (hash(x)=137541932)
+2200 val loss 6.4726
+2200 val perplexity 647.1771
+2200 train 6.316986 (lr=2.7269e-05) (hash(x)=137541932)
+2300 val loss 6.5292
+2300 val perplexity 684.8694
+2300 train 6.481870 (lr=3.1443e-05) (hash(x)=150149692)
+2300 val loss 6.4415
+2300 val perplexity 627.3774
+2300 train 6.394355 (lr=2.6951e-05) (hash(x)=150149692)
+2400 val loss 6.4797
+2400 val perplexity 651.7456
+2400 train 6.469861 (lr=3.1054e-05) (hash(x)=151730720)
+2400 val loss 6.3974
+2400 val perplexity 600.2633
+2400 train 6.381145 (lr=2.6618e-05) (hash(x)=151730720)
+2500 val loss 6.4501
+2500 val perplexity 632.7767
+2500 train 6.218892 (lr=3.0649e-05) (hash(x)=143406752)
+2500 val loss 6.3688
+2500 val perplexity 583.3505
+2500 train 6.147860 (lr=2.6270e-05) (hash(x)=143406752)
+2600 val loss 6.4203
+2600 val perplexity 614.1707
+2600 train 6.254517 (lr=3.0227e-05) (hash(x)=157272496)
+2600 val loss 6.3524
+2600 val perplexity 573.8466
+2600 train 6.186916 (lr=2.5909e-05) (hash(x)=157272496)
+2700 val loss 6.3945
+2700 val perplexity 598.5561
+2700 train 6.410750 (lr=2.9789e-05) (hash(x)=155342327)
+2700 val loss 6.3262
+2700 val perplexity 559.0328
+2700 train 6.334075 (lr=2.5533e-05) (hash(x)=155342327)
+2800 val loss 6.3544
+2800 val perplexity 575.0129
+2800 train 6.225688 (lr=2.9336e-05) (hash(x)=140626679)
+2800 val loss 6.2908
+2800 val perplexity 539.5964
+2800 train 6.168956 (lr=2.5145e-05) (hash(x)=140626679)
+2900 val loss 6.3277
+2900 val perplexity 559.8731
+2900 train 6.188353 (lr=2.8868e-05) (hash(x)=144953350)
+2900 val loss 6.2653
+2900 val perplexity 526.0153
+2900 train 6.125029 (lr=2.4744e-05) (hash(x)=144953350)
+3000 val loss 6.3080
+3000 val perplexity 548.9646
+3000 train 6.216789 (lr=2.8386e-05) (hash(x)=172449837)
+3000 val loss 6.2493
+3000 val perplexity 517.6444
+3000 train 6.155919 (lr=2.4331e-05) (hash(x)=172449837)
+3100 val loss 6.2954
+3100 val perplexity 542.0637
+3100 train 6.085942 (lr=2.7891e-05) (hash(x)=141710086)
+3100 val loss 6.2444
+3100 val perplexity 515.1201
+3100 train 6.041224 (lr=2.3906e-05) (hash(x)=141710086)
+3200 val loss 6.2588
+3200 val perplexity 522.5837
+3200 train 6.189919 (lr=2.7383e-05) (hash(x)=151299772)
+3200 val loss 6.2087
+3200 val perplexity 497.0476
+3200 train 6.142445 (lr=2.3471e-05) (hash(x)=151299772)
+3300 val loss 6.2412
+3300 val perplexity 513.4838
+3300 train 6.105106 (lr=2.6864e-05) (hash(x)=146473110)
+3300 val loss 6.1987
+3300 val perplexity 492.0873
+3300 train 6.058009 (lr=2.3026e-05) (hash(x)=146473110)
+3400 val loss 6.2167
+3400 val perplexity 501.0693
+3400 train 6.293976 (lr=2.6333e-05) (hash(x)=153954157)
+3400 val loss 6.1735
+3400 val perplexity 479.8751
+3400 train 6.249593 (lr=2.2572e-05) (hash(x)=153954157)
+3500 val loss 6.2010
+3500 val perplexity 493.2191
+3500 train 6.127666 (lr=2.5793e-05) (hash(x)=153717336)
+3500 val loss 6.1570
+3500 val perplexity 471.9926
+3500 train 6.084462 (lr=2.2108e-05) (hash(x)=153717336)
+3600 val loss 6.1805
+3600 val perplexity 483.2234
+3600 train 5.934934 (lr=2.5243e-05) (hash(x)=144965161)
+3600 val loss 6.1449
+3600 val perplexity 466.3374
+3600 train 5.896233 (lr=2.1637e-05) (hash(x)=144965161)
+3700 val loss 6.1731
+3700 val perplexity 479.6843
+3700 train 5.946197 (lr=2.4684e-05) (hash(x)=125969741)
+3700 val loss 6.1403
+3700 val perplexity 464.2038
+3700 train 5.915022 (lr=2.1158e-05) (hash(x)=125969741)
+3800 val loss 6.1465
+3800 val perplexity 467.0876
+3800 train 6.011291 (lr=2.4117e-05) (hash(x)=155070487)
+3800 val loss 6.1142
+3800 val perplexity 452.2469
+3800 train 5.975008 (lr=2.0672e-05) (hash(x)=155070487)
+3900 val loss 6.1278
+3900 val perplexity 458.4420
+3900 train 6.005039 (lr=2.3543e-05) (hash(x)=149444644)
+3900 val loss 6.1017
+3900 val perplexity 446.5967
+3900 train 5.975531 (lr=2.0180e-05) (hash(x)=149444644)
+4000 val loss 6.1155
+4000 val perplexity 452.8252
+4000 train 5.972252 (lr=2.2963e-05) (hash(x)=151663033)
+4000 val loss 6.0910
+4000 val perplexity 441.8456
+4000 train 5.942688 (lr=1.9683e-05) (hash(x)=151663033)
+4100 val loss 6.1077
+4100 val perplexity 449.3011
+4100 train 6.060684 (lr=2.2378e-05) (hash(x)=143688282)
+4100 val loss 6.0840
+4100 val perplexity 438.7943
+4100 train 6.035672 (lr=1.9181e-05) (hash(x)=143688282)
+4200 val loss 6.0846
+4200 val perplexity 439.0362
+4200 train 6.078417 (lr=2.1788e-05) (hash(x)=163361651)
+4200 val loss 6.0626
+4200 val perplexity 429.4702
+4200 train 6.058801 (lr=1.8675e-05) (hash(x)=163361651)
+4300 val loss 6.0690
+4300 val perplexity 432.2372
+4300 train 6.133219 (lr=2.1194e-05) (hash(x)=153619361)
+4300 val loss 6.0485
+4300 val perplexity 423.4790
+4300 train 6.109252 (lr=1.8166e-05) (hash(x)=153619361)
+4400 val loss 6.0636
+4400 val perplexity 429.9185
+4400 train 6.355730 (lr=2.0598e-05) (hash(x)=168527064)
+4400 val loss 6.0480
+4400 val perplexity 423.2458
+4400 train 6.335233 (lr=1.7655e-05) (hash(x)=168527064)
+4500 val loss 6.0544
+4500 val perplexity 425.9846
+4500 train 5.854112 (lr=1.9999e-05) (hash(x)=125588037)
+4500 val loss 6.0369
+4500 val perplexity 418.6092
+4500 train 5.830371 (lr=1.7142e-05) (hash(x)=125588037)
+4600 val loss 6.0342
+4600 val perplexity 417.4467
+4600 train 5.998290 (lr=1.9400e-05) (hash(x)=143710941)
+4600 val loss 6.0200
+4600 val perplexity 411.5641
+4600 train 5.977354 (lr=1.6629e-05) (hash(x)=143710941)
+4700 val loss 6.0192
+4700 val perplexity 411.2676
+4700 train 5.844175 (lr=1.8800e-05) (hash(x)=150952742)
+4700 val loss 6.0082
+4700 val perplexity 406.7620
+4700 train 5.829061 (lr=1.6114e-05) (hash(x)=150952742)
+4800 val loss 6.0118
+4800 val perplexity 408.2310
+4800 train 5.876356 (lr=1.8201e-05) (hash(x)=145323659)
+4800 val loss 6.0027
+4800 val perplexity 404.5156
+4800 train 5.861185 (lr=1.5601e-05) (hash(x)=145323659)
+4900 val loss 6.0072
+4900 val perplexity 406.3518
+4900 train 6.022637 (lr=1.7604e-05) (hash(x)=153151397)
+4900 val loss 5.9937
+4900 val perplexity 400.8849
+4900 train 6.009444 (lr=1.5089e-05) (hash(x)=153151397)
+5000 val loss 5.9942
+5000 val perplexity 401.1065
+5000 train 5.895233 (lr=1.7009e-05) (hash(x)=143182059)
+5000 val loss 5.9870
+5000 val perplexity 398.2089
+5000 train 5.890842 (lr=1.4579e-05) (hash(x)=143182059)
+5100 val loss 5.9844
+5100 val perplexity 397.1822
+5100 train 6.032875 (lr=1.6417e-05) (hash(x)=170083586)
+5100 val loss 5.9762
+5100 val perplexity 393.9346
+5100 train 6.027209 (lr=1.4071e-05) (hash(x)=170083586)
+5200 val loss 5.9702
+5200 val perplexity 391.5924
+5200 train 5.865292 (lr=1.5829e-05) (hash(x)=149363919)
+5200 val loss 5.9649
+5200 val perplexity 389.5172
+5200 train 5.861600 (lr=1.3568e-05) (hash(x)=149363919)
+5300 val loss 5.9669
+5300 val perplexity 390.3069
+5300 train 5.877049 (lr=1.5246e-05) (hash(x)=152033784)
+5300 val loss 5.9612
+5300 val perplexity 388.0795
+5300 train 5.867223 (lr=1.3068e-05) (hash(x)=152033784)
+5400 val loss 5.9501
+5400 val perplexity 383.8001
+5400 train 6.054345 (lr=1.4669e-05) (hash(x)=154614289)
+5400 val loss 5.9471
+5400 val perplexity 382.6569
+5400 train 6.046638 (lr=1.2573e-05) (hash(x)=154614289)
+5500 val loss 5.9431
+5500 val perplexity 381.0967
+5500 train 6.028569 (lr=1.4099e-05) (hash(x)=157745174)
+5500 val loss 5.9410
+5500 val perplexity 380.3125
+5500 train 6.025530 (lr=1.2085e-05) (hash(x)=157745174)
+5600 val loss 5.9373
+5600 val perplexity 378.9089
+5600 train 5.829021 (lr=1.3536e-05) (hash(x)=147693222)
+5600 val loss 5.9377
+5600 val perplexity 379.0604
+5600 train 5.827706 (lr=1.1602e-05) (hash(x)=147693222)
+5700 val loss 5.9251
+5700 val perplexity 374.3267
+5700 train 5.866521 (lr=1.2981e-05) (hash(x)=149784627)
+5700 val loss 5.9253
+5700 val perplexity 374.3747
+5700 train 5.866199 (lr=1.1127e-05) (hash(x)=149784627)
+5800 val loss 5.9247
+5800 val perplexity 374.1822
+5800 train 5.834700 (lr=1.2436e-05) (hash(x)=158620729)
+5800 val loss 5.9247
+5800 val perplexity 374.1743
+5800 train 5.826473 (lr=1.0659e-05) (hash(x)=158620729)
+5900 val loss 5.9146
+5900 val perplexity 370.3900
+5900 train 5.858362 (lr=1.1900e-05) (hash(x)=159763910)
+5900 val loss 5.9180
+5900 val perplexity 371.6826
+5900 train 5.861243 (lr=1.0200e-05) (hash(x)=159763910)
+6000 val loss 5.9043
+6000 val perplexity 366.6024
+6000 train 5.836517 (lr=1.1375e-05) (hash(x)=147640561)
+6000 val loss 5.9081
+6000 val perplexity 367.9886
+6000 train 5.841051 (lr=9.7500e-06) (hash(x)=147640561)
+6100 val loss 5.8966
+6100 val perplexity 363.7880
+6100 train 5.878016 (lr=1.0861e-05) (hash(x)=156613394)
+6100 val loss 5.9015
+6100 val perplexity 365.5672
+6100 train 5.885981 (lr=9.3098e-06) (hash(x)=156613394)
+6200 val loss 5.8921
+6200 val perplexity 362.1524
+6200 train 5.853373 (lr=1.0360e-05) (hash(x)=186221290)
+6200 val loss 5.8976
+6200 val perplexity 364.1776
+6200 train 5.857404 (lr=8.8800e-06) (hash(x)=186221290)
+6300 val loss 5.8814
+6300 val perplexity 358.3256
+6300 train 5.760697 (lr=9.8715e-06) (hash(x)=152081419)
+6300 val loss 5.8885
+6300 val perplexity 360.8471
+6300 train 5.763207 (lr=8.4613e-06) (hash(x)=152081419)
+6400 val loss 5.8804
+6400 val perplexity 357.9507
+6400 train 5.825439 (lr=9.3966e-06) (hash(x)=154808349)
+6400 val loss 5.8885
+6400 val perplexity 360.8786
+6400 train 5.834856 (lr=8.0542e-06) (hash(x)=154808349)
+6500 val loss 5.8733
+6500 val perplexity 355.4034
+6500 train 5.840936 (lr=8.9359e-06) (hash(x)=159437208)
+6500 val loss 5.8832
+6500 val perplexity 358.9690
+6500 train 5.854115 (lr=7.6594e-06) (hash(x)=159437208)
+6600 val loss 5.8716
+6600 val perplexity 354.8281
+6600 train 5.681755 (lr=8.4903e-06) (hash(x)=157933074)
+6600 val loss 5.8805
+6600 val perplexity 358.0042
+6600 train 5.683859 (lr=7.2774e-06) (hash(x)=157933074)
+6700 val loss 5.8631
+6700 val perplexity 351.8213
+6700 train 5.846561 (lr=8.0602e-06) (hash(x)=161560240)
+6700 val loss 5.8725
+6700 val perplexity 355.1426
+6700 train 5.855772 (lr=6.9087e-06) (hash(x)=161560240)
+6800 val loss 5.8610
+6800 val perplexity 351.0881
+6800 train 5.835786 (lr=7.6463e-06) (hash(x)=155424292)
+6800 val loss 5.8702
+6800 val perplexity 354.3057
+6800 train 5.845886 (lr=6.5540e-06) (hash(x)=155424292)
+6900 val loss 5.8587
+6900 val perplexity 350.2836
+6900 train 5.851926 (lr=7.2493e-06) (hash(x)=148561470)
+6900 val loss 5.8693
+6900 val perplexity 354.0038
+6900 train 5.859293 (lr=6.2137e-06) (hash(x)=148561470)
+7000 val loss 5.8522
+7000 val perplexity 348.0006
+7000 train 5.749753 (lr=6.8697e-06) (hash(x)=141527450)
+7000 val loss 5.8612
+7000 val perplexity 351.1305
+7000 train 5.753252 (lr=5.8883e-06) (hash(x)=141527450)
+7100 val loss 5.8464
+7100 val perplexity 345.9703
+7100 train 5.800970 (lr=6.5080e-06) (hash(x)=151066339)
+7100 val loss 5.8581
+7100 val perplexity 350.0529
+7100 train 5.809224 (lr=5.5783e-06) (hash(x)=151066339)
+7200 val loss 5.8468
+7200 val perplexity 346.1266
+7200 train 5.652260 (lr=6.1648e-06) (hash(x)=155231264)
+7200 val loss 5.8590
+7200 val perplexity 350.3668
+7200 train 5.663008 (lr=5.2841e-06) (hash(x)=155231264)
+7300 val loss 5.8382
+7300 val perplexity 343.1572
+7300 train 5.854438 (lr=5.8405e-06) (hash(x)=150281149)
+7300 val loss 5.8514
+7300 val perplexity 347.7130
+7300 train 5.861528 (lr=5.0062e-06) (hash(x)=150281149)
+7400 val loss 5.8352
+7400 val perplexity 342.1290
+7400 train 5.717260 (lr=5.5357e-06) (hash(x)=148421717)
+7400 val loss 5.8498
+7400 val perplexity 347.1809
+7400 train 5.727182 (lr=4.7449e-06) (hash(x)=148421717)
+7500 val loss 5.8349
+7500 val perplexity 342.0240
+7500 train 5.600042 (lr=5.2508e-06) (hash(x)=146921118)
+7500 val loss 5.8495
+7500 val perplexity 347.0692
+7500 train 5.604131 (lr=4.5007e-06) (hash(x)=146921118)
+7600 val loss 5.8285
+7600 val perplexity 339.8405
+7600 train 5.879191 (lr=4.9862e-06) (hash(x)=150660048)
+7600 val loss 5.8426
+7600 val perplexity 344.6835
+7600 train 5.886945 (lr=4.2739e-06) (hash(x)=150660048)
+7700 val loss 5.8274
+7700 val perplexity 339.4743
+7700 train 5.699795 (lr=4.7423e-06) (hash(x)=148059852)
+7700 val loss 5.8417
+7700 val perplexity 344.3644
+7700 train 5.711916 (lr=4.0648e-06) (hash(x)=148059852)
+7800 val loss 5.8256
+7800 val perplexity 338.8620
+7800 train 5.527908 (lr=4.5194e-06) (hash(x)=148331002)
+7800 val loss 5.8412
+7800 val perplexity 344.1924
+7800 train 5.541048 (lr=3.8738e-06) (hash(x)=148331002)
+7900 val loss 5.8206
+7900 val perplexity 337.1732
+7900 train 5.777098 (lr=4.3179e-06) (hash(x)=164923883)
+7900 val loss 5.8355
+7900 val perplexity 342.2498
+7900 train 5.794849 (lr=3.7010e-06) (hash(x)=164923883)
+8000 val loss 5.8183
+8000 val perplexity 336.3914
+8000 train 5.699238 (lr=4.1380e-06) (hash(x)=143545384)
+8000 val loss 5.8344
+8000 val perplexity 341.8687
+8000 train 5.713335 (lr=3.5468e-06) (hash(x)=143545384)
+8100 val loss 5.8173
+8100 val perplexity 336.0612
+8100 train 5.634319 (lr=3.9800e-06) (hash(x)=160686959)
+8100 val loss 5.8336
+8100 val perplexity 341.5852
+8100 train 5.655682 (lr=3.4114e-06) (hash(x)=160686959)
+8200 val loss 5.8150
+8200 val perplexity 335.3045
+8200 train 5.773064 (lr=3.8442e-06) (hash(x)=156501889)
+8200 val loss 5.8326
+8200 val perplexity 341.2524
+8200 train 5.789817 (lr=3.2950e-06) (hash(x)=156501889)
+8300 val loss 5.8130
+8300 val perplexity 334.6131
+8300 train 5.656518 (lr=3.7307e-06) (hash(x)=142716875)
+8300 val loss 5.8295
+8300 val perplexity 340.2042
+8300 train 5.667881 (lr=3.1977e-06) (hash(x)=142716875)
+8400 val loss 5.8083
+8400 val perplexity 333.0655
+8400 train 5.715852 (lr=3.6397e-06) (hash(x)=154436684)
+8400 val loss 5.8256
+8400 val perplexity 338.8665
+8400 train 5.732801 (lr=3.1197e-06) (hash(x)=154436684)
+8500 val loss 5.8068
+8500 val perplexity 332.5389
+8500 train 5.878377 (lr=3.5713e-06) (hash(x)=147965839)
+8500 val loss 5.8247
+8500 val perplexity 338.5674
+8500 train 5.897286 (lr=3.0611e-06) (hash(x)=147965839)
+8600 val loss 5.8056
+8600 val perplexity 332.1485
+8600 train 5.608908 (lr=3.5257e-06) (hash(x)=145228097)
+8600 val loss 5.8239
+8600 val perplexity 338.2750
+8600 train 5.626842 (lr=3.0220e-06) (hash(x)=145228097)
+8700 val loss 5.8016
+8700 val perplexity 330.8403
+8700 train 5.983572 (lr=3.5029e-06) (hash(x)=152910357)
+8700 val loss 5.8198
+8700 val perplexity 336.9131
+8700 train 5.998275 (lr=3.0024e-06) (hash(x)=152910357)
+8749 val loss 5.8021
+8749 val perplexity 330.9829
+8749 val loss 5.8203
+8749 val perplexity 337.0700

attention_kindselective_n_heads4_seed1338/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d9aea93b1a4a003ae1bb2610aa8e99a57ab1cbb4737ed231f282363c6467892
+oid sha256:f0ca05b3344aa66408b045c0838e43612e6fde6f89e7dfea8f7fe75e490896e0
 size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8ad9c642a2e4de42b4ccaecdd362a1673e76da42d6280c1a1635b4fe2515f05
+oid sha256:70de7c3fffdc7ca40ec0743f8447c426842a43ad5a3f91f2f8dfdeb161f89769
 size 179406214