Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_61440_4_1341", "n_embd": 256}

attention_kindselective_n_heads4_seed1341/log2.txt

@@ -1,476 +1,534 @@
 max_steps: 8750
-1900 val loss 6.4169
-1900 val perplexity 612.1060
-1900 train 6.250515 (lr=3.7502e-05) (hash(x)=149434659)
+max_steps: 8750
+0 val loss 11.3095
+0 val perplexity 81594.2500
 0 val loss 11.3095
 0 val perplexity 81594.2500
-2000 val loss 6.3543
-2000 val perplexity 574.9713
-2000 train 6.255687 (lr=3.7143e-05) (hash(x)=152285486)
-0 train 11.313040 (lr=1.0000e-07) (hash(x)=145079536)
-2100 val loss 6.3165
-2100 val perplexity 553.6381
-2100 train 6.064629 (lr=3.6761e-05) (hash(x)=144294295)
-100 val loss 10.0523
-100 val perplexity 23209.7812
-100 train 10.045820 (lr=1.0100e-05) (hash(x)=155800595)
-2200 val loss 6.2910
-2200 val perplexity 539.7109
-2200 train 6.295984 (lr=3.6358e-05) (hash(x)=175030215)
-200 val loss 9.5099
-200 val perplexity 13492.4014
-200 train 9.462710 (lr=2.0100e-05) (hash(x)=145606733)
-2300 val loss 6.2671
-2300 val perplexity 526.9221
-2300 train 6.077068 (lr=3.5935e-05) (hash(x)=150831428)
-300 val loss 8.3310
-300 val perplexity 4150.4053
-300 train 8.320391 (lr=3.0100e-05) (hash(x)=150367139)
-2400 val loss 6.2306
-2400 val perplexity 508.0812
-2400 train 6.415508 (lr=3.5491e-05) (hash(x)=140808297)
-400 val loss 7.6708
-400 val perplexity 2144.7644
-400 train 7.970377 (lr=4.0100e-05) (hash(x)=155747374)
-2500 val loss 6.2058
-2500 val perplexity 495.6054
-2500 train 6.186115 (lr=3.5027e-05) (hash(x)=153160275)
-500 val loss 7.5037
-500 val perplexity 1814.6550
-500 train 7.391925 (lr=5.0000e-05) (hash(x)=140604760)
-2600 val loss 6.1802
-2600 val perplexity 483.0939
-2600 train 5.962604 (lr=3.4545e-05) (hash(x)=133990623)
-600 val loss 7.4009
-600 val perplexity 1637.5042
-600 train 7.347689 (lr=4.9984e-05) (hash(x)=148404734)
-2700 val loss 6.1672
-2700 val perplexity 476.8687
-2700 train 6.135607 (lr=3.4044e-05) (hash(x)=142860944)
-700 val loss 7.3296
-700 val perplexity 1524.7316
-700 train 7.283709 (lr=4.9935e-05) (hash(x)=148115934)
-2800 val loss 6.1343
-2800 val perplexity 461.4319
-2800 train 6.149925 (lr=3.3526e-05) (hash(x)=137959511)
-800 val loss 7.2290
-800 val perplexity 1378.7905
-800 train 7.112580 (lr=4.9853e-05) (hash(x)=137464699)
-2900 val loss 6.1097
-2900 val perplexity 450.2144
-2900 train 6.063270 (lr=3.2992e-05) (hash(x)=147009873)
-900 val loss 7.1585
-900 val perplexity 1284.9253
-900 train 6.972766 (lr=4.9739e-05) (hash(x)=143886042)
-3000 val loss 6.0910
-3000 val perplexity 441.8452
-3000 train 5.950695 (lr=3.2441e-05) (hash(x)=158264841)
-1000 val loss 7.0824
-1000 val perplexity 1190.8246
-1000 train 7.084828 (lr=4.9593e-05) (hash(x)=163799796)
-3100 val loss 6.0868
-3100 val perplexity 440.0080
-3100 train 5.932418 (lr=3.1875e-05) (hash(x)=139232251)
-1100 val loss 7.0104
-1100 val perplexity 1108.0588
-1100 train 6.966660 (lr=4.9415e-05) (hash(x)=144592844)
-3200 val loss 6.0559
-3200 val perplexity 426.6234
-3200 train 6.098489 (lr=3.1295e-05) (hash(x)=153436104)
-1200 val loss 6.9273
-1200 val perplexity 1019.7122
-1200 train 7.257290 (lr=4.9205e-05) (hash(x)=204706354)
-3300 val loss 6.0380
-3300 val perplexity 419.0717
-3300 train 5.860431 (lr=3.0702e-05) (hash(x)=149681831)
-1300 val loss 6.8551
-1300 val perplexity 948.7275
-1300 train 6.785756 (lr=4.8964e-05) (hash(x)=150862210)
-3400 val loss 6.0271
-3400 val perplexity 414.5083
-3400 train 5.904068 (lr=3.0095e-05) (hash(x)=168425516)
-1400 val loss 6.7796
-1400 val perplexity 879.7139
-1400 train 6.770991 (lr=4.8691e-05) (hash(x)=147766811)
-3500 val loss 6.0053
-3500 val perplexity 405.5786
-3500 train 6.031878 (lr=2.9477e-05) (hash(x)=163104338)
-1500 val loss 6.6920
-1500 val perplexity 805.9559
-1500 train 6.509122 (lr=4.8388e-05) (hash(x)=135925327)
-3600 val loss 5.9910
-3600 val perplexity 399.8305
-3600 train 5.969259 (lr=2.8849e-05) (hash(x)=165109772)
-1600 val loss 6.6294
-1600 val perplexity 757.0620
-1600 train 6.598938 (lr=4.8055e-05) (hash(x)=160440642)
-3700 val loss 5.9877
-3700 val perplexity 398.4793
-3700 train 5.721743 (lr=2.8210e-05) (hash(x)=153420306)
-1700 val loss 6.5596
-1700 val perplexity 705.9973
-1700 train 6.660162 (lr=4.7691e-05) (hash(x)=151184106)
-3800 val loss 5.9577
-3800 val perplexity 386.7325
-3800 train 5.977221 (lr=2.7562e-05) (hash(x)=160168863)
-1800 val loss 6.5071
-1800 val perplexity 669.8593
-1800 train 6.396586 (lr=4.7299e-05) (hash(x)=148100580)
-3900 val loss 5.9497
-3900 val perplexity 383.6512
-3900 train 5.866309 (lr=2.6907e-05) (hash(x)=153906073)
-1900 val loss 6.4710
-1900 val perplexity 646.0992
-1900 train 6.298414 (lr=4.6878e-05) (hash(x)=149434659)
-4000 val loss 5.9363
-4000 val perplexity 378.5279
-4000 train 5.876613 (lr=2.6244e-05) (hash(x)=151055067)
-2000 val loss 6.4085
-2000 val perplexity 606.9621
-2000 train 6.308722 (lr=4.6428e-05) (hash(x)=152285486)
-4100 val loss 5.9223
-4100 val perplexity 373.2578
-4100 train 5.903517 (lr=2.5575e-05) (hash(x)=149629830)
-2100 val loss 6.3651
-2100 val perplexity 581.1993
-2100 train 6.112900 (lr=4.5951e-05) (hash(x)=144294295)
-4200 val loss 5.9090
-4200 val perplexity 368.3522
-4200 train 5.792145 (lr=2.4900e-05) (hash(x)=143101381)
-2200 val loss 6.3310
-2200 val perplexity 561.7220
-2200 train 6.346328 (lr=4.5448e-05) (hash(x)=175030215)
-4300 val loss 5.9024
-4300 val perplexity 365.9272
-4300 train 5.808477 (lr=2.4222e-05) (hash(x)=149712044)
-2300 val loss 6.3039
-2300 val perplexity 546.6758
-2300 train 6.113389 (lr=4.4918e-05) (hash(x)=150831428)
-4400 val loss 5.8847
-4400 val perplexity 359.4937
-4400 train 5.809499 (lr=2.3540e-05) (hash(x)=153446449)
-2400 val loss 6.2631
-2400 val perplexity 524.8505
-2400 train 6.434492 (lr=4.4363e-05) (hash(x)=140808297)
-4500 val loss 5.8707
-4500 val perplexity 354.4794
-4500 train 5.820339 (lr=2.2856e-05) (hash(x)=146086947)
-2500 val loss 6.2334
-2500 val perplexity 509.4646
-2500 train 6.218014 (lr=4.3784e-05) (hash(x)=153160275)
-4600 val loss 5.8615
-4600 val perplexity 351.2458
-4600 train 5.702044 (lr=2.2171e-05) (hash(x)=153800173)
-2600 val loss 6.2117
-2600 val perplexity 498.5354
-2600 train 5.992179 (lr=4.3181e-05) (hash(x)=133990623)
-4700 val loss 5.8508
-4700 val perplexity 347.5017
-4700 train 5.917328 (lr=2.1486e-05) (hash(x)=155962726)
-2700 val loss 6.1884
-2700 val perplexity 487.0828
-2700 train 6.157396 (lr=4.2555e-05) (hash(x)=142860944)
-4800 val loss 5.8399
-4800 val perplexity 343.7307
-4800 train 6.078052 (lr=2.0801e-05) (hash(x)=142045616)
-2800 val loss 6.1528
-2800 val perplexity 470.0521
-2800 train 6.166156 (lr=4.1908e-05) (hash(x)=137959511)
-4900 val loss 5.8263
-4900 val perplexity 339.1056
-4900 train 5.799638 (lr=2.0118e-05) (hash(x)=143418248)
-2900 val loss 6.1300
-2900 val perplexity 459.4257
-2900 train 6.085161 (lr=4.1240e-05) (hash(x)=147009873)
-5000 val loss 5.8207
-5000 val perplexity 337.2140
-5000 train 5.740273 (lr=1.9438e-05) (hash(x)=145789790)
-3000 val loss 6.1100
-3000 val perplexity 450.3489
-3000 train 5.963087 (lr=4.0551e-05) (hash(x)=158264841)
-5100 val loss 5.8133
-5100 val perplexity 334.7339
-5100 train 5.792392 (lr=1.8762e-05) (hash(x)=137795633)
-3100 val loss 6.0980
-3100 val perplexity 444.9706
-3100 train 5.941044 (lr=3.9844e-05) (hash(x)=139232251)
-5200 val loss 5.8031
-5200 val perplexity 331.3376
-5200 train 5.965540 (lr=1.8090e-05) (hash(x)=148907132)
-3200 val loss 6.0665
-3200 val perplexity 431.1898
-3200 train 6.106855 (lr=3.9119e-05) (hash(x)=153436104)
-5300 val loss 5.7981
-5300 val perplexity 329.6798
-5300 train 5.675566 (lr=1.7424e-05) (hash(x)=152343580)
-3300 val loss 6.0532
-3300 val perplexity 425.4629
-3300 train 5.876822 (lr=3.8377e-05) (hash(x)=149681831)
-5400 val loss 5.7863
-5400 val perplexity 325.8111
-5400 train 5.640113 (lr=1.6765e-05) (hash(x)=148578264)
-3400 val loss 6.0428
-3400 val perplexity 421.0752
-3400 train 5.926132 (lr=3.7619e-05) (hash(x)=168425516)
-5500 val loss 5.7738
-5500 val perplexity 321.7622
-5500 train 5.940225 (lr=1.6113e-05) (hash(x)=145635833)
-3500 val loss 6.0154
-3500 val perplexity 409.6693
-3500 train 6.041699 (lr=3.6847e-05) (hash(x)=163104338)
-5600 val loss 5.7682
-5600 val perplexity 319.9536
-5600 train 5.752505 (lr=1.5469e-05) (hash(x)=156337844)
-3600 val loss 6.0012
-3600 val perplexity 403.8958
-3600 train 5.974601 (lr=3.6061e-05) (hash(x)=165109772)
-5700 val loss 5.7646
-5700 val perplexity 318.8028
-5700 train 5.645874 (lr=1.4836e-05) (hash(x)=147168506)
-3700 val loss 5.9942
-3700 val perplexity 401.1147
-3700 train 5.726319 (lr=3.5263e-05) (hash(x)=153420306)
-5800 val loss 5.7550
-5800 val perplexity 315.7605
-5800 train 5.812964 (lr=1.4212e-05) (hash(x)=159566920)
-3800 val loss 5.9667
-3800 val perplexity 390.2202
-3800 train 5.985947 (lr=3.4453e-05) (hash(x)=160168863)
-5900 val loss 5.7469
-5900 val perplexity 313.2336
-5900 train 5.812284 (lr=1.3600e-05) (hash(x)=158273929)
-3900 val loss 5.9539
-3900 val perplexity 385.2505
-3900 train 5.868129 (lr=3.3633e-05) (hash(x)=153906073)
-6000 val loss 5.7454
-6000 val perplexity 312.7557
-6000 train 5.718481 (lr=1.3000e-05) (hash(x)=156649749)
-4000 val loss 5.9410
-4000 val perplexity 380.3042
-4000 train 5.883380 (lr=3.2805e-05) (hash(x)=151055067)
-6100 val loss 5.7351
-6100 val perplexity 309.5356
-6100 train 5.649320 (lr=1.2413e-05) (hash(x)=146812388)
-4100 val loss 5.9261
-4100 val perplexity 374.7030
-4100 train 5.906900 (lr=3.1968e-05) (hash(x)=149629830)
-6200 val loss 5.7306
-6200 val perplexity 308.1406
-6200 train 5.558667 (lr=1.1840e-05) (hash(x)=143522146)
-4200 val loss 5.9100
-4200 val perplexity 368.6896
-4200 train 5.794271 (lr=3.1126e-05) (hash(x)=143101381)
-6300 val loss 5.7261
-6300 val perplexity 306.7715
-6300 train 5.550650 (lr=1.1282e-05) (hash(x)=150124474)
-4300 val loss 5.9004
-4300 val perplexity 365.2003
-4300 train 5.797774 (lr=3.0277e-05) (hash(x)=149712044)
-6400 val loss 5.7211
-6400 val perplexity 305.2509
-6400 train 5.749485 (lr=1.0739e-05) (hash(x)=141242117)
-4400 val loss 5.8842
-4400 val perplexity 359.3067
-4400 train 5.809937 (lr=2.9425e-05) (hash(x)=153446449)
-6500 val loss 5.7133
-6500 val perplexity 302.8625
-6500 train 5.593424 (lr=1.0213e-05) (hash(x)=143529762)
-4500 val loss 5.8706
-4500 val perplexity 354.4476
-4500 train 5.820799 (lr=2.8571e-05) (hash(x)=146086947)
-6600 val loss 5.7079
-6600 val perplexity 301.2269
-6600 train 5.496843 (lr=9.7032e-06) (hash(x)=136948374)
-4600 val loss 5.8618
-4600 val perplexity 351.3521
-4600 train 5.701613 (lr=2.7714e-05) (hash(x)=153800173)
-6700 val loss 5.7075
-6700 val perplexity 301.1216
-6700 train 5.509098 (lr=9.2116e-06) (hash(x)=146268592)
-4700 val loss 5.8520
-4700 val perplexity 347.9245
-4700 train 5.926853 (lr=2.6857e-05) (hash(x)=155962726)
-6800 val loss 5.6996
-6800 val perplexity 298.7610
-6800 train 5.658401 (lr=8.7387e-06) (hash(x)=152676836)
-4800 val loss 5.8400
-4800 val perplexity 343.7773
-4800 train 6.078588 (lr=2.6002e-05) (hash(x)=142045616)
-6900 val loss 5.6922
-6900 val perplexity 296.5384
-6900 train 5.611579 (lr=8.2849e-06) (hash(x)=134657776)
-4900 val loss 5.8270
-4900 val perplexity 339.3542
-4900 train 5.798401 (lr=2.5148e-05) (hash(x)=143418248)
-7000 val loss 5.6908
-7000 val perplexity 296.1258
-7000 train 5.645500 (lr=7.8510e-06) (hash(x)=166721861)
-5000 val loss 5.8158
-5000 val perplexity 335.5614
-5000 train 5.741745 (lr=2.4298e-05) (hash(x)=145789790)
-7100 val loss 5.6882
-7100 val perplexity 295.3699
-7100 train 5.487141 (lr=7.4377e-06) (hash(x)=135496702)
-5100 val loss 5.8126
-5100 val perplexity 334.4930
-5100 train 5.793215 (lr=2.3452e-05) (hash(x)=137795633)
-7200 val loss 5.6852
-7200 val perplexity 294.4621
-7200 train 5.796127 (lr=7.0455e-06) (hash(x)=155567461)
-5200 val loss 5.7978
-5200 val perplexity 329.5794
-5200 train 5.952859 (lr=2.2613e-05) (hash(x)=148907132)
-7300 val loss 5.6786
-7300 val perplexity 292.5464
-7300 train 5.477703 (lr=6.6749e-06) (hash(x)=142803829)
-5300 val loss 5.7882
-5300 val perplexity 326.4250
-5300 train 5.663875 (lr=2.1780e-05) (hash(x)=152343580)
-7400 val loss 5.6780
-7400 val perplexity 292.3659
-7400 train 5.497990 (lr=6.3266e-06) (hash(x)=145294178)
-5400 val loss 5.7794
-5400 val perplexity 323.5801
-5400 train 5.630389 (lr=2.0956e-05) (hash(x)=148578264)
-7500 val loss 5.6748
-7500 val perplexity 291.4342
-7500 train 5.431237 (lr=6.0010e-06) (hash(x)=150573713)
-5500 val loss 5.7663
-5500 val perplexity 319.3484
-5500 train 5.930579 (lr=2.0141e-05) (hash(x)=145635833)
-7600 val loss 5.6735
-7600 val perplexity 291.0639
-7600 train 5.689526 (lr=5.6986e-06) (hash(x)=142771511)
-5600 val loss 5.7618
-5600 val perplexity 317.9220
-5600 train 5.749690 (lr=1.9337e-05) (hash(x)=156337844)
-7700 val loss 5.6666
-7700 val perplexity 289.0573
-7700 train 5.571441 (lr=5.4198e-06) (hash(x)=143602175)
-5700 val loss 5.7559
-5700 val perplexity 316.0397
-5700 train 5.634243 (lr=1.8545e-05) (hash(x)=147168506)
-7800 val loss 5.6639
-7800 val perplexity 288.2749
-7800 train 5.717690 (lr=5.1650e-06) (hash(x)=152379862)
-5800 val loss 5.7454
-5800 val perplexity 312.7545
-5800 train 5.805884 (lr=1.7765e-05) (hash(x)=159566920)
-7900 val loss 5.6640
-7900 val perplexity 288.2939
-7900 train 5.486330 (lr=4.9347e-06) (hash(x)=146655921)
-5900 val loss 5.7365
-5900 val perplexity 309.9833
-5900 train 5.803931 (lr=1.7000e-05) (hash(x)=158273929)
-8000 val loss 5.6640
-8000 val perplexity 288.3034
-8000 train 5.793463 (lr=4.7291e-06) (hash(x)=148262482)
-6000 val loss 5.7346
-6000 val perplexity 309.3907
-6000 train 5.703155 (lr=1.6250e-05) (hash(x)=156649749)
-8100 val loss 5.6561
-8100 val perplexity 286.0415
-8100 train 5.593901 (lr=4.5486e-06) (hash(x)=147683655)
-6100 val loss 5.7216
-6100 val perplexity 305.3945
-6100 train 5.637345 (lr=1.5516e-05) (hash(x)=146812388)
-8200 val loss 5.6568
-8200 val perplexity 286.2420
-8200 train 5.767795 (lr=4.3933e-06) (hash(x)=157312987)
-6200 val loss 5.7177
-6200 val perplexity 304.2039
-6200 train 5.546375 (lr=1.4800e-05) (hash(x)=143522146)
-8300 val loss 5.6541
-8300 val perplexity 285.4726
-8300 train 5.617525 (lr=4.2636e-06) (hash(x)=141107543)
-6300 val loss 5.7147
-6300 val perplexity 303.2979
-6300 train 5.535329 (lr=1.4102e-05) (hash(x)=150124474)
-8400 val loss 5.6495
-8400 val perplexity 284.1558
-8400 train 5.641396 (lr=4.1596e-06) (hash(x)=141323024)
-6400 val loss 5.7066
-6400 val perplexity 300.8557
-6400 train 5.735716 (lr=1.3424e-05) (hash(x)=141242117)
-8500 val loss 5.6482
-8500 val perplexity 283.7680
-8500 train 5.618681 (lr=4.0815e-06) (hash(x)=150696521)
-6500 val loss 5.6979
-6500 val perplexity 298.2270
-6500 train 5.574231 (lr=1.2766e-05) (hash(x)=143529762)
-8600 val loss 5.6481
-8600 val perplexity 283.7611
-8600 train 5.595167 (lr=4.0294e-06) (hash(x)=162288191)
-6600 val loss 5.6934
-6600 val perplexity 296.9074
-6600 train 5.482853 (lr=1.2129e-05) (hash(x)=136948374)
-8700 val loss 5.6443
-8700 val perplexity 282.6739
-8700 train 5.511504 (lr=4.0033e-06) (hash(x)=152860941)
-8749 val loss 5.6436
-8749 val perplexity 282.4701
-6700 val loss 5.6928
-6700 val perplexity 296.7115
-6700 train 5.493250 (lr=1.1515e-05) (hash(x)=146268592)
-6800 val loss 5.6825
-6800 val perplexity 293.6945
-6800 train 5.628151 (lr=1.0923e-05) (hash(x)=152676836)
-6900 val loss 5.6746
-6900 val perplexity 291.3775
-6900 train 5.592096 (lr=1.0356e-05) (hash(x)=134657776)
-7000 val loss 5.6738
-7000 val perplexity 291.1277
-7000 train 5.625089 (lr=9.8138e-06) (hash(x)=166721861)
-7100 val loss 5.6712
-7100 val perplexity 290.3784
-7100 train 5.471138 (lr=9.2971e-06) (hash(x)=135496702)
-7200 val loss 5.6675
-7200 val perplexity 289.3047
-7200 train 5.778218 (lr=8.8068e-06) (hash(x)=155567461)
-7300 val loss 5.6589
-7300 val perplexity 286.8447
-7300 train 5.459130 (lr=8.3436e-06) (hash(x)=142803829)
-7400 val loss 5.6582
-7400 val perplexity 286.6350
-7400 train 5.478490 (lr=7.9082e-06) (hash(x)=145294178)
-7500 val loss 5.6553
-7500 val perplexity 285.8107
-7500 train 5.411656 (lr=7.5012e-06) (hash(x)=150573713)
-7600 val loss 5.6530
-7600 val perplexity 285.1458
-7600 train 5.668416 (lr=7.1232e-06) (hash(x)=142771511)
-7700 val loss 5.6466
-7700 val perplexity 283.3324
-7700 train 5.552392 (lr=6.7747e-06) (hash(x)=143602175)
-7800 val loss 5.6435
-7800 val perplexity 282.4496
-7800 train 5.698727 (lr=6.4563e-06) (hash(x)=152379862)
-7900 val loss 5.6427
-7900 val perplexity 282.2134
-7900 train 5.463721 (lr=6.1684e-06) (hash(x)=146655921)
-8000 val loss 5.6430
-8000 val perplexity 282.2946
-8000 train 5.779183 (lr=5.9114e-06) (hash(x)=148262482)
-8100 val loss 5.6351
-8100 val perplexity 280.0877
-8100 train 5.579781 (lr=5.6857e-06) (hash(x)=147683655)
-8200 val loss 5.6347
-8200 val perplexity 279.9642
-8200 train 5.747372 (lr=5.4917e-06) (hash(x)=157312987)
-8300 val loss 5.6320
-8300 val perplexity 279.2189
-8300 train 5.594587 (lr=5.3295e-06) (hash(x)=141107543)
-8400 val loss 5.6274
-8400 val perplexity 277.9336
-8400 train 5.622123 (lr=5.1995e-06) (hash(x)=141323024)
-8500 val loss 5.6254
-8500 val perplexity 277.3963
-8500 train 5.598778 (lr=5.1019e-06) (hash(x)=150696521)
-8600 val loss 5.6246
-8600 val perplexity 277.1568
-8600 train 5.568236 (lr=5.0367e-06) (hash(x)=162288191)
-8700 val loss 5.6214
-8700 val perplexity 276.2682
-8700 train 5.487337 (lr=5.0041e-06) (hash(x)=152860941)
-8749 val loss 5.6205
-8749 val perplexity 276.0402
+0 train 11.313038 (lr=7.0000e-08) (hash(x)=145079536)
+0 train 11.313038 (lr=6.0000e-08) (hash(x)=145079536)
+100 val loss 10.1630
+100 val perplexity 25926.1094
+100 val loss 10.2343
+100 val perplexity 27841.4590
+100 train 10.156528 (lr=7.0700e-06) (hash(x)=155800595)
+100 train 10.225678 (lr=6.0600e-06) (hash(x)=155800595)
+200 val loss 9.7494
+200 val perplexity 17144.4492
+200 train 9.706524 (lr=1.4070e-05) (hash(x)=145606733)
+200 val loss 9.7756
+200 val perplexity 17599.2559
+200 train 9.732876 (lr=1.2060e-05) (hash(x)=145606733)
+300 val loss 8.8013
+300 val perplexity 6643.1152
+300 train 8.798988 (lr=2.1070e-05) (hash(x)=150367139)
+300 val loss 8.9184
+300 val perplexity 7468.2007
+300 train 8.917364 (lr=1.8060e-05) (hash(x)=150367139)
+400 val loss 7.9646
+400 val perplexity 2877.3264
+400 train 8.234877 (lr=2.8070e-05) (hash(x)=155747374)
+400 val loss 7.9782
+400 val perplexity 2916.8208
+400 train 8.244305 (lr=2.4060e-05) (hash(x)=155747374)
+500 val loss 7.6135
+500 val perplexity 2025.2765
+500 train 7.510717 (lr=3.5000e-05) (hash(x)=140604760)
+500 val loss 7.5787
+500 val perplexity 1955.9950
+500 train 7.475321 (lr=3.0000e-05) (hash(x)=140604760)
+600 val loss 7.4823
+600 val perplexity 1776.2891
+600 train 7.433376 (lr=3.4989e-05) (hash(x)=148404734)
+600 val loss 7.4607
+600 val perplexity 1738.3960
+600 train 7.414741 (lr=2.9990e-05) (hash(x)=148404734)
+700 val loss 7.4270
+700 val perplexity 1680.7137
+700 train 7.384828 (lr=3.4954e-05) (hash(x)=148115934)
+700 val loss 7.3964
+700 val perplexity 1630.0658
+700 train 7.354086 (lr=2.9961e-05) (hash(x)=148115934)
+800 val loss 7.3625
+800 val perplexity 1575.7120
+800 train 7.258959 (lr=3.4897e-05) (hash(x)=137464699)
+800 val loss 7.3292
+800 val perplexity 1524.1631
+800 train 7.219800 (lr=2.9912e-05) (hash(x)=137464699)
+900 val loss 7.3188
+900 val perplexity 1508.4202
+900 train 7.139495 (lr=3.4818e-05) (hash(x)=143886042)
+900 val loss 7.2721
+900 val perplexity 1439.5078
+900 train 7.090878 (lr=2.9844e-05) (hash(x)=143886042)
+1000 val loss 7.2571
+1000 val perplexity 1418.1678
+1000 train 7.259868 (lr=3.4715e-05) (hash(x)=163799796)
+1000 val loss 7.2204
+1000 val perplexity 1367.0923
+1000 train 7.224873 (lr=2.9756e-05) (hash(x)=163799796)
+1100 val loss 7.2004
+1100 val perplexity 1339.9773
+1100 train 7.151253 (lr=3.4591e-05) (hash(x)=144592844)
+1100 val loss 7.1528
+1100 val perplexity 1277.6487
+1100 train 7.103980 (lr=2.9649e-05) (hash(x)=144592844)
+1200 val loss 7.1496
+1200 val perplexity 1273.5430
+1200 train 7.510290 (lr=3.4444e-05) (hash(x)=204706354)
+1200 val loss 7.0939
+1200 val perplexity 1204.5594
+1200 train 7.442509 (lr=2.9523e-05) (hash(x)=204706354)
+1300 val loss 7.0922
+1300 val perplexity 1202.5422
+1300 train 7.020781 (lr=3.4275e-05) (hash(x)=150862210)
+1300 val loss 7.0370
+1300 val perplexity 1138.0204
+1300 train 6.966258 (lr=2.9378e-05) (hash(x)=150862210)
+1400 val loss 7.0167
+1400 val perplexity 1115.1079
+1400 train 6.995105 (lr=3.4084e-05) (hash(x)=147766811)
+1400 val loss 6.9707
+1400 val perplexity 1064.9824
+1400 train 6.951100 (lr=2.9215e-05) (hash(x)=147766811)
+1500 val loss 6.9622
+1500 val perplexity 1056.0016
+1500 train 6.773920 (lr=3.3872e-05) (hash(x)=135925327)
+1500 val loss 6.9144
+1500 val perplexity 1006.6741
+1500 train 6.724929 (lr=2.9033e-05) (hash(x)=135925327)
+1600 val loss 6.9194
+1600 val perplexity 1011.7564
+1600 train 6.889316 (lr=3.3638e-05) (hash(x)=160440642)
+1600 val loss 6.8620
+1600 val perplexity 955.2529
+1600 train 6.835199 (lr=2.8833e-05) (hash(x)=160440642)
+1700 val loss 6.8600
+1700 val perplexity 953.3604
+1700 train 6.953665 (lr=3.3384e-05) (hash(x)=151184106)
+1700 val loss 6.7968
+1700 val perplexity 894.9904
+1700 train 6.890760 (lr=2.8615e-05) (hash(x)=151184106)
+1800 val loss 6.8067
+1800 val perplexity 903.8531
+1800 train 6.697411 (lr=3.3109e-05) (hash(x)=148100580)
+1800 val loss 6.7426
+1800 val perplexity 847.7204
+1800 train 6.634793 (lr=2.8379e-05) (hash(x)=148100580)
+1900 val loss 6.7725
+1900 val perplexity 873.4525
+1900 train 6.604763 (lr=3.2814e-05) (hash(x)=149434659)
+1900 val loss 6.6984
+1900 val perplexity 811.1066
+1900 train 6.530216 (lr=2.8127e-05) (hash(x)=149434659)
+2000 val loss 6.7098
+2000 val perplexity 820.4353
+2000 train 6.604181 (lr=3.2500e-05) (hash(x)=152285486)
+2000 val loss 6.6355
+2000 val perplexity 761.6394
+2000 train 6.533551 (lr=2.7857e-05) (hash(x)=152285486)
+2100 val loss 6.6650
+2100 val perplexity 784.4327
+2100 train 6.413668 (lr=3.2166e-05) (hash(x)=144294295)
+2100 val loss 6.5839
+2100 val perplexity 723.3694
+2100 train 6.331763 (lr=2.7571e-05) (hash(x)=144294295)
+2200 val loss 6.6268
+2200 val perplexity 755.0834
+2200 train 6.659666 (lr=3.1813e-05) (hash(x)=175030215)
+2200 val loss 6.5444
+2200 val perplexity 695.3740
+2200 train 6.575781 (lr=2.7269e-05) (hash(x)=175030215)
+2300 val loss 6.5935
+2300 val perplexity 730.3160
+2300 train 6.395243 (lr=3.1443e-05) (hash(x)=150831428)
+2300 val loss 6.5118
+2300 val perplexity 673.0463
+2300 train 6.314830 (lr=2.6951e-05) (hash(x)=150831428)
+2400 val loss 6.5340
+2400 val perplexity 688.1121
+2400 train 6.689769 (lr=3.1054e-05) (hash(x)=140808297)
+2400 val loss 6.4649
+2400 val perplexity 642.2308
+2400 train 6.624811 (lr=2.6618e-05) (hash(x)=140808297)
+2500 val loss 6.4948
+2500 val perplexity 661.7064
+2500 train 6.484925 (lr=3.0649e-05) (hash(x)=153160275)
+2500 val loss 6.4345
+2500 val perplexity 622.9711
+2500 train 6.418693 (lr=2.6270e-05) (hash(x)=153160275)
+2600 val loss 6.4673
+2600 val perplexity 643.7227
+2600 train 6.227062 (lr=3.0227e-05) (hash(x)=133990623)
+2600 val loss 6.4079
+2600 val perplexity 606.6450
+2600 train 6.165543 (lr=2.5909e-05) (hash(x)=133990623)
+2700 val loss 6.4393
+2700 val perplexity 625.9974
+2700 train 6.414563 (lr=2.9789e-05) (hash(x)=142860944)
+2700 val loss 6.3846
+2700 val perplexity 592.6437
+2700 train 6.360803 (lr=2.5533e-05) (hash(x)=142860944)
+2800 val loss 6.4029
+2800 val perplexity 603.5896
+2800 train 6.432872 (lr=2.9336e-05) (hash(x)=137959511)
+2800 val loss 6.3519
+2800 val perplexity 573.5657
+2800 train 6.378989 (lr=2.5145e-05) (hash(x)=137959511)
+2900 val loss 6.3718
+2900 val perplexity 585.1338
+2900 train 6.311280 (lr=2.8868e-05) (hash(x)=147009873)
+2900 val loss 6.3265
+2900 val perplexity 559.1986
+2900 train 6.269898 (lr=2.4744e-05) (hash(x)=147009873)
+3000 val loss 6.3476
+3000 val perplexity 571.1144
+3000 train 6.218025 (lr=2.8386e-05) (hash(x)=158264841)
+3000 val loss 6.3063
+3000 val perplexity 548.0392
+3000 train 6.173168 (lr=2.4331e-05) (hash(x)=158264841)
+3100 val loss 6.3292
+3100 val perplexity 560.6805
+3100 train 6.188198 (lr=2.7891e-05) (hash(x)=139232251)
+3100 val loss 6.2925
+3100 val perplexity 540.5228
+3100 train 6.155460 (lr=2.3906e-05) (hash(x)=139232251)
+3200 val loss 6.2932
+3200 val perplexity 540.8548
+3200 train 6.341791 (lr=2.7383e-05) (hash(x)=153436104)
+3200 val loss 6.2591
+3200 val perplexity 522.7567
+3200 train 6.305866 (lr=2.3471e-05) (hash(x)=153436104)
+3300 val loss 6.2764
+3300 val perplexity 531.8832
+3300 train 6.092842 (lr=2.6864e-05) (hash(x)=149681831)
+3300 val loss 6.2423
+3300 val perplexity 514.0164
+3300 train 6.057350 (lr=2.3026e-05) (hash(x)=149681831)
+3400 val loss 6.2633
+3400 val perplexity 524.9594
+3400 train 6.166925 (lr=2.6333e-05) (hash(x)=168425516)
+3400 val loss 6.2299
+3400 val perplexity 507.6933
+3400 train 6.136177 (lr=2.2572e-05) (hash(x)=168425516)
+3500 val loss 6.2348
+3500 val perplexity 510.2036
+3500 train 6.263444 (lr=2.5793e-05) (hash(x)=163104338)
+3500 val loss 6.2078
+3500 val perplexity 496.5847
+3500 train 6.231250 (lr=2.2108e-05) (hash(x)=163104338)
+3600 val loss 6.2157
+3600 val perplexity 500.5353
+3600 train 6.190797 (lr=2.5243e-05) (hash(x)=165109772)
+3600 val loss 6.1890
+3600 val perplexity 487.3521
+3600 train 6.160804 (lr=2.1637e-05) (hash(x)=165109772)
+3700 val loss 6.2076
+3700 val perplexity 496.5286
+3700 train 5.940827 (lr=2.4684e-05) (hash(x)=153420306)
+3700 val loss 6.1816
+3700 val perplexity 483.7829
+3700 train 5.910549 (lr=2.1158e-05) (hash(x)=153420306)
+3800 val loss 6.1786
+3800 val perplexity 482.2998
+3800 train 6.204558 (lr=2.4117e-05) (hash(x)=160168863)
+3800 val loss 6.1568
+3800 val perplexity 471.9337
+3800 train 6.183776 (lr=2.0672e-05) (hash(x)=160168863)
+3900 val loss 6.1666
+3900 val perplexity 476.5572
+3900 train 6.095548 (lr=2.3543e-05) (hash(x)=153906073)
+3900 val loss 6.1428
+3900 val perplexity 465.3320
+3900 train 6.070639 (lr=2.0180e-05) (hash(x)=153906073)
+4000 val loss 6.1527
+4000 val perplexity 469.9752
+4000 train 6.081241 (lr=2.2963e-05) (hash(x)=151055067)
+4000 val loss 6.1338
+4000 val perplexity 461.1803
+4000 train 6.064647 (lr=1.9683e-05) (hash(x)=151055067)
+4100 val loss 6.1350
+4100 val perplexity 461.7586
+4100 train 6.116509 (lr=2.2378e-05) (hash(x)=149629830)
+4100 val loss 6.1163
+4100 val perplexity 453.1846
+4100 train 6.090425 (lr=1.9181e-05) (hash(x)=149629830)
+4200 val loss 6.1249
+4200 val perplexity 457.1209
+4200 train 6.003567 (lr=2.1788e-05) (hash(x)=143101381)
+4200 val loss 6.1033
+4200 val perplexity 447.3311
+4200 train 5.984151 (lr=1.8675e-05) (hash(x)=143101381)
+4300 val loss 6.1115
+4300 val perplexity 451.0039
+4300 train 6.010469 (lr=2.1194e-05) (hash(x)=149712044)
+4300 val loss 6.0949
+4300 val perplexity 443.5709
+4300 train 5.993714 (lr=1.8166e-05) (hash(x)=149712044)
+4400 val loss 6.0941
+4400 val perplexity 443.2506
+4400 train 6.020243 (lr=2.0598e-05) (hash(x)=153446449)
+4400 val loss 6.0809
+4400 val perplexity 437.4159
+4400 train 6.007243 (lr=1.7655e-05) (hash(x)=153446449)
+4500 val loss 6.0824
+4500 val perplexity 438.0891
+4500 train 6.027428 (lr=1.9999e-05) (hash(x)=146086947)
+4500 val loss 6.0681
+4500 val perplexity 431.8431
+4500 train 6.013597 (lr=1.7142e-05) (hash(x)=146086947)
+4600 val loss 6.0720
+4600 val perplexity 433.5492
+4600 train 5.910559 (lr=1.9400e-05) (hash(x)=153800173)
+4600 val loss 6.0581
+4600 val perplexity 427.5742
+4600 train 5.898770 (lr=1.6629e-05) (hash(x)=153800173)
+4700 val loss 6.0597
+4700 val perplexity 428.2572
+4700 train 6.110185 (lr=1.8800e-05) (hash(x)=155962726)
+4700 val loss 6.0479
+4700 val perplexity 423.2194
+4700 train 6.100060 (lr=1.6114e-05) (hash(x)=155962726)
+4800 val loss 6.0515
+4800 val perplexity 424.7603
+4800 train 6.275963 (lr=1.8201e-05) (hash(x)=142045616)
+4800 val loss 6.0388
+4800 val perplexity 419.4092
+4800 train 6.270424 (lr=1.5601e-05) (hash(x)=142045616)
+4900 val loss 6.0377
+4900 val perplexity 418.9119
+4900 train 6.005858 (lr=1.7604e-05) (hash(x)=143418248)
+4900 val loss 6.0267
+4900 val perplexity 414.3510
+4900 train 5.989192 (lr=1.5089e-05) (hash(x)=143418248)
+5000 val loss 6.0290
+5000 val perplexity 415.2997
+5000 train 5.954850 (lr=1.7009e-05) (hash(x)=145789790)
+5000 val loss 6.0175
+5000 val perplexity 410.5454
+5000 train 5.938465 (lr=1.4579e-05) (hash(x)=145789790)
+5100 val loss 6.0238
+5100 val perplexity 413.1402
+5100 train 5.980156 (lr=1.6417e-05) (hash(x)=137795633)
+5100 val loss 6.0147
+5100 val perplexity 409.3998
+5100 train 5.970935 (lr=1.4071e-05) (hash(x)=137795633)
+5200 val loss 6.0137
+5200 val perplexity 408.9803
+5200 train 6.178001 (lr=1.5829e-05) (hash(x)=148907132)
+5200 val loss 6.0025
+5200 val perplexity 404.4547
+5200 train 6.166307 (lr=1.3568e-05) (hash(x)=148907132)
+5300 val loss 6.0028
+5300 val perplexity 404.5762
+5300 train 5.882075 (lr=1.5246e-05) (hash(x)=152343580)
+5300 val loss 5.9958
+5300 val perplexity 401.7314
+5300 train 5.877028 (lr=1.3068e-05) (hash(x)=152343580)
+5400 val loss 5.9968
+5400 val perplexity 402.1420
+5400 train 5.843998 (lr=1.4669e-05) (hash(x)=148578264)
+5400 val loss 5.9889
+5400 val perplexity 398.9933
+5400 train 5.840799 (lr=1.2573e-05) (hash(x)=148578264)
+5500 val loss 5.9842
+5500 val perplexity 397.1019
+5500 train 6.141342 (lr=1.4099e-05) (hash(x)=145635833)
+5500 val loss 5.9769
+5500 val perplexity 394.1999
+5500 train 6.139304 (lr=1.2085e-05) (hash(x)=145635833)
+5600 val loss 5.9796
+5600 val perplexity 395.2935
+5600 train 5.955497 (lr=1.3536e-05) (hash(x)=156337844)
+5600 val loss 5.9735
+5600 val perplexity 392.8972
+5600 train 5.955674 (lr=1.1602e-05) (hash(x)=156337844)
+5700 val loss 5.9731
+5700 val perplexity 392.7069
+5700 train 5.856874 (lr=1.2981e-05) (hash(x)=147168506)
+5700 val loss 5.9688
+5700 val perplexity 391.0504
+5700 train 5.857741 (lr=1.1127e-05) (hash(x)=147168506)
+5800 val loss 5.9644
+5800 val perplexity 389.3232
+5800 train 6.032385 (lr=1.2436e-05) (hash(x)=159566920)
+5800 val loss 5.9585
+5800 val perplexity 387.0450
+5800 train 6.023722 (lr=1.0659e-05) (hash(x)=159566920)
+5900 val loss 5.9578
+5900 val perplexity 386.7631
+5900 train 6.025391 (lr=1.1900e-05) (hash(x)=158273929)
+5900 val loss 5.9536
+5900 val perplexity 385.1432
+5900 train 6.027411 (lr=1.0200e-05) (hash(x)=158273929)
+6000 val loss 5.9532
+6000 val perplexity 384.9910
+6000 train 5.921682 (lr=1.1375e-05) (hash(x)=156649749)
+6000 val loss 5.9499
+6000 val perplexity 383.6991
+6000 train 5.916996 (lr=9.7500e-06) (hash(x)=156649749)
+6100 val loss 5.9437
+6100 val perplexity 381.3258
+6100 train 5.860867 (lr=1.0861e-05) (hash(x)=146812388)
+6100 val loss 5.9397
+6100 val perplexity 379.8224
+6100 train 5.858076 (lr=9.3098e-06) (hash(x)=146812388)
+6200 val loss 5.9380
+6200 val perplexity 379.1947
+6200 train 5.760242 (lr=1.0360e-05) (hash(x)=143522146)
+6200 val loss 5.9360
+6200 val perplexity 378.4272
+6200 train 5.755511 (lr=8.8800e-06) (hash(x)=143522146)
+6300 val loss 5.9354
+6300 val perplexity 378.1945
+6300 train 5.772961 (lr=9.8715e-06) (hash(x)=150124474)
+6300 val loss 5.9314
+6300 val perplexity 376.6789
+6300 train 5.768746 (lr=8.4613e-06) (hash(x)=150124474)
+6400 val loss 5.9273
+6400 val perplexity 375.1227
+6400 train 5.950421 (lr=9.3966e-06) (hash(x)=141242117)
+6400 val loss 5.9284
+6400 val perplexity 375.5501
+6400 train 5.955993 (lr=8.0542e-06) (hash(x)=141242117)
+6500 val loss 5.9207
+6500 val perplexity 372.6563
+6500 train 5.801312 (lr=8.9359e-06) (hash(x)=143529762)
+6500 val loss 5.9189
+6500 val perplexity 371.9922
+6500 train 5.801332 (lr=7.6594e-06) (hash(x)=143529762)
+6600 val loss 5.9155
+6600 val perplexity 370.7257
+6600 train 5.697849 (lr=8.4903e-06) (hash(x)=136948374)
+6600 val loss 5.9148
+6600 val perplexity 370.4690
+6600 train 5.696297 (lr=7.2774e-06) (hash(x)=136948374)
+6700 val loss 5.9141
+6700 val perplexity 370.2074
+6700 train 5.733417 (lr=8.0602e-06) (hash(x)=146268592)
+6700 val loss 5.9131
+6700 val perplexity 369.8498
+6700 train 5.730753 (lr=6.9087e-06) (hash(x)=146268592)
+6800 val loss 5.9055
+6800 val perplexity 367.0687
+6800 train 5.868842 (lr=7.6463e-06) (hash(x)=152676836)
+6800 val loss 5.9058
+6800 val perplexity 367.1629
+6800 train 5.863817 (lr=6.5540e-06) (hash(x)=152676836)
+6900 val loss 5.8984
+6900 val perplexity 364.4714
+6900 train 5.812114 (lr=7.2493e-06) (hash(x)=134657776)
+6900 val loss 5.8995
+6900 val perplexity 364.8668
+6900 train 5.811509 (lr=6.2137e-06) (hash(x)=134657776)
+7000 val loss 5.8971
+7000 val perplexity 363.9915
+7000 train 5.862196 (lr=6.8697e-06) (hash(x)=166721861)
+7000 val loss 5.8977
+7000 val perplexity 364.2080
+7000 train 5.862389 (lr=5.8883e-06) (hash(x)=166721861)
+7100 val loss 5.8936
+7100 val perplexity 362.6912
+7100 train 5.697666 (lr=6.5080e-06) (hash(x)=135496702)
+7100 val loss 5.8958
+7100 val perplexity 363.5053
+7100 train 5.697424 (lr=5.5783e-06) (hash(x)=135496702)
+7200 val loss 5.8906
+7200 val perplexity 361.6083
+7200 train 5.993858 (lr=6.1648e-06) (hash(x)=155567461)
+7200 val loss 5.8928
+7200 val perplexity 362.4025
+7200 train 5.994301 (lr=5.2841e-06) (hash(x)=155567461)
+7300 val loss 5.8846
+7300 val perplexity 359.4441
+7300 train 5.676643 (lr=5.8405e-06) (hash(x)=142803829)
+7300 val loss 5.8867
+7300 val perplexity 360.1984
+7300 train 5.685822 (lr=5.0062e-06) (hash(x)=142803829)
+7400 val loss 5.8824
+7400 val perplexity 358.6774
+7400 train 5.699541 (lr=5.5357e-06) (hash(x)=145294178)
+7400 val loss 5.8848
+7400 val perplexity 359.5374
+7400 train 5.707546 (lr=4.7449e-06) (hash(x)=145294178)
+7500 val loss 5.8797
+7500 val perplexity 357.6940
+7500 train 5.642534 (lr=5.2508e-06) (hash(x)=150573713)
+7500 val loss 5.8828
+7500 val perplexity 358.8115
+7500 train 5.639037 (lr=4.5007e-06) (hash(x)=150573713)
+7600 val loss 5.8773
+7600 val perplexity 356.8476
+7600 train 5.904793 (lr=4.9862e-06) (hash(x)=142771511)
+7600 val loss 5.8806
+7600 val perplexity 358.0397
+7600 train 5.905676 (lr=4.2739e-06) (hash(x)=142771511)
+7700 val loss 5.8716
+7700 val perplexity 354.8214
+7700 train 5.766538 (lr=4.7423e-06) (hash(x)=143602175)
+7700 val loss 5.8750
+7700 val perplexity 356.0248
+7700 train 5.774282 (lr=4.0648e-06) (hash(x)=143602175)
+7800 val loss 5.8698
+7800 val perplexity 354.1783
+7800 train 5.892739 (lr=4.5194e-06) (hash(x)=152379862)
+7800 val loss 5.8722
+7800 val perplexity 355.0424
+7800 train 5.895572 (lr=3.8738e-06) (hash(x)=152379862)
+7900 val loss 5.8686
+7900 val perplexity 353.7581
+7900 train 5.703426 (lr=4.3179e-06) (hash(x)=146655921)
+7900 val loss 5.8714
+7900 val perplexity 354.7439
+7900 train 5.703874 (lr=3.7010e-06) (hash(x)=146655921)
+8000 val loss 5.8693
+8000 val perplexity 353.9875
+8000 train 5.982882 (lr=4.1380e-06) (hash(x)=148262482)
+8000 val loss 5.8727
+8000 val perplexity 355.2155
+8000 train 5.985026 (lr=3.5468e-06) (hash(x)=148262482)
+8100 val loss 5.8610
+8100 val perplexity 351.0891
+8100 train 5.792439 (lr=3.9800e-06) (hash(x)=147683655)
+8100 val loss 5.8643
+8100 val perplexity 352.2198
+8100 train 5.793932 (lr=3.4114e-06) (hash(x)=147683655)
+8200 val loss 5.8610
+8200 val perplexity 351.0704
+8200 train 5.954844 (lr=3.8442e-06) (hash(x)=157312987)
+8200 val loss 5.8644
+8200 val perplexity 352.2767
+8200 train 5.959354 (lr=3.2950e-06) (hash(x)=157312987)
+8300 val loss 5.8584
+8300 val perplexity 350.1542
+8300 train 5.827306 (lr=3.7307e-06) (hash(x)=141107543)
+8300 val loss 5.8617
+8300 val perplexity 351.3268
+8300 train 5.833569 (lr=3.1977e-06) (hash(x)=141107543)
+8400 val loss 5.8546
+8400 val perplexity 348.8233
+8400 train 5.829935 (lr=3.6397e-06) (hash(x)=141323024)
+8400 val loss 5.8581
+8400 val perplexity 350.0630
+8400 train 5.836152 (lr=3.1197e-06) (hash(x)=141323024)
+8500 val loss 5.8523
+8500 val perplexity 348.0277
+8500 train 5.828456 (lr=3.5713e-06) (hash(x)=150696521)
+8500 val loss 5.8561
+8500 val perplexity 349.3654
+8500 train 5.834352 (lr=3.0611e-06) (hash(x)=150696521)
+8600 val loss 5.8522
+8600 val perplexity 348.0011
+8600 train 5.811702 (lr=3.5257e-06) (hash(x)=162288191)
+8600 val loss 5.8554
+8600 val perplexity 349.1052
+8600 train 5.809893 (lr=3.0220e-06) (hash(x)=162288191)
+8700 val loss 5.8482
+8700 val perplexity 346.6077
+8700 train 5.718023 (lr=3.5029e-06) (hash(x)=152860941)
+8700 val loss 5.8523
+8700 val perplexity 348.0176
+8700 train 5.726460 (lr=3.0024e-06) (hash(x)=152860941)
+8749 val loss 5.8467
+8749 val perplexity 346.0782
+8749 val loss 5.8518
+8749 val perplexity 347.8583

attention_kindselective_n_heads4_seed1341/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed0dcb5d343458d407779e590b050be53e5b8310ad8f034f32c18a7b9fe44301
+oid sha256:395f6caf00fc6931495f915a58e51ee5930a2f4ca2b3f84a36b624a6ae25a106
 size 92843394

attention_kindselective_n_heads4_seed1341/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7c58c6893213b5eb99fd4135d3c6ce96bb74c5d8052222ff21a834154e88aea
+oid sha256:9059975da2d811a2ec619a9cd6f63d985d15ff163e8a4becb0434b514ed8062b
 size 179406214