Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.75e-5_30720_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_61440_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,1056 +1,105 @@
-max_steps: 17500
-max_steps: 17500
-0 val loss 11.2663
-0 val perplexity 78147.0000
-0 val loss 11.2663
-0 val perplexity 78147.0000
-0 train 11.263341 (lr=1.8750e-07) (hash(x)=150327452)
-0 train 11.263341 (lr=2.0000e-07) (hash(x)=150327452)
-100 val loss 9.8595
-100 val perplexity 19138.5859
-100 val loss 9.8359
-100 val perplexity 18693.2441
-100 train 9.868124 (lr=1.8937e-05) (hash(x)=166441190)
-100 train 9.845812 (lr=2.0200e-05) (hash(x)=166441190)
-200 val loss 8.5181
-200 val perplexity 5004.7261
-200 val loss 8.6538
-200 val perplexity 5731.6953
-200 train 8.651850 (lr=4.0000e-05) (hash(x)=166780046)
-200 train 8.781153 (lr=3.7500e-05) (hash(x)=166780046)
-300 val loss 7.7057
-300 val perplexity 2221.0286
-300 val loss 7.6969
-300 val perplexity 2201.4924
-300 train 7.656827 (lr=3.9997e-05) (hash(x)=159835303)
-300 train 7.646784 (lr=3.7497e-05) (hash(x)=159835303)
-400 val loss 7.5543
-400 val perplexity 1909.0188
-400 train 7.335799 (lr=3.9988e-05) (hash(x)=155040610)
-400 val loss 7.5167
-400 val perplexity 1838.4224
-400 train 7.293813 (lr=3.7489e-05) (hash(x)=155040610)
-500 val loss 7.4826
-500 val perplexity 1776.7881
-500 train 7.209811 (lr=3.9973e-05) (hash(x)=130190460)
-500 val loss 7.4408
-500 val perplexity 1704.1182
-500 train 7.158998 (lr=3.7475e-05) (hash(x)=130190460)
-600 val loss 7.4172
-600 val perplexity 1664.2971
-600 train 7.436842 (lr=3.9953e-05) (hash(x)=155504036)
-600 val loss 7.3542
-600 val perplexity 1562.7886
-600 train 7.372247 (lr=3.7456e-05) (hash(x)=155504036)
-700 val loss 7.3468
-700 val perplexity 1551.1505
-700 train 7.196675 (lr=3.9926e-05) (hash(x)=137347213)
-700 val loss 7.2823
-700 val perplexity 1454.3055
-700 train 7.140363 (lr=3.7430e-05) (hash(x)=137347213)
-800 val loss 7.2940
-800 val perplexity 1471.5103
-800 train 7.142735 (lr=3.9893e-05) (hash(x)=143823248)
-800 val loss 7.2112
-800 val perplexity 1354.4867
-800 train 7.056271 (lr=3.7400e-05) (hash(x)=143823248)
-900 val loss 7.2423
-900 val perplexity 1397.3634
-900 train 7.346307 (lr=3.9855e-05) (hash(x)=156260416)
-900 val loss 7.1369
-900 val perplexity 1257.5696
-900 train 7.245272 (lr=3.7364e-05) (hash(x)=156260416)
-1000 val loss 7.1949
-1000 val perplexity 1332.5796
-1000 train 7.155627 (lr=3.9810e-05) (hash(x)=143734685)
-1000 val loss 7.0832
-1000 val perplexity 1191.7954
-1000 train 7.037784 (lr=3.7322e-05) (hash(x)=143734685)
-1100 val loss 7.1500
-1100 val perplexity 1274.1158
-1100 train 7.072702 (lr=3.9760e-05) (hash(x)=160013925)
-1100 val loss 7.0295
-1100 val perplexity 1129.5018
-1100 train 6.946764 (lr=3.7275e-05) (hash(x)=160013925)
-1200 val loss 7.1062
-1200 val perplexity 1219.4874
-1200 train 6.836085 (lr=3.9704e-05) (hash(x)=150678249)
-1200 val loss 6.9870
-1200 val perplexity 1082.4991
-1200 train 6.713125 (lr=3.7223e-05) (hash(x)=150678249)
-1300 val loss 7.0673
-1300 val perplexity 1173.0056
-1300 train 6.833193 (lr=3.9642e-05) (hash(x)=149073315)
-1300 val loss 6.9427
-1300 val perplexity 1035.5397
-1300 train 6.705812 (lr=3.7164e-05) (hash(x)=149073315)
-1400 val loss 7.0057
-1400 val perplexity 1102.8519
-1400 train 7.326665 (lr=3.9574e-05) (hash(x)=175802021)
-1400 val loss 6.8794
-1400 val perplexity 972.0729
-1400 train 7.181681 (lr=3.7101e-05) (hash(x)=175802021)
-1500 val loss 6.9568
-1500 val perplexity 1050.2328
-1500 train 7.220825 (lr=3.9501e-05) (hash(x)=171034639)
-1500 val loss 6.8245
-1500 val perplexity 920.0974
-1500 train 7.093277 (lr=3.7032e-05) (hash(x)=171034639)
-1600 val loss 6.9040
-1600 val perplexity 996.2525
-1600 train 7.110297 (lr=3.9421e-05) (hash(x)=158681215)
-1600 val loss 6.7638
-1600 val perplexity 865.9183
-1600 train 6.971634 (lr=3.6958e-05) (hash(x)=158681215)
-1700 val loss 6.8712
-1700 val perplexity 964.1384
-1700 train 6.815831 (lr=3.9336e-05) (hash(x)=152116061)
-1700 val loss 6.7173
-1700 val perplexity 826.6052
-1700 train 6.662481 (lr=3.6878e-05) (hash(x)=152116061)
-1800 val loss 6.8333
-1800 val perplexity 928.2429
-1800 train 6.769077 (lr=3.9246e-05) (hash(x)=146108145)
-1800 val loss 6.6703
-1800 val perplexity 788.5938
-1800 train 6.601736 (lr=3.6793e-05) (hash(x)=146108145)
-1900 val loss 6.7823
-1900 val perplexity 882.0527
-1900 train 6.609385 (lr=3.9149e-05) (hash(x)=147598108)
-1900 val loss 6.6343
-1900 val perplexity 760.7567
-1900 train 6.457673 (lr=3.6702e-05) (hash(x)=147598108)
-2000 val loss 6.7541
-2000 val perplexity 857.5994
-2000 train 6.593875 (lr=3.9047e-05) (hash(x)=154996086)
-2000 val loss 6.6140
-2000 val perplexity 745.4609
-2000 train 6.439816 (lr=3.6606e-05) (hash(x)=154996086)
-2100 val loss 6.7144
-2100 val perplexity 824.1536
-2100 train 6.490150 (lr=3.8939e-05) (hash(x)=153396183)
-2100 val loss 6.5849
-2100 val perplexity 724.1075
-2100 train 6.354748 (lr=3.6505e-05) (hash(x)=153396183)
-2200 val loss 6.6532
-2200 val perplexity 775.2857
-2200 train 6.646795 (lr=3.8826e-05) (hash(x)=153885445)
-2200 val loss 6.5269
-2200 val perplexity 683.2792
-2200 train 6.532281 (lr=3.6399e-05) (hash(x)=153885445)
-2300 val loss 6.6119
-2300 val perplexity 743.8623
-2300 train 6.614337 (lr=3.8707e-05) (hash(x)=159666385)
-2300 val loss 6.5068
-2300 val perplexity 669.6613
-2300 train 6.508514 (lr=3.6288e-05) (hash(x)=159666385)
-2400 val loss 6.5667
-2400 val perplexity 711.0148
-2400 train 6.558314 (lr=3.8583e-05) (hash(x)=142353087)
-2400 val loss 6.4607
-2400 val perplexity 639.5293
-2400 train 6.453071 (lr=3.6171e-05) (hash(x)=142353087)
-2500 val loss 6.5303
-2500 val perplexity 685.6010
-2500 train 6.551023 (lr=3.8453e-05) (hash(x)=146491718)
-2500 val loss 6.4310
-2500 val perplexity 620.7972
-2500 train 6.445704 (lr=3.6049e-05) (hash(x)=146491718)
-2600 val loss 6.4981
-2600 val perplexity 663.8701
-2600 train 6.420842 (lr=3.8317e-05) (hash(x)=150750353)
-2600 val loss 6.4089
-2600 val perplexity 607.2394
-2600 train 6.333244 (lr=3.5923e-05) (hash(x)=150750353)
-2700 val loss 6.4712
-2700 val perplexity 646.2736
-2700 train 6.197730 (lr=3.8177e-05) (hash(x)=129849193)
-2700 val loss 6.3935
-2700 val perplexity 597.9317
-2700 train 6.113802 (lr=3.5791e-05) (hash(x)=129849193)
-2800 val loss 6.4585
-2800 val perplexity 638.0903
-2800 train 6.284863 (lr=3.8031e-05) (hash(x)=152767913)
-2800 val loss 6.3859
-2800 val perplexity 593.3999
-2800 train 6.207295 (lr=3.5654e-05) (hash(x)=152767913)
-2900 val loss 6.4376
-2900 val perplexity 624.8848
-2900 train 6.206501 (lr=3.7879e-05) (hash(x)=146531140)
-2900 val loss 6.3696
-2900 val perplexity 583.8331
-2900 train 6.140223 (lr=3.5512e-05) (hash(x)=146531140)
-3000 val loss 6.4011
-3000 val perplexity 602.4949
-3000 train 6.483201 (lr=3.7723e-05) (hash(x)=151562048)
-3000 val loss 6.3378
-3000 val perplexity 565.5319
-3000 train 6.426512 (lr=3.5365e-05) (hash(x)=151562048)
-3100 val loss 6.3668
-3100 val perplexity 582.2114
-3100 train 6.322643 (lr=3.7561e-05) (hash(x)=146001424)
-3100 val loss 6.3160
-3100 val perplexity 553.3704
-3100 train 6.270426 (lr=3.5214e-05) (hash(x)=146001424)
-3200 val loss 6.3596
-3200 val perplexity 577.9986
-3200 train 6.421289 (lr=3.7394e-05) (hash(x)=166486165)
-3200 val loss 6.3009
-3200 val perplexity 545.0633
-3200 train 6.371137 (lr=3.5057e-05) (hash(x)=166486165)
-3300 val loss 6.3258
-3300 val perplexity 558.8276
-3300 train 6.260159 (lr=3.7222e-05) (hash(x)=150866680)
-3300 val loss 6.2683
-3300 val perplexity 527.5996
-3300 train 6.203961 (lr=3.4896e-05) (hash(x)=150866680)
-3400 val loss 6.3106
-3400 val perplexity 550.3920
-3400 train 6.202477 (lr=3.7045e-05) (hash(x)=143900419)
-3400 val loss 6.2639
-3400 val perplexity 525.2636
-3400 train 6.162298 (lr=3.4730e-05) (hash(x)=143900419)
-3500 val loss 6.2926
-3500 val perplexity 540.5385
-3500 train 6.108777 (lr=3.6864e-05) (hash(x)=148845794)
-3500 val loss 6.2436
-3500 val perplexity 514.7196
-3500 train 6.060290 (lr=3.4560e-05) (hash(x)=148845794)
-3600 val loss 6.2743
-3600 val perplexity 530.7532
-3600 train 6.098810 (lr=3.6677e-05) (hash(x)=145667796)
-3600 val loss 6.2313
-3600 val perplexity 508.4273
-3600 train 6.051695 (lr=3.4384e-05) (hash(x)=145667796)
-3700 val loss 6.2667
-3700 val perplexity 526.7236
-3700 train 6.283004 (lr=3.6485e-05) (hash(x)=163563851)
-3700 val loss 6.2186
-3700 val perplexity 502.0200
-3700 train 6.234773 (lr=3.4205e-05) (hash(x)=163563851)
-3800 val loss 6.2501
-3800 val perplexity 518.0671
-3800 train 6.205328 (lr=3.6289e-05) (hash(x)=147488689)
-3800 val loss 6.2043
-3800 val perplexity 494.8778
-3800 train 6.156865 (lr=3.4021e-05) (hash(x)=147488689)
-3900 val loss 6.2277
-3900 val perplexity 506.5876
-3900 train 6.239246 (lr=3.6088e-05) (hash(x)=148186608)
-3900 val loss 6.1811
-3900 val perplexity 483.5410
-3900 train 6.190732 (lr=3.3832e-05) (hash(x)=148186608)
-4000 val loss 6.2055
-4000 val perplexity 495.4650
-4000 train 6.086553 (lr=3.5882e-05) (hash(x)=142970187)
-4000 val loss 6.1604
-4000 val perplexity 473.5967
-4000 train 6.046245 (lr=3.3639e-05) (hash(x)=142970187)
-4100 val loss 6.1877
-4100 val perplexity 486.7348
-4100 train 6.322111 (lr=3.5671e-05) (hash(x)=141584883)
-4100 val loss 6.1480
-4100 val perplexity 467.8040
-4100 train 6.294571 (lr=3.3442e-05) (hash(x)=141584883)
-4200 val loss 6.1772
-4200 val perplexity 481.6568
-4200 train 5.979122 (lr=3.5457e-05) (hash(x)=145664585)
-4200 val loss 6.1330
-4200 val perplexity 460.8101
-4200 train 5.931549 (lr=3.3240e-05) (hash(x)=145664585)
-4300 val loss 6.1621
-4300 val perplexity 474.4248
-4300 train 6.036442 (lr=3.5237e-05) (hash(x)=143736499)
-4300 val loss 6.1214
-4300 val perplexity 455.5243
-4300 train 5.990649 (lr=3.3035e-05) (hash(x)=143736499)
-4400 val loss 6.1566
-4400 val perplexity 471.8342
-4400 train 5.982193 (lr=3.5014e-05) (hash(x)=151883322)
-4400 val loss 6.1144
-4400 val perplexity 452.3284
-4400 train 5.935634 (lr=3.2825e-05) (hash(x)=151883322)
-4500 val loss 6.1477
-4500 val perplexity 467.6193
-4500 train 6.027667 (lr=3.4786e-05) (hash(x)=153904871)
-4500 val loss 6.1101
-4500 val perplexity 450.3830
-4500 train 5.990096 (lr=3.2611e-05) (hash(x)=153904871)
-4600 val loss 6.1353
-4600 val perplexity 461.8768
-4600 train 6.231039 (lr=3.4553e-05) (hash(x)=154893521)
-4600 val loss 6.0971
-4600 val perplexity 444.5587
-4600 train 6.197966 (lr=3.2394e-05) (hash(x)=154893521)
-4700 val loss 6.1131
-4700 val perplexity 451.7544
-4700 train 6.154427 (lr=3.4317e-05) (hash(x)=152323949)
-4700 val loss 6.0800
-4700 val perplexity 437.0204
-4700 train 6.121953 (lr=3.2172e-05) (hash(x)=152323949)
-4800 val loss 6.0982
-4800 val perplexity 445.0459
-4800 train 6.050216 (lr=3.4077e-05) (hash(x)=154104619)
-4800 val loss 6.0606
-4800 val perplexity 428.6257
-4800 train 6.015411 (lr=3.1947e-05) (hash(x)=154104619)
-4900 val loss 6.0874
-4900 val perplexity 440.2722
-4900 train 6.218273 (lr=3.3832e-05) (hash(x)=146311426)
-4900 val loss 6.0524
-4900 val perplexity 425.1423
-4900 train 6.163857 (lr=3.1718e-05) (hash(x)=146311426)
-5000 val loss 6.0780
-5000 val perplexity 436.1702
-5000 train 6.054106 (lr=3.3584e-05) (hash(x)=156741847)
-5000 val loss 6.0432
-5000 val perplexity 421.2397
-5000 train 6.020451 (lr=3.1485e-05) (hash(x)=156741847)
-5100 val loss 6.0687
-5100 val perplexity 432.1014
-5100 train 5.851731 (lr=3.3332e-05) (hash(x)=142086346)
-5100 val loss 6.0378
-5100 val perplexity 418.9780
-5100 train 5.812495 (lr=3.1249e-05) (hash(x)=142086346)
-5200 val loss 6.0645
-5200 val perplexity 430.2945
-5200 train 5.838616 (lr=3.3076e-05) (hash(x)=150265428)
-5200 val loss 6.0325
-5200 val perplexity 416.7517
-5200 train 5.801290 (lr=3.1009e-05) (hash(x)=150265428)
-5300 val loss 6.0519
-5300 val perplexity 424.9258
-5300 train 5.920601 (lr=3.2817e-05) (hash(x)=151339108)
-5300 val loss 6.0259
-5300 val perplexity 414.0215
-5300 train 5.896479 (lr=3.0766e-05) (hash(x)=151339108)
-5400 val loss 6.0438
-5400 val perplexity 421.4719
-5400 train 6.020870 (lr=3.2554e-05) (hash(x)=154654372)
-5400 val loss 6.0132
-5400 val perplexity 408.7832
-5400 train 5.971983 (lr=3.0519e-05) (hash(x)=154654372)
-5500 val loss 6.0302
-5500 val perplexity 415.8119
-5500 train 6.037389 (lr=3.2287e-05) (hash(x)=150575051)
-5500 val loss 5.9965
-5500 val perplexity 402.0139
-5500 train 6.008774 (lr=3.0269e-05) (hash(x)=150575051)
-5600 val loss 6.0248
-5600 val perplexity 413.5508
-5600 train 5.938440 (lr=3.2017e-05) (hash(x)=140396423)
-5600 val loss 5.9856
-5600 val perplexity 397.6622
-5600 train 5.897523 (lr=3.0016e-05) (hash(x)=140396423)
-5700 val loss 6.0045
-5700 val perplexity 405.2670
-5700 train 5.853642 (lr=3.1744e-05) (hash(x)=144678758)
-5700 val loss 5.9772
-5700 val perplexity 394.3266
-5700 train 5.828709 (lr=2.9760e-05) (hash(x)=144678758)
-5800 val loss 5.9977
-5800 val perplexity 402.4905
-5800 train 6.057874 (lr=3.1468e-05) (hash(x)=151992743)
-5800 val loss 5.9700
-5800 val perplexity 391.5067
-5800 train 6.027828 (lr=2.9501e-05) (hash(x)=151992743)
-5900 val loss 5.9894
-5900 val perplexity 399.1632
-5900 train 5.782817 (lr=3.1188e-05) (hash(x)=144396927)
-5900 val loss 5.9604
-5900 val perplexity 387.7543
-5900 train 5.755085 (lr=2.9239e-05) (hash(x)=144396927)
-6000 val loss 5.9792
-6000 val perplexity 395.1375
-6000 train 5.933915 (lr=3.0905e-05) (hash(x)=165478625)
-6000 val loss 5.9519
-6000 val perplexity 384.4901
-6000 train 5.896648 (lr=2.8974e-05) (hash(x)=165478625)
-6100 val loss 5.9793
-6100 val perplexity 395.1680
-6100 train 5.708070 (lr=3.0620e-05) (hash(x)=147088621)
-6100 val loss 5.9592
-6100 val perplexity 387.3049
-6100 train 5.687679 (lr=2.8706e-05) (hash(x)=147088621)
-6200 val loss 5.9699
-6200 val perplexity 391.4770
-6200 train 5.730008 (lr=3.0332e-05) (hash(x)=140794994)
-6200 val loss 5.9432
-6200 val perplexity 381.1576
-6200 train 5.709146 (lr=2.8436e-05) (hash(x)=140794994)
-6300 val loss 5.9605
-6300 val perplexity 387.8083
-6300 train 5.902456 (lr=3.0040e-05) (hash(x)=134780906)
-6300 val loss 5.9288
-6300 val perplexity 375.7183
-6300 train 5.868412 (lr=2.8163e-05) (hash(x)=134780906)
-6400 val loss 5.9421
-6400 val perplexity 380.7487
-6400 train 5.926373 (lr=2.9747e-05) (hash(x)=149023655)
-6400 val loss 5.9124
-6400 val perplexity 369.5739
-6400 train 5.901180 (lr=2.7888e-05) (hash(x)=149023655)
-6500 val loss 5.9334
-6500 val perplexity 377.4468
-6500 train 5.781084 (lr=2.9450e-05) (hash(x)=147497796)
-6500 val loss 5.9058
-6500 val perplexity 367.1653
-6500 train 5.750584 (lr=2.7610e-05) (hash(x)=147497796)
-6600 val loss 5.9301
-6600 val perplexity 376.1914
-6600 train 5.772248 (lr=2.9152e-05) (hash(x)=152902689)
-6600 val loss 5.9013
-6600 val perplexity 365.4952
-6600 train 5.752711 (lr=2.7330e-05) (hash(x)=152902689)
-6700 val loss 5.9384
-6700 val perplexity 379.3110
-6700 train 5.978984 (lr=2.8851e-05) (hash(x)=153846046)
-6700 val loss 5.9061
-6700 val perplexity 367.2744
-6700 train 5.948668 (lr=2.7047e-05) (hash(x)=153846046)
-6800 val loss 5.9072
-6800 val perplexity 367.6755
-6800 train 6.038209 (lr=2.8547e-05) (hash(x)=158512738)
-6800 val loss 5.8796
-6800 val perplexity 357.6506
-6800 train 6.010082 (lr=2.6763e-05) (hash(x)=158512738)
-6900 val loss 5.9006
-6900 val perplexity 365.2498
-6900 train 6.420359 (lr=2.8242e-05) (hash(x)=156849968)
-6900 val loss 5.8750
-6900 val perplexity 356.0296
-6900 train 6.411304 (lr=2.6477e-05) (hash(x)=156849968)
-7000 val loss 5.8930
-7000 val perplexity 362.4887
-7000 train 5.751428 (lr=2.7934e-05) (hash(x)=142395855)
-7000 val loss 5.8643
-7000 val perplexity 352.2307
-7000 train 5.722020 (lr=2.6188e-05) (hash(x)=142395855)
-7100 val loss 5.8898
-7100 val perplexity 361.3358
-7100 train 5.744161 (lr=2.7624e-05) (hash(x)=147114884)
-7100 val loss 5.8620
-7100 val perplexity 351.4352
-7100 train 5.708766 (lr=2.5898e-05) (hash(x)=147114884)
-7200 val loss 5.8776
-7200 val perplexity 356.9494
-7200 train 5.911930 (lr=2.7313e-05) (hash(x)=156979839)
-7200 val loss 5.8497
-7200 val perplexity 347.1398
-7200 train 5.874716 (lr=2.5606e-05) (hash(x)=156979839)
-7300 val loss 5.8737
-7300 val perplexity 355.5622
-7300 train 5.665551 (lr=2.7000e-05) (hash(x)=145584373)
-7300 val loss 5.8484
-7300 val perplexity 346.6742
-7300 train 5.648200 (lr=2.5312e-05) (hash(x)=145584373)
-7400 val loss 5.8650
-7400 val perplexity 352.4915
-7400 train 5.609486 (lr=2.6685e-05) (hash(x)=141508204)
-7400 val loss 5.8448
-7400 val perplexity 345.4498
-7400 train 5.591662 (lr=2.5017e-05) (hash(x)=141508204)
-7500 val loss 5.8531
-7500 val perplexity 348.3254
-7500 train 5.891374 (lr=2.6369e-05) (hash(x)=148803965)
-7500 val loss 5.8303
-7500 val perplexity 340.4442
-7500 train 5.863412 (lr=2.4721e-05) (hash(x)=148803965)
-7600 val loss 5.8480
-7600 val perplexity 346.5572
-7600 train 5.902753 (lr=2.6051e-05) (hash(x)=151019676)
-7600 val loss 5.8243
-7600 val perplexity 338.4339
-7600 train 5.884450 (lr=2.4423e-05) (hash(x)=151019676)
-7700 val loss 5.8308
-7700 val perplexity 340.6423
-7700 train 5.933746 (lr=2.5732e-05) (hash(x)=143155750)
-7700 val loss 5.8114
-7700 val perplexity 334.0776
-7700 train 5.916495 (lr=2.4124e-05) (hash(x)=143155750)
-7800 val loss 5.8294
-7800 val perplexity 340.1667
-7800 train 5.860056 (lr=2.5411e-05) (hash(x)=152569653)
-7800 val loss 5.8093
-7800 val perplexity 333.3885
-7800 train 5.832789 (lr=2.3823e-05) (hash(x)=152569653)
-7900 val loss 5.8226
-7900 val perplexity 337.8345
-7900 train 5.699649 (lr=2.5090e-05) (hash(x)=143519455)
-7900 val loss 5.8013
-7900 val perplexity 330.7151
-7900 train 5.672275 (lr=2.3522e-05) (hash(x)=143519455)
-8000 val loss 5.8151
-8000 val perplexity 335.3357
-8000 train 5.913954 (lr=2.4767e-05) (hash(x)=161180944)
-8000 val loss 5.7937
-8000 val perplexity 328.2259
-8000 train 5.883524 (lr=2.3219e-05) (hash(x)=161180944)
-8100 val loss 5.8179
-8100 val perplexity 336.2599
-8100 train 5.844440 (lr=2.4444e-05) (hash(x)=154107345)
-8100 val loss 5.7921
-8100 val perplexity 327.6842
-8100 train 5.819099 (lr=2.2916e-05) (hash(x)=154107345)
-8200 val loss 5.8122
-8200 val perplexity 334.3447
-8200 train 5.851220 (lr=2.4120e-05) (hash(x)=152486517)
-8200 val loss 5.7916
-8200 val perplexity 327.5419
-8200 train 5.825037 (lr=2.2612e-05) (hash(x)=152486517)
-8300 val loss 5.7987
-8300 val perplexity 329.8811
-8300 train 5.804880 (lr=2.3795e-05) (hash(x)=156167749)
-8300 val loss 5.7805
-8300 val perplexity 323.9186
-8300 train 5.781146 (lr=2.2308e-05) (hash(x)=156167749)
-8400 val loss 5.7949
-8400 val perplexity 328.6116
-8400 train 5.884000 (lr=2.3469e-05) (hash(x)=149155006)
-8400 val loss 5.7753
-8400 val perplexity 322.2279
-8400 train 5.864207 (lr=2.2002e-05) (hash(x)=149155006)
-8500 val loss 5.7817
-8500 val perplexity 324.3089
-8500 train 5.722820 (lr=2.3143e-05) (hash(x)=147844390)
-8500 val loss 5.7626
-8500 val perplexity 318.1658
-8500 train 5.712843 (lr=2.1697e-05) (hash(x)=147844390)
-8600 val loss 5.7772
-8600 val perplexity 322.8451
-8600 train 6.027016 (lr=2.2817e-05) (hash(x)=165753320)
-8600 val loss 5.7581
-8600 val perplexity 316.7603
-8600 train 6.013570 (lr=2.1391e-05) (hash(x)=165753320)
-8700 val loss 5.7711
-8700 val perplexity 320.8813
-8700 train 5.550282 (lr=2.2490e-05) (hash(x)=146079979)
-8700 val loss 5.7549
-8700 val perplexity 315.7386
-8700 train 5.528018 (lr=2.1085e-05) (hash(x)=146079979)
-8800 val loss 5.7729
-8800 val perplexity 321.4843
-8800 train 5.959742 (lr=2.2163e-05) (hash(x)=172259509)
-8800 val loss 5.7516
-8800 val perplexity 314.7053
-8800 train 5.941445 (lr=2.0778e-05) (hash(x)=172259509)
-8900 val loss 5.7657
-8900 val perplexity 319.1769
-8900 train 5.439955 (lr=2.1837e-05) (hash(x)=145148314)
-8900 val loss 5.7465
-8900 val perplexity 313.0956
-8900 train 5.426799 (lr=2.0472e-05) (hash(x)=145148314)
-9000 val loss 5.7599
-9000 val perplexity 317.3039
-9000 train 5.682996 (lr=2.1510e-05) (hash(x)=144250633)
-9000 val loss 5.7424
-9000 val perplexity 311.8057
-9000 train 5.664256 (lr=2.0165e-05) (hash(x)=144250633)
-9100 val loss 5.7545
-9100 val perplexity 315.6111
-9100 train 5.857194 (lr=2.1183e-05) (hash(x)=157219797)
-9100 val loss 5.7381
-9100 val perplexity 310.4626
-9100 train 5.840177 (lr=1.9859e-05) (hash(x)=157219797)
-9200 val loss 5.7413
-9200 val perplexity 311.4770
-9200 train 5.797914 (lr=2.0857e-05) (hash(x)=142743778)
-9200 val loss 5.7290
-9200 val perplexity 307.6659
-9200 train 5.779316 (lr=1.9553e-05) (hash(x)=142743778)
-9300 val loss 5.7385
-9300 val perplexity 310.6078
-9300 train 5.666136 (lr=2.0531e-05) (hash(x)=139669771)
-9300 val loss 5.7255
-9300 val perplexity 306.5905
-9300 train 5.659318 (lr=1.9248e-05) (hash(x)=139669771)
-9400 val loss 5.7292
-9400 val perplexity 307.7287
-9400 train 5.776366 (lr=2.0205e-05) (hash(x)=145916843)
-9400 val loss 5.7158
-9400 val perplexity 303.6291
-9400 train 5.751498 (lr=1.8942e-05) (hash(x)=145916843)
-9500 val loss 5.7304
-9500 val perplexity 308.0824
-9500 train 5.645638 (lr=1.9880e-05) (hash(x)=150196125)
-9500 val loss 5.7119
-9500 val perplexity 302.4328
-9500 train 5.636480 (lr=1.8638e-05) (hash(x)=150196125)
-9600 val loss 5.7219
-9600 val perplexity 305.4899
-9600 train 6.033727 (lr=1.9556e-05) (hash(x)=160041419)
-9600 val loss 5.7099
-9600 val perplexity 301.8288
-9600 train 6.023571 (lr=1.8334e-05) (hash(x)=160041419)
-9700 val loss 5.7220
-9700 val perplexity 305.5088
-9700 train 5.503003 (lr=1.9233e-05) (hash(x)=139931627)
-9700 val loss 5.7082
-9700 val perplexity 301.3137
-9700 train 5.491812 (lr=1.8031e-05) (hash(x)=139931627)
-9800 val loss 5.7155
-9800 val perplexity 303.5482
-9800 train 5.696881 (lr=1.8910e-05) (hash(x)=150370792)
-9800 val loss 5.7005
-9800 val perplexity 299.0040
-9800 train 5.680792 (lr=1.7728e-05) (hash(x)=150370792)
-9900 val loss 5.7093
-9900 val perplexity 301.6725
-9900 train 5.542067 (lr=1.8589e-05) (hash(x)=153014886)
-9900 val loss 5.6953
-9900 val perplexity 297.4720
-9900 train 5.523115 (lr=1.7427e-05) (hash(x)=153014886)
-10000 val loss 5.7047
-10000 val perplexity 300.2856
-10000 train 5.607134 (lr=1.8268e-05) (hash(x)=141915265)
-10000 val loss 5.6927
-10000 val perplexity 296.6991
-10000 train 5.594075 (lr=1.7126e-05) (hash(x)=141915265)
-10100 val loss 5.6994
-10100 val perplexity 298.7018
-10100 train 6.204209 (lr=1.7949e-05) (hash(x)=197770952)
-10100 val loss 5.6883
-10100 val perplexity 295.3954
-10100 train 6.189380 (lr=1.6827e-05) (hash(x)=197770952)
-10200 val loss 5.6916
-10200 val perplexity 296.3653
-10200 train 5.666861 (lr=1.7631e-05) (hash(x)=156806005)
-10200 val loss 5.6819
-10200 val perplexity 293.5147
-10200 train 5.647396 (lr=1.6529e-05) (hash(x)=156806005)
-10300 val loss 5.6888
-10300 val perplexity 295.5269
-10300 train 5.500148 (lr=1.7315e-05) (hash(x)=145378216)
-10300 val loss 5.6795
-10300 val perplexity 292.8058
-10300 train 5.484988 (lr=1.6233e-05) (hash(x)=145378216)
-10400 val loss 5.6817
-10400 val perplexity 293.4449
-10400 train 5.593282 (lr=1.7000e-05) (hash(x)=145125407)
-10400 val loss 5.6708
-10400 val perplexity 290.2617
-10400 train 5.582484 (lr=1.5938e-05) (hash(x)=145125407)
-10500 val loss 5.6802
-10500 val perplexity 293.0004
-10500 train 5.493242 (lr=1.6687e-05) (hash(x)=147553745)
-10500 val loss 5.6710
-10500 val perplexity 290.3318
-10500 train 5.494811 (lr=1.5644e-05) (hash(x)=147553745)
-10600 val loss 5.6777
-10600 val perplexity 292.2679
-10600 train 5.317801 (lr=1.6376e-05) (hash(x)=134894889)
-10600 val loss 5.6662
-10600 val perplexity 288.9396
-10600 train 5.318324 (lr=1.5352e-05) (hash(x)=134894889)
-10700 val loss 5.6693
-10700 val perplexity 289.8371
-10700 train 5.593366 (lr=1.6066e-05) (hash(x)=151041040)
-10700 val loss 5.6609
-10700 val perplexity 287.4037
-10700 train 5.582285 (lr=1.5062e-05) (hash(x)=151041040)
-10800 val loss 5.6634
-10800 val perplexity 288.1177
-10800 train 5.595318 (lr=1.5758e-05) (hash(x)=145685829)
-10800 val loss 5.6543
-10800 val perplexity 285.5227
-10800 train 5.583758 (lr=1.4773e-05) (hash(x)=145685829)
-10900 val loss 5.6585
-10900 val perplexity 286.7199
-10900 train 5.739454 (lr=1.5453e-05) (hash(x)=156000716)
-10900 val loss 5.6492
-10900 val perplexity 284.0699
-10900 train 5.734696 (lr=1.4487e-05) (hash(x)=156000716)
-11000 val loss 5.6547
-11000 val perplexity 285.6243
-11000 train 5.665429 (lr=1.5149e-05) (hash(x)=149858163)
-11000 val loss 5.6467
-11000 val perplexity 283.3628
-11000 train 5.654157 (lr=1.4203e-05) (hash(x)=149858163)
-11100 val loss 5.6523
-11100 val perplexity 284.9481
-11100 train 5.932511 (lr=1.4848e-05) (hash(x)=164873575)
-11100 val loss 5.6453
-11100 val perplexity 282.9635
-11100 train 5.932080 (lr=1.3920e-05) (hash(x)=164873575)
-11200 val loss 5.6532
-11200 val perplexity 285.2096
-11200 train 5.536577 (lr=1.4550e-05) (hash(x)=156370069)
-11200 val loss 5.6454
-11200 val perplexity 282.9827
-11200 train 5.519896 (lr=1.3640e-05) (hash(x)=156370069)
-11300 val loss 5.6495
-11300 val perplexity 284.1511
-11300 train 5.738060 (lr=1.4253e-05) (hash(x)=160356892)
-11300 val loss 5.6433
-11300 val perplexity 282.3891
-11300 train 5.722073 (lr=1.3362e-05) (hash(x)=160356892)
-11400 val loss 5.6395
-11400 val perplexity 281.3151
-11400 train 5.707570 (lr=1.3960e-05) (hash(x)=158955430)
-11400 val loss 5.6331
-11400 val perplexity 279.5309
-11400 train 5.699232 (lr=1.3087e-05) (hash(x)=158955430)
-11500 val loss 5.6344
-11500 val perplexity 279.8896
-11500 train 5.614134 (lr=1.3668e-05) (hash(x)=146598377)
-11500 val loss 5.6283
-11500 val perplexity 278.2024
-11500 train 5.589320 (lr=1.2814e-05) (hash(x)=146598377)
-11600 val loss 5.6628
-11600 val perplexity 287.9396
-11600 train 5.808537 (lr=1.3380e-05) (hash(x)=156045795)
-11600 val loss 5.6480
-11600 val perplexity 283.7278
-11600 train 5.784801 (lr=1.2544e-05) (hash(x)=156045795)
-11700 val loss 5.6311
-11700 val perplexity 278.9812
-11700 train 5.494103 (lr=1.3095e-05) (hash(x)=143963334)
-11700 val loss 5.6257
-11700 val perplexity 277.4742
-11700 train 5.477342 (lr=1.2276e-05) (hash(x)=143963334)
-11800 val loss 5.6293
-11800 val perplexity 278.4727
-11800 train 5.468873 (lr=1.2812e-05) (hash(x)=145831579)
-11800 val loss 5.6245
-11800 val perplexity 277.1250
-11800 train 5.454970 (lr=1.2011e-05) (hash(x)=145831579)
-11900 val loss 5.6252
-11900 val perplexity 277.3227
-11900 train 5.793966 (lr=1.2532e-05) (hash(x)=165154141)
-11900 val loss 5.6200
-11900 val perplexity 275.9032
-11900 train 5.778847 (lr=1.1749e-05) (hash(x)=165154141)
-12000 val loss 5.6199
-12000 val perplexity 275.8593
-12000 train 5.701312 (lr=1.2256e-05) (hash(x)=160373794)
-12000 val loss 5.6135
-12000 val perplexity 274.1089
-12000 train 5.691423 (lr=1.1490e-05) (hash(x)=160373794)
-12100 val loss 5.6162
-12100 val perplexity 274.8550
-12100 train 5.406276 (lr=1.1983e-05) (hash(x)=141476346)
-12100 val loss 5.6107
-12100 val perplexity 273.3370
-12100 train 5.404443 (lr=1.1234e-05) (hash(x)=141476346)
-12200 val loss 5.6134
-12200 val perplexity 274.0763
-12200 train 5.868688 (lr=1.1713e-05) (hash(x)=164508058)
-12200 val loss 5.6091
-12200 val perplexity 272.9045
-12200 train 5.864333 (lr=1.0981e-05) (hash(x)=164508058)
-12300 val loss 5.6108
-12300 val perplexity 273.3618
-12300 train 5.597710 (lr=1.1446e-05) (hash(x)=140407664)
-12300 val loss 5.6059
-12300 val perplexity 272.0320
-12300 train 5.588737 (lr=1.0731e-05) (hash(x)=140407664)
-12400 val loss 5.6079
-12400 val perplexity 272.5839
-12400 train 5.461676 (lr=1.1183e-05) (hash(x)=149526189)
-12400 val loss 5.6054
-12400 val perplexity 271.8886
-12400 train 5.460255 (lr=1.0484e-05) (hash(x)=149526189)
-12500 val loss 5.6030
-12500 val perplexity 271.2485
-12500 train 5.449765 (lr=1.0924e-05) (hash(x)=142060916)
-12500 val loss 5.6003
-12500 val perplexity 270.4955
-12500 train 5.456053 (lr=1.0241e-05) (hash(x)=142060916)
-12600 val loss 5.5984
-12600 val perplexity 269.9859
-12600 train 5.778718 (lr=1.0668e-05) (hash(x)=168650848)
-12600 val loss 5.5952
-12600 val perplexity 269.1323
-12600 train 5.773306 (lr=1.0001e-05) (hash(x)=168650848)
-12700 val loss 5.5969
-12700 val perplexity 269.5833
-12700 train 5.618319 (lr=1.0416e-05) (hash(x)=154353018)
-12700 val loss 5.5934
-12700 val perplexity 268.6388
-12700 train 5.614925 (lr=9.7649e-06) (hash(x)=154353018)
-12800 val loss 5.5980
-12800 val perplexity 269.8842
-12800 train 5.762591 (lr=1.0168e-05) (hash(x)=156115265)
-12800 val loss 5.5941
-12800 val perplexity 268.8290
-12900 val loss 5.5944
-12900 val perplexity 268.9202
-12800 train 5.749104 (lr=9.5322e-06) (hash(x)=156115265)
-12900 train 5.519612 (lr=9.9233e-06) (hash(x)=143378175)
-13000 val loss 5.5921
-13000 val perplexity 268.3089
-12900 val loss 5.5926
-12900 val perplexity 268.4366
-13000 train 5.694238 (lr=9.6829e-06) (hash(x)=166124864)
-12900 train 5.517610 (lr=9.3031e-06) (hash(x)=143378175)
-13000 val loss 5.5889
-13000 val perplexity 267.4538
-13000 train 5.684518 (lr=9.0777e-06) (hash(x)=166124864)
-13100 val loss 5.5895
-13100 val perplexity 267.5899
-13100 train 5.472170 (lr=9.4466e-06) (hash(x)=146615385)
-13200 val loss 5.5914
-13200 val perplexity 268.1238
-13200 train 5.491530 (lr=9.2144e-06) (hash(x)=149236047)
-13100 val loss 5.5882
-13100 val perplexity 267.2526
-13100 train 5.465880 (lr=8.8562e-06) (hash(x)=146615385)
-13300 val loss 5.5855
-13300 val perplexity 266.5218
-13300 train 5.513978 (lr=8.9865e-06) (hash(x)=151346972)
-13200 val loss 5.5862
-13200 val perplexity 266.7275
-13200 train 5.484173 (lr=8.6385e-06) (hash(x)=149236047)
-13400 val loss 5.5827
-13400 val perplexity 265.7754
-13400 train 5.675163 (lr=8.7628e-06) (hash(x)=159167093)
-13300 val loss 5.5833
-13300 val perplexity 265.9412
-13300 train 5.509476 (lr=8.4248e-06) (hash(x)=151346972)
-13500 val loss 5.5806
-13500 val perplexity 265.2330
-13500 train 5.302473 (lr=8.5435e-06) (hash(x)=135054650)
-13400 val loss 5.5798
-13400 val perplexity 265.0265
-13400 train 5.674298 (lr=8.2151e-06) (hash(x)=159167093)
-13600 val loss 5.5801
-13600 val perplexity 265.0899
-13600 train 5.463277 (lr=8.3286e-06) (hash(x)=151033927)
-13500 val loss 5.5785
-13500 val perplexity 264.6842
-13500 train 5.312832 (lr=8.0095e-06) (hash(x)=135054650)
-13700 val loss 5.5792
-13700 val perplexity 264.8670
-13700 train 5.272966 (lr=8.1183e-06) (hash(x)=141119172)
-13600 val loss 5.5776
-13600 val perplexity 264.4442
-13600 train 5.465922 (lr=7.8081e-06) (hash(x)=151033927)
-13800 val loss 5.5764
-13800 val perplexity 264.1136
-13800 train 5.289057 (lr=7.9125e-06) (hash(x)=144197766)
-13700 val loss 5.5762
-13700 val perplexity 264.0586
-13700 train 5.273306 (lr=7.6109e-06) (hash(x)=141119172)
-13900 val loss 5.5724
-13900 val perplexity 263.0594
-13900 train 5.510831 (lr=7.7114e-06) (hash(x)=145148158)
-13800 val loss 5.5757
-13800 val perplexity 263.9417
-13800 train 5.286690 (lr=7.4180e-06) (hash(x)=144197766)
-14000 val loss 5.5687
-14000 val perplexity 262.0901
-14000 train 6.069955 (lr=7.5149e-06) (hash(x)=179068751)
-13900 val loss 5.5698
-13900 val perplexity 262.3841
-13900 train 5.509115 (lr=7.2294e-06) (hash(x)=145148158)
-14100 val loss 5.5677
-14100 val perplexity 261.8275
-14100 train 5.435109 (lr=7.3233e-06) (hash(x)=144988373)
-14000 val loss 5.5673
-14000 val perplexity 261.7274
-14000 train 6.063221 (lr=7.0452e-06) (hash(x)=179068751)
-14200 val loss 5.5658
-14200 val perplexity 261.3357
-14200 train 5.512167 (lr=7.1365e-06) (hash(x)=133379696)
-14100 val loss 5.5657
-14100 val perplexity 261.2974
-14100 train 5.440147 (lr=6.8656e-06) (hash(x)=144988373)
-14300 val loss 5.5669
-14300 val perplexity 261.6292
-14300 train 5.442722 (lr=6.9546e-06) (hash(x)=149883422)
-14200 val loss 5.5650
-14200 val perplexity 261.1154
-14200 train 5.502961 (lr=6.6904e-06) (hash(x)=133379696)
-14400 val loss 5.5665
-14400 val perplexity 261.5081
-14400 train 5.428222 (lr=6.7776e-06) (hash(x)=145709198)
-14300 val loss 5.5660
-14300 val perplexity 261.3931
-14300 train 5.445783 (lr=6.5199e-06) (hash(x)=149883422)
-14500 val loss 5.5604
-14500 val perplexity 259.9273
-14500 train 5.575936 (lr=6.6057e-06) (hash(x)=148016579)
-14400 val loss 5.5656
-14400 val perplexity 261.2873
-14400 train 5.421428 (lr=6.3540e-06) (hash(x)=145709198)
-14600 val loss 5.5589
-14600 val perplexity 259.5357
-14600 train 5.529923 (lr=6.4389e-06) (hash(x)=150218222)
-14500 val loss 5.5601
-14500 val perplexity 259.8371
-14500 train 5.578542 (lr=6.1928e-06) (hash(x)=148016579)
-14700 val loss 5.5592
-14700 val perplexity 259.6161
-14700 train 5.484089 (lr=6.2771e-06) (hash(x)=142258737)
-14600 val loss 5.5581
-14600 val perplexity 259.3359
-14600 train 5.534735 (lr=6.0364e-06) (hash(x)=150218222)
-14800 val loss 5.5551
-14800 val perplexity 258.5652
-14800 train 5.453934 (lr=6.1206e-06) (hash(x)=147628003)
-14700 val loss 5.5577
-14700 val perplexity 259.2366
-14700 train 5.493864 (lr=5.8848e-06) (hash(x)=142258737)
-14900 val loss 5.5566
-14900 val perplexity 258.9344
-14900 train 5.411365 (lr=5.9693e-06) (hash(x)=149144036)
-14800 val loss 5.5555
-14800 val perplexity 258.6584
-14800 train 5.448818 (lr=5.7381e-06) (hash(x)=147628003)
-15000 val loss 5.5561
-15000 val perplexity 258.8178
-15000 train 5.489982 (lr=5.8233e-06) (hash(x)=145925409)
-14900 val loss 5.5569
-14900 val perplexity 259.0222
-14900 train 5.407426 (lr=5.5962e-06) (hash(x)=149144036)
-15100 val loss 5.5518
-15100 val perplexity 257.7099
-15100 train 5.436214 (lr=5.6826e-06) (hash(x)=144555751)
-15000 val loss 5.5554
-15000 val perplexity 258.6341
-15000 train 5.492520 (lr=5.4593e-06) (hash(x)=145925409)
-15200 val loss 5.5504
-15200 val perplexity 257.3345
-15200 train 5.364831 (lr=5.5473e-06) (hash(x)=143448852)
-15100 val loss 5.5514
-15100 val perplexity 257.6073
-15100 train 5.432653 (lr=5.3275e-06) (hash(x)=144555751)
-15300 val loss 5.5487
-15300 val perplexity 256.9110
-15300 train 5.554673 (lr=5.4175e-06) (hash(x)=156347253)
-15200 val loss 5.5495
-15200 val perplexity 257.0970
-15200 train 5.369192 (lr=5.2006e-06) (hash(x)=143448852)
-15400 val loss 5.5467
-15400 val perplexity 256.3846
-15400 train 5.494298 (lr=5.2931e-06) (hash(x)=153010179)
-15300 val loss 5.5487
-15300 val perplexity 256.9073
-15300 train 5.551206 (lr=5.0789e-06) (hash(x)=156347253)
-15500 val loss 5.5477
-15500 val perplexity 256.6517
-15500 train 5.325893 (lr=5.1742e-06) (hash(x)=144066445)
-15400 val loss 5.5468
-15400 val perplexity 256.4174
-15400 train 5.497810 (lr=4.9622e-06) (hash(x)=153010179)
-15600 val loss 5.5476
-15600 val perplexity 256.6144
-15600 train 5.283917 (lr=5.0608e-06) (hash(x)=146107722)
-15500 val loss 5.5471
-15500 val perplexity 256.4917
-15500 train 5.329374 (lr=4.8508e-06) (hash(x)=144066445)
-15700 val loss 5.5433
-15700 val perplexity 255.5193
-15700 train 5.703694 (lr=4.9531e-06) (hash(x)=152533799)
-15600 val loss 5.5457
-15600 val perplexity 256.1370
-15600 train 5.281954 (lr=4.7445e-06) (hash(x)=146107722)
-15800 val loss 5.5401
-15800 val perplexity 254.7081
-15800 train 5.594624 (lr=4.8509e-06) (hash(x)=151756366)
-15700 val loss 5.5431
-15700 val perplexity 255.4666
-15700 train 5.690786 (lr=4.6435e-06) (hash(x)=152533799)
-15900 val loss 5.5398
-15900 val perplexity 254.6376
-15900 train 5.522333 (lr=4.7545e-06) (hash(x)=151735039)
-15800 val loss 5.5402
-15800 val perplexity 254.7284
-15800 train 5.592268 (lr=4.5478e-06) (hash(x)=151756366)
-16000 val loss 5.5389
-16000 val perplexity 254.4079
-16000 train 5.564443 (lr=4.6637e-06) (hash(x)=154633226)
-15900 val loss 5.5406
-15900 val perplexity 254.8195
-15900 train 5.530737 (lr=4.4573e-06) (hash(x)=151735039)
-16100 val loss 5.5407
-16100 val perplexity 254.8598
-16100 train 5.373457 (lr=4.5786e-06) (hash(x)=158094333)
-16000 val loss 5.5398
-16000 val perplexity 254.6348
-16000 train 5.566914 (lr=4.3722e-06) (hash(x)=154633226)
-16200 val loss 5.5395
-16200 val perplexity 254.5566
-16200 train 5.301944 (lr=4.4993e-06) (hash(x)=142772927)
-16100 val loss 5.5413
-16100 val perplexity 255.0077
-16100 train 5.377645 (lr=4.2924e-06) (hash(x)=158094333)
-16300 val loss 5.5382
-16300 val perplexity 254.2214
-16300 train 5.442211 (lr=4.4257e-06) (hash(x)=148042896)
-16200 val loss 5.5397
-16200 val perplexity 254.6087
-16200 train 5.298206 (lr=4.2180e-06) (hash(x)=142772927)
-16400 val loss 5.5343
-16400 val perplexity 253.2240
-16400 train 6.030413 (lr=4.3579e-06) (hash(x)=170306399)
-16300 val loss 5.5383
-16300 val perplexity 254.2502
-16300 train 5.440192 (lr=4.1491e-06) (hash(x)=148042896)
-16500 val loss 5.5344
-16500 val perplexity 253.2674
-16500 train 5.555060 (lr=4.2960e-06) (hash(x)=141049320)
-16400 val loss 5.5351
-16400 val perplexity 253.4445
-16400 train 6.030590 (lr=4.0856e-06) (hash(x)=170306399)
-16600 val loss 5.5348
-16600 val perplexity 253.3616
-16600 train 5.383412 (lr=4.2399e-06) (hash(x)=150512136)
-16500 val loss 5.5352
-16500 val perplexity 253.4678
-16500 train 5.552647 (lr=4.0275e-06) (hash(x)=141049320)
-16700 val loss 5.5323
-16700 val perplexity 252.7309
-16700 train 5.544337 (lr=4.1896e-06) (hash(x)=151843645)
-16600 val loss 5.5355
-16600 val perplexity 253.5231
-16600 train 5.393153 (lr=3.9749e-06) (hash(x)=150512136)
-16800 val loss 5.5298
-16800 val perplexity 252.1039
-16800 train 5.617894 (lr=4.1452e-06) (hash(x)=140073659)
-16700 val loss 5.5328
-16700 val perplexity 252.8530
-16700 train 5.542039 (lr=3.9278e-06) (hash(x)=151843645)
-16900 val loss 5.5283
-16900 val perplexity 251.7074
-16900 train 5.680115 (lr=4.1067e-06) (hash(x)=171157212)
-16800 val loss 5.5304
-16800 val perplexity 252.2500
-16800 train 5.605481 (lr=3.8862e-06) (hash(x)=140073659)
-17000 val loss 5.5268
-17000 val perplexity 251.3435
-17000 train 5.455764 (lr=4.0741e-06) (hash(x)=144553276)
-16900 val loss 5.5295
-16900 val perplexity 252.0072
-16900 train 5.689977 (lr=3.8501e-06) (hash(x)=171157212)
-17100 val loss 5.5280
-17100 val perplexity 251.6405
-17100 train 5.518610 (lr=4.0475e-06) (hash(x)=157717565)
-17000 val loss 5.5295
-17000 val perplexity 252.0275
-17000 train 5.469572 (lr=3.8195e-06) (hash(x)=144553276)
-17200 val loss 5.5286
-17200 val perplexity 251.8003
-17200 train 5.387378 (lr=4.0267e-06) (hash(x)=149843860)
-17100 val loss 5.5289
-17100 val perplexity 251.8564
-17100 train 5.516634 (lr=3.7945e-06) (hash(x)=157717565)
-17300 val loss 5.5249
-17300 val perplexity 250.8722
-17300 train 5.581902 (lr=4.0119e-06) (hash(x)=162171979)
-17200 val loss 5.5297
-17200 val perplexity 252.0703
-17200 train 5.394426 (lr=3.7750e-06) (hash(x)=149843860)
-17400 val loss 5.5241
-17400 val perplexity 250.6583
-17400 train 5.499109 (lr=4.0030e-06) (hash(x)=148400702)
-17300 val loss 5.5271
-17300 val perplexity 251.4217
-17300 train 5.585364 (lr=3.7611e-06) (hash(x)=162171979)
-17499 val loss 5.5230
-17499 val perplexity 250.3805
-17400 val loss 5.5252
-17400 val perplexity 250.9298
-17400 train 5.501467 (lr=3.7528e-06) (hash(x)=148400702)
-17499 val loss 5.5247
-17499 val perplexity 250.8136
+max_steps: 8750
+7100 val loss 5.9287
+7100 val perplexity 375.6843
+7100 train 5.880068 (lr=6.5080e-06) (hash(x)=151066339)
+0 val loss 11.2647
+0 val perplexity 78014.9766
+7200 val loss 5.9281
+7200 val perplexity 375.4359
+7200 train 5.731686 (lr=6.1648e-06) (hash(x)=155231264)
+0 train 11.267745 (lr=8.0000e-08) (hash(x)=150327452)
+7300 val loss 5.9206
+7300 val perplexity 372.6359
+7300 train 5.929327 (lr=5.8405e-06) (hash(x)=150281149)
+100 val loss 10.0255
+100 val perplexity 22595.3477
+100 train 10.075964 (lr=8.0800e-06) (hash(x)=166780046)
+7400 val loss 5.9179
+7400 val perplexity 371.6358
+7400 train 5.795485 (lr=5.5357e-06) (hash(x)=148421717)
+200 val loss 9.5219
+200 val perplexity 13655.3779
+200 train 9.486035 (lr=1.6080e-05) (hash(x)=155040610)
+7500 val loss 5.9176
+7500 val perplexity 371.5118
+7500 train 5.672206 (lr=5.2508e-06) (hash(x)=146921118)
+300 val loss 8.5181
+300 val perplexity 5004.6064
+300 train 8.538123 (lr=2.4080e-05) (hash(x)=155504036)
+7600 val loss 5.9100
+7600 val perplexity 368.6994
+7600 train 5.960232 (lr=4.9862e-06) (hash(x)=150660048)
+400 val loss 7.7858
+400 val perplexity 2406.2373
+400 train 7.670135 (lr=3.2080e-05) (hash(x)=143823248)
+7700 val loss 5.9087
+7700 val perplexity 368.2145
+7700 train 5.777826 (lr=4.7423e-06) (hash(x)=148059852)
+500 val loss 7.5398
+500 val perplexity 1881.4100
+500 train 7.509302 (lr=4.0000e-05) (hash(x)=143734685)
+7800 val loss 5.9071
+7800 val perplexity 367.6471
+7800 train 5.606631 (lr=4.5194e-06) (hash(x)=148331002)
+600 val loss 7.4574
+600 val perplexity 1732.6132
+600 train 7.243900 (lr=3.9987e-05) (hash(x)=150678249)
+7900 val loss 5.9006
+7900 val perplexity 365.2534
+7900 train 5.859887 (lr=4.3179e-06) (hash(x)=164923883)
+700 val loss 7.3848
+700 val perplexity 1611.2378
+700 train 7.528392 (lr=3.9948e-05) (hash(x)=175802021)
+8000 val loss 5.8995
+8000 val perplexity 364.8397
+8000 train 5.780609 (lr=4.1380e-06) (hash(x)=143545384)
+800 val loss 7.2813
+800 val perplexity 1452.8223
+800 train 7.292555 (lr=3.9883e-05) (hash(x)=158681215)
+8100 val loss 5.8987
+8100 val perplexity 364.5598
+8100 train 5.720141 (lr=3.9800e-06) (hash(x)=160686959)
+900 val loss 7.1971
+900 val perplexity 1335.4854
+900 train 7.089118 (lr=3.9792e-05) (hash(x)=146108145)
+8200 val loss 5.8961
+8200 val perplexity 363.6072
+8200 train 5.852425 (lr=3.8442e-06) (hash(x)=156501889)
+1000 val loss 7.1234
+1000 val perplexity 1240.7209
+1000 train 6.964436 (lr=3.9675e-05) (hash(x)=154996086)
+8300 val loss 5.8938
+8300 val perplexity 362.7742
+8300 train 5.731362 (lr=3.7307e-06) (hash(x)=142716875)
+1100 val loss 7.0250
+1100 val perplexity 1124.4392
+1100 train 7.009098 (lr=3.9532e-05) (hash(x)=153885445)
+8400 val loss 5.8888
+8400 val perplexity 360.9724
+8400 train 5.790694 (lr=3.6397e-06) (hash(x)=154436684)
+1200 val loss 6.9352
+1200 val perplexity 1027.8256
+1200 train 6.891309 (lr=3.9364e-05) (hash(x)=142353087)
+8500 val loss 5.8872
+8500 val perplexity 360.4104
+8500 train 5.953193 (lr=3.5713e-06) (hash(x)=147965839)
+1300 val loss 6.8669
+1300 val perplexity 959.9401
+1300 train 6.776668 (lr=3.9171e-05) (hash(x)=150750353)
+8600 val loss 5.8862
+8600 val perplexity 360.0428
+8600 train 5.690831 (lr=3.5257e-06) (hash(x)=145228097)
+1400 val loss 6.8045
+1400 val perplexity 901.8710
+1400 train 6.626483 (lr=3.8953e-05) (hash(x)=152767913)
+8700 val loss 5.8822
+8700 val perplexity 358.5815
+8700 train 6.058657 (lr=3.5029e-06) (hash(x)=152910357)
+1500 val loss 6.7235
+1500 val perplexity 831.7086
+1500 train 6.764162 (lr=3.8711e-05) (hash(x)=151562048)
+8749 val loss 5.8818
+8749 val perplexity 358.4548
+1600 val loss 6.6411
+1600 val perplexity 765.9436
+1600 train 6.638767 (lr=3.8444e-05) (hash(x)=166486165)

attention_kindselective_n_heads4_seed1338/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1bac2e88faa57534b7341e4dc64586426fcadfd3c4f0cca0c9fa558bf960b18
+size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffaba6e69d30b87c050e2bc7acc71bb66c4439938865e3f0d226c5e8348474a9
+size 179406214