Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1344/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_61440_4_1344", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_61440_4_1344", "n_embd": 256}

attention_kindselective_n_heads4_seed1344/log2.txt

@@ -1,494 +1,534 @@
 max_steps: 8750
-1300 val loss 6.6895
-1300 val perplexity 803.9285
-1300 train 6.773464 (lr=4.8964e-05) (hash(x)=173675040)
+max_steps: 8750
+0 val loss 11.2831
+0 val perplexity 79466.0234
 0 val loss 11.2831
-0 val perplexity 79465.5000
-1400 val loss 6.6173
-1400 val perplexity 747.9091
-1400 train 6.590247 (lr=4.8691e-05) (hash(x)=162475906)
-0 train 11.285261 (lr=8.0000e-08) (hash(x)=145830960)
-1500 val loss 6.5603
-1500 val perplexity 706.4640
-1500 train 6.387834 (lr=4.8388e-05) (hash(x)=152785612)
-100 val loss 10.1076
-100 val perplexity 24529.0078
-100 train 10.141041 (lr=8.0800e-06) (hash(x)=144157453)
-1600 val loss 6.4984
-1600 val perplexity 664.0759
-1600 train 6.445701 (lr=4.8055e-05) (hash(x)=151987383)
-200 val loss 9.6421
-200 val perplexity 15400.2119
-200 train 9.681087 (lr=1.6080e-05) (hash(x)=146764602)
-1700 val loss 6.4370
-1700 val perplexity 624.5526
-1700 train 6.434201 (lr=4.7691e-05) (hash(x)=143781605)
-300 val loss 8.5994
-300 val perplexity 5428.6685
-300 train 8.501122 (lr=2.4080e-05) (hash(x)=140559124)
-1800 val loss 6.4001
-1800 val perplexity 601.8904
-1800 train 6.390086 (lr=4.7299e-05) (hash(x)=151040203)
-1900 val loss 6.3648
-1900 val perplexity 581.0037
-1900 train 6.261320 (lr=4.6878e-05) (hash(x)=149414572)
-400 val loss 7.7656
-400 val perplexity 2358.1003
-400 train 8.115546 (lr=3.2080e-05) (hash(x)=166024176)
-2000 val loss 6.3277
-2000 val perplexity 559.8475
-2000 train 6.281135 (lr=4.6428e-05) (hash(x)=156479674)
-500 val loss 7.5214
-500 val perplexity 1847.1365
-500 train 7.493972 (lr=4.0000e-05) (hash(x)=161040668)
-2100 val loss 6.2903
-2100 val perplexity 539.3040
-2100 train 6.165673 (lr=4.5951e-05) (hash(x)=137861481)
-600 val loss 7.4516
-600 val perplexity 1722.6665
-600 train 7.479457 (lr=3.9987e-05) (hash(x)=148878990)
-2200 val loss 6.2634
-2200 val perplexity 524.9904
-2200 train 6.324560 (lr=4.5448e-05) (hash(x)=155937443)
-700 val loss 7.3861
-700 val perplexity 1613.4520
-700 train 7.367638 (lr=3.9948e-05) (hash(x)=150420695)
-2300 val loss 6.2612
-2300 val perplexity 523.8252
-2300 train 5.928801 (lr=4.4918e-05) (hash(x)=131035715)
-800 val loss 7.3272
-800 val perplexity 1521.0419
-800 train 7.300409 (lr=3.9883e-05) (hash(x)=151995229)
-2400 val loss 6.2118
-2400 val perplexity 498.5806
-2400 train 6.209708 (lr=4.4363e-05) (hash(x)=146568981)
-900 val loss 7.2836
-900 val perplexity 1456.1638
-900 train 7.042968 (lr=3.9792e-05) (hash(x)=139853932)
-2500 val loss 6.1798
-2500 val perplexity 482.9110
-2500 train 6.160361 (lr=4.3784e-05) (hash(x)=163317586)
-1000 val loss 7.2349
-1000 val perplexity 1386.9896
-1000 train 7.431800 (lr=3.9675e-05) (hash(x)=174104207)
-2600 val loss 6.1566
-2600 val perplexity 471.8229
-2600 train 5.974750 (lr=4.3181e-05) (hash(x)=144201060)
-1100 val loss 7.1852
-1100 val perplexity 1319.7642
-1100 train 7.018353 (lr=3.9532e-05) (hash(x)=146275038)
-2700 val loss 6.1323
-2700 val perplexity 460.5000
-2700 train 6.032785 (lr=4.2555e-05) (hash(x)=141825701)
-1200 val loss 7.1329
-1200 val perplexity 1252.4744
-1200 train 6.912647 (lr=3.9364e-05) (hash(x)=141403655)
-2800 val loss 6.1118
-2800 val perplexity 451.1669
-2800 train 6.219571 (lr=4.1908e-05) (hash(x)=160561627)
-1300 val loss 7.0764
-1300 val perplexity 1183.7563
-1300 train 7.156072 (lr=3.9171e-05) (hash(x)=173675040)
-2900 val loss 6.0842
-2900 val perplexity 438.8623
-2900 train 6.021353 (lr=4.1240e-05) (hash(x)=151758176)
-1400 val loss 7.0171
-1400 val perplexity 1115.5658
-1400 train 6.980552 (lr=3.8953e-05) (hash(x)=162475906)
-3000 val loss 6.0656
-3000 val perplexity 430.7632
-3000 train 6.088512 (lr=4.0551e-05) (hash(x)=155815751)
-1500 val loss 6.9736
-1500 val perplexity 1068.0918
-1500 train 6.806849 (lr=3.8711e-05) (hash(x)=152785612)
-3100 val loss 6.0532
-3100 val perplexity 425.4607
-3100 train 5.723763 (lr=3.9844e-05) (hash(x)=142307043)
-1600 val loss 6.9146
-1600 val perplexity 1006.8359
-1600 train 6.869276 (lr=3.8444e-05) (hash(x)=151987383)
-3200 val loss 6.0339
-3200 val perplexity 417.3221
-3200 train 6.141800 (lr=3.9119e-05) (hash(x)=156310690)
-1700 val loss 6.8514
-1700 val perplexity 945.2108
-1700 train 6.829998 (lr=3.8153e-05) (hash(x)=143781605)
-3300 val loss 6.0083
-3300 val perplexity 406.8000
-3300 train 6.009042 (lr=3.8377e-05) (hash(x)=151344506)
-1800 val loss 6.8108
-1800 val perplexity 907.5953
-1800 train 6.784783 (lr=3.7839e-05) (hash(x)=151040203)
-3400 val loss 5.9890
-3400 val perplexity 399.0146
-3400 train 6.022029 (lr=3.7619e-05) (hash(x)=162911881)
-1900 val loss 6.7506
-1900 val perplexity 854.5989
-1900 train 6.654410 (lr=3.7502e-05) (hash(x)=149414572)
-3500 val loss 5.9747
-3500 val perplexity 393.3347
-3500 train 5.916205 (lr=3.6847e-05) (hash(x)=142889971)
-2000 val loss 6.6949
-2000 val perplexity 808.3021
-2000 train 6.656959 (lr=3.7143e-05) (hash(x)=156479674)
-3600 val loss 5.9525
-3600 val perplexity 384.7280
-3600 train 5.941948 (lr=3.6061e-05) (hash(x)=149470354)
-2100 val loss 6.6422
-2100 val perplexity 766.7804
-2100 train 6.508312 (lr=3.6761e-05) (hash(x)=137861481)
-3700 val loss 5.9335
-3700 val perplexity 377.4560
-3700 train 5.878082 (lr=3.5263e-05) (hash(x)=154194821)
-2200 val loss 6.5950
-2200 val perplexity 731.4640
-2200 train 6.654806 (lr=3.6358e-05) (hash(x)=155937443)
-3800 val loss 5.9256
-3800 val perplexity 374.5177
-3800 train 5.851969 (lr=3.4453e-05) (hash(x)=148885848)
-2300 val loss 6.5656
-2300 val perplexity 710.2440
-2300 train 6.250451 (lr=3.5935e-05) (hash(x)=131035715)
-3900 val loss 5.9066
-3900 val perplexity 367.4492
-3900 train 5.823092 (lr=3.3633e-05) (hash(x)=148308484)
-2400 val loss 6.5147
-2400 val perplexity 675.0030
-2400 train 6.513992 (lr=3.5491e-05) (hash(x)=146568981)
-4000 val loss 5.8898
-4000 val perplexity 361.3280
-4000 train 5.728326 (lr=3.2805e-05) (hash(x)=139828564)
-4100 val loss 5.8785
-4100 val perplexity 357.2629
-4100 train 5.783032 (lr=3.1968e-05) (hash(x)=139981997)
-2500 val loss 6.4747
-2500 val perplexity 648.5444
-2500 train 6.444137 (lr=3.5027e-05) (hash(x)=163317586)
-4200 val loss 5.8653
-4200 val perplexity 352.5873
-4200 train 6.020027 (lr=3.1126e-05) (hash(x)=150738447)
-2600 val loss 6.4492
-2600 val perplexity 632.2130
-2600 train 6.273980 (lr=3.4545e-05) (hash(x)=144201060)
-4300 val loss 5.8484
-4300 val perplexity 346.6892
-4300 train 5.620984 (lr=3.0277e-05) (hash(x)=142198107)
-2700 val loss 6.4168
-2700 val perplexity 612.0558
-2700 train 6.313120 (lr=3.4044e-05) (hash(x)=141825701)
-4400 val loss 5.8384
-4400 val perplexity 343.2307
-4400 train 5.713844 (lr=2.9425e-05) (hash(x)=142731201)
-2800 val loss 6.3883
-2800 val perplexity 594.8422
-2800 train 6.510801 (lr=3.3526e-05) (hash(x)=160561627)
-4500 val loss 5.8210
-4500 val perplexity 337.3009
-4500 train 5.849483 (lr=2.8571e-05) (hash(x)=154814426)
-2900 val loss 6.3587
-2900 val perplexity 577.5201
-2900 train 6.310970 (lr=3.2992e-05) (hash(x)=151758176)
-4600 val loss 5.8097
-4600 val perplexity 333.5216
-4600 train 5.787569 (lr=2.7714e-05) (hash(x)=155922230)
-3000 val loss 6.3336
-3000 val perplexity 563.1740
-3000 train 6.341168 (lr=3.2441e-05) (hash(x)=155815751)
-4700 val loss 5.8002
-4700 val perplexity 330.3598
-4700 train 5.552835 (lr=2.6857e-05) (hash(x)=139398510)
-3100 val loss 6.3187
-3100 val perplexity 554.8702
-3100 train 5.986887 (lr=3.1875e-05) (hash(x)=142307043)
-4800 val loss 5.7928
-4800 val perplexity 327.9305
-4800 train 5.851633 (lr=2.6002e-05) (hash(x)=140893236)
-3200 val loss 6.2920
-3200 val perplexity 540.2058
-3200 train 6.404789 (lr=3.1295e-05) (hash(x)=156310690)
-4900 val loss 5.7760
-4900 val perplexity 322.4515
-4900 train 5.668627 (lr=2.5148e-05) (hash(x)=153747830)
-3300 val loss 6.2681
-3300 val perplexity 527.4542
-3300 train 6.271742 (lr=3.0702e-05) (hash(x)=151344506)
-5000 val loss 5.7650
-5000 val perplexity 318.9458
-5000 train 6.023503 (lr=2.4298e-05) (hash(x)=148919005)
-3400 val loss 6.2469
-3400 val perplexity 516.4142
-3400 train 6.282847 (lr=3.0095e-05) (hash(x)=162911881)
-5100 val loss 5.7579
-5100 val perplexity 316.6893
-5100 train 5.549688 (lr=2.3452e-05) (hash(x)=142281936)
-3500 val loss 6.2319
-3500 val perplexity 508.7363
-3500 train 6.170129 (lr=2.9477e-05) (hash(x)=142889971)
-5200 val loss 5.7594
-5200 val perplexity 317.1647
-5200 train 5.555599 (lr=2.2613e-05) (hash(x)=143162650)
-3600 val loss 6.2112
-3600 val perplexity 498.2821
-3600 train 6.204837 (lr=2.8849e-05) (hash(x)=149470354)
-5300 val loss 5.7368
-5300 val perplexity 310.0715
-5300 train 5.798581 (lr=2.1780e-05) (hash(x)=148546849)
-3700 val loss 6.1918
-3700 val perplexity 488.7092
-3700 train 6.136409 (lr=2.8210e-05) (hash(x)=154194821)
-5400 val loss 5.7275
-5400 val perplexity 307.2088
-5400 train 5.588672 (lr=2.0956e-05) (hash(x)=143492259)
-3800 val loss 6.1763
-3800 val perplexity 481.2229
-3800 train 6.109374 (lr=2.7562e-05) (hash(x)=148885848)
-5500 val loss 5.7216
-5500 val perplexity 305.4041
-5500 train 5.477193 (lr=2.0141e-05) (hash(x)=141023941)
-3900 val loss 6.1563
-3900 val perplexity 471.6664
-3900 train 6.070787 (lr=2.6907e-05) (hash(x)=148308484)
-5600 val loss 5.7205
-5600 val perplexity 305.0673
-5600 train 5.603661 (lr=1.9337e-05) (hash(x)=142065021)
-4000 val loss 6.1397
-4000 val perplexity 463.9084
-4000 train 5.980125 (lr=2.6244e-05) (hash(x)=139828564)
-5700 val loss 5.7037
-5700 val perplexity 299.9660
-5700 train 5.590576 (lr=1.8545e-05) (hash(x)=145749913)
-4100 val loss 6.1250
-4100 val perplexity 457.1491
-4100 train 6.032627 (lr=2.5575e-05) (hash(x)=139981997)
-5800 val loss 5.6974
-5800 val perplexity 298.0915
-5800 train 5.381054 (lr=1.7765e-05) (hash(x)=140035522)
-4200 val loss 6.1090
-4200 val perplexity 449.9003
-4200 train 6.255210 (lr=2.4900e-05) (hash(x)=150738447)
-5900 val loss 5.6954
-5900 val perplexity 297.5097
-5900 train 5.436858 (lr=1.7000e-05) (hash(x)=154752726)
-4300 val loss 6.0933
-4300 val perplexity 442.8800
-4300 train 5.871885 (lr=2.4222e-05) (hash(x)=142198107)
-6000 val loss 5.6885
-6000 val perplexity 295.4404
-6000 train 5.713228 (lr=1.6250e-05) (hash(x)=158715824)
-4400 val loss 6.0813
-4400 val perplexity 437.6097
-4400 train 5.954550 (lr=2.3540e-05) (hash(x)=142731201)
-6100 val loss 5.6800
-6100 val perplexity 292.9566
-6100 train 5.699704 (lr=1.5516e-05) (hash(x)=137413820)
-4500 val loss 6.0630
-4500 val perplexity 429.6517
-4500 train 6.074458 (lr=2.2856e-05) (hash(x)=154814426)
-6200 val loss 5.6700
-6200 val perplexity 290.0225
-6200 train 5.589405 (lr=1.4800e-05) (hash(x)=151507523)
-4600 val loss 6.0509
-4600 val perplexity 424.4775
-4600 train 6.017695 (lr=2.2171e-05) (hash(x)=155922230)
-6300 val loss 5.6700
-6300 val perplexity 290.0232
-6300 train 5.439530 (lr=1.4102e-05) (hash(x)=147514617)
-4700 val loss 6.0449
-4700 val perplexity 421.9507
-4700 train 5.787765 (lr=2.1486e-05) (hash(x)=139398510)
-6400 val loss 5.6652
-6400 val perplexity 288.6526
-6400 train 5.666666 (lr=1.3424e-05) (hash(x)=151604465)
-6500 val loss 5.6557
-6500 val perplexity 285.9176
-4800 val loss 6.0293
-4800 val perplexity 415.4446
-6500 train 5.715902 (lr=1.2766e-05) (hash(x)=144515881)
-4800 train 6.103447 (lr=2.0801e-05) (hash(x)=140893236)
-6600 val loss 5.6484
-6600 val perplexity 283.8284
-6600 train 5.447026 (lr=1.2129e-05) (hash(x)=136948374)
-4900 val loss 6.0135
-4900 val perplexity 408.9137
-4900 train 5.920414 (lr=2.0118e-05) (hash(x)=153747830)
-6700 val loss 5.6471
-6700 val perplexity 283.4608
-6700 train 5.448362 (lr=1.1515e-05) (hash(x)=146268592)
-5000 val loss 6.0050
-5000 val perplexity 405.4520
-5000 train 6.230835 (lr=1.9438e-05) (hash(x)=148919005)
-6800 val loss 5.6381
-6800 val perplexity 280.9316
-6800 train 5.587202 (lr=1.0923e-05) (hash(x)=152676836)
-5100 val loss 5.9941
-5100 val perplexity 401.0715
-5100 train 5.782193 (lr=1.8762e-05) (hash(x)=142281936)
-6900 val loss 5.6313
-6900 val perplexity 279.0120
-6900 train 5.562898 (lr=1.0356e-05) (hash(x)=134657776)
-5200 val loss 5.9957
-5200 val perplexity 401.7111
-5200 train 5.804429 (lr=1.8090e-05) (hash(x)=143162650)
-7000 val loss 5.6291
-7000 val perplexity 278.4016
-7000 train 5.589009 (lr=9.8138e-06) (hash(x)=166721861)
-5300 val loss 5.9733
-5300 val perplexity 392.7957
-5300 train 6.033432 (lr=1.7424e-05) (hash(x)=148546849)
-7100 val loss 5.6279
-7100 val perplexity 278.0805
-7100 train 5.430983 (lr=9.2971e-06) (hash(x)=135496702)
-5400 val loss 5.9609
-5400 val perplexity 387.9548
-5400 train 5.819170 (lr=1.6765e-05) (hash(x)=143492259)
-7200 val loss 5.6235
-7200 val perplexity 276.8535
-7200 train 5.736270 (lr=8.8068e-06) (hash(x)=155567461)
-5500 val loss 5.9559
-5500 val perplexity 386.0132
-5500 train 5.713650 (lr=1.6113e-05) (hash(x)=141023941)
-7300 val loss 5.6167
-7300 val perplexity 274.9700
-7300 train 5.422538 (lr=8.3436e-06) (hash(x)=142803829)
-5600 val loss 5.9542
-5600 val perplexity 385.3673
-5600 train 5.828371 (lr=1.5469e-05) (hash(x)=142065021)
-7400 val loss 5.6160
-7400 val perplexity 274.8009
-7400 train 5.435742 (lr=7.9082e-06) (hash(x)=145294178)
-5700 val loss 5.9355
-5700 val perplexity 378.2421
-5700 train 5.798361 (lr=1.4836e-05) (hash(x)=145749913)
-7500 val loss 5.6122
-7500 val perplexity 273.7481
-7500 train 5.373355 (lr=7.5012e-06) (hash(x)=150573713)
-5800 val loss 5.9275
-5800 val perplexity 375.2310
-5800 train 5.613351 (lr=1.4212e-05) (hash(x)=140035522)
-7600 val loss 5.6107
-7600 val perplexity 273.3234
-7600 train 5.632406 (lr=7.1232e-06) (hash(x)=142771511)
-5900 val loss 5.9241
-5900 val perplexity 373.9262
-5900 train 5.678686 (lr=1.3600e-05) (hash(x)=154752726)
-7700 val loss 5.6042
-7700 val perplexity 271.5583
-7700 train 5.505773 (lr=6.7747e-06) (hash(x)=143602175)
-6000 val loss 5.9181
-6000 val perplexity 371.7014
-6000 train 5.941476 (lr=1.3000e-05) (hash(x)=158715824)
-7800 val loss 5.6010
-7800 val perplexity 270.6951
-7800 train 5.663585 (lr=6.4563e-06) (hash(x)=152379862)
-6100 val loss 5.9064
-6100 val perplexity 367.3962
-6100 train 5.913001 (lr=1.2413e-05) (hash(x)=137413820)
-7900 val loss 5.5999
-7900 val perplexity 270.4090
-7900 train 5.425761 (lr=6.1684e-06) (hash(x)=146655921)
-6200 val loss 5.8991
-6200 val perplexity 364.7029
-6200 train 5.816253 (lr=1.1840e-05) (hash(x)=151507523)
-8000 val loss 5.6012
-8000 val perplexity 270.7527
-8000 train 5.731220 (lr=5.9114e-06) (hash(x)=148262482)
-6300 val loss 5.8957
-6300 val perplexity 363.4883
-6300 train 5.667901 (lr=1.1282e-05) (hash(x)=147514617)
-8100 val loss 5.5923
-8100 val perplexity 268.3542
-8100 train 5.538779 (lr=5.6857e-06) (hash(x)=147683655)
-6400 val loss 5.8926
-6400 val perplexity 362.3358
-6400 train 5.901004 (lr=1.0739e-05) (hash(x)=151604465)
-8200 val loss 5.5916
-8200 val perplexity 268.1735
-8200 train 5.658997 (lr=5.4917e-06) (hash(x)=164975934)
-6500 val loss 5.8813
-6500 val perplexity 358.2624
-6500 train 5.937473 (lr=1.0213e-05) (hash(x)=144515881)
-8300 val loss 5.5882
-8300 val perplexity 267.2640
-8300 train 5.499679 (lr=5.3295e-06) (hash(x)=145300550)
-6600 val loss 5.8753
-6600 val perplexity 356.1367
-6600 train 5.660358 (lr=9.7032e-06) (hash(x)=136948374)
-8400 val loss 5.5866
-8400 val perplexity 266.8259
-8400 train 5.493410 (lr=5.1995e-06) (hash(x)=150679400)
-6700 val loss 5.8745
-6700 val perplexity 355.8448
-6700 train 5.690690 (lr=9.2116e-06) (hash(x)=146268592)
-8500 val loss 5.5882
-8500 val perplexity 267.2431
-8500 train 5.576384 (lr=5.1019e-06) (hash(x)=164109401)
-6800 val loss 5.8652
-6800 val perplexity 352.5601
-6800 train 5.824735 (lr=8.7387e-06) (hash(x)=152676836)
-8600 val loss 5.5841
-8600 val perplexity 266.1479
-8600 train 5.520129 (lr=5.0367e-06) (hash(x)=161036376)
-6900 val loss 5.8585
-6900 val perplexity 350.1941
-6900 train 5.778625 (lr=8.2849e-06) (hash(x)=134657776)
-8700 val loss 5.5798
-8700 val perplexity 265.0147
-8700 train 5.501758 (lr=5.0041e-06) (hash(x)=153828820)
-8749 val loss 5.5778
-8749 val perplexity 264.4979
-7000 val loss 5.8552
-7000 val perplexity 349.0465
-7000 train 5.827059 (lr=7.8510e-06) (hash(x)=166721861)
-7100 val loss 5.8537
-7100 val perplexity 348.5362
-7100 train 5.664058 (lr=7.4377e-06) (hash(x)=135496702)
-7200 val loss 5.8496
-7200 val perplexity 347.0883
-7200 train 5.947722 (lr=7.0455e-06) (hash(x)=155567461)
-7300 val loss 5.8428
-7300 val perplexity 344.7538
-7300 train 5.637713 (lr=6.6749e-06) (hash(x)=142803829)
-7400 val loss 5.8403
-7400 val perplexity 343.8778
-7400 train 5.662089 (lr=6.3266e-06) (hash(x)=145294178)
-7500 val loss 5.8383
-7500 val perplexity 343.1886
-7500 train 5.605683 (lr=6.0010e-06) (hash(x)=150573713)
-7600 val loss 5.8350
-7600 val perplexity 342.0600
-7600 train 5.852287 (lr=5.6986e-06) (hash(x)=142771511)
-7700 val loss 5.8286
-7700 val perplexity 339.8656
-7700 train 5.725527 (lr=5.4198e-06) (hash(x)=143602175)
-7800 val loss 5.8250
-7800 val perplexity 338.6624
-7800 train 5.857786 (lr=5.1650e-06) (hash(x)=152379862)
-7900 val loss 5.8238
-7900 val perplexity 338.2581
-7900 train 5.664768 (lr=4.9347e-06) (hash(x)=146655921)
-8000 val loss 5.8242
-8000 val perplexity 338.4018
-8000 train 5.935122 (lr=4.7291e-06) (hash(x)=148262482)
-8100 val loss 5.8166
-8100 val perplexity 335.8375
-8100 train 5.753369 (lr=4.5486e-06) (hash(x)=147683655)
-8200 val loss 5.8155
-8200 val perplexity 335.4515
-8200 train 5.876520 (lr=4.3933e-06) (hash(x)=164975934)
-8300 val loss 5.8120
-8300 val perplexity 334.2799
-8300 train 5.729677 (lr=4.2636e-06) (hash(x)=145300550)
-8400 val loss 5.8100
-8400 val perplexity 333.6333
-8400 train 5.721456 (lr=4.1596e-06) (hash(x)=150679400)
-8500 val loss 5.8102
-8500 val perplexity 333.6947
-8500 train 5.822725 (lr=4.0815e-06) (hash(x)=164109401)
-8600 val loss 5.8071
-8600 val perplexity 332.6415
-8600 train 5.800099 (lr=4.0294e-06) (hash(x)=161036376)
-8700 val loss 5.8029
-8700 val perplexity 331.2553
-8700 train 5.736811 (lr=4.0033e-06) (hash(x)=153828820)
-8749 val loss 5.8012
-8749 val perplexity 330.7012
+0 val perplexity 79466.0234
+0 train 11.285278 (lr=7.0000e-08) (hash(x)=145830960)
+0 train 11.285278 (lr=6.0000e-08) (hash(x)=145830960)
+100 val loss 10.1647
+100 val perplexity 25971.0234
+100 train 10.198525 (lr=7.0700e-06) (hash(x)=144157453)
+100 val loss 10.2511
+100 val perplexity 28312.6055
+100 train 10.282244 (lr=6.0600e-06) (hash(x)=144157453)
+200 val loss 9.7081
+200 val perplexity 16450.9492
+200 train 9.747164 (lr=1.4070e-05) (hash(x)=146764602)
+200 val loss 9.8092
+200 val perplexity 18200.1836
+200 train 9.849621 (lr=1.2060e-05) (hash(x)=146764602)
+300 val loss 8.8063
+300 val perplexity 6676.1025
+300 train 8.721718 (lr=2.1070e-05) (hash(x)=140559124)
+300 val loss 9.0431
+300 val perplexity 8460.3369
+300 train 8.977057 (lr=1.8060e-05) (hash(x)=140559124)
+400 val loss 7.9303
+400 val perplexity 2780.2043
+400 train 8.263488 (lr=2.8070e-05) (hash(x)=166024176)
+400 val loss 8.1200
+400 val perplexity 3361.0332
+400 train 8.433102 (lr=2.4060e-05) (hash(x)=166024176)
+500 val loss 7.5507
+500 val perplexity 1902.1611
+500 train 7.518233 (lr=3.5000e-05) (hash(x)=161040668)
+500 val loss 7.6197
+500 val perplexity 2037.9108
+500 train 7.589498 (lr=3.0000e-05) (hash(x)=161040668)
+600 val loss 7.4388
+600 val perplexity 1700.7233
+600 train 7.468059 (lr=3.4989e-05) (hash(x)=148878990)
+600 val loss 7.4876
+600 val perplexity 1785.7812
+600 train 7.517530 (lr=2.9990e-05) (hash(x)=148878990)
+700 val loss 7.3485
+700 val perplexity 1553.8051
+700 train 7.329822 (lr=3.4954e-05) (hash(x)=150420695)
+700 val loss 7.4002
+700 val perplexity 1636.2662
+700 train 7.380228 (lr=2.9961e-05) (hash(x)=150420695)
+800 val loss 7.2750
+800 val perplexity 1443.7603
+800 train 7.251266 (lr=3.4897e-05) (hash(x)=151995229)
+800 val loss 7.3320
+800 val perplexity 1528.5052
+800 train 7.302983 (lr=2.9912e-05) (hash(x)=151995229)
+900 val loss 7.2145
+900 val perplexity 1358.9803
+900 train 6.968035 (lr=3.4818e-05) (hash(x)=139853932)
+900 val loss 7.2744
+900 val perplexity 1442.9172
+900 train 7.033135 (lr=2.9844e-05) (hash(x)=139853932)
+1000 val loss 7.1335
+1000 val perplexity 1253.2953
+1000 train 7.335016 (lr=3.4715e-05) (hash(x)=174104207)
+1000 val loss 7.2135
+1000 val perplexity 1357.5917
+1000 train 7.412142 (lr=2.9756e-05) (hash(x)=174104207)
+1100 val loss 7.0682
+1100 val perplexity 1174.0667
+1100 train 6.900183 (lr=3.4591e-05) (hash(x)=146275038)
+1100 val loss 7.1555
+1100 val perplexity 1281.1627
+1100 train 6.990688 (lr=2.9649e-05) (hash(x)=146275038)
+1200 val loss 7.0004
+1200 val perplexity 1097.0662
+1200 train 6.771210 (lr=3.4444e-05) (hash(x)=141403655)
+1200 val loss 7.1005
+1200 val perplexity 1212.6289
+1200 train 6.876398 (lr=2.9523e-05) (hash(x)=141403655)
+1300 val loss 6.9292
+1300 val perplexity 1021.7067
+1300 train 7.012178 (lr=3.4275e-05) (hash(x)=173675040)
+1300 val loss 7.0356
+1300 val perplexity 1136.3947
+1300 train 7.122135 (lr=2.9378e-05) (hash(x)=173675040)
+1400 val loss 6.8715
+1400 val perplexity 964.4235
+1400 train 6.844200 (lr=3.4084e-05) (hash(x)=162475906)
+1400 val loss 6.9720
+1400 val perplexity 1066.3026
+1400 train 6.941983 (lr=2.9215e-05) (hash(x)=162475906)
+1500 val loss 6.8163
+1500 val perplexity 912.6173
+1500 train 6.646946 (lr=3.3872e-05) (hash(x)=152785612)
+1500 val loss 6.9207
+1500 val perplexity 1013.0270
+1500 train 6.750599 (lr=2.9033e-05) (hash(x)=152785612)
+1600 val loss 6.7411
+1600 val perplexity 846.5029
+1600 train 6.695332 (lr=3.3638e-05) (hash(x)=151987383)
+1600 val loss 6.8430
+1600 val perplexity 937.3285
+1600 train 6.799621 (lr=2.8833e-05) (hash(x)=151987383)
+1700 val loss 6.6802
+1700 val perplexity 796.5001
+1700 val loss 6.7791
+1700 val perplexity 879.2971
+1700 train 6.662879 (lr=3.3384e-05) (hash(x)=143781605)
+1700 train 6.760727 (lr=2.8615e-05) (hash(x)=143781605)
+1800 val loss 6.6307
+1800 val perplexity 758.0203
+1800 val loss 6.7255
+1800 val perplexity 833.4037
+1800 train 6.614835 (lr=3.3109e-05) (hash(x)=151040203)
+1800 train 6.708007 (lr=2.8379e-05) (hash(x)=151040203)
+1900 val loss 6.5788
+1900 val perplexity 719.6393
+1900 val loss 6.6702
+1900 val perplexity 788.5310
+1900 train 6.482689 (lr=3.2814e-05) (hash(x)=149414572)
+1900 train 6.571493 (lr=2.8127e-05) (hash(x)=149414572)
+2000 val loss 6.6195
+2000 val perplexity 749.5918
+2000 val loss 6.5286
+2000 val perplexity 684.4636
+2000 train 6.576480 (lr=2.7857e-05) (hash(x)=156479674)
+2000 train 6.482908 (lr=3.2500e-05) (hash(x)=156479674)
+2100 val loss 6.4825
+2100 val perplexity 653.5756
+2100 val loss 6.5620
+2100 val perplexity 707.6946
+2100 train 6.345187 (lr=3.2166e-05) (hash(x)=137861481)
+2100 train 6.429942 (lr=2.7571e-05) (hash(x)=137861481)
+2200 val loss 6.4416
+2200 val perplexity 627.3801
+2200 val loss 6.5179
+2200 val perplexity 677.1461
+2200 train 6.499350 (lr=3.1813e-05) (hash(x)=155937443)
+2200 train 6.576139 (lr=2.7269e-05) (hash(x)=155937443)
+2300 val loss 6.4203
+2300 val perplexity 614.1733
+2300 val loss 6.4995
+2300 val perplexity 664.7992
+2300 train 6.101424 (lr=3.1443e-05) (hash(x)=131035715)
+2300 train 6.185195 (lr=2.6951e-05) (hash(x)=131035715)
+2400 val loss 6.3757
+2400 val perplexity 587.4105
+2400 val loss 6.4501
+2400 val perplexity 632.7535
+2400 train 6.374401 (lr=3.1054e-05) (hash(x)=146568981)
+2400 train 6.447587 (lr=2.6618e-05) (hash(x)=146568981)
+2500 val loss 6.3448
+2500 val perplexity 569.5289
+2500 val loss 6.4192
+2500 val perplexity 613.5393
+2500 train 6.317100 (lr=3.0649e-05) (hash(x)=163317586)
+2500 train 6.389849 (lr=2.6270e-05) (hash(x)=163317586)
+2600 val loss 6.3181
+2600 val perplexity 554.5211
+2600 val loss 6.3867
+2600 val perplexity 593.9120
+2600 train 6.141277 (lr=3.0227e-05) (hash(x)=144201060)
+2600 train 6.207941 (lr=2.5909e-05) (hash(x)=144201060)
+2700 val loss 6.2931
+2700 val perplexity 540.8275
+2700 val loss 6.3637
+2700 val perplexity 580.3699
+2700 train 6.193167 (lr=2.9789e-05) (hash(x)=141825701)
+2700 train 6.259410 (lr=2.5533e-05) (hash(x)=141825701)
+2800 val loss 6.2649
+2800 val perplexity 525.7846
+2800 val loss 6.3332
+2800 val perplexity 562.9761
+2800 train 6.384350 (lr=2.9336e-05) (hash(x)=160561627)
+2800 train 6.452592 (lr=2.5145e-05) (hash(x)=160561627)
+2900 val loss 6.2419
+2900 val perplexity 513.8532
+2900 val loss 6.3035
+2900 val perplexity 546.5074
+2900 train 6.189714 (lr=2.8868e-05) (hash(x)=151758176)
+2900 train 6.255837 (lr=2.4744e-05) (hash(x)=151758176)
+3000 val loss 6.2147
+3000 val perplexity 500.0573
+3000 val loss 6.2803
+3000 val perplexity 533.9685
+3000 train 6.226311 (lr=2.8386e-05) (hash(x)=155815751)
+3000 train 6.286588 (lr=2.4331e-05) (hash(x)=155815751)
+3100 val loss 6.2016
+3100 val perplexity 493.5247
+3100 val loss 6.2672
+3100 val perplexity 527.0150
+3100 train 5.858310 (lr=2.7891e-05) (hash(x)=142307043)
+3100 train 5.926589 (lr=2.3906e-05) (hash(x)=142307043)
+3200 val loss 6.1807
+3200 val perplexity 483.3375
+3200 train 6.293739 (lr=2.7383e-05) (hash(x)=156310690)
+3200 val loss 6.2449
+3200 val perplexity 515.3918
+3200 train 6.361027 (lr=2.3471e-05) (hash(x)=156310690)
+3300 val loss 6.2148
+3300 val perplexity 500.1102
+3300 val loss 6.1538
+3300 val perplexity 470.4862
+3300 train 6.216784 (lr=2.3026e-05) (hash(x)=151344506)
+3300 train 6.156497 (lr=2.6864e-05) (hash(x)=151344506)
+3400 val loss 6.2043
+3400 val perplexity 494.8538
+3400 val loss 6.1388
+3400 val perplexity 463.4761
+3400 train 6.240551 (lr=2.2572e-05) (hash(x)=162911881)
+3400 train 6.174221 (lr=2.6333e-05) (hash(x)=162911881)
+3500 val loss 6.1855
+3500 val perplexity 485.6743
+3500 val loss 6.1274
+3500 val perplexity 458.2211
+3500 train 6.123961 (lr=2.2108e-05) (hash(x)=142889971)
+3500 train 6.059972 (lr=2.5793e-05) (hash(x)=142889971)
+3600 val loss 6.1675
+3600 val perplexity 476.9710
+3600 val loss 6.1050
+3600 val perplexity 448.1018
+3600 train 6.160964 (lr=2.1637e-05) (hash(x)=149470354)
+3600 train 6.092852 (lr=2.5243e-05) (hash(x)=149470354)
+3700 val loss 6.1523
+3700 val perplexity 469.7756
+3700 val loss 6.0880
+3700 val perplexity 440.5436
+3700 train 6.096142 (lr=2.1158e-05) (hash(x)=154194821)
+3700 train 6.030409 (lr=2.4684e-05) (hash(x)=154194821)
+3800 val loss 6.1387
+3800 val perplexity 463.4522
+3800 val loss 6.0779
+3800 val perplexity 436.1298
+3800 train 6.069734 (lr=2.0672e-05) (hash(x)=148885848)
+3800 train 6.011635 (lr=2.4117e-05) (hash(x)=148885848)
+3900 val loss 6.1210
+3900 val perplexity 455.3399
+3900 val loss 6.0583
+3900 val perplexity 427.6662
+3900 train 6.032857 (lr=2.0180e-05) (hash(x)=148308484)
+3900 train 5.971268 (lr=2.3543e-05) (hash(x)=148308484)
+4000 val loss 6.1042
+4000 val perplexity 447.7170
+4000 val loss 6.0434
+4000 val perplexity 421.3136
+4000 train 5.941088 (lr=1.9683e-05) (hash(x)=139828564)
+4000 train 5.881995 (lr=2.2963e-05) (hash(x)=139828564)
+4100 val loss 6.0952
+4100 val perplexity 443.7090
+4100 val loss 6.0326
+4100 val perplexity 416.8078
+4100 train 6.002620 (lr=1.9181e-05) (hash(x)=139981997)
+4100 train 5.940226 (lr=2.2378e-05) (hash(x)=139981997)
+4200 val loss 6.0805
+4200 val perplexity 437.2441
+4200 train 6.236762 (lr=1.8675e-05) (hash(x)=150738447)
+4200 val loss 6.0164
+4200 val perplexity 410.0989
+4200 train 6.170493 (lr=2.1788e-05) (hash(x)=150738447)
+4300 val loss 6.0659
+4300 val perplexity 430.9070
+4300 train 5.848683 (lr=1.8166e-05) (hash(x)=142198107)
+4300 val loss 6.0055
+4300 val perplexity 405.6398
+4300 train 5.784900 (lr=2.1194e-05) (hash(x)=142198107)
+4400 val loss 6.0590
+4400 val perplexity 427.9448
+4400 train 5.930000 (lr=1.7655e-05) (hash(x)=142731201)
+4400 val loss 5.9963
+4400 val perplexity 401.9518
+4400 train 5.867516 (lr=2.0598e-05) (hash(x)=142731201)
+4500 val loss 6.0420
+4500 val perplexity 420.7414
+4500 train 6.054802 (lr=1.7142e-05) (hash(x)=154814426)
+4500 val loss 5.9799
+4500 val perplexity 395.4100
+4500 train 5.997611 (lr=1.9999e-05) (hash(x)=154814426)
+4600 val loss 6.0305
+4600 val perplexity 415.9094
+4600 train 5.998842 (lr=1.6629e-05) (hash(x)=155922230)
+4600 val loss 5.9672
+4600 val perplexity 390.4033
+4600 train 5.934934 (lr=1.9400e-05) (hash(x)=155922230)
+4700 val loss 6.0251
+4700 val perplexity 413.6869
+4700 train 5.771210 (lr=1.6114e-05) (hash(x)=139398510)
+4700 val loss 5.9632
+4700 val perplexity 388.8658
+4700 train 5.711072 (lr=1.8800e-05) (hash(x)=139398510)
+4800 val loss 6.0116
+4800 val perplexity 408.1422
+4800 train 6.080204 (lr=1.5601e-05) (hash(x)=140893236)
+4800 val loss 5.9466
+4800 val perplexity 382.4628
+4800 train 6.004228 (lr=1.8201e-05) (hash(x)=140893236)
+4900 val loss 5.9966
+4900 val perplexity 402.0549
+4900 val loss 5.9329
+4900 val perplexity 377.2361
+4900 train 5.900732 (lr=1.5089e-05) (hash(x)=153747830)
+4900 train 5.829324 (lr=1.7604e-05) (hash(x)=153747830)
+5000 val loss 5.9886
+5000 val perplexity 398.8546
+5000 val loss 5.9249
+5000 val perplexity 374.2589
+5000 train 6.216745 (lr=1.4579e-05) (hash(x)=148919005)
+5000 train 6.162242 (lr=1.7009e-05) (hash(x)=148919005)
+5100 val loss 5.9818
+5100 val perplexity 396.1589
+5100 val loss 5.9171
+5100 val perplexity 371.3396
+5100 train 5.771548 (lr=1.4071e-05) (hash(x)=142281936)
+5100 train 5.707696 (lr=1.6417e-05) (hash(x)=142281936)
+5200 val loss 5.9826
+5200 val perplexity 396.4724
+5200 val loss 5.9180
+5200 val perplexity 371.6624
+5200 train 5.790337 (lr=1.3568e-05) (hash(x)=143162650)
+5200 train 5.723374 (lr=1.5829e-05) (hash(x)=143162650)
+5300 val loss 5.9621
+5300 val perplexity 388.4120
+5300 val loss 5.8968
+5300 val perplexity 363.8678
+5300 train 6.020540 (lr=1.3068e-05) (hash(x)=148546849)
+5300 train 5.958957 (lr=1.5246e-05) (hash(x)=148546849)
+5400 val loss 5.9524
+5400 val perplexity 384.6609
+5400 val loss 5.8871
+5400 val perplexity 360.3507
+5400 train 5.812173 (lr=1.2573e-05) (hash(x)=143492259)
+5400 train 5.746771 (lr=1.4669e-05) (hash(x)=143492259)
+5500 val loss 5.9471
+5500 val perplexity 382.6500
+5500 val loss 5.8826
+5500 val perplexity 358.7458
+5500 train 5.701526 (lr=1.2085e-05) (hash(x)=141023941)
+5500 train 5.634879 (lr=1.4099e-05) (hash(x)=141023941)
+5600 val loss 5.9471
+5600 val perplexity 382.6252
+5600 val loss 5.8808
+5600 val perplexity 358.0882
+5600 train 5.817373 (lr=1.1602e-05) (hash(x)=142065021)
+5600 train 5.754811 (lr=1.3536e-05) (hash(x)=142065021)
+5700 val loss 5.9307
+5700 val perplexity 376.4344
+5700 val loss 5.8645
+5700 val perplexity 352.3100
+5700 train 5.791829 (lr=1.1127e-05) (hash(x)=145749913)
+5700 train 5.731421 (lr=1.2981e-05) (hash(x)=145749913)
+5800 val loss 5.8578
+5800 val perplexity 349.9652
+5800 val loss 5.9241
+5800 val perplexity 373.9264
+5800 train 5.537057 (lr=1.2436e-05) (hash(x)=140035522)
+5800 train 5.610334 (lr=1.0659e-05) (hash(x)=140035522)
+5900 val loss 5.8524
+5900 val perplexity 348.0614
+5900 val loss 5.9200
+5900 val perplexity 372.4236
+5900 train 5.614779 (lr=1.1900e-05) (hash(x)=154752726)
+5900 train 5.682038 (lr=1.0200e-05) (hash(x)=154752726)
+6000 val loss 5.8496
+6000 val perplexity 347.1107
+6000 val loss 5.9171
+6000 val perplexity 371.3166
+6000 train 5.874280 (lr=1.1375e-05) (hash(x)=158715824)
+6000 train 5.940979 (lr=9.7500e-06) (hash(x)=158715824)
+6100 val loss 5.8378
+6100 val perplexity 343.0087
+6100 val loss 5.9049
+6100 val perplexity 366.8322
+6100 train 5.851554 (lr=1.0861e-05) (hash(x)=137413820)
+6100 train 5.914152 (lr=9.3098e-06) (hash(x)=137413820)
+6200 val loss 5.8303
+6200 val perplexity 340.4680
+6200 train 5.751894 (lr=1.0360e-05) (hash(x)=151507523)
+6200 val loss 5.8989
+6200 val perplexity 364.6396
+6200 train 5.823660 (lr=8.8800e-06) (hash(x)=151507523)
+6300 val loss 5.8296
+6300 val perplexity 340.2184
+6300 train 5.600849 (lr=9.8715e-06) (hash(x)=147514617)
+6300 val loss 5.8977
+6300 val perplexity 364.1924
+6300 train 5.668419 (lr=8.4613e-06) (hash(x)=147514617)
+6400 val loss 5.8260
+6400 val perplexity 338.9898
+6400 train 5.836993 (lr=9.3966e-06) (hash(x)=151604465)
+6400 val loss 5.8942
+6400 val perplexity 362.9248
+6400 train 5.905121 (lr=8.0542e-06) (hash(x)=151604465)
+6500 val loss 5.8152
+6500 val perplexity 335.3742
+6500 train 5.872994 (lr=8.9359e-06) (hash(x)=144515881)
+6500 val loss 5.8838
+6500 val perplexity 359.1624
+6500 train 5.940874 (lr=7.6594e-06) (hash(x)=144515881)
+6600 val loss 5.8092
+6600 val perplexity 333.3445
+6600 train 5.602404 (lr=8.4903e-06) (hash(x)=136948374)
+6600 val loss 5.8793
+6600 val perplexity 357.5739
+6600 train 5.664325 (lr=7.2774e-06) (hash(x)=136948374)
+6700 val loss 5.8089
+6700 val perplexity 333.2539
+6700 train 5.620818 (lr=8.0602e-06) (hash(x)=146268592)
+6700 val loss 5.8775
+6700 val perplexity 356.8992
+6700 train 5.695412 (lr=6.9087e-06) (hash(x)=146268592)
+6800 val loss 5.8009
+6800 val perplexity 330.5903
+6800 train 5.757844 (lr=7.6463e-06) (hash(x)=152676836)
+6800 val loss 5.8705
+6800 val perplexity 354.4432
+6800 train 5.824782 (lr=6.5540e-06) (hash(x)=152676836)
+6900 val loss 5.7935
+6900 val perplexity 328.1473
+6900 train 5.711613 (lr=7.2493e-06) (hash(x)=134657776)
+6900 val loss 5.8644
+6900 val perplexity 352.2704
+6900 train 5.785072 (lr=6.2137e-06) (hash(x)=134657776)
+7000 val loss 5.7921
+7000 val perplexity 327.6984
+7000 train 5.767048 (lr=6.8697e-06) (hash(x)=166721861)
+7000 val loss 5.8625
+7000 val perplexity 351.6026
+7000 train 5.835593 (lr=5.8883e-06) (hash(x)=166721861)
+7100 val loss 5.7891
+7100 val perplexity 326.7293
+7100 train 5.594516 (lr=6.5080e-06) (hash(x)=135496702)
+7100 val loss 5.8591
+7100 val perplexity 350.4164
+7100 train 5.666256 (lr=5.5783e-06) (hash(x)=135496702)
+7200 val loss 5.7857
+7200 val perplexity 325.5949
+7200 train 5.894931 (lr=6.1648e-06) (hash(x)=155567461)
+7200 val loss 5.8553
+7200 val perplexity 349.0879
+7200 train 5.959052 (lr=5.2841e-06) (hash(x)=155567461)
+7300 val loss 5.7788
+7300 val perplexity 323.3828
+7300 train 5.581634 (lr=5.8405e-06) (hash(x)=142803829)
+7300 val loss 5.8498
+7300 val perplexity 347.1809
+7300 train 5.649696 (lr=5.0062e-06) (hash(x)=142803829)
+7400 val loss 5.7771
+7400 val perplexity 322.8163
+7400 train 5.600485 (lr=5.5357e-06) (hash(x)=145294178)
+7400 val loss 5.8478
+7400 val perplexity 346.4585
+7400 train 5.666378 (lr=4.7449e-06) (hash(x)=145294178)
+7500 val loss 5.7761
+7500 val perplexity 322.5064
+7500 train 5.544384 (lr=5.2508e-06) (hash(x)=150573713)
+7500 val loss 5.8469
+7500 val perplexity 346.1439
+7500 train 5.614801 (lr=4.5007e-06) (hash(x)=150573713)
+7600 val loss 5.7728
+7600 val perplexity 321.4253
+7600 train 5.784178 (lr=4.9862e-06) (hash(x)=142771511)
+7600 val loss 5.8438
+7600 val perplexity 345.0785
+7600 train 5.856913 (lr=4.2739e-06) (hash(x)=142771511)
+7700 val loss 5.7669
+7700 val perplexity 319.5430
+7700 train 5.666105 (lr=4.7423e-06) (hash(x)=143602175)
+7700 val loss 5.8380
+7700 val perplexity 343.1030
+7700 train 5.733759 (lr=4.0648e-06) (hash(x)=143602175)
+7800 val loss 5.7637
+7800 val perplexity 318.5268
+7800 train 5.804039 (lr=4.5194e-06) (hash(x)=152379862)
+7800 val loss 5.8356
+7800 val perplexity 342.2685
+7800 train 5.870350 (lr=3.8738e-06) (hash(x)=152379862)
+7900 val loss 5.7638
+7900 val perplexity 318.5619
+7900 train 5.605504 (lr=4.3179e-06) (hash(x)=146655921)
+7900 val loss 5.8343
+7900 val perplexity 341.8145
+7900 train 5.679904 (lr=3.7010e-06) (hash(x)=146655921)
+8000 val loss 5.7638
+8000 val perplexity 318.5710
+8000 train 5.879199 (lr=4.1380e-06) (hash(x)=148262482)
+8000 val loss 5.8350
+8000 val perplexity 342.0742
+8000 train 5.947587 (lr=3.5468e-06) (hash(x)=148262482)
+8100 val loss 5.7564
+8100 val perplexity 316.1968
+8100 train 5.696403 (lr=3.9800e-06) (hash(x)=147683655)
+8100 val loss 5.8278
+8100 val perplexity 339.6119
+8100 train 5.762560 (lr=3.4114e-06) (hash(x)=147683655)
+8200 val loss 5.8263
+8200 val perplexity 339.1142
+8200 train 5.888067 (lr=3.2950e-06) (hash(x)=164975934)
+8200 val loss 5.7551
+8200 val perplexity 315.8080
+8200 train 5.818132 (lr=3.8442e-06) (hash(x)=164975934)
+8300 val loss 5.8229
+8300 val perplexity 337.9619
+8300 train 5.741323 (lr=3.1977e-06) (hash(x)=145300550)
+8300 val loss 5.7525
+8300 val perplexity 314.9815
+8300 train 5.667071 (lr=3.7307e-06) (hash(x)=145300550)
+8400 val loss 5.8219
+8400 val perplexity 337.6240
+8400 train 5.738000 (lr=3.1197e-06) (hash(x)=150679400)
+8400 val loss 5.7511
+8400 val perplexity 314.5375
+8400 train 5.666170 (lr=3.6397e-06) (hash(x)=150679400)
+8500 val loss 5.8224
+8500 val perplexity 337.7816
+8500 train 5.843013 (lr=3.0611e-06) (hash(x)=164109401)
+8500 val loss 5.7511
+8500 val perplexity 314.5462
+8500 train 5.767133 (lr=3.5713e-06) (hash(x)=164109401)
+8600 val loss 5.8202
+8600 val perplexity 337.0385
+8600 train 5.823380 (lr=3.0220e-06) (hash(x)=161036376)
+8600 val loss 5.7481
+8600 val perplexity 313.5803
+8600 train 5.730533 (lr=3.5257e-06) (hash(x)=161036376)
+8700 val loss 5.8156
+8700 val perplexity 335.5001
+8700 train 5.750320 (lr=3.0024e-06) (hash(x)=153828820)
+8700 val loss 5.7438
+8700 val perplexity 312.2529
+8700 train 5.674262 (lr=3.5029e-06) (hash(x)=153828820)
+8749 val loss 5.8145
+8749 val perplexity 335.1370
+8749 val loss 5.7424
+8749 val perplexity 311.8204

attention_kindselective_n_heads4_seed1344/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:883f9a6eef276c05beb952634c52bbc6e83e399d790fd34ec66cd002ab63319e
+oid sha256:05d476d1b1967d89cb078db0cb8dde81e6a201530c206f968474e1bd60409cec
 size 92843394

attention_kindselective_n_heads4_seed1344/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:11a8da35040e3dcb60b93f1d0e06cd32200bd13e83ffd949d3626688fcd9dd31
+oid sha256:0bd43931f26ce626d3345e60608b25bfbdb809636a0d94748839af3204037042
 size 179406214