Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1342/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2.75e-5_30720_4_1342", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.75e-5_30720_4_1342", "n_embd": 256}

attention_kindselective_n_heads4_seed1342/log2.txt

@@ -1,528 +1,1056 @@
 max_steps: 17500
+max_steps: 17500
+0 val loss 11.2907
+0 val perplexity 80077.4453
 0 val loss 11.2907
 0 val perplexity 80077.4453
-0 train 11.286213 (lr=1.3750e-07) (hash(x)=150332693)
-100 val loss 9.9514
-100 val perplexity 20981.1719
-100 train 9.972456 (lr=1.3888e-05) (hash(x)=148948662)
-200 val loss 9.0084
-200 val perplexity 8171.4985
-200 train 8.930535 (lr=2.7500e-05) (hash(x)=149277319)
-300 val loss 7.9062
-300 val perplexity 2714.1672
-300 train 7.803435 (lr=2.7498e-05) (hash(x)=146429920)
-400 val loss 7.5790
-400 val perplexity 1956.7255
-400 train 7.696357 (lr=2.7492e-05) (hash(x)=155934075)
-500 val loss 7.4743
-500 val perplexity 1762.2222
-500 train 7.482019 (lr=2.7482e-05) (hash(x)=157510366)
-600 val loss 7.4041
-600 val perplexity 1642.7472
-600 train 7.518789 (lr=2.7467e-05) (hash(x)=165402628)
-700 val loss 7.3399
-700 val perplexity 1540.5265
-700 train 7.399743 (lr=2.7449e-05) (hash(x)=166427433)
-800 val loss 7.2851
-800 val perplexity 1458.4285
-800 train 7.083396 (lr=2.7427e-05) (hash(x)=139168939)
-900 val loss 7.2414
-900 val perplexity 1396.0408
-900 train 7.144016 (lr=2.7400e-05) (hash(x)=144698667)
-1000 val loss 7.1911
-1000 val perplexity 1327.6097
-1000 train 7.171625 (lr=2.7370e-05) (hash(x)=147134338)
-1100 val loss 7.1443
-1100 val perplexity 1266.8485
-1100 train 7.110559 (lr=2.7335e-05) (hash(x)=147172640)
-1200 val loss 7.1005
-1200 val perplexity 1212.5867
-1200 train 7.100369 (lr=2.7297e-05) (hash(x)=159086459)
-1300 val loss 7.0613
-1300 val perplexity 1165.9291
-1300 train 6.819506 (lr=2.7254e-05) (hash(x)=141578940)
-1400 val loss 7.0270
-1400 val perplexity 1126.5918
-1400 train 7.090215 (lr=2.7207e-05) (hash(x)=152399099)
-1500 val loss 6.9761
-1500 val perplexity 1070.7262
-1500 train 7.055522 (lr=2.7157e-05) (hash(x)=158569712)
-1600 val loss 6.9304
-1600 val perplexity 1022.9405
-1600 train 7.023970 (lr=2.7102e-05) (hash(x)=156548843)
-1700 val loss 6.8955
-1700 val perplexity 987.8098
-1700 train 6.901347 (lr=2.7044e-05) (hash(x)=152608173)
-1800 val loss 6.8543
-1800 val perplexity 947.9863
-1800 train 6.928975 (lr=2.6981e-05) (hash(x)=157889911)
-1900 val loss 6.8293
-1900 val perplexity 924.5248
-1900 train 6.787612 (lr=2.6915e-05) (hash(x)=140839139)
-2000 val loss 6.7897
-2000 val perplexity 888.6237
-2000 train 6.679601 (lr=2.6845e-05) (hash(x)=142943707)
-2100 val loss 6.7547
-2100 val perplexity 858.0652
-2100 train 6.727079 (lr=2.6771e-05) (hash(x)=151025344)
-2200 val loss 6.7160
-2200 val perplexity 825.4786
-2200 train 6.709481 (lr=2.6693e-05) (hash(x)=148938689)
-2300 val loss 6.6813
-2300 val perplexity 797.3430
-2300 train 6.643854 (lr=2.6611e-05) (hash(x)=150321226)
-2400 val loss 6.6369
-2400 val perplexity 762.7330
-2400 train 6.865366 (lr=2.6525e-05) (hash(x)=161962903)
-2500 val loss 6.6012
-2500 val perplexity 735.9520
-2500 train 6.529111 (lr=2.6436e-05) (hash(x)=153085222)
-2600 val loss 6.5678
-2600 val perplexity 711.7835
-2600 train 6.628248 (lr=2.6343e-05) (hash(x)=145979235)
-2700 val loss 6.5422
-2700 val perplexity 693.8163
-2700 train 6.600425 (lr=2.6246e-05) (hash(x)=149091945)
-2800 val loss 6.5175
-2800 val perplexity 676.8688
-2800 train 6.596137 (lr=2.6146e-05) (hash(x)=159157507)
-2900 val loss 6.5001
-2900 val perplexity 665.1902
-2900 train 6.509790 (lr=2.6042e-05) (hash(x)=155377499)
-3000 val loss 6.4763
-3000 val perplexity 649.5642
-3000 train 6.439157 (lr=2.5934e-05) (hash(x)=145943130)
-3100 val loss 6.4548
-3100 val perplexity 635.7664
-3100 train 6.511821 (lr=2.5823e-05) (hash(x)=153264973)
-3200 val loss 6.4241
-3200 val perplexity 616.5283
-3200 train 6.256510 (lr=2.5709e-05) (hash(x)=143249173)
-3300 val loss 6.4090
-3300 val perplexity 607.2606
-3300 train 6.416599 (lr=2.5590e-05) (hash(x)=155558767)
+0 train 11.286213 (lr=2.0000e-07) (hash(x)=150332693)
+0 train 11.286213 (lr=1.8750e-07) (hash(x)=150332693)
+100 val loss 9.8768
+100 val perplexity 19473.3750
+100 train 9.899889 (lr=2.0200e-05) (hash(x)=148948662)
+100 val loss 9.8862
+100 val perplexity 19657.5391
+100 train 9.909360 (lr=1.8937e-05) (hash(x)=148948662)
+200 val loss 8.6349
+200 val perplexity 5624.3892
+200 train 8.549774 (lr=4.0000e-05) (hash(x)=149277319)
+200 val loss 8.6742
+200 val perplexity 5850.1021
+200 train 8.591024 (lr=3.7500e-05) (hash(x)=149277319)
+300 val loss 7.7933
+300 val perplexity 2424.3916
+300 train 7.673641 (lr=3.9997e-05) (hash(x)=146429920)
+300 val loss 7.7411
+300 val perplexity 2300.9182
+300 train 7.615427 (lr=3.7497e-05) (hash(x)=146429920)
+400 val loss 7.5715
+400 val perplexity 1942.0657
+400 train 7.687140 (lr=3.9988e-05) (hash(x)=155934075)
+400 val loss 7.5523
+400 val perplexity 1905.0985
+400 train 7.665451 (lr=3.7489e-05) (hash(x)=155934075)
+500 val loss 7.4870
+500 val perplexity 1784.6857
+500 train 7.494686 (lr=3.9973e-05) (hash(x)=157510366)
+500 val loss 7.4642
+500 val perplexity 1744.5452
+500 train 7.467919 (lr=3.7475e-05) (hash(x)=157510366)
+600 val loss 7.4308
+600 val perplexity 1687.1985
+600 train 7.547135 (lr=3.9953e-05) (hash(x)=165402628)
+600 val loss 7.3924
+600 val perplexity 1623.6310
+600 train 7.507157 (lr=3.7456e-05) (hash(x)=165402628)
+700 val loss 7.3856
+700 val perplexity 1612.6390
+700 train 7.444174 (lr=3.9926e-05) (hash(x)=166427433)
+700 val loss 7.3212
+700 val perplexity 1512.0294
+700 train 7.385526 (lr=3.7430e-05) (hash(x)=166427433)
+800 val loss 7.3425
+800 val perplexity 1544.5521
+800 train 7.137204 (lr=3.9893e-05) (hash(x)=139168939)
+800 val loss 7.2604
+800 val perplexity 1422.8741
+800 train 7.058400 (lr=3.7400e-05) (hash(x)=139168939)
+900 val loss 7.3042
+900 val perplexity 1486.5560
+900 train 7.209212 (lr=3.9855e-05) (hash(x)=144698667)
+900 val loss 7.2081
+900 val perplexity 1350.3298
+900 train 7.108169 (lr=3.7364e-05) (hash(x)=144698667)
+1000 val loss 7.2667
+1000 val perplexity 1431.8467
+1000 val loss 7.1451
+1000 val perplexity 1267.8656
+1000 train 7.247368 (lr=3.9810e-05) (hash(x)=147134338)
+1000 train 7.127390 (lr=3.7322e-05) (hash(x)=147134338)
+1100 val loss 7.0904
+1100 val perplexity 1200.3669
+1100 val loss 7.2316
+1100 val perplexity 1382.4969
+1100 train 7.058753 (lr=3.7275e-05) (hash(x)=147172640)
+1100 train 7.195152 (lr=3.9760e-05) (hash(x)=147172640)
+1200 val loss 7.0314
+1200 val perplexity 1131.6199
+1200 train 7.031106 (lr=3.7223e-05) (hash(x)=159086459)
+1200 val loss 7.2021
+1200 val perplexity 1342.2770
+1200 train 7.207237 (lr=3.9704e-05) (hash(x)=159086459)
+1300 val loss 6.9726
+1300 val perplexity 1067.0111
+1300 train 6.723200 (lr=3.7164e-05) (hash(x)=141578940)
+1300 val loss 7.1625
+1300 val perplexity 1290.1248
+1300 train 6.937757 (lr=3.9642e-05) (hash(x)=141578940)
+1400 val loss 6.9289
+1400 val perplexity 1021.3248
+1400 train 6.988761 (lr=3.7101e-05) (hash(x)=152399099)
+1400 val loss 7.1186
+1400 val perplexity 1234.7281
+1400 train 7.181373 (lr=3.9574e-05) (hash(x)=152399099)
+1500 val loss 6.8611
+1500 val perplexity 954.4279
+1500 train 6.944953 (lr=3.7032e-05) (hash(x)=158569712)
+1500 val loss 7.0568
+1500 val perplexity 1160.7614
+1500 train 7.136433 (lr=3.9501e-05) (hash(x)=158569712)
+1600 val loss 6.8073
+1600 val perplexity 904.4675
+1600 train 6.901510 (lr=3.6958e-05) (hash(x)=156548843)
+1600 val loss 7.0105
+1600 val perplexity 1108.2358
+1600 train 7.107373 (lr=3.9421e-05) (hash(x)=156548843)
+1700 val loss 6.7516
+1700 val perplexity 855.4005
+1700 train 6.759593 (lr=3.6878e-05) (hash(x)=152608173)
+1700 val loss 6.9768
+1700 val perplexity 1071.5275
+1700 train 6.989625 (lr=3.9336e-05) (hash(x)=152608173)
+1800 val loss 6.7063
+1800 val perplexity 817.5263
+1800 train 6.779020 (lr=3.6793e-05) (hash(x)=157889911)
+1800 val loss 6.9349
+1800 val perplexity 1027.5291
+1800 train 7.009769 (lr=3.9246e-05) (hash(x)=157889911)
+1900 val loss 6.6729
+1900 val perplexity 790.6511
+1900 train 6.639544 (lr=3.6702e-05) (hash(x)=140839139)
+1900 val loss 6.9128
+1900 val perplexity 1005.0913
+1900 train 6.870525 (lr=3.9149e-05) (hash(x)=140839139)
+2000 val loss 6.6417
+2000 val perplexity 766.3823
+2000 train 6.528852 (lr=3.6606e-05) (hash(x)=142943707)
+2000 val loss 6.8705
+2000 val perplexity 963.4188
+2000 train 6.761942 (lr=3.9047e-05) (hash(x)=142943707)
+2100 val loss 6.6061
+2100 val perplexity 739.6149
+2100 train 6.573959 (lr=3.6505e-05) (hash(x)=151025344)
+2100 val loss 6.8408
+2100 val perplexity 935.2663
+2100 train 6.814910 (lr=3.8939e-05) (hash(x)=151025344)
+2200 val loss 6.5747
+2200 val perplexity 716.7421
+2200 train 6.582545 (lr=3.6399e-05) (hash(x)=148938689)
+2200 val loss 6.8176
+2200 val perplexity 913.7847
+2200 train 6.824997 (lr=3.8826e-05) (hash(x)=148938689)
+2300 val loss 6.5303
+2300 val perplexity 685.6262
+2300 train 6.494017 (lr=3.6288e-05) (hash(x)=150321226)
+2300 val loss 6.7555
+2300 val perplexity 858.7787
+2300 train 6.711614 (lr=3.8707e-05) (hash(x)=150321226)
+2400 val loss 6.4979
+2400 val perplexity 663.7457
+2400 train 6.736341 (lr=3.6171e-05) (hash(x)=161962903)
+2400 val loss 6.7165
+2400 val perplexity 825.9503
+2400 train 6.929638 (lr=3.8583e-05) (hash(x)=161962903)
+2500 val loss 6.4677
+2500 val perplexity 643.9689
+2500 train 6.404032 (lr=3.6049e-05) (hash(x)=153085222)
+2500 val loss 6.6707
+2500 val perplexity 788.9158
+2500 train 6.590450 (lr=3.8453e-05) (hash(x)=153085222)
+2600 val loss 6.4401
+2600 val perplexity 626.4914
+2600 train 6.491509 (lr=3.5923e-05) (hash(x)=145979235)
+2600 val loss 6.6330
+2600 val perplexity 759.7482
+2600 train 6.684030 (lr=3.8317e-05) (hash(x)=145979235)
+2700 val loss 6.4106
+2700 val perplexity 608.2325
+2700 train 6.479415 (lr=3.5791e-05) (hash(x)=149091945)
+2700 val loss 6.5907
+2700 val perplexity 728.3218
+2700 train 6.650594 (lr=3.8177e-05) (hash(x)=149091945)
+2800 val loss 6.3838
+2800 val perplexity 592.1553
+2800 train 6.473955 (lr=3.5654e-05) (hash(x)=159157507)
+2800 val loss 6.5587
+2800 val perplexity 705.3589
+2800 train 6.634536 (lr=3.8031e-05) (hash(x)=159157507)
+2900 val loss 6.3717
+2900 val perplexity 585.0361
+2900 train 6.383224 (lr=3.5512e-05) (hash(x)=155377499)
+2900 val loss 6.5366
+2900 val perplexity 689.9300
+2900 train 6.539573 (lr=3.7879e-05) (hash(x)=155377499)
+3000 val loss 6.3532
+3000 val perplexity 574.3139
+3000 train 6.320881 (lr=3.5365e-05) (hash(x)=145943130)
+3000 val loss 6.5047
+3000 val perplexity 668.2750
+3000 train 6.473698 (lr=3.7723e-05) (hash(x)=145943130)
+3100 val loss 6.3280
+3100 val perplexity 560.0309
+3100 train 6.382774 (lr=3.5214e-05) (hash(x)=153264973)
+3100 val loss 6.4704
+3100 val perplexity 645.7274
+3100 train 6.523706 (lr=3.7561e-05) (hash(x)=153264973)
+3200 val loss 6.3052
+3200 val perplexity 547.4370
+3200 train 6.144718 (lr=3.5057e-05) (hash(x)=143249173)
+3200 val loss 6.4375
+3200 val perplexity 624.8588
+3200 train 6.272245 (lr=3.7394e-05) (hash(x)=143249173)
+3300 val loss 6.2948
+3300 val perplexity 541.7688
+3300 train 6.303265 (lr=3.4896e-05) (hash(x)=155558767)
+3300 val loss 6.4252
+3300 val perplexity 617.2025
+3300 train 6.432137 (lr=3.7222e-05) (hash(x)=155558767)
+3400 val loss 6.2653
+3400 val perplexity 526.0201
+3400 train 6.206819 (lr=3.4730e-05) (hash(x)=144137265)
 3400 val loss 6.3850
-3400 val perplexity 592.9050
-3400 train 6.332426 (lr=2.5469e-05) (hash(x)=144137265)
-3500 val loss 6.3681
-3500 val perplexity 582.9590
-3500 train 6.559334 (lr=2.5344e-05) (hash(x)=154379201)
-3600 val loss 6.3557
-3600 val perplexity 575.7748
-3600 train 6.476018 (lr=2.5215e-05) (hash(x)=158728918)
-3700 val loss 6.3384
-3700 val perplexity 565.8771
-3700 train 6.080819 (lr=2.5083e-05) (hash(x)=139902844)
-3800 val loss 6.3307
-3800 val perplexity 561.5490
-3800 train 6.400692 (lr=2.4948e-05) (hash(x)=160738928)
-3900 val loss 6.3023
-3900 val perplexity 545.8097
-3900 train 6.346352 (lr=2.4810e-05) (hash(x)=156122233)
-4000 val loss 6.2826
-4000 val perplexity 535.1808
-4000 train 6.109324 (lr=2.4669e-05) (hash(x)=136691174)
-4100 val loss 6.2762
-4100 val perplexity 531.7452
-4100 train 6.242570 (lr=2.4524e-05) (hash(x)=150135523)
-4200 val loss 6.2686
-4200 val perplexity 527.7131
-4200 train 6.256784 (lr=2.4376e-05) (hash(x)=154053307)
-4300 val loss 6.2503
-4300 val perplexity 518.1600
-4300 train 6.028021 (lr=2.4226e-05) (hash(x)=136076439)
-4400 val loss 6.2345
-4400 val perplexity 510.0574
-4400 train 6.124469 (lr=2.4072e-05) (hash(x)=147368042)
-4500 val loss 6.2168
-4500 val perplexity 501.1109
-4500 train 6.318371 (lr=2.3915e-05) (hash(x)=163726147)
-4600 val loss 6.2070
-4600 val perplexity 496.2216
-4600 train 6.221811 (lr=2.3755e-05) (hash(x)=151996388)
-4700 val loss 6.1921
-4700 val perplexity 488.8610
-4700 train 6.235697 (lr=2.3593e-05) (hash(x)=148952763)
-4800 val loss 6.1823
-4800 val perplexity 484.1152
-4800 train 6.098751 (lr=2.3428e-05) (hash(x)=139747769)
-4900 val loss 6.1781
-4900 val perplexity 482.0607
-4900 train 6.045214 (lr=2.3260e-05) (hash(x)=145011383)
-5000 val loss 6.1633
-5000 val perplexity 474.9830
-5000 train 6.185411 (lr=2.3089e-05) (hash(x)=153449428)
-5100 val loss 6.1545
-5100 val perplexity 470.8437
-5100 train 6.195387 (lr=2.2916e-05) (hash(x)=161397672)
-5200 val loss 6.1344
-5200 val perplexity 461.4544
-5200 train 6.126793 (lr=2.2740e-05) (hash(x)=164813156)
-5300 val loss 6.1258
-5300 val perplexity 457.5305
-5300 train 6.177435 (lr=2.2561e-05) (hash(x)=146155759)
-5400 val loss 6.1213
-5400 val perplexity 455.4430
-5400 train 6.183184 (lr=2.2381e-05) (hash(x)=152337652)
-5500 val loss 6.1119
-5500 val perplexity 451.1878
-5500 train 5.973808 (lr=2.2197e-05) (hash(x)=148346631)
-5600 val loss 6.1063
-5600 val perplexity 448.6686
-5600 train 6.012882 (lr=2.2012e-05) (hash(x)=146652463)
-5700 val loss 6.1002
-5700 val perplexity 445.9608
-5700 train 6.034775 (lr=2.1824e-05) (hash(x)=147553632)
-5800 val loss 6.0846
-5800 val perplexity 439.0377
-5800 train 6.185885 (lr=2.1634e-05) (hash(x)=153512314)
-5900 val loss 6.0728
-5900 val perplexity 433.8759
-5900 train 6.044499 (lr=2.1442e-05) (hash(x)=137351707)
-6000 val loss 6.0726
-6000 val perplexity 433.7897
-6000 train 6.099359 (lr=2.1248e-05) (hash(x)=143375055)
-6100 val loss 6.0634
-6100 val perplexity 429.8267
-6100 train 6.073904 (lr=2.1051e-05) (hash(x)=144781857)
-6200 val loss 6.0519
-6200 val perplexity 424.9072
-6200 train 6.308413 (lr=2.0853e-05) (hash(x)=167556812)
-6300 val loss 6.0446
-6300 val perplexity 421.8202
-6300 train 6.198883 (lr=2.0653e-05) (hash(x)=162290597)
-6400 val loss 6.0359
-6400 val perplexity 418.1750
-6400 train 5.934332 (lr=2.0451e-05) (hash(x)=144836361)
-6500 val loss 6.0226
-6500 val perplexity 412.6547
-6500 train 5.907002 (lr=2.0247e-05) (hash(x)=150949131)
-6600 val loss 6.0350
-6600 val perplexity 417.7888
-6600 train 6.098232 (lr=2.0042e-05) (hash(x)=151386312)
-6700 val loss 6.0056
-6700 val perplexity 405.7001
-6700 train 6.086429 (lr=1.9835e-05) (hash(x)=159840718)
-6800 val loss 5.9969
-6800 val perplexity 402.1692
-6800 train 6.088245 (lr=1.9626e-05) (hash(x)=158843242)
-6900 val loss 5.9943
-6900 val perplexity 401.1250
-6900 train 6.230082 (lr=1.9416e-05) (hash(x)=160544793)
-7000 val loss 5.9862
-7000 val perplexity 397.9082
-7000 train 5.832255 (lr=1.9205e-05) (hash(x)=139031423)
-7100 val loss 5.9803
-7100 val perplexity 395.5639
-7100 train 5.924754 (lr=1.8992e-05) (hash(x)=148444150)
-7200 val loss 5.9745
-7200 val perplexity 393.2809
-7200 train 6.303175 (lr=1.8778e-05) (hash(x)=162348298)
-7300 val loss 5.9657
-7300 val perplexity 389.8229
-7300 train 6.031035 (lr=1.8562e-05) (hash(x)=160070640)
-7400 val loss 5.9595
-7400 val perplexity 387.4172
-7400 train 5.878331 (lr=1.8346e-05) (hash(x)=142907242)
-7500 val loss 5.9573
-7500 val perplexity 386.5767
-7500 train 5.959059 (lr=1.8128e-05) (hash(x)=155567735)
-7600 val loss 5.9498
-7600 val perplexity 383.6834
-7600 train 5.799048 (lr=1.7910e-05) (hash(x)=149026190)
-7700 val loss 5.9417
-7700 val perplexity 380.5897
-7700 train 6.468088 (lr=1.7691e-05) (hash(x)=172973386)
-7800 val loss 5.9373
-7800 val perplexity 378.9236
-7800 train 5.907558 (lr=1.7470e-05) (hash(x)=148166384)
-7900 val loss 5.9337
-7900 val perplexity 377.5314
-7900 train 6.369796 (lr=1.7249e-05) (hash(x)=174580801)
-8000 val loss 5.9233
-8000 val perplexity 373.6542
-8000 train 5.941860 (lr=1.7028e-05) (hash(x)=153576391)
-8100 val loss 5.9166
-8100 val perplexity 371.1408
-8100 train 5.815281 (lr=1.6805e-05) (hash(x)=143483378)
-8200 val loss 5.9127
-8200 val perplexity 369.7064
-8200 train 6.327120 (lr=1.6582e-05) (hash(x)=125210829)
-8300 val loss 5.9088
-8300 val perplexity 368.2632
-8300 train 5.830599 (lr=1.6359e-05) (hash(x)=152432601)
-8400 val loss 5.9119
-8400 val perplexity 369.3940
-8400 train 5.790779 (lr=1.6135e-05) (hash(x)=148759990)
-8500 val loss 5.8946
-8500 val perplexity 363.0729
-8500 train 5.893033 (lr=1.5911e-05) (hash(x)=156974533)
-8600 val loss 5.8877
-8600 val perplexity 360.5785
-8600 train 5.997952 (lr=1.5687e-05) (hash(x)=155441439)
-8700 val loss 5.8845
-8700 val perplexity 359.4311
-8700 train 5.866372 (lr=1.5462e-05) (hash(x)=149001504)
-8800 val loss 5.8765
-8800 val perplexity 356.5466
-8800 train 5.753472 (lr=1.5237e-05) (hash(x)=144721722)
-8900 val loss 5.8741
-8900 val perplexity 355.7068
-8900 train 5.780489 (lr=1.5013e-05) (hash(x)=151789511)
-9000 val loss 5.8679
-9000 val perplexity 353.4932
-9000 train 5.797716 (lr=1.4788e-05) (hash(x)=151307797)
-9100 val loss 5.8624
-9100 val perplexity 351.5597
-9100 train 5.990904 (lr=1.4563e-05) (hash(x)=155534554)
-9200 val loss 5.8534
-9200 val perplexity 348.4138
-9200 train 5.737578 (lr=1.4339e-05) (hash(x)=138467356)
-9300 val loss 5.8519
-9300 val perplexity 347.8787
-9300 train 5.780663 (lr=1.4115e-05) (hash(x)=148622510)
-9400 val loss 5.8451
-9400 val perplexity 345.5435
-9400 train 5.971584 (lr=1.3891e-05) (hash(x)=154343672)
-9500 val loss 5.8430
-9500 val perplexity 344.8152
-9500 train 5.831597 (lr=1.3668e-05) (hash(x)=156831628)
-9600 val loss 5.8395
-9600 val perplexity 343.6098
-9600 train 5.615783 (lr=1.3445e-05) (hash(x)=142190812)
-9700 val loss 5.8304
-9700 val perplexity 340.5052
-9700 train 5.740179 (lr=1.3222e-05) (hash(x)=146487376)
-9800 val loss 5.8253
-9800 val perplexity 338.7470
-9800 train 5.915164 (lr=1.3001e-05) (hash(x)=164178945)
-9900 val loss 5.8230
-9900 val perplexity 337.9937
-9900 train 5.688615 (lr=1.2780e-05) (hash(x)=142224918)
-10000 val loss 5.8229
-10000 val perplexity 337.9587
-10000 train 5.753917 (lr=1.2559e-05) (hash(x)=148509740)
-10100 val loss 5.8134
-10100 val perplexity 334.7476
-10100 train 5.879743 (lr=1.2340e-05) (hash(x)=153536520)
-10200 val loss 5.8054
-10200 val perplexity 332.1046
-10200 train 5.738307 (lr=1.2122e-05) (hash(x)=153740168)
-10300 val loss 5.8274
-10300 val perplexity 339.4831
-10300 train 5.786556 (lr=1.1904e-05) (hash(x)=160410260)
-10400 val loss 5.7976
-10400 val perplexity 329.5164
-10400 train 5.951182 (lr=1.1688e-05) (hash(x)=136788294)
-10500 val loss 5.7929
-10500 val perplexity 327.9620
-10500 train 5.652078 (lr=1.1472e-05) (hash(x)=153988760)
-10600 val loss 5.7913
-10600 val perplexity 327.4321
-10600 train 5.665859 (lr=1.1258e-05) (hash(x)=141896522)
-10700 val loss 5.7906
-10700 val perplexity 327.2158
-10700 train 6.129151 (lr=1.1045e-05) (hash(x)=147057702)
-10800 val loss 5.7819
-10800 val perplexity 324.3701
-10800 train 5.848338 (lr=1.0834e-05) (hash(x)=157542287)
-10900 val loss 5.7773
-10900 val perplexity 322.8757
-10900 train 5.841738 (lr=1.0624e-05) (hash(x)=153860040)
-11000 val loss 5.7728
-11000 val perplexity 321.4455
-11000 train 5.637451 (lr=1.0415e-05) (hash(x)=142932555)
-11100 val loss 5.7708
-11100 val perplexity 320.7993
-11100 train 5.806400 (lr=1.0208e-05) (hash(x)=146260760)
-11200 val loss 5.7700
-11200 val perplexity 320.5267
-11200 train 5.858836 (lr=1.0003e-05) (hash(x)=148970533)
-11300 val loss 5.7650
-11300 val perplexity 318.9247
-11300 train 5.781595 (lr=9.7991e-06) (hash(x)=149421645)
-11400 val loss 5.7578
-11400 val perplexity 316.6360
-11400 train 5.698096 (lr=9.5972e-06) (hash(x)=158728429)
-11500 val loss 5.7544
-11500 val perplexity 315.5844
-11500 train 5.843369 (lr=9.3970e-06) (hash(x)=139524335)
-11600 val loss 5.7539
-11600 val perplexity 315.4190
-11600 train 5.796008 (lr=9.1988e-06) (hash(x)=152760174)
-11700 val loss 5.7492
-11700 val perplexity 313.9311
-11700 train 5.653507 (lr=9.0025e-06) (hash(x)=140740682)
-11800 val loss 5.7480
-11800 val perplexity 313.5658
-11800 train 5.604234 (lr=8.8082e-06) (hash(x)=153665407)
-11900 val loss 5.7477
-11900 val perplexity 313.4580
-11900 train 5.717497 (lr=8.6160e-06) (hash(x)=163068424)
-12000 val loss 5.7406
-12000 val perplexity 311.2459
-12000 train 5.797648 (lr=8.4260e-06) (hash(x)=145166541)
-12100 val loss 5.7374
-12100 val perplexity 310.2444
-12100 train 5.691184 (lr=8.2381e-06) (hash(x)=143228042)
-12200 val loss 5.7353
-12200 val perplexity 309.5932
-12200 train 5.842823 (lr=8.0526e-06) (hash(x)=159222680)
-12300 val loss 5.7341
-12300 val perplexity 309.2215
-12300 train 5.674417 (lr=7.8694e-06) (hash(x)=145196157)
-12400 val loss 5.7328
-12400 val perplexity 308.8237
-12400 train 5.636087 (lr=7.6885e-06) (hash(x)=143033742)
-12500 val loss 5.7302
-12500 val perplexity 308.0446
-12500 train 5.762257 (lr=7.5101e-06) (hash(x)=151133346)
-12600 val loss 5.7278
-12600 val perplexity 307.2995
-12600 train 5.752061 (lr=7.3342e-06) (hash(x)=153607184)
-12700 val loss 5.7241
-12700 val perplexity 306.1727
-12700 train 5.640927 (lr=7.1609e-06) (hash(x)=136931911)
-12800 val loss 5.7206
-12800 val perplexity 305.0846
-12800 train 5.832143 (lr=6.9903e-06) (hash(x)=155105402)
-12900 val loss 5.7191
-12900 val perplexity 304.6164
-12900 train 5.547976 (lr=6.8223e-06) (hash(x)=144141636)
-13000 val loss 5.7202
-13000 val perplexity 304.9768
-13000 train 5.634468 (lr=6.6570e-06) (hash(x)=139380218)
-13100 val loss 5.7123
-13100 val perplexity 302.5593
-13100 train 5.798109 (lr=6.4945e-06) (hash(x)=157695174)
-13200 val loss 5.7105
-13200 val perplexity 302.0205
-13200 train 5.648192 (lr=6.3349e-06) (hash(x)=143697560)
-13300 val loss 5.7079
-13300 val perplexity 301.2352
-13300 train 5.550069 (lr=6.1782e-06) (hash(x)=156314952)
-13400 val loss 5.7082
-13400 val perplexity 301.3254
-13400 train 5.694297 (lr=6.0244e-06) (hash(x)=136720279)
-13500 val loss 5.7072
-13500 val perplexity 301.0287
-13500 train 5.658339 (lr=5.8737e-06) (hash(x)=142579957)
-13600 val loss 5.7082
-13600 val perplexity 301.3250
-13600 train 5.420486 (lr=5.7259e-06) (hash(x)=140719440)
-13700 val loss 5.7064
-13700 val perplexity 300.7962
-13700 train 5.813454 (lr=5.5813e-06) (hash(x)=164396003)
-13800 val loss 5.7019
-13800 val perplexity 299.4438
-13800 train 5.761009 (lr=5.4398e-06) (hash(x)=152712862)
-13900 val loss 5.6972
-13900 val perplexity 298.0294
-13900 train 5.723367 (lr=5.3016e-06) (hash(x)=160985139)
-14000 val loss 5.6961
-14000 val perplexity 297.7088
-14000 train 5.682368 (lr=5.1665e-06) (hash(x)=147429379)
-14100 val loss 5.6939
-14100 val perplexity 297.0557
-14100 train 5.748112 (lr=5.0348e-06) (hash(x)=131830470)
-14200 val loss 5.6948
-14200 val perplexity 297.3153
-14200 train 5.442815 (lr=4.9063e-06) (hash(x)=146383028)
-14300 val loss 5.6928
-14300 val perplexity 296.7108
-14300 train 5.738539 (lr=4.7813e-06) (hash(x)=157158747)
-14400 val loss 5.6954
-14400 val perplexity 297.4931
-14400 train 5.549876 (lr=4.6596e-06) (hash(x)=143911520)
-14500 val loss 5.6943
-14500 val perplexity 297.1754
-14500 train 5.474485 (lr=4.5414e-06) (hash(x)=159555347)
-14600 val loss 5.6888
-14600 val perplexity 295.5297
-14600 train 5.689527 (lr=4.4267e-06) (hash(x)=152957062)
-14700 val loss 5.6849
-14700 val perplexity 294.3972
-14700 train 5.756591 (lr=4.3155e-06) (hash(x)=156001594)
-14800 val loss 5.6850
-14800 val perplexity 294.4080
-14800 train 5.783759 (lr=4.2079e-06) (hash(x)=145885458)
-14900 val loss 5.6814
-14900 val perplexity 293.3562
-14900 train 5.734191 (lr=4.1039e-06) (hash(x)=144544291)
-15000 val loss 5.6825
-15000 val perplexity 293.6803
-15000 train 5.689342 (lr=4.0035e-06) (hash(x)=154606709)
-15100 val loss 5.6817
-15100 val perplexity 293.4573
-15100 train 5.734373 (lr=3.9068e-06) (hash(x)=156005324)
-15200 val loss 5.6824
-15200 val perplexity 293.6678
-15200 train 5.786275 (lr=3.8138e-06) (hash(x)=158847373)
-15300 val loss 5.6821
-15300 val perplexity 293.5554
-15300 train 5.474887 (lr=3.7245e-06) (hash(x)=159609594)
-15400 val loss 5.6780
-15400 val perplexity 292.3782
-15400 train 5.767415 (lr=3.6390e-06) (hash(x)=152428492)
-15500 val loss 5.6752
-15500 val perplexity 291.5412
-15500 train 5.741548 (lr=3.5572e-06) (hash(x)=162251192)
-15600 val loss 5.6734
-15600 val perplexity 291.0144
-15600 train 5.920865 (lr=3.4793e-06) (hash(x)=155072002)
-15700 val loss 5.6726
-15700 val perplexity 290.8026
-15700 train 5.688998 (lr=3.4052e-06) (hash(x)=151203094)
-15800 val loss 5.6722
-15800 val perplexity 290.6788
-15800 train 5.592441 (lr=3.3350e-06) (hash(x)=146125932)
-15900 val loss 5.6719
-15900 val perplexity 290.5904
-15900 train 5.568439 (lr=3.2687e-06) (hash(x)=148060195)
-16000 val loss 5.6720
-16000 val perplexity 290.6114
-16000 train 5.368674 (lr=3.2063e-06) (hash(x)=137406416)
-16100 val loss 5.6725
-16100 val perplexity 290.7514
-16100 train 5.737967 (lr=3.1478e-06) (hash(x)=160149755)
-16200 val loss 5.6678
-16200 val perplexity 289.3838
-16200 train 5.834690 (lr=3.0932e-06) (hash(x)=151371894)
-16300 val loss 5.6665
-16300 val perplexity 289.0163
-16300 train 5.719723 (lr=3.0427e-06) (hash(x)=151810252)
-16400 val loss 5.6650
-16400 val perplexity 288.5883
-16400 train 5.651065 (lr=2.9961e-06) (hash(x)=153088303)
-16500 val loss 5.6643
-16500 val perplexity 288.3983
-16500 train 5.553295 (lr=2.9535e-06) (hash(x)=151706615)
-16600 val loss 5.6628
-16600 val perplexity 287.9627
-16600 train 5.548329 (lr=2.9149e-06) (hash(x)=142110626)
-16700 val loss 5.6636
-16700 val perplexity 288.1895
-16700 train 5.557631 (lr=2.8804e-06) (hash(x)=153222439)
-16800 val loss 5.6631
-16800 val perplexity 288.0464
-16800 train 5.597987 (lr=2.8498e-06) (hash(x)=154964902)
-16900 val loss 5.6597
-16900 val perplexity 287.0678
-16900 train 5.819545 (lr=2.8234e-06) (hash(x)=158893204)
-17000 val loss 5.6579
-17000 val perplexity 286.5460
-17000 train 5.582509 (lr=2.8010e-06) (hash(x)=148929955)
-17100 val loss 5.6586
-17100 val perplexity 286.7525
-17100 train 5.656537 (lr=2.7826e-06) (hash(x)=147818677)
-17200 val loss 5.6572
-17200 val perplexity 286.3454
-17200 train 5.592340 (lr=2.7684e-06) (hash(x)=147119446)
-17300 val loss 5.6573
-17300 val perplexity 286.3883
-17300 train 5.699736 (lr=2.7582e-06) (hash(x)=150494142)
-17400 val loss 5.6580
-17400 val perplexity 286.5887
-17400 train 5.689650 (lr=2.7520e-06) (hash(x)=155918058)
-17499 val loss 5.6557
-17499 val perplexity 285.9162
+3400 val perplexity 592.8885
+3400 train 6.325558 (lr=3.7045e-05) (hash(x)=144137265)
+3500 val loss 6.2389
+3500 val perplexity 512.2823
+3500 train 6.433990 (lr=3.4560e-05) (hash(x)=154379201)
+3500 val loss 6.3609
+3500 val perplexity 578.7686
+3500 train 6.544823 (lr=3.6864e-05) (hash(x)=154379201)
+3600 val loss 6.2298
+3600 val perplexity 507.6419
+3600 train 6.364053 (lr=3.4384e-05) (hash(x)=158728918)
+3600 val loss 6.3463
+3600 val perplexity 570.3826
+3600 train 6.472357 (lr=3.6677e-05) (hash(x)=158728918)
+3700 val loss 6.2122
+3700 val perplexity 498.8144
+3700 train 5.966904 (lr=3.4205e-05) (hash(x)=139902844)
+3700 val loss 6.3319
+3700 val perplexity 562.2496
+3700 train 6.076136 (lr=3.6485e-05) (hash(x)=139902844)
+3800 val loss 6.2045
+3800 val perplexity 494.9547
+3800 train 6.286787 (lr=3.4021e-05) (hash(x)=160738928)
+3800 val loss 6.3220
+3800 val perplexity 556.6725
+3800 train 6.395666 (lr=3.6289e-05) (hash(x)=160738928)
+3900 val loss 6.1780
+3900 val perplexity 482.0338
+3900 train 6.223629 (lr=3.3832e-05) (hash(x)=156122233)
+3900 val loss 6.2837
+3900 val perplexity 535.7934
+3900 train 6.330824 (lr=3.6088e-05) (hash(x)=156122233)
+4000 val loss 6.1577
+4000 val perplexity 472.3254
+4000 train 5.994511 (lr=3.3639e-05) (hash(x)=136691174)
+4000 val loss 6.2657
+4000 val perplexity 526.1896
+4000 train 6.095181 (lr=3.5882e-05) (hash(x)=136691174)
+4100 val loss 6.1589
+4100 val perplexity 472.9088
+4100 train 6.126831 (lr=3.3442e-05) (hash(x)=150135523)
+4100 val loss 6.2577
+4100 val perplexity 522.0146
+4100 train 6.226404 (lr=3.5671e-05) (hash(x)=150135523)
+4200 val loss 6.1453
+4200 val perplexity 466.4993
+4200 train 6.137742 (lr=3.3240e-05) (hash(x)=154053307)
+4200 val loss 6.2370
+4200 val perplexity 511.2998
+4200 train 6.229482 (lr=3.5457e-05) (hash(x)=154053307)
+4300 val loss 6.1309
+4300 val perplexity 459.8623
+4300 train 5.923357 (lr=3.3035e-05) (hash(x)=136076439)
+4300 val loss 6.2232
+4300 val perplexity 504.3218
+4300 train 6.012386 (lr=3.5237e-05) (hash(x)=136076439)
+4400 val loss 6.1126
+4400 val perplexity 451.4940
+4400 train 6.003034 (lr=3.2825e-05) (hash(x)=147368042)
+4400 val loss 6.2035
+4400 val perplexity 494.4707
+4400 train 6.094748 (lr=3.5014e-05) (hash(x)=147368042)
+4500 val loss 6.1012
+4500 val perplexity 446.3787
+4500 train 6.190740 (lr=3.2611e-05) (hash(x)=163726147)
+4500 val loss 6.1846
+4500 val perplexity 485.2155
+4500 train 6.276649 (lr=3.4786e-05) (hash(x)=163726147)
+4600 val loss 6.0856
+4600 val perplexity 439.5045
+4600 train 6.098429 (lr=3.2394e-05) (hash(x)=151996388)
+4600 val loss 6.1658
+4600 val perplexity 476.1843
+4600 train 6.189225 (lr=3.4553e-05) (hash(x)=151996388)
+4700 val loss 6.0705
+4700 val perplexity 432.9117
+4700 train 6.129060 (lr=3.2172e-05) (hash(x)=148952763)
+4700 val loss 6.1439
+4700 val perplexity 465.8540
+4700 train 6.196408 (lr=3.4317e-05) (hash(x)=148952763)
+4800 val loss 6.0627
+4800 val perplexity 429.5331
+4800 train 5.990180 (lr=3.1947e-05) (hash(x)=139747769)
+4800 val loss 6.1356
+4800 val perplexity 462.0074
+4800 train 6.060421 (lr=3.4077e-05) (hash(x)=139747769)
+4900 val loss 6.0544
+4900 val perplexity 425.9879
+4900 train 5.932305 (lr=3.1718e-05) (hash(x)=145011383)
+4900 val loss 6.1256
+4900 val perplexity 457.3985
+4900 train 6.002694 (lr=3.3832e-05) (hash(x)=145011383)
+5000 val loss 6.0411
+5000 val perplexity 420.3618
+5000 train 6.063803 (lr=3.1485e-05) (hash(x)=153449428)
+5000 val loss 6.1166
+5000 val perplexity 453.3174
+5000 train 6.139397 (lr=3.3584e-05) (hash(x)=153449428)
+5100 val loss 6.0310
+5100 val perplexity 416.1304
+5100 train 6.080400 (lr=3.1249e-05) (hash(x)=161397672)
+5100 val loss 6.0984
+5100 val perplexity 445.1656
+5100 train 6.141521 (lr=3.3332e-05) (hash(x)=161397672)
+5200 val loss 6.0118
+5200 val perplexity 408.2232
+5200 train 5.999507 (lr=3.1009e-05) (hash(x)=164813156)
+5200 val loss 6.0819
+5200 val perplexity 437.8560
+5200 train 6.071654 (lr=3.3076e-05) (hash(x)=164813156)
+5300 val loss 5.9973
+5300 val perplexity 402.3416
+5300 train 6.054934 (lr=3.0766e-05) (hash(x)=146155759)
+5300 val loss 6.0705
+5300 val perplexity 432.8826
+5300 train 6.119602 (lr=3.2817e-05) (hash(x)=146155759)
+5400 val loss 5.9907
+5400 val perplexity 399.7007
+5400 train 6.072926 (lr=3.0519e-05) (hash(x)=152337652)
+5400 val loss 6.0599
+5400 val perplexity 428.3523
+5400 train 6.129765 (lr=3.2554e-05) (hash(x)=152337652)
+5500 val loss 5.9846
+5500 val perplexity 397.2581
+5500 train 5.847214 (lr=3.0269e-05) (hash(x)=148346631)
+5500 val loss 6.0540
+5500 val perplexity 425.8023
+5500 train 5.916514 (lr=3.2287e-05) (hash(x)=148346631)
+5600 val loss 5.9770
+5600 val perplexity 394.2721
+5600 train 5.882283 (lr=3.0016e-05) (hash(x)=146652463)
+5600 val loss 6.0421
+5600 val perplexity 420.7834
+5600 train 5.945309 (lr=3.2017e-05) (hash(x)=146652463)
+5700 val loss 5.9691
+5700 val perplexity 391.1408
+5700 train 5.921083 (lr=2.9760e-05) (hash(x)=147553632)
+5700 val loss 6.0329
+5700 val perplexity 416.9108
+5700 train 5.973693 (lr=3.1744e-05) (hash(x)=147553632)
+5800 val loss 5.9471
+5800 val perplexity 382.6452
+5800 train 6.046297 (lr=2.9501e-05) (hash(x)=153512314)
+5800 val loss 6.0122
+5800 val perplexity 408.3965
+5800 train 6.112329 (lr=3.1468e-05) (hash(x)=153512314)
+5900 val loss 5.9382
+5900 val perplexity 379.2424
+5900 train 5.917999 (lr=2.9239e-05) (hash(x)=137351707)
+5900 val loss 5.9983
+5900 val perplexity 402.7321
+5900 train 5.978490 (lr=3.1188e-05) (hash(x)=137351707)
+6000 val loss 5.9298
+6000 val perplexity 376.0757
+6000 train 5.960650 (lr=2.8974e-05) (hash(x)=143375055)
+6000 val loss 5.9937
+6000 val perplexity 400.8940
+6000 train 6.022861 (lr=3.0905e-05) (hash(x)=143375055)
+6100 val loss 5.9277
+6100 val perplexity 375.2952
+6100 train 5.960719 (lr=2.8706e-05) (hash(x)=144781857)
+6100 val loss 5.9934
+6100 val perplexity 400.7845
+6100 train 6.023008 (lr=3.0620e-05) (hash(x)=144781857)
+6200 val loss 5.9126
+6200 val perplexity 369.6685
+6200 train 6.142590 (lr=2.8436e-05) (hash(x)=167556812)
+6200 val loss 5.9725
+6200 val perplexity 392.4781
+6200 train 6.209857 (lr=3.0332e-05) (hash(x)=167556812)
+6300 val loss 5.9028
+6300 val perplexity 366.0782
+6300 train 6.067983 (lr=2.8163e-05) (hash(x)=162290597)
+6300 val loss 5.9647
+6300 val perplexity 389.4418
+6300 train 6.122912 (lr=3.0040e-05) (hash(x)=162290597)
+6400 val loss 5.8906
+6400 val perplexity 361.6111
+6400 train 5.784720 (lr=2.7888e-05) (hash(x)=144836361)
+6400 val loss 5.9561
+6400 val perplexity 386.1159
+6400 train 5.860235 (lr=2.9747e-05) (hash(x)=144836361)
+6500 val loss 5.8787
+6500 val perplexity 357.3279
+6500 train 5.761406 (lr=2.7610e-05) (hash(x)=150949131)
+6500 val loss 5.9379
+6500 val perplexity 379.1356
+6500 train 5.818823 (lr=2.9450e-05) (hash(x)=150949131)
+6600 val loss 5.8904
+6600 val perplexity 361.5419
+6600 train 5.956506 (lr=2.7330e-05) (hash(x)=151386312)
+6600 val loss 5.9497
+6600 val perplexity 383.6486
+6600 train 6.014431 (lr=2.9152e-05) (hash(x)=151386312)
+6700 val loss 5.8572
+6700 val perplexity 349.7287
+6700 train 5.944593 (lr=2.7047e-05) (hash(x)=159840718)
+6700 val loss 5.9174
+6700 val perplexity 371.4353
+6700 train 5.998442 (lr=2.8851e-05) (hash(x)=159840718)
+6800 val loss 5.8479
+6800 val perplexity 346.5221
+6800 train 5.949664 (lr=2.6763e-05) (hash(x)=158843242)
+6800 val loss 5.9085
+6800 val perplexity 368.1369
+6800 train 6.006040 (lr=2.8547e-05) (hash(x)=158843242)
+6900 val loss 5.8432
+6900 val perplexity 344.8760
+6900 train 6.077638 (lr=2.6477e-05) (hash(x)=160544793)
+6900 val loss 5.9060
+6900 val perplexity 367.2513
+6900 train 6.131098 (lr=2.8242e-05) (hash(x)=160544793)
+7000 val loss 5.8336
+7000 val perplexity 341.5859
+7000 train 5.691507 (lr=2.6188e-05) (hash(x)=139031423)
+7000 val loss 5.8963
+7000 val perplexity 363.6898
+7000 train 5.744294 (lr=2.7934e-05) (hash(x)=139031423)
+7100 val loss 5.8322
+7100 val perplexity 341.1207
+7100 train 5.789629 (lr=2.5898e-05) (hash(x)=148444150)
+7100 val loss 5.8901
+7100 val perplexity 361.4375
+7100 train 5.837803 (lr=2.7624e-05) (hash(x)=148444150)
+7200 val loss 5.8216
+7200 val perplexity 337.5242
+7200 train 6.172296 (lr=2.5606e-05) (hash(x)=162348298)
+7200 val loss 5.8795
+7200 val perplexity 357.6189
+7200 train 6.223478 (lr=2.7313e-05) (hash(x)=162348298)
+7300 val loss 5.8094
+7300 val perplexity 333.4139
+7300 train 5.849937 (lr=2.5312e-05) (hash(x)=160070640)
+7300 val loss 5.8669
+7300 val perplexity 353.1607
+7300 train 5.919557 (lr=2.7000e-05) (hash(x)=160070640)
+7400 val loss 5.8009
+7400 val perplexity 330.6072
+7400 train 5.728707 (lr=2.5017e-05) (hash(x)=142907242)
+7400 val loss 5.8618
+7400 val perplexity 351.3591
+7400 train 5.789402 (lr=2.6685e-05) (hash(x)=142907242)
+7500 val loss 5.8015
+7500 val perplexity 330.8050
+7500 train 5.783610 (lr=2.4721e-05) (hash(x)=155567735)
+7500 val loss 5.8515
+7500 val perplexity 347.7528
+7500 train 5.846130 (lr=2.6369e-05) (hash(x)=155567735)
+7600 val loss 5.7887
+7600 val perplexity 326.5765
+7600 train 5.645206 (lr=2.4423e-05) (hash(x)=149026190)
+7600 val loss 5.8432
+7600 val perplexity 344.8734
+7600 train 5.700372 (lr=2.6051e-05) (hash(x)=149026190)
+7700 val loss 5.7812
+7700 val perplexity 324.1628
+7700 train 6.316875 (lr=2.4124e-05) (hash(x)=172973386)
+7700 val loss 5.8359
+7700 val perplexity 342.3663
+7700 train 6.377451 (lr=2.5732e-05) (hash(x)=172973386)
+7800 val loss 5.7737
+7800 val perplexity 321.7310
+7800 train 5.766070 (lr=2.3823e-05) (hash(x)=148166384)
+7800 val loss 5.8271
+7800 val perplexity 339.3716
+7800 train 5.799076 (lr=2.5411e-05) (hash(x)=148166384)
+7900 val loss 5.7741
+7900 val perplexity 321.8507
+7900 train 6.198745 (lr=2.3522e-05) (hash(x)=174580801)
+7900 val loss 5.8263
+7900 val perplexity 339.0856
+7900 train 6.266204 (lr=2.5090e-05) (hash(x)=174580801)
+8000 val loss 5.7541
+8000 val perplexity 315.4787
+8000 train 5.774290 (lr=2.3219e-05) (hash(x)=153576391)
+8000 val loss 5.8086
+8000 val perplexity 333.1376
+8000 train 5.828216 (lr=2.4767e-05) (hash(x)=153576391)
+8100 val loss 5.7513
+8100 val perplexity 314.5843
+8100 train 5.638414 (lr=2.2916e-05) (hash(x)=143483378)
+8100 val loss 5.8040
+8100 val perplexity 331.6256
+8100 train 5.701946 (lr=2.4444e-05) (hash(x)=143483378)
+8200 val loss 5.7457
+8200 val perplexity 312.8474
+8200 train 6.278048 (lr=2.2612e-05) (hash(x)=125210829)
+8200 val loss 5.7982
+8200 val perplexity 329.7017
+8200 train 6.284613 (lr=2.4120e-05) (hash(x)=125210829)
+8300 val loss 5.7391
+8300 val perplexity 310.7990
+8300 train 5.668536 (lr=2.2308e-05) (hash(x)=152432601)
+8300 val loss 5.7916
+8300 val perplexity 327.5516
+8300 train 5.714247 (lr=2.3795e-05) (hash(x)=152432601)
+8400 val loss 5.7390
+8400 val perplexity 310.7404
+8400 train 5.612227 (lr=2.2002e-05) (hash(x)=148759990)
+8400 val loss 5.7956
+8400 val perplexity 328.8651
+8400 train 5.674462 (lr=2.3469e-05) (hash(x)=148759990)
+8500 val loss 5.7250
+8500 val perplexity 306.4205
+8500 train 5.687849 (lr=2.1697e-05) (hash(x)=156974533)
+8500 val loss 5.7785
+8500 val perplexity 323.2726
+8500 train 5.754704 (lr=2.3143e-05) (hash(x)=156974533)
+8600 val loss 5.7160
+8600 val perplexity 303.6733
+8600 train 5.836418 (lr=2.1391e-05) (hash(x)=155441439)
+8600 val loss 5.7721
+8600 val perplexity 321.2024
+8600 train 5.887455 (lr=2.2817e-05) (hash(x)=155441439)
+8700 val loss 5.7128
+8700 val perplexity 302.7141
+8700 train 5.697697 (lr=2.1085e-05) (hash(x)=149001504)
+8700 val loss 5.7634
+8700 val perplexity 318.4435
+8700 train 5.748235 (lr=2.2490e-05) (hash(x)=149001504)
+8800 val loss 5.7084
+8800 val perplexity 301.3885
+8800 train 5.586840 (lr=2.0778e-05) (hash(x)=144721722)
+8800 val loss 5.7601
+8800 val perplexity 317.3691
+8800 train 5.637744 (lr=2.2163e-05) (hash(x)=144721722)
+8900 val loss 5.7057
+8900 val perplexity 300.5775
+8900 train 5.609116 (lr=2.0472e-05) (hash(x)=151789511)
+8900 val loss 5.7535
+8900 val perplexity 315.2899
+8900 train 5.661813 (lr=2.1837e-05) (hash(x)=151789511)
+9000 val loss 5.6970
+9000 val perplexity 297.9864
+9000 train 5.631220 (lr=2.0165e-05) (hash(x)=151307797)
+9000 val loss 5.7489
+9000 val perplexity 313.8452
+9000 train 5.678300 (lr=2.1510e-05) (hash(x)=151307797)
+9100 val loss 5.6909
+9100 val perplexity 296.1675
+9100 train 5.818474 (lr=1.9859e-05) (hash(x)=155534554)
+9100 val loss 5.7447
+9100 val perplexity 312.5339
+9100 train 5.884479 (lr=2.1183e-05) (hash(x)=155534554)
+9200 val loss 5.6822
+9200 val perplexity 293.5884
+9200 train 5.587260 (lr=1.9553e-05) (hash(x)=138467356)
+9200 val loss 5.7322
+9200 val perplexity 308.6354
+9200 train 5.624728 (lr=2.0857e-05) (hash(x)=138467356)
+9300 val loss 5.6811
+9300 val perplexity 293.2634
+9300 train 5.622063 (lr=1.9248e-05) (hash(x)=148622510)
+9300 val loss 5.7291
+9300 val perplexity 307.7005
+9300 train 5.665057 (lr=2.0531e-05) (hash(x)=148622510)
+9400 val loss 5.6739
+9400 val perplexity 291.1561
+9400 train 5.816579 (lr=1.8942e-05) (hash(x)=154343672)
+9400 val loss 5.7260
+9400 val perplexity 306.7281
+9400 train 5.859208 (lr=2.0205e-05) (hash(x)=154343672)
+9500 val loss 5.6728
+9500 val perplexity 290.8406
+9500 train 5.656345 (lr=1.8638e-05) (hash(x)=156831628)
+9500 val loss 5.7223
+9500 val perplexity 305.6166
+9500 train 5.709967 (lr=1.9880e-05) (hash(x)=156831628)
+9600 val loss 5.6668
+9600 val perplexity 289.1156
+9600 train 5.442144 (lr=1.8334e-05) (hash(x)=142190812)
+9600 val loss 5.7179
+9600 val perplexity 304.2583
+9600 train 5.493146 (lr=1.9556e-05) (hash(x)=142190812)
+9700 val loss 5.6603
+9700 val perplexity 287.2379
+9700 train 5.565337 (lr=1.8031e-05) (hash(x)=146487376)
+9700 val loss 5.7078
+9700 val perplexity 301.1933
+9700 train 5.615256 (lr=1.9233e-05) (hash(x)=146487376)
+9800 val loss 5.6522
+9800 val perplexity 284.9193
+9800 train 5.710732 (lr=1.7728e-05) (hash(x)=164178945)
+9800 val loss 5.7028
+9800 val perplexity 299.6926
+9800 train 5.779359 (lr=1.8910e-05) (hash(x)=164178945)
+9900 val loss 5.6502
+9900 val perplexity 284.3600
+9900 train 5.516235 (lr=1.7427e-05) (hash(x)=142224918)
+9900 val loss 5.6991
+9900 val perplexity 298.5944
+9900 train 5.561543 (lr=1.8589e-05) (hash(x)=142224918)
+10000 val loss 5.6520
+10000 val perplexity 284.8605
+10000 train 5.579762 (lr=1.7126e-05) (hash(x)=148509740)
+10000 val loss 5.6977
+10000 val perplexity 298.1938
+10000 train 5.623291 (lr=1.8268e-05) (hash(x)=148509740)
+10100 val loss 5.6415
+10100 val perplexity 281.8954
+10100 train 5.708606 (lr=1.6827e-05) (hash(x)=153536520)
+10100 val loss 5.6884
+10100 val perplexity 295.4208
+10100 train 5.755403 (lr=1.7949e-05) (hash(x)=153536520)
+10200 val loss 5.6333
+10200 val perplexity 279.5924
+10200 train 5.571514 (lr=1.6529e-05) (hash(x)=153740168)
+10200 val loss 5.6808
+10200 val perplexity 293.1710
+10200 train 5.616961 (lr=1.7631e-05) (hash(x)=153740168)
+10300 val loss 5.6513
+10300 val perplexity 284.6588
+10300 train 5.605822 (lr=1.6233e-05) (hash(x)=160410260)
+10300 val loss 5.6982
+10300 val perplexity 298.3185
+10300 train 5.657148 (lr=1.7315e-05) (hash(x)=160410260)
+10400 val loss 5.6261
+10400 val perplexity 277.5663
+10400 train 5.775826 (lr=1.5938e-05) (hash(x)=136788294)
+10400 val loss 5.6720
+10400 val perplexity 290.6160
+10400 train 5.818732 (lr=1.7000e-05) (hash(x)=136788294)
+10500 val loss 5.6201
+10500 val perplexity 275.9232
+10500 train 5.476357 (lr=1.5644e-05) (hash(x)=153988760)
+10500 val loss 5.6673
+10500 val perplexity 289.2501
+10500 train 5.514534 (lr=1.6687e-05) (hash(x)=153988760)
+10600 val loss 5.6196
+10600 val perplexity 275.7818
+10600 train 5.509031 (lr=1.5352e-05) (hash(x)=141896522)
+10600 val loss 5.6644
+10600 val perplexity 288.4057
+10600 train 5.548376 (lr=1.6376e-05) (hash(x)=141896522)
+10700 val loss 5.6178
+10700 val perplexity 275.2754
+10700 train 5.978009 (lr=1.5062e-05) (hash(x)=147057702)
+10700 val loss 5.6629
+10700 val perplexity 287.9748
+10700 train 6.012309 (lr=1.6066e-05) (hash(x)=147057702)
+10800 val loss 5.6056
+10800 val perplexity 271.9341
+10800 train 5.675177 (lr=1.4773e-05) (hash(x)=157542287)
+10800 val loss 5.6528
+10800 val perplexity 285.0970
+10800 train 5.723511 (lr=1.5758e-05) (hash(x)=157542287)
+10900 val loss 5.6030
+10900 val perplexity 271.2358
+10900 train 5.687401 (lr=1.4487e-05) (hash(x)=153860040)
+10900 val loss 5.6480
+10900 val perplexity 283.7203
+10900 train 5.729506 (lr=1.5453e-05) (hash(x)=153860040)
+11000 val loss 5.5985
+11000 val perplexity 270.0329
+11000 train 5.466104 (lr=1.4203e-05) (hash(x)=142932555)
+11000 val loss 5.6457
+11000 val perplexity 283.0762
+11000 train 5.509791 (lr=1.5149e-05) (hash(x)=142932555)
+11100 val loss 5.5975
+11100 val perplexity 269.7550
+11100 train 5.635029 (lr=1.3920e-05) (hash(x)=146260760)
+11100 val loss 5.6413
+11100 val perplexity 281.8367
+11100 train 5.682206 (lr=1.4848e-05) (hash(x)=146260760)
+11200 val loss 5.5972
+11200 val perplexity 269.6688
+11200 train 5.709498 (lr=1.3640e-05) (hash(x)=148970533)
+11200 val loss 5.6417
+11200 val perplexity 281.9330
+11200 train 5.746831 (lr=1.4550e-05) (hash(x)=148970533)
+11300 val loss 5.5887
+11300 val perplexity 267.3885
+11300 train 5.623042 (lr=1.3362e-05) (hash(x)=149421645)
+11300 val loss 5.6346
+11300 val perplexity 279.9398
+11300 train 5.655313 (lr=1.4253e-05) (hash(x)=149421645)
+11400 val loss 5.5828
+11400 val perplexity 265.8127
+11400 train 5.517864 (lr=1.3087e-05) (hash(x)=158728429)
+11400 val loss 5.6288
+11400 val perplexity 278.3406
+11400 train 5.564662 (lr=1.3960e-05) (hash(x)=158728429)
+11500 val loss 5.5789
+11500 val perplexity 264.7920
+11500 train 5.688905 (lr=1.2814e-05) (hash(x)=139524335)
+11500 val loss 5.6251
+11500 val perplexity 277.3092
+11500 train 5.728657 (lr=1.3668e-05) (hash(x)=139524335)
+11600 val loss 5.5777
+11600 val perplexity 264.4737
+11600 train 5.614056 (lr=1.2544e-05) (hash(x)=152760174)
+11600 val loss 5.6231
+11600 val perplexity 276.7369
+11600 train 5.658471 (lr=1.3380e-05) (hash(x)=152760174)
+11700 val loss 5.5742
+11700 val perplexity 263.5510
+11700 train 5.470773 (lr=1.2276e-05) (hash(x)=140740682)
+11700 val loss 5.6185
+11700 val perplexity 275.4809
+11700 train 5.515319 (lr=1.3095e-05) (hash(x)=140740682)
+11800 val loss 5.5725
+11800 val perplexity 263.0792
+11800 train 5.420375 (lr=1.2011e-05) (hash(x)=153665407)
+11800 val loss 5.6179
+11800 val perplexity 275.3106
+11800 train 5.472673 (lr=1.2812e-05) (hash(x)=153665407)
+11900 val loss 5.5704
+11900 val perplexity 262.5393
+11900 train 5.499734 (lr=1.1749e-05) (hash(x)=163068424)
+11900 val loss 5.6158
+11900 val perplexity 274.7328
+11900 train 5.570469 (lr=1.2532e-05) (hash(x)=163068424)
+12000 val loss 5.5642
+12000 val perplexity 260.9184
+12000 train 5.614580 (lr=1.1490e-05) (hash(x)=145166541)
+12000 val loss 5.6081
+12000 val perplexity 272.6255
+12000 train 5.665174 (lr=1.2256e-05) (hash(x)=145166541)
+12100 val loss 5.5611
+12100 val perplexity 260.0986
+12100 train 5.521405 (lr=1.1234e-05) (hash(x)=143228042)
+12100 val loss 5.6050
+12100 val perplexity 271.7805
+12100 train 5.560250 (lr=1.1983e-05) (hash(x)=143228042)
+12200 val loss 5.5597
+12200 val perplexity 259.7433
+12200 train 5.660483 (lr=1.0981e-05) (hash(x)=159222680)
+12200 val loss 5.6033
+12200 val perplexity 271.3172
+12200 train 5.700716 (lr=1.1713e-05) (hash(x)=159222680)
+12300 val loss 5.5570
+12300 val perplexity 259.0396
+12300 train 5.514515 (lr=1.0731e-05) (hash(x)=145196157)
+12300 val loss 5.6015
+12300 val perplexity 270.8249
+12300 train 5.550727 (lr=1.1446e-05) (hash(x)=145196157)
+12400 val loss 5.5549
+12400 val perplexity 258.4948
+12400 train 5.481500 (lr=1.0484e-05) (hash(x)=143033742)
+12400 val loss 5.5994
+12400 val perplexity 270.2646
+12400 train 5.521342 (lr=1.1183e-05) (hash(x)=143033742)
+12500 val loss 5.5534
+12500 val perplexity 258.1117
+12500 train 5.594991 (lr=1.0241e-05) (hash(x)=151133346)
+12500 val loss 5.5983
+12500 val perplexity 269.9767
+12500 train 5.637483 (lr=1.0924e-05) (hash(x)=151133346)
+12600 val loss 5.5497
+12600 val perplexity 257.1644
+12600 train 5.575035 (lr=1.0001e-05) (hash(x)=153607184)
+12600 val loss 5.5948
+12600 val perplexity 269.0154
+12600 train 5.616836 (lr=1.0668e-05) (hash(x)=153607184)
+12700 val loss 5.5448
+12700 val perplexity 255.9025
+12700 train 5.456029 (lr=9.7649e-06) (hash(x)=136931911)
+12700 val loss 5.5893
+12700 val perplexity 267.5548
+12700 train 5.501020 (lr=1.0416e-05) (hash(x)=136931911)
+12800 val loss 5.5424
+12800 val perplexity 255.2786
+12800 train 5.652829 (lr=9.5322e-06) (hash(x)=155105402)
+12800 val loss 5.5863
+12800 val perplexity 266.7446
+12800 train 5.701947 (lr=1.0168e-05) (hash(x)=155105402)
+12900 val loss 5.5416
+12900 val perplexity 255.0922
+12900 train 5.377511 (lr=9.3031e-06) (hash(x)=144141636)
+12900 val loss 5.5850
+12900 val perplexity 266.3893
+12900 train 5.414651 (lr=9.9233e-06) (hash(x)=144141636)
+13000 val loss 5.5418
+13000 val perplexity 255.1259
+13000 train 5.474589 (lr=9.0777e-06) (hash(x)=139380218)
+13000 val loss 5.5863
+13000 val perplexity 266.7417
+13000 train 5.515913 (lr=9.6829e-06) (hash(x)=139380218)
+13100 val loss 5.5343
+13100 val perplexity 253.2362
+13100 train 5.632472 (lr=8.8562e-06) (hash(x)=157695174)
+13100 val loss 5.5780
+13100 val perplexity 264.5459
+13100 train 5.674640 (lr=9.4466e-06) (hash(x)=157695174)
+13200 val loss 5.5346
+13200 val perplexity 253.3157
+13200 train 5.482560 (lr=8.6385e-06) (hash(x)=143697560)
+13200 val loss 5.5764
+13200 val perplexity 264.1287
+13200 train 5.523151 (lr=9.2144e-06) (hash(x)=143697560)
+13300 val loss 5.5307
+13300 val perplexity 252.3304
+13300 train 5.361203 (lr=8.4248e-06) (hash(x)=156314952)
+13300 val loss 5.5741
+13300 val perplexity 263.5179
+13300 train 5.405429 (lr=8.9865e-06) (hash(x)=156314952)
+13400 val loss 5.5296
+13400 val perplexity 252.0309
+13400 train 5.499445 (lr=8.2151e-06) (hash(x)=136720279)
+13400 val loss 5.5738
+13400 val perplexity 263.4412
+13400 train 5.526761 (lr=8.7628e-06) (hash(x)=136720279)
+13500 val loss 5.5286
+13500 val perplexity 251.7948
+13500 train 5.488736 (lr=8.0095e-06) (hash(x)=142579957)
+13500 val loss 5.5725
+13500 val perplexity 263.0993
+13500 train 5.532460 (lr=8.5435e-06) (hash(x)=142579957)
+13600 val loss 5.5321
+13600 val perplexity 252.6708
+13600 train 5.248423 (lr=7.8081e-06) (hash(x)=140719440)
+13600 val loss 5.5737
+13600 val perplexity 263.4108
+13600 train 5.283875 (lr=8.3286e-06) (hash(x)=140719440)
+13700 val loss 5.5288
+13700 val perplexity 251.8322
+13700 train 5.619996 (lr=7.6109e-06) (hash(x)=164396003)
+13700 val loss 5.5726
+13700 val perplexity 263.1302
+13700 train 5.667033 (lr=8.1183e-06) (hash(x)=164396003)
+13800 val loss 5.5220
+13800 val perplexity 250.1227
+13800 train 5.578545 (lr=7.4180e-06) (hash(x)=152712862)
+13800 val loss 5.5655
+13800 val perplexity 261.2472
+13800 train 5.624051 (lr=7.9125e-06) (hash(x)=152712862)
+13900 val loss 5.5178
+13900 val perplexity 249.0800
+13900 train 5.540569 (lr=7.2294e-06) (hash(x)=160985139)
+13900 val loss 5.5627
+13900 val perplexity 260.5220
+13900 train 5.587029 (lr=7.7114e-06) (hash(x)=160985139)
+14000 val loss 5.5167
+14000 val perplexity 248.8097
+14000 train 5.520864 (lr=7.0452e-06) (hash(x)=147429379)
+14000 val loss 5.5613
+14000 val perplexity 260.1528
+14000 train 5.558174 (lr=7.5149e-06) (hash(x)=147429379)
+14100 val loss 5.5154
+14100 val perplexity 248.4913
+14100 train 5.592762 (lr=6.8656e-06) (hash(x)=131830470)
+14100 val loss 5.5583
+14100 val perplexity 259.3858
+14100 train 5.637152 (lr=7.3233e-06) (hash(x)=131830470)
+14200 val loss 5.5162
+14200 val perplexity 248.6958
+14200 train 5.260109 (lr=6.6904e-06) (hash(x)=146383028)
+14200 val loss 5.5595
+14200 val perplexity 259.6955
+14200 train 5.301928 (lr=7.1365e-06) (hash(x)=146383028)
+14300 val loss 5.5127
+14300 val perplexity 247.8100
+14300 train 5.558857 (lr=6.5199e-06) (hash(x)=157158747)
+14300 val loss 5.5570
+14300 val perplexity 259.0358
+14300 train 5.601773 (lr=6.9546e-06) (hash(x)=157158747)
+14400 val loss 5.5163
+14400 val perplexity 248.7070
+14400 train 5.372640 (lr=6.3540e-06) (hash(x)=143911520)
+14400 val loss 5.5601
+14400 val perplexity 259.8445
+14400 train 5.416796 (lr=6.7776e-06) (hash(x)=143911520)
+14500 val loss 5.5152
+14500 val perplexity 248.4450
+14500 train 5.278310 (lr=6.1928e-06) (hash(x)=159555347)
+14500 val loss 5.5598
+14500 val perplexity 259.7617
+14500 train 5.331186 (lr=6.6057e-06) (hash(x)=159555347)
+14600 val loss 5.5092
+14600 val perplexity 246.9438
+14600 train 5.506528 (lr=6.0364e-06) (hash(x)=152957062)
+14600 val loss 5.5535
+14600 val perplexity 258.1299
+14600 train 5.551717 (lr=6.4389e-06) (hash(x)=152957062)
+14700 val loss 5.5050
+14700 val perplexity 245.9211
+14700 train 5.580899 (lr=5.8848e-06) (hash(x)=156001594)
+14700 val loss 5.5486
+14700 val perplexity 256.8655
+14700 train 5.622464 (lr=6.2771e-06) (hash(x)=156001594)
+14800 val loss 5.5047
+14800 val perplexity 245.8406
+14800 train 5.615385 (lr=5.7381e-06) (hash(x)=145885458)
+14800 val loss 5.5474
+14800 val perplexity 256.5798
+14800 train 5.659868 (lr=6.1206e-06) (hash(x)=145885458)
+14900 val loss 5.5016
+14900 val perplexity 245.0851
+14900 train 5.578045 (lr=5.5962e-06) (hash(x)=144544291)
+14900 val loss 5.5439
+14900 val perplexity 255.6789
+14900 train 5.613038 (lr=5.9693e-06) (hash(x)=144544291)
+15000 val loss 5.5013
+15000 val perplexity 245.0193
+15000 train 5.507167 (lr=5.4593e-06) (hash(x)=154606709)
+15000 val loss 5.5454
+15000 val perplexity 256.0479
+15000 train 5.549639 (lr=5.8233e-06) (hash(x)=154606709)
+15100 val loss 5.5017
+15100 val perplexity 245.1157
+15100 train 5.538658 (lr=5.3275e-06) (hash(x)=156005324)
+15100 val loss 5.5445
+15100 val perplexity 255.8389
+15100 train 5.584861 (lr=5.6826e-06) (hash(x)=156005324)
+15200 val loss 5.5032
+15200 val perplexity 245.4764
+15200 train 5.607048 (lr=5.2006e-06) (hash(x)=158847373)
+15200 val loss 5.5465
+15200 val perplexity 256.3338
+15200 train 5.648160 (lr=5.5473e-06) (hash(x)=158847373)
+15300 val loss 5.5038
+15300 val perplexity 245.6311
+15300 train 5.283220 (lr=5.0789e-06) (hash(x)=159609594)
+15300 val loss 5.5466
+15300 val perplexity 256.3549
+15300 train 5.323767 (lr=5.4175e-06) (hash(x)=159609594)
+15400 val loss 5.4993
+15400 val perplexity 244.5235
+15400 train 5.592093 (lr=4.9622e-06) (hash(x)=152428492)
+15400 val loss 5.5411
+15400 val perplexity 254.9683
+15400 train 5.637389 (lr=5.2931e-06) (hash(x)=152428492)
+15500 val loss 5.4957
+15500 val perplexity 243.6322
+15500 train 5.560111 (lr=4.8508e-06) (hash(x)=162251192)
+15500 val loss 5.5371
+15500 val perplexity 253.9309
+15500 train 5.607491 (lr=5.1742e-06) (hash(x)=162251192)
+15600 val loss 5.4948
+15600 val perplexity 243.4196
+15600 train 5.757086 (lr=4.7445e-06) (hash(x)=155072002)
+15600 val loss 5.5366
+15600 val perplexity 253.8090
+15600 train 5.794656 (lr=5.0608e-06) (hash(x)=155072002)
+15700 val loss 5.4938
+15700 val perplexity 243.1737
+15700 train 5.524041 (lr=4.6435e-06) (hash(x)=151203094)
+15700 val loss 5.5359
+15700 val perplexity 253.6246
+15700 train 5.563150 (lr=4.9531e-06) (hash(x)=151203094)
+15800 val loss 5.4922
+15800 val perplexity 242.7884
+15800 train 5.428759 (lr=4.5478e-06) (hash(x)=146125932)
+15800 val loss 5.5348
+15800 val perplexity 253.3518
+15800 train 5.461537 (lr=4.8509e-06) (hash(x)=146125932)
+15900 val loss 5.4931
+15900 val perplexity 243.0073
+15900 train 5.364917 (lr=4.4573e-06) (hash(x)=148060195)
+15900 val loss 5.5345
+15900 val perplexity 253.2838
+15900 train 5.415287 (lr=4.7545e-06) (hash(x)=148060195)
+16000 val loss 5.4935
+16000 val perplexity 243.1048
+16000 train 5.201401 (lr=4.3722e-06) (hash(x)=137406416)
+16000 val loss 5.5355
+16000 val perplexity 253.5310
+16000 train 5.242233 (lr=4.6637e-06) (hash(x)=137406416)
+16100 val loss 5.4938
+16100 val perplexity 243.1702
+16100 train 5.557673 (lr=4.2924e-06) (hash(x)=160149755)
+16100 val loss 5.5355
+16100 val perplexity 253.5390
+16100 train 5.597098 (lr=4.5786e-06) (hash(x)=160149755)
+16200 val loss 5.4883
+16200 val perplexity 241.8399
+16200 train 5.672664 (lr=4.2180e-06) (hash(x)=151371894)
+16200 val loss 5.5305
+16200 val perplexity 252.2687
+16200 train 5.709032 (lr=4.4993e-06) (hash(x)=151371894)
+16300 val loss 5.4856
+16300 val perplexity 241.1907
+16300 train 5.551252 (lr=4.1491e-06) (hash(x)=151810252)
+16300 val loss 5.5286
+16300 val perplexity 251.7811
+16300 train 5.583846 (lr=4.4257e-06) (hash(x)=151810252)
+16400 val loss 5.4843
+16400 val perplexity 240.8846
+16400 train 5.470860 (lr=4.0856e-06) (hash(x)=153088303)
+16400 val loss 5.5272
+16400 val perplexity 251.4453
+16400 train 5.514785 (lr=4.3579e-06) (hash(x)=153088303)
+16500 val loss 5.4839
+16500 val perplexity 240.7784
+16500 train 5.350903 (lr=4.0275e-06) (hash(x)=151706615)
+16500 val loss 5.5262
+16500 val perplexity 251.1843
+16500 train 5.398167 (lr=4.2960e-06) (hash(x)=151706615)
+16600 val loss 5.4829
+16600 val perplexity 240.5481
+16600 train 5.380322 (lr=3.9749e-06) (hash(x)=142110626)
+16600 val loss 5.5248
+16600 val perplexity 250.8432
+16600 train 5.422053 (lr=4.2399e-06) (hash(x)=142110626)
+16700 val loss 5.4836
+16700 val perplexity 240.7059
+16700 train 5.376807 (lr=3.9278e-06) (hash(x)=153222439)
+16700 val loss 5.5262
+16700 val perplexity 251.1830
+16700 train 5.419888 (lr=4.1896e-06) (hash(x)=153222439)
+16800 val loss 5.4831
+16800 val perplexity 240.5931
+16800 train 5.391656 (lr=3.8862e-06) (hash(x)=154964902)
+16800 val loss 5.5252
+16800 val perplexity 250.9311
+16800 train 5.448435 (lr=4.1452e-06) (hash(x)=154964902)
+16900 val loss 5.4795
+16900 val perplexity 239.7348
+16900 train 5.667234 (lr=3.8501e-06) (hash(x)=158893204)
+16900 val loss 5.5213
+16900 val perplexity 249.9718
+16900 train 5.701284 (lr=4.1067e-06) (hash(x)=158893204)
+17000 val loss 5.4779
+17000 val perplexity 239.3489
+17000 train 5.411547 (lr=3.8195e-06) (hash(x)=148929955)
+17000 val loss 5.5203
+17000 val perplexity 249.7003
+17000 train 5.451448 (lr=4.0741e-06) (hash(x)=148929955)
+17100 val loss 5.4779
+17100 val perplexity 239.3535
+17100 train 5.500988 (lr=3.7945e-06) (hash(x)=147818677)
+17100 val loss 5.5206
+17100 val perplexity 249.7858
+17100 train 5.537355 (lr=4.0475e-06) (hash(x)=147818677)
+17200 val loss 5.4761
+17200 val perplexity 238.9149
+17200 train 5.405346 (lr=3.7750e-06) (hash(x)=147119446)
+17200 val loss 5.5186
+17200 val perplexity 249.2853
+17200 train 5.448709 (lr=4.0267e-06) (hash(x)=147119446)
+17300 val loss 5.4766
+17300 val perplexity 239.0285
+17300 train 5.538480 (lr=3.7611e-06) (hash(x)=150494142)
+17300 val loss 5.5197
+17300 val perplexity 249.5582
+17300 train 5.575846 (lr=4.0119e-06) (hash(x)=150494142)
+17400 val loss 5.4770
+17400 val perplexity 239.1375
+17400 train 5.496178 (lr=3.7528e-06) (hash(x)=155918058)
+17400 val loss 5.5196
+17400 val perplexity 249.5304
+17400 train 5.545844 (lr=4.0030e-06) (hash(x)=155918058)
+17499 val loss 5.4740
+17499 val perplexity 238.4194
+17499 val loss 5.5160
+17499 val perplexity 248.6281

attention_kindselective_n_heads4_seed1342/model_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9def388577ca6684b4d2318222575b30520f226958f22a1a955916b8dc34087
+oid sha256:fc3e5e9b1c20d56afc6bd9b1dc72192198afc3101d5b705ad19a4f9f086907ad
 size 92843394

attention_kindselective_n_heads4_seed1342/optimizer_17499.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3631c59d915628c94b2e90c73fd41a51ac337ccb57c78c9ceccbe5cf36299c38
+oid sha256:1e29c784964341b5240e50204165212914ec7ea50ca24a1748ff5c0632e38d84
 size 179406214