Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1342/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.75e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.75e-5_30720_4_1342", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_61440_4_1342", "n_embd": 256}

attention_kindselective_n_heads4_seed1342/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

attention_kindselective_n_heads4_seed1342/log2.txt

@@ -1,1056 +1,105 @@
-max_steps: 17500
-max_steps: 17500
-0 val loss 11.2907
-0 val perplexity 80077.4453
-0 val loss 11.2907
-0 val perplexity 80077.4453
-0 train 11.286213 (lr=2.0000e-07) (hash(x)=150332693)
-0 train 11.286213 (lr=1.8750e-07) (hash(x)=150332693)
-100 val loss 9.8768
-100 val perplexity 19473.3750
-100 train 9.899889 (lr=2.0200e-05) (hash(x)=148948662)
-100 val loss 9.8862
-100 val perplexity 19657.5391
-100 train 9.909360 (lr=1.8937e-05) (hash(x)=148948662)
-200 val loss 8.6349
-200 val perplexity 5624.3892
-200 train 8.549774 (lr=4.0000e-05) (hash(x)=149277319)
-200 val loss 8.6742
-200 val perplexity 5850.1021
-200 train 8.591024 (lr=3.7500e-05) (hash(x)=149277319)
-300 val loss 7.7933
-300 val perplexity 2424.3916
-300 train 7.673641 (lr=3.9997e-05) (hash(x)=146429920)
-300 val loss 7.7411
-300 val perplexity 2300.9182
-300 train 7.615427 (lr=3.7497e-05) (hash(x)=146429920)
-400 val loss 7.5715
-400 val perplexity 1942.0657
-400 train 7.687140 (lr=3.9988e-05) (hash(x)=155934075)
-400 val loss 7.5523
-400 val perplexity 1905.0985
-400 train 7.665451 (lr=3.7489e-05) (hash(x)=155934075)
-500 val loss 7.4870
-500 val perplexity 1784.6857
-500 train 7.494686 (lr=3.9973e-05) (hash(x)=157510366)
-500 val loss 7.4642
-500 val perplexity 1744.5452
-500 train 7.467919 (lr=3.7475e-05) (hash(x)=157510366)
-600 val loss 7.4308
-600 val perplexity 1687.1985
-600 train 7.547135 (lr=3.9953e-05) (hash(x)=165402628)
-600 val loss 7.3924
-600 val perplexity 1623.6310
-600 train 7.507157 (lr=3.7456e-05) (hash(x)=165402628)
-700 val loss 7.3856
-700 val perplexity 1612.6390
-700 train 7.444174 (lr=3.9926e-05) (hash(x)=166427433)
-700 val loss 7.3212
-700 val perplexity 1512.0294
-700 train 7.385526 (lr=3.7430e-05) (hash(x)=166427433)
-800 val loss 7.3425
-800 val perplexity 1544.5521
-800 train 7.137204 (lr=3.9893e-05) (hash(x)=139168939)
-800 val loss 7.2604
-800 val perplexity 1422.8741
-800 train 7.058400 (lr=3.7400e-05) (hash(x)=139168939)
-900 val loss 7.3042
-900 val perplexity 1486.5560
-900 train 7.209212 (lr=3.9855e-05) (hash(x)=144698667)
-900 val loss 7.2081
-900 val perplexity 1350.3298
-900 train 7.108169 (lr=3.7364e-05) (hash(x)=144698667)
-1000 val loss 7.2667
-1000 val perplexity 1431.8467
-1000 val loss 7.1451
-1000 val perplexity 1267.8656
-1000 train 7.247368 (lr=3.9810e-05) (hash(x)=147134338)
-1000 train 7.127390 (lr=3.7322e-05) (hash(x)=147134338)
-1100 val loss 7.0904
-1100 val perplexity 1200.3669
-1100 val loss 7.2316
-1100 val perplexity 1382.4969
-1100 train 7.058753 (lr=3.7275e-05) (hash(x)=147172640)
-1100 train 7.195152 (lr=3.9760e-05) (hash(x)=147172640)
-1200 val loss 7.0314
-1200 val perplexity 1131.6199
-1200 train 7.031106 (lr=3.7223e-05) (hash(x)=159086459)
-1200 val loss 7.2021
-1200 val perplexity 1342.2770
-1200 train 7.207237 (lr=3.9704e-05) (hash(x)=159086459)
-1300 val loss 6.9726
-1300 val perplexity 1067.0111
-1300 train 6.723200 (lr=3.7164e-05) (hash(x)=141578940)
-1300 val loss 7.1625
-1300 val perplexity 1290.1248
-1300 train 6.937757 (lr=3.9642e-05) (hash(x)=141578940)
-1400 val loss 6.9289
-1400 val perplexity 1021.3248
-1400 train 6.988761 (lr=3.7101e-05) (hash(x)=152399099)
-1400 val loss 7.1186
-1400 val perplexity 1234.7281
-1400 train 7.181373 (lr=3.9574e-05) (hash(x)=152399099)
-1500 val loss 6.8611
-1500 val perplexity 954.4279
-1500 train 6.944953 (lr=3.7032e-05) (hash(x)=158569712)
-1500 val loss 7.0568
-1500 val perplexity 1160.7614
-1500 train 7.136433 (lr=3.9501e-05) (hash(x)=158569712)
-1600 val loss 6.8073
-1600 val perplexity 904.4675
-1600 train 6.901510 (lr=3.6958e-05) (hash(x)=156548843)
-1600 val loss 7.0105
-1600 val perplexity 1108.2358
-1600 train 7.107373 (lr=3.9421e-05) (hash(x)=156548843)
-1700 val loss 6.7516
-1700 val perplexity 855.4005
-1700 train 6.759593 (lr=3.6878e-05) (hash(x)=152608173)
-1700 val loss 6.9768
-1700 val perplexity 1071.5275
-1700 train 6.989625 (lr=3.9336e-05) (hash(x)=152608173)
-1800 val loss 6.7063
-1800 val perplexity 817.5263
-1800 train 6.779020 (lr=3.6793e-05) (hash(x)=157889911)
-1800 val loss 6.9349
-1800 val perplexity 1027.5291
-1800 train 7.009769 (lr=3.9246e-05) (hash(x)=157889911)
-1900 val loss 6.6729
-1900 val perplexity 790.6511
-1900 train 6.639544 (lr=3.6702e-05) (hash(x)=140839139)
-1900 val loss 6.9128
-1900 val perplexity 1005.0913
-1900 train 6.870525 (lr=3.9149e-05) (hash(x)=140839139)
-2000 val loss 6.6417
-2000 val perplexity 766.3823
-2000 train 6.528852 (lr=3.6606e-05) (hash(x)=142943707)
-2000 val loss 6.8705
-2000 val perplexity 963.4188
-2000 train 6.761942 (lr=3.9047e-05) (hash(x)=142943707)
-2100 val loss 6.6061
-2100 val perplexity 739.6149
-2100 train 6.573959 (lr=3.6505e-05) (hash(x)=151025344)
-2100 val loss 6.8408
-2100 val perplexity 935.2663
-2100 train 6.814910 (lr=3.8939e-05) (hash(x)=151025344)
-2200 val loss 6.5747
-2200 val perplexity 716.7421
-2200 train 6.582545 (lr=3.6399e-05) (hash(x)=148938689)
-2200 val loss 6.8176
-2200 val perplexity 913.7847
-2200 train 6.824997 (lr=3.8826e-05) (hash(x)=148938689)
-2300 val loss 6.5303
-2300 val perplexity 685.6262
-2300 train 6.494017 (lr=3.6288e-05) (hash(x)=150321226)
-2300 val loss 6.7555
-2300 val perplexity 858.7787
-2300 train 6.711614 (lr=3.8707e-05) (hash(x)=150321226)
-2400 val loss 6.4979
-2400 val perplexity 663.7457
-2400 train 6.736341 (lr=3.6171e-05) (hash(x)=161962903)
-2400 val loss 6.7165
-2400 val perplexity 825.9503
-2400 train 6.929638 (lr=3.8583e-05) (hash(x)=161962903)
-2500 val loss 6.4677
-2500 val perplexity 643.9689
-2500 train 6.404032 (lr=3.6049e-05) (hash(x)=153085222)
-2500 val loss 6.6707
-2500 val perplexity 788.9158
-2500 train 6.590450 (lr=3.8453e-05) (hash(x)=153085222)
-2600 val loss 6.4401
-2600 val perplexity 626.4914
-2600 train 6.491509 (lr=3.5923e-05) (hash(x)=145979235)
-2600 val loss 6.6330
-2600 val perplexity 759.7482
-2600 train 6.684030 (lr=3.8317e-05) (hash(x)=145979235)
-2700 val loss 6.4106
-2700 val perplexity 608.2325
-2700 train 6.479415 (lr=3.5791e-05) (hash(x)=149091945)
-2700 val loss 6.5907
-2700 val perplexity 728.3218
-2700 train 6.650594 (lr=3.8177e-05) (hash(x)=149091945)
-2800 val loss 6.3838
-2800 val perplexity 592.1553
-2800 train 6.473955 (lr=3.5654e-05) (hash(x)=159157507)
-2800 val loss 6.5587
-2800 val perplexity 705.3589
-2800 train 6.634536 (lr=3.8031e-05) (hash(x)=159157507)
-2900 val loss 6.3717
-2900 val perplexity 585.0361
-2900 train 6.383224 (lr=3.5512e-05) (hash(x)=155377499)
-2900 val loss 6.5366
-2900 val perplexity 689.9300
-2900 train 6.539573 (lr=3.7879e-05) (hash(x)=155377499)
-3000 val loss 6.3532
-3000 val perplexity 574.3139
-3000 train 6.320881 (lr=3.5365e-05) (hash(x)=145943130)
-3000 val loss 6.5047
-3000 val perplexity 668.2750
-3000 train 6.473698 (lr=3.7723e-05) (hash(x)=145943130)
-3100 val loss 6.3280
-3100 val perplexity 560.0309
-3100 train 6.382774 (lr=3.5214e-05) (hash(x)=153264973)
-3100 val loss 6.4704
-3100 val perplexity 645.7274
-3100 train 6.523706 (lr=3.7561e-05) (hash(x)=153264973)
-3200 val loss 6.3052
-3200 val perplexity 547.4370
-3200 train 6.144718 (lr=3.5057e-05) (hash(x)=143249173)
-3200 val loss 6.4375
-3200 val perplexity 624.8588
-3200 train 6.272245 (lr=3.7394e-05) (hash(x)=143249173)
-3300 val loss 6.2948
-3300 val perplexity 541.7688
-3300 train 6.303265 (lr=3.4896e-05) (hash(x)=155558767)
-3300 val loss 6.4252
-3300 val perplexity 617.2025
-3300 train 6.432137 (lr=3.7222e-05) (hash(x)=155558767)
-3400 val loss 6.2653
-3400 val perplexity 526.0201
-3400 train 6.206819 (lr=3.4730e-05) (hash(x)=144137265)
-3400 val loss 6.3850
-3400 val perplexity 592.8885
-3400 train 6.325558 (lr=3.7045e-05) (hash(x)=144137265)
-3500 val loss 6.2389
-3500 val perplexity 512.2823
-3500 train 6.433990 (lr=3.4560e-05) (hash(x)=154379201)
-3500 val loss 6.3609
-3500 val perplexity 578.7686
-3500 train 6.544823 (lr=3.6864e-05) (hash(x)=154379201)
-3600 val loss 6.2298
-3600 val perplexity 507.6419
-3600 train 6.364053 (lr=3.4384e-05) (hash(x)=158728918)
-3600 val loss 6.3463
-3600 val perplexity 570.3826
-3600 train 6.472357 (lr=3.6677e-05) (hash(x)=158728918)
-3700 val loss 6.2122
-3700 val perplexity 498.8144
-3700 train 5.966904 (lr=3.4205e-05) (hash(x)=139902844)
-3700 val loss 6.3319
-3700 val perplexity 562.2496
-3700 train 6.076136 (lr=3.6485e-05) (hash(x)=139902844)
-3800 val loss 6.2045
-3800 val perplexity 494.9547
-3800 train 6.286787 (lr=3.4021e-05) (hash(x)=160738928)
-3800 val loss 6.3220
-3800 val perplexity 556.6725
-3800 train 6.395666 (lr=3.6289e-05) (hash(x)=160738928)
-3900 val loss 6.1780
-3900 val perplexity 482.0338
-3900 train 6.223629 (lr=3.3832e-05) (hash(x)=156122233)
-3900 val loss 6.2837
-3900 val perplexity 535.7934
-3900 train 6.330824 (lr=3.6088e-05) (hash(x)=156122233)
-4000 val loss 6.1577
-4000 val perplexity 472.3254
-4000 train 5.994511 (lr=3.3639e-05) (hash(x)=136691174)
-4000 val loss 6.2657
-4000 val perplexity 526.1896
-4000 train 6.095181 (lr=3.5882e-05) (hash(x)=136691174)
-4100 val loss 6.1589
-4100 val perplexity 472.9088
-4100 train 6.126831 (lr=3.3442e-05) (hash(x)=150135523)
-4100 val loss 6.2577
-4100 val perplexity 522.0146
-4100 train 6.226404 (lr=3.5671e-05) (hash(x)=150135523)
-4200 val loss 6.1453
-4200 val perplexity 466.4993
-4200 train 6.137742 (lr=3.3240e-05) (hash(x)=154053307)
-4200 val loss 6.2370
-4200 val perplexity 511.2998
-4200 train 6.229482 (lr=3.5457e-05) (hash(x)=154053307)
-4300 val loss 6.1309
-4300 val perplexity 459.8623
-4300 train 5.923357 (lr=3.3035e-05) (hash(x)=136076439)
-4300 val loss 6.2232
-4300 val perplexity 504.3218
-4300 train 6.012386 (lr=3.5237e-05) (hash(x)=136076439)
-4400 val loss 6.1126
-4400 val perplexity 451.4940
-4400 train 6.003034 (lr=3.2825e-05) (hash(x)=147368042)
-4400 val loss 6.2035
-4400 val perplexity 494.4707
-4400 train 6.094748 (lr=3.5014e-05) (hash(x)=147368042)
-4500 val loss 6.1012
-4500 val perplexity 446.3787
-4500 train 6.190740 (lr=3.2611e-05) (hash(x)=163726147)
-4500 val loss 6.1846
-4500 val perplexity 485.2155
-4500 train 6.276649 (lr=3.4786e-05) (hash(x)=163726147)
-4600 val loss 6.0856
-4600 val perplexity 439.5045
-4600 train 6.098429 (lr=3.2394e-05) (hash(x)=151996388)
-4600 val loss 6.1658
-4600 val perplexity 476.1843
-4600 train 6.189225 (lr=3.4553e-05) (hash(x)=151996388)
-4700 val loss 6.0705
-4700 val perplexity 432.9117
-4700 train 6.129060 (lr=3.2172e-05) (hash(x)=148952763)
-4700 val loss 6.1439
-4700 val perplexity 465.8540
-4700 train 6.196408 (lr=3.4317e-05) (hash(x)=148952763)
-4800 val loss 6.0627
-4800 val perplexity 429.5331
-4800 train 5.990180 (lr=3.1947e-05) (hash(x)=139747769)
-4800 val loss 6.1356
-4800 val perplexity 462.0074
-4800 train 6.060421 (lr=3.4077e-05) (hash(x)=139747769)
-4900 val loss 6.0544
-4900 val perplexity 425.9879
-4900 train 5.932305 (lr=3.1718e-05) (hash(x)=145011383)
-4900 val loss 6.1256
-4900 val perplexity 457.3985
-4900 train 6.002694 (lr=3.3832e-05) (hash(x)=145011383)
-5000 val loss 6.0411
-5000 val perplexity 420.3618
-5000 train 6.063803 (lr=3.1485e-05) (hash(x)=153449428)
-5000 val loss 6.1166
-5000 val perplexity 453.3174
-5000 train 6.139397 (lr=3.3584e-05) (hash(x)=153449428)
-5100 val loss 6.0310
-5100 val perplexity 416.1304
-5100 train 6.080400 (lr=3.1249e-05) (hash(x)=161397672)
-5100 val loss 6.0984
-5100 val perplexity 445.1656
-5100 train 6.141521 (lr=3.3332e-05) (hash(x)=161397672)
-5200 val loss 6.0118
-5200 val perplexity 408.2232
-5200 train 5.999507 (lr=3.1009e-05) (hash(x)=164813156)
-5200 val loss 6.0819
-5200 val perplexity 437.8560
-5200 train 6.071654 (lr=3.3076e-05) (hash(x)=164813156)
-5300 val loss 5.9973
-5300 val perplexity 402.3416
-5300 train 6.054934 (lr=3.0766e-05) (hash(x)=146155759)
-5300 val loss 6.0705
-5300 val perplexity 432.8826
-5300 train 6.119602 (lr=3.2817e-05) (hash(x)=146155759)
-5400 val loss 5.9907
-5400 val perplexity 399.7007
-5400 train 6.072926 (lr=3.0519e-05) (hash(x)=152337652)
-5400 val loss 6.0599
-5400 val perplexity 428.3523
-5400 train 6.129765 (lr=3.2554e-05) (hash(x)=152337652)
-5500 val loss 5.9846
-5500 val perplexity 397.2581
-5500 train 5.847214 (lr=3.0269e-05) (hash(x)=148346631)
-5500 val loss 6.0540
-5500 val perplexity 425.8023
-5500 train 5.916514 (lr=3.2287e-05) (hash(x)=148346631)
-5600 val loss 5.9770
-5600 val perplexity 394.2721
-5600 train 5.882283 (lr=3.0016e-05) (hash(x)=146652463)
-5600 val loss 6.0421
-5600 val perplexity 420.7834
-5600 train 5.945309 (lr=3.2017e-05) (hash(x)=146652463)
-5700 val loss 5.9691
-5700 val perplexity 391.1408
-5700 train 5.921083 (lr=2.9760e-05) (hash(x)=147553632)
-5700 val loss 6.0329
-5700 val perplexity 416.9108
-5700 train 5.973693 (lr=3.1744e-05) (hash(x)=147553632)
-5800 val loss 5.9471
-5800 val perplexity 382.6452
-5800 train 6.046297 (lr=2.9501e-05) (hash(x)=153512314)
-5800 val loss 6.0122
-5800 val perplexity 408.3965
-5800 train 6.112329 (lr=3.1468e-05) (hash(x)=153512314)
-5900 val loss 5.9382
-5900 val perplexity 379.2424
-5900 train 5.917999 (lr=2.9239e-05) (hash(x)=137351707)
-5900 val loss 5.9983
-5900 val perplexity 402.7321
-5900 train 5.978490 (lr=3.1188e-05) (hash(x)=137351707)
-6000 val loss 5.9298
-6000 val perplexity 376.0757
-6000 train 5.960650 (lr=2.8974e-05) (hash(x)=143375055)
-6000 val loss 5.9937
-6000 val perplexity 400.8940
-6000 train 6.022861 (lr=3.0905e-05) (hash(x)=143375055)
-6100 val loss 5.9277
-6100 val perplexity 375.2952
-6100 train 5.960719 (lr=2.8706e-05) (hash(x)=144781857)
-6100 val loss 5.9934
-6100 val perplexity 400.7845
-6100 train 6.023008 (lr=3.0620e-05) (hash(x)=144781857)
-6200 val loss 5.9126
-6200 val perplexity 369.6685
-6200 train 6.142590 (lr=2.8436e-05) (hash(x)=167556812)
-6200 val loss 5.9725
-6200 val perplexity 392.4781
-6200 train 6.209857 (lr=3.0332e-05) (hash(x)=167556812)
-6300 val loss 5.9028
-6300 val perplexity 366.0782
-6300 train 6.067983 (lr=2.8163e-05) (hash(x)=162290597)
-6300 val loss 5.9647
-6300 val perplexity 389.4418
-6300 train 6.122912 (lr=3.0040e-05) (hash(x)=162290597)
-6400 val loss 5.8906
-6400 val perplexity 361.6111
-6400 train 5.784720 (lr=2.7888e-05) (hash(x)=144836361)
-6400 val loss 5.9561
-6400 val perplexity 386.1159
-6400 train 5.860235 (lr=2.9747e-05) (hash(x)=144836361)
-6500 val loss 5.8787
-6500 val perplexity 357.3279
-6500 train 5.761406 (lr=2.7610e-05) (hash(x)=150949131)
-6500 val loss 5.9379
-6500 val perplexity 379.1356
-6500 train 5.818823 (lr=2.9450e-05) (hash(x)=150949131)
-6600 val loss 5.8904
-6600 val perplexity 361.5419
-6600 train 5.956506 (lr=2.7330e-05) (hash(x)=151386312)
-6600 val loss 5.9497
-6600 val perplexity 383.6486
-6600 train 6.014431 (lr=2.9152e-05) (hash(x)=151386312)
-6700 val loss 5.8572
-6700 val perplexity 349.7287
-6700 train 5.944593 (lr=2.7047e-05) (hash(x)=159840718)
-6700 val loss 5.9174
-6700 val perplexity 371.4353
-6700 train 5.998442 (lr=2.8851e-05) (hash(x)=159840718)
-6800 val loss 5.8479
-6800 val perplexity 346.5221
-6800 train 5.949664 (lr=2.6763e-05) (hash(x)=158843242)
-6800 val loss 5.9085
-6800 val perplexity 368.1369
-6800 train 6.006040 (lr=2.8547e-05) (hash(x)=158843242)
-6900 val loss 5.8432
-6900 val perplexity 344.8760
-6900 train 6.077638 (lr=2.6477e-05) (hash(x)=160544793)
-6900 val loss 5.9060
-6900 val perplexity 367.2513
-6900 train 6.131098 (lr=2.8242e-05) (hash(x)=160544793)
-7000 val loss 5.8336
-7000 val perplexity 341.5859
-7000 train 5.691507 (lr=2.6188e-05) (hash(x)=139031423)
-7000 val loss 5.8963
-7000 val perplexity 363.6898
-7000 train 5.744294 (lr=2.7934e-05) (hash(x)=139031423)
-7100 val loss 5.8322
-7100 val perplexity 341.1207
-7100 train 5.789629 (lr=2.5898e-05) (hash(x)=148444150)
-7100 val loss 5.8901
-7100 val perplexity 361.4375
-7100 train 5.837803 (lr=2.7624e-05) (hash(x)=148444150)
-7200 val loss 5.8216
-7200 val perplexity 337.5242
-7200 train 6.172296 (lr=2.5606e-05) (hash(x)=162348298)
-7200 val loss 5.8795
-7200 val perplexity 357.6189
-7200 train 6.223478 (lr=2.7313e-05) (hash(x)=162348298)
-7300 val loss 5.8094
-7300 val perplexity 333.4139
-7300 train 5.849937 (lr=2.5312e-05) (hash(x)=160070640)
-7300 val loss 5.8669
-7300 val perplexity 353.1607
-7300 train 5.919557 (lr=2.7000e-05) (hash(x)=160070640)
-7400 val loss 5.8009
-7400 val perplexity 330.6072
-7400 train 5.728707 (lr=2.5017e-05) (hash(x)=142907242)
-7400 val loss 5.8618
-7400 val perplexity 351.3591
-7400 train 5.789402 (lr=2.6685e-05) (hash(x)=142907242)
-7500 val loss 5.8015
-7500 val perplexity 330.8050
-7500 train 5.783610 (lr=2.4721e-05) (hash(x)=155567735)
-7500 val loss 5.8515
-7500 val perplexity 347.7528
-7500 train 5.846130 (lr=2.6369e-05) (hash(x)=155567735)
-7600 val loss 5.7887
-7600 val perplexity 326.5765
-7600 train 5.645206 (lr=2.4423e-05) (hash(x)=149026190)
-7600 val loss 5.8432
-7600 val perplexity 344.8734
-7600 train 5.700372 (lr=2.6051e-05) (hash(x)=149026190)
-7700 val loss 5.7812
-7700 val perplexity 324.1628
-7700 train 6.316875 (lr=2.4124e-05) (hash(x)=172973386)
-7700 val loss 5.8359
-7700 val perplexity 342.3663
-7700 train 6.377451 (lr=2.5732e-05) (hash(x)=172973386)
-7800 val loss 5.7737
-7800 val perplexity 321.7310
-7800 train 5.766070 (lr=2.3823e-05) (hash(x)=148166384)
-7800 val loss 5.8271
-7800 val perplexity 339.3716
-7800 train 5.799076 (lr=2.5411e-05) (hash(x)=148166384)
-7900 val loss 5.7741
-7900 val perplexity 321.8507
-7900 train 6.198745 (lr=2.3522e-05) (hash(x)=174580801)
-7900 val loss 5.8263
-7900 val perplexity 339.0856
-7900 train 6.266204 (lr=2.5090e-05) (hash(x)=174580801)
-8000 val loss 5.7541
-8000 val perplexity 315.4787
-8000 train 5.774290 (lr=2.3219e-05) (hash(x)=153576391)
-8000 val loss 5.8086
-8000 val perplexity 333.1376
-8000 train 5.828216 (lr=2.4767e-05) (hash(x)=153576391)
-8100 val loss 5.7513
-8100 val perplexity 314.5843
-8100 train 5.638414 (lr=2.2916e-05) (hash(x)=143483378)
-8100 val loss 5.8040
-8100 val perplexity 331.6256
-8100 train 5.701946 (lr=2.4444e-05) (hash(x)=143483378)
-8200 val loss 5.7457
-8200 val perplexity 312.8474
-8200 train 6.278048 (lr=2.2612e-05) (hash(x)=125210829)
-8200 val loss 5.7982
-8200 val perplexity 329.7017
-8200 train 6.284613 (lr=2.4120e-05) (hash(x)=125210829)
-8300 val loss 5.7391
-8300 val perplexity 310.7990
-8300 train 5.668536 (lr=2.2308e-05) (hash(x)=152432601)
-8300 val loss 5.7916
-8300 val perplexity 327.5516
-8300 train 5.714247 (lr=2.3795e-05) (hash(x)=152432601)
-8400 val loss 5.7390
-8400 val perplexity 310.7404
-8400 train 5.612227 (lr=2.2002e-05) (hash(x)=148759990)
-8400 val loss 5.7956
-8400 val perplexity 328.8651
-8400 train 5.674462 (lr=2.3469e-05) (hash(x)=148759990)
-8500 val loss 5.7250
-8500 val perplexity 306.4205
-8500 train 5.687849 (lr=2.1697e-05) (hash(x)=156974533)
-8500 val loss 5.7785
-8500 val perplexity 323.2726
-8500 train 5.754704 (lr=2.3143e-05) (hash(x)=156974533)
-8600 val loss 5.7160
-8600 val perplexity 303.6733
-8600 train 5.836418 (lr=2.1391e-05) (hash(x)=155441439)
-8600 val loss 5.7721
-8600 val perplexity 321.2024
-8600 train 5.887455 (lr=2.2817e-05) (hash(x)=155441439)
-8700 val loss 5.7128
-8700 val perplexity 302.7141
-8700 train 5.697697 (lr=2.1085e-05) (hash(x)=149001504)
-8700 val loss 5.7634
-8700 val perplexity 318.4435
-8700 train 5.748235 (lr=2.2490e-05) (hash(x)=149001504)
-8800 val loss 5.7084
-8800 val perplexity 301.3885
-8800 train 5.586840 (lr=2.0778e-05) (hash(x)=144721722)
-8800 val loss 5.7601
-8800 val perplexity 317.3691
-8800 train 5.637744 (lr=2.2163e-05) (hash(x)=144721722)
-8900 val loss 5.7057
-8900 val perplexity 300.5775
-8900 train 5.609116 (lr=2.0472e-05) (hash(x)=151789511)
-8900 val loss 5.7535
-8900 val perplexity 315.2899
-8900 train 5.661813 (lr=2.1837e-05) (hash(x)=151789511)
-9000 val loss 5.6970
-9000 val perplexity 297.9864
-9000 train 5.631220 (lr=2.0165e-05) (hash(x)=151307797)
-9000 val loss 5.7489
-9000 val perplexity 313.8452
-9000 train 5.678300 (lr=2.1510e-05) (hash(x)=151307797)
-9100 val loss 5.6909
-9100 val perplexity 296.1675
-9100 train 5.818474 (lr=1.9859e-05) (hash(x)=155534554)
-9100 val loss 5.7447
-9100 val perplexity 312.5339
-9100 train 5.884479 (lr=2.1183e-05) (hash(x)=155534554)
-9200 val loss 5.6822
-9200 val perplexity 293.5884
-9200 train 5.587260 (lr=1.9553e-05) (hash(x)=138467356)
-9200 val loss 5.7322
-9200 val perplexity 308.6354
-9200 train 5.624728 (lr=2.0857e-05) (hash(x)=138467356)
-9300 val loss 5.6811
-9300 val perplexity 293.2634
-9300 train 5.622063 (lr=1.9248e-05) (hash(x)=148622510)
-9300 val loss 5.7291
-9300 val perplexity 307.7005
-9300 train 5.665057 (lr=2.0531e-05) (hash(x)=148622510)
-9400 val loss 5.6739
-9400 val perplexity 291.1561
-9400 train 5.816579 (lr=1.8942e-05) (hash(x)=154343672)
-9400 val loss 5.7260
-9400 val perplexity 306.7281
-9400 train 5.859208 (lr=2.0205e-05) (hash(x)=154343672)
-9500 val loss 5.6728
-9500 val perplexity 290.8406
-9500 train 5.656345 (lr=1.8638e-05) (hash(x)=156831628)
-9500 val loss 5.7223
-9500 val perplexity 305.6166
-9500 train 5.709967 (lr=1.9880e-05) (hash(x)=156831628)
-9600 val loss 5.6668
-9600 val perplexity 289.1156
-9600 train 5.442144 (lr=1.8334e-05) (hash(x)=142190812)
-9600 val loss 5.7179
-9600 val perplexity 304.2583
-9600 train 5.493146 (lr=1.9556e-05) (hash(x)=142190812)
-9700 val loss 5.6603
-9700 val perplexity 287.2379
-9700 train 5.565337 (lr=1.8031e-05) (hash(x)=146487376)
-9700 val loss 5.7078
-9700 val perplexity 301.1933
-9700 train 5.615256 (lr=1.9233e-05) (hash(x)=146487376)
-9800 val loss 5.6522
-9800 val perplexity 284.9193
-9800 train 5.710732 (lr=1.7728e-05) (hash(x)=164178945)
-9800 val loss 5.7028
-9800 val perplexity 299.6926
-9800 train 5.779359 (lr=1.8910e-05) (hash(x)=164178945)
-9900 val loss 5.6502
-9900 val perplexity 284.3600
-9900 train 5.516235 (lr=1.7427e-05) (hash(x)=142224918)
-9900 val loss 5.6991
-9900 val perplexity 298.5944
-9900 train 5.561543 (lr=1.8589e-05) (hash(x)=142224918)
-10000 val loss 5.6520
-10000 val perplexity 284.8605
-10000 train 5.579762 (lr=1.7126e-05) (hash(x)=148509740)
-10000 val loss 5.6977
-10000 val perplexity 298.1938
-10000 train 5.623291 (lr=1.8268e-05) (hash(x)=148509740)
-10100 val loss 5.6415
-10100 val perplexity 281.8954
-10100 train 5.708606 (lr=1.6827e-05) (hash(x)=153536520)
-10100 val loss 5.6884
-10100 val perplexity 295.4208
-10100 train 5.755403 (lr=1.7949e-05) (hash(x)=153536520)
-10200 val loss 5.6333
-10200 val perplexity 279.5924
-10200 train 5.571514 (lr=1.6529e-05) (hash(x)=153740168)
-10200 val loss 5.6808
-10200 val perplexity 293.1710
-10200 train 5.616961 (lr=1.7631e-05) (hash(x)=153740168)
-10300 val loss 5.6513
-10300 val perplexity 284.6588
-10300 train 5.605822 (lr=1.6233e-05) (hash(x)=160410260)
-10300 val loss 5.6982
-10300 val perplexity 298.3185
-10300 train 5.657148 (lr=1.7315e-05) (hash(x)=160410260)
-10400 val loss 5.6261
-10400 val perplexity 277.5663
-10400 train 5.775826 (lr=1.5938e-05) (hash(x)=136788294)
-10400 val loss 5.6720
-10400 val perplexity 290.6160
-10400 train 5.818732 (lr=1.7000e-05) (hash(x)=136788294)
-10500 val loss 5.6201
-10500 val perplexity 275.9232
-10500 train 5.476357 (lr=1.5644e-05) (hash(x)=153988760)
-10500 val loss 5.6673
-10500 val perplexity 289.2501
-10500 train 5.514534 (lr=1.6687e-05) (hash(x)=153988760)
-10600 val loss 5.6196
-10600 val perplexity 275.7818
-10600 train 5.509031 (lr=1.5352e-05) (hash(x)=141896522)
-10600 val loss 5.6644
-10600 val perplexity 288.4057
-10600 train 5.548376 (lr=1.6376e-05) (hash(x)=141896522)
-10700 val loss 5.6178
-10700 val perplexity 275.2754
-10700 train 5.978009 (lr=1.5062e-05) (hash(x)=147057702)
-10700 val loss 5.6629
-10700 val perplexity 287.9748
-10700 train 6.012309 (lr=1.6066e-05) (hash(x)=147057702)
-10800 val loss 5.6056
-10800 val perplexity 271.9341
-10800 train 5.675177 (lr=1.4773e-05) (hash(x)=157542287)
-10800 val loss 5.6528
-10800 val perplexity 285.0970
-10800 train 5.723511 (lr=1.5758e-05) (hash(x)=157542287)
-10900 val loss 5.6030
-10900 val perplexity 271.2358
-10900 train 5.687401 (lr=1.4487e-05) (hash(x)=153860040)
-10900 val loss 5.6480
-10900 val perplexity 283.7203
-10900 train 5.729506 (lr=1.5453e-05) (hash(x)=153860040)
-11000 val loss 5.5985
-11000 val perplexity 270.0329
-11000 train 5.466104 (lr=1.4203e-05) (hash(x)=142932555)
-11000 val loss 5.6457
-11000 val perplexity 283.0762
-11000 train 5.509791 (lr=1.5149e-05) (hash(x)=142932555)
-11100 val loss 5.5975
-11100 val perplexity 269.7550
-11100 train 5.635029 (lr=1.3920e-05) (hash(x)=146260760)
-11100 val loss 5.6413
-11100 val perplexity 281.8367
-11100 train 5.682206 (lr=1.4848e-05) (hash(x)=146260760)
-11200 val loss 5.5972
-11200 val perplexity 269.6688
-11200 train 5.709498 (lr=1.3640e-05) (hash(x)=148970533)
-11200 val loss 5.6417
-11200 val perplexity 281.9330
-11200 train 5.746831 (lr=1.4550e-05) (hash(x)=148970533)
-11300 val loss 5.5887
-11300 val perplexity 267.3885
-11300 train 5.623042 (lr=1.3362e-05) (hash(x)=149421645)
-11300 val loss 5.6346
-11300 val perplexity 279.9398
-11300 train 5.655313 (lr=1.4253e-05) (hash(x)=149421645)
-11400 val loss 5.5828
-11400 val perplexity 265.8127
-11400 train 5.517864 (lr=1.3087e-05) (hash(x)=158728429)
-11400 val loss 5.6288
-11400 val perplexity 278.3406
-11400 train 5.564662 (lr=1.3960e-05) (hash(x)=158728429)
-11500 val loss 5.5789
-11500 val perplexity 264.7920
-11500 train 5.688905 (lr=1.2814e-05) (hash(x)=139524335)
-11500 val loss 5.6251
-11500 val perplexity 277.3092
-11500 train 5.728657 (lr=1.3668e-05) (hash(x)=139524335)
-11600 val loss 5.5777
-11600 val perplexity 264.4737
-11600 train 5.614056 (lr=1.2544e-05) (hash(x)=152760174)
-11600 val loss 5.6231
-11600 val perplexity 276.7369
-11600 train 5.658471 (lr=1.3380e-05) (hash(x)=152760174)
-11700 val loss 5.5742
-11700 val perplexity 263.5510
-11700 train 5.470773 (lr=1.2276e-05) (hash(x)=140740682)
-11700 val loss 5.6185
-11700 val perplexity 275.4809
-11700 train 5.515319 (lr=1.3095e-05) (hash(x)=140740682)
-11800 val loss 5.5725
-11800 val perplexity 263.0792
-11800 train 5.420375 (lr=1.2011e-05) (hash(x)=153665407)
-11800 val loss 5.6179
-11800 val perplexity 275.3106
-11800 train 5.472673 (lr=1.2812e-05) (hash(x)=153665407)
-11900 val loss 5.5704
-11900 val perplexity 262.5393
-11900 train 5.499734 (lr=1.1749e-05) (hash(x)=163068424)
-11900 val loss 5.6158
-11900 val perplexity 274.7328
-11900 train 5.570469 (lr=1.2532e-05) (hash(x)=163068424)
-12000 val loss 5.5642
-12000 val perplexity 260.9184
-12000 train 5.614580 (lr=1.1490e-05) (hash(x)=145166541)
-12000 val loss 5.6081
-12000 val perplexity 272.6255
-12000 train 5.665174 (lr=1.2256e-05) (hash(x)=145166541)
-12100 val loss 5.5611
-12100 val perplexity 260.0986
-12100 train 5.521405 (lr=1.1234e-05) (hash(x)=143228042)
-12100 val loss 5.6050
-12100 val perplexity 271.7805
-12100 train 5.560250 (lr=1.1983e-05) (hash(x)=143228042)
-12200 val loss 5.5597
-12200 val perplexity 259.7433
-12200 train 5.660483 (lr=1.0981e-05) (hash(x)=159222680)
-12200 val loss 5.6033
-12200 val perplexity 271.3172
-12200 train 5.700716 (lr=1.1713e-05) (hash(x)=159222680)
-12300 val loss 5.5570
-12300 val perplexity 259.0396
-12300 train 5.514515 (lr=1.0731e-05) (hash(x)=145196157)
-12300 val loss 5.6015
-12300 val perplexity 270.8249
-12300 train 5.550727 (lr=1.1446e-05) (hash(x)=145196157)
-12400 val loss 5.5549
-12400 val perplexity 258.4948
-12400 train 5.481500 (lr=1.0484e-05) (hash(x)=143033742)
-12400 val loss 5.5994
-12400 val perplexity 270.2646
-12400 train 5.521342 (lr=1.1183e-05) (hash(x)=143033742)
-12500 val loss 5.5534
-12500 val perplexity 258.1117
-12500 train 5.594991 (lr=1.0241e-05) (hash(x)=151133346)
-12500 val loss 5.5983
-12500 val perplexity 269.9767
-12500 train 5.637483 (lr=1.0924e-05) (hash(x)=151133346)
-12600 val loss 5.5497
-12600 val perplexity 257.1644
-12600 train 5.575035 (lr=1.0001e-05) (hash(x)=153607184)
-12600 val loss 5.5948
-12600 val perplexity 269.0154
-12600 train 5.616836 (lr=1.0668e-05) (hash(x)=153607184)
-12700 val loss 5.5448
-12700 val perplexity 255.9025
-12700 train 5.456029 (lr=9.7649e-06) (hash(x)=136931911)
-12700 val loss 5.5893
-12700 val perplexity 267.5548
-12700 train 5.501020 (lr=1.0416e-05) (hash(x)=136931911)
-12800 val loss 5.5424
-12800 val perplexity 255.2786
-12800 train 5.652829 (lr=9.5322e-06) (hash(x)=155105402)
-12800 val loss 5.5863
-12800 val perplexity 266.7446
-12800 train 5.701947 (lr=1.0168e-05) (hash(x)=155105402)
-12900 val loss 5.5416
-12900 val perplexity 255.0922
-12900 train 5.377511 (lr=9.3031e-06) (hash(x)=144141636)
-12900 val loss 5.5850
-12900 val perplexity 266.3893
-12900 train 5.414651 (lr=9.9233e-06) (hash(x)=144141636)
-13000 val loss 5.5418
-13000 val perplexity 255.1259
-13000 train 5.474589 (lr=9.0777e-06) (hash(x)=139380218)
-13000 val loss 5.5863
-13000 val perplexity 266.7417
-13000 train 5.515913 (lr=9.6829e-06) (hash(x)=139380218)
-13100 val loss 5.5343
-13100 val perplexity 253.2362
-13100 train 5.632472 (lr=8.8562e-06) (hash(x)=157695174)
-13100 val loss 5.5780
-13100 val perplexity 264.5459
-13100 train 5.674640 (lr=9.4466e-06) (hash(x)=157695174)
-13200 val loss 5.5346
-13200 val perplexity 253.3157
-13200 train 5.482560 (lr=8.6385e-06) (hash(x)=143697560)
-13200 val loss 5.5764
-13200 val perplexity 264.1287
-13200 train 5.523151 (lr=9.2144e-06) (hash(x)=143697560)
-13300 val loss 5.5307
-13300 val perplexity 252.3304
-13300 train 5.361203 (lr=8.4248e-06) (hash(x)=156314952)
-13300 val loss 5.5741
-13300 val perplexity 263.5179
-13300 train 5.405429 (lr=8.9865e-06) (hash(x)=156314952)
-13400 val loss 5.5296
-13400 val perplexity 252.0309
-13400 train 5.499445 (lr=8.2151e-06) (hash(x)=136720279)
-13400 val loss 5.5738
-13400 val perplexity 263.4412
-13400 train 5.526761 (lr=8.7628e-06) (hash(x)=136720279)
-13500 val loss 5.5286
-13500 val perplexity 251.7948
-13500 train 5.488736 (lr=8.0095e-06) (hash(x)=142579957)
-13500 val loss 5.5725
-13500 val perplexity 263.0993
-13500 train 5.532460 (lr=8.5435e-06) (hash(x)=142579957)
-13600 val loss 5.5321
-13600 val perplexity 252.6708
-13600 train 5.248423 (lr=7.8081e-06) (hash(x)=140719440)
-13600 val loss 5.5737
-13600 val perplexity 263.4108
-13600 train 5.283875 (lr=8.3286e-06) (hash(x)=140719440)
-13700 val loss 5.5288
-13700 val perplexity 251.8322
-13700 train 5.619996 (lr=7.6109e-06) (hash(x)=164396003)
-13700 val loss 5.5726
-13700 val perplexity 263.1302
-13700 train 5.667033 (lr=8.1183e-06) (hash(x)=164396003)
-13800 val loss 5.5220
-13800 val perplexity 250.1227
-13800 train 5.578545 (lr=7.4180e-06) (hash(x)=152712862)
-13800 val loss 5.5655
-13800 val perplexity 261.2472
-13800 train 5.624051 (lr=7.9125e-06) (hash(x)=152712862)
-13900 val loss 5.5178
-13900 val perplexity 249.0800
-13900 train 5.540569 (lr=7.2294e-06) (hash(x)=160985139)
-13900 val loss 5.5627
-13900 val perplexity 260.5220
-13900 train 5.587029 (lr=7.7114e-06) (hash(x)=160985139)
-14000 val loss 5.5167
-14000 val perplexity 248.8097
-14000 train 5.520864 (lr=7.0452e-06) (hash(x)=147429379)
-14000 val loss 5.5613
-14000 val perplexity 260.1528
-14000 train 5.558174 (lr=7.5149e-06) (hash(x)=147429379)
-14100 val loss 5.5154
-14100 val perplexity 248.4913
-14100 train 5.592762 (lr=6.8656e-06) (hash(x)=131830470)
-14100 val loss 5.5583
-14100 val perplexity 259.3858
-14100 train 5.637152 (lr=7.3233e-06) (hash(x)=131830470)
-14200 val loss 5.5162
-14200 val perplexity 248.6958
-14200 train 5.260109 (lr=6.6904e-06) (hash(x)=146383028)
-14200 val loss 5.5595
-14200 val perplexity 259.6955
-14200 train 5.301928 (lr=7.1365e-06) (hash(x)=146383028)
-14300 val loss 5.5127
-14300 val perplexity 247.8100
-14300 train 5.558857 (lr=6.5199e-06) (hash(x)=157158747)
-14300 val loss 5.5570
-14300 val perplexity 259.0358
-14300 train 5.601773 (lr=6.9546e-06) (hash(x)=157158747)
-14400 val loss 5.5163
-14400 val perplexity 248.7070
-14400 train 5.372640 (lr=6.3540e-06) (hash(x)=143911520)
-14400 val loss 5.5601
-14400 val perplexity 259.8445
-14400 train 5.416796 (lr=6.7776e-06) (hash(x)=143911520)
-14500 val loss 5.5152
-14500 val perplexity 248.4450
-14500 train 5.278310 (lr=6.1928e-06) (hash(x)=159555347)
-14500 val loss 5.5598
-14500 val perplexity 259.7617
-14500 train 5.331186 (lr=6.6057e-06) (hash(x)=159555347)
-14600 val loss 5.5092
-14600 val perplexity 246.9438
-14600 train 5.506528 (lr=6.0364e-06) (hash(x)=152957062)
-14600 val loss 5.5535
-14600 val perplexity 258.1299
-14600 train 5.551717 (lr=6.4389e-06) (hash(x)=152957062)
-14700 val loss 5.5050
-14700 val perplexity 245.9211
-14700 train 5.580899 (lr=5.8848e-06) (hash(x)=156001594)
-14700 val loss 5.5486
-14700 val perplexity 256.8655
-14700 train 5.622464 (lr=6.2771e-06) (hash(x)=156001594)
-14800 val loss 5.5047
-14800 val perplexity 245.8406
-14800 train 5.615385 (lr=5.7381e-06) (hash(x)=145885458)
-14800 val loss 5.5474
-14800 val perplexity 256.5798
-14800 train 5.659868 (lr=6.1206e-06) (hash(x)=145885458)
-14900 val loss 5.5016
-14900 val perplexity 245.0851
-14900 train 5.578045 (lr=5.5962e-06) (hash(x)=144544291)
-14900 val loss 5.5439
-14900 val perplexity 255.6789
-14900 train 5.613038 (lr=5.9693e-06) (hash(x)=144544291)
-15000 val loss 5.5013
-15000 val perplexity 245.0193
-15000 train 5.507167 (lr=5.4593e-06) (hash(x)=154606709)
-15000 val loss 5.5454
-15000 val perplexity 256.0479
-15000 train 5.549639 (lr=5.8233e-06) (hash(x)=154606709)
-15100 val loss 5.5017
-15100 val perplexity 245.1157
-15100 train 5.538658 (lr=5.3275e-06) (hash(x)=156005324)
-15100 val loss 5.5445
-15100 val perplexity 255.8389
-15100 train 5.584861 (lr=5.6826e-06) (hash(x)=156005324)
-15200 val loss 5.5032
-15200 val perplexity 245.4764
-15200 train 5.607048 (lr=5.2006e-06) (hash(x)=158847373)
-15200 val loss 5.5465
-15200 val perplexity 256.3338
-15200 train 5.648160 (lr=5.5473e-06) (hash(x)=158847373)
-15300 val loss 5.5038
-15300 val perplexity 245.6311
-15300 train 5.283220 (lr=5.0789e-06) (hash(x)=159609594)
-15300 val loss 5.5466
-15300 val perplexity 256.3549
-15300 train 5.323767 (lr=5.4175e-06) (hash(x)=159609594)
-15400 val loss 5.4993
-15400 val perplexity 244.5235
-15400 train 5.592093 (lr=4.9622e-06) (hash(x)=152428492)
-15400 val loss 5.5411
-15400 val perplexity 254.9683
-15400 train 5.637389 (lr=5.2931e-06) (hash(x)=152428492)
-15500 val loss 5.4957
-15500 val perplexity 243.6322
-15500 train 5.560111 (lr=4.8508e-06) (hash(x)=162251192)
-15500 val loss 5.5371
-15500 val perplexity 253.9309
-15500 train 5.607491 (lr=5.1742e-06) (hash(x)=162251192)
-15600 val loss 5.4948
-15600 val perplexity 243.4196
-15600 train 5.757086 (lr=4.7445e-06) (hash(x)=155072002)
-15600 val loss 5.5366
-15600 val perplexity 253.8090
-15600 train 5.794656 (lr=5.0608e-06) (hash(x)=155072002)
-15700 val loss 5.4938
-15700 val perplexity 243.1737
-15700 train 5.524041 (lr=4.6435e-06) (hash(x)=151203094)
-15700 val loss 5.5359
-15700 val perplexity 253.6246
-15700 train 5.563150 (lr=4.9531e-06) (hash(x)=151203094)
-15800 val loss 5.4922
-15800 val perplexity 242.7884
-15800 train 5.428759 (lr=4.5478e-06) (hash(x)=146125932)
-15800 val loss 5.5348
-15800 val perplexity 253.3518
-15800 train 5.461537 (lr=4.8509e-06) (hash(x)=146125932)
-15900 val loss 5.4931
-15900 val perplexity 243.0073
-15900 train 5.364917 (lr=4.4573e-06) (hash(x)=148060195)
-15900 val loss 5.5345
-15900 val perplexity 253.2838
-15900 train 5.415287 (lr=4.7545e-06) (hash(x)=148060195)
-16000 val loss 5.4935
-16000 val perplexity 243.1048
-16000 train 5.201401 (lr=4.3722e-06) (hash(x)=137406416)
-16000 val loss 5.5355
-16000 val perplexity 253.5310
-16000 train 5.242233 (lr=4.6637e-06) (hash(x)=137406416)
-16100 val loss 5.4938
-16100 val perplexity 243.1702
-16100 train 5.557673 (lr=4.2924e-06) (hash(x)=160149755)
-16100 val loss 5.5355
-16100 val perplexity 253.5390
-16100 train 5.597098 (lr=4.5786e-06) (hash(x)=160149755)
-16200 val loss 5.4883
-16200 val perplexity 241.8399
-16200 train 5.672664 (lr=4.2180e-06) (hash(x)=151371894)
-16200 val loss 5.5305
-16200 val perplexity 252.2687
-16200 train 5.709032 (lr=4.4993e-06) (hash(x)=151371894)
-16300 val loss 5.4856
-16300 val perplexity 241.1907
-16300 train 5.551252 (lr=4.1491e-06) (hash(x)=151810252)
-16300 val loss 5.5286
-16300 val perplexity 251.7811
-16300 train 5.583846 (lr=4.4257e-06) (hash(x)=151810252)
-16400 val loss 5.4843
-16400 val perplexity 240.8846
-16400 train 5.470860 (lr=4.0856e-06) (hash(x)=153088303)
-16400 val loss 5.5272
-16400 val perplexity 251.4453
-16400 train 5.514785 (lr=4.3579e-06) (hash(x)=153088303)
-16500 val loss 5.4839
-16500 val perplexity 240.7784
-16500 train 5.350903 (lr=4.0275e-06) (hash(x)=151706615)
-16500 val loss 5.5262
-16500 val perplexity 251.1843
-16500 train 5.398167 (lr=4.2960e-06) (hash(x)=151706615)
-16600 val loss 5.4829
-16600 val perplexity 240.5481
-16600 train 5.380322 (lr=3.9749e-06) (hash(x)=142110626)
-16600 val loss 5.5248
-16600 val perplexity 250.8432
-16600 train 5.422053 (lr=4.2399e-06) (hash(x)=142110626)
-16700 val loss 5.4836
-16700 val perplexity 240.7059
-16700 train 5.376807 (lr=3.9278e-06) (hash(x)=153222439)
-16700 val loss 5.5262
-16700 val perplexity 251.1830
-16700 train 5.419888 (lr=4.1896e-06) (hash(x)=153222439)
-16800 val loss 5.4831
-16800 val perplexity 240.5931
-16800 train 5.391656 (lr=3.8862e-06) (hash(x)=154964902)
-16800 val loss 5.5252
-16800 val perplexity 250.9311
-16800 train 5.448435 (lr=4.1452e-06) (hash(x)=154964902)
-16900 val loss 5.4795
-16900 val perplexity 239.7348
-16900 train 5.667234 (lr=3.8501e-06) (hash(x)=158893204)
-16900 val loss 5.5213
-16900 val perplexity 249.9718
-16900 train 5.701284 (lr=4.1067e-06) (hash(x)=158893204)
-17000 val loss 5.4779
-17000 val perplexity 239.3489
-17000 train 5.411547 (lr=3.8195e-06) (hash(x)=148929955)
-17000 val loss 5.5203
-17000 val perplexity 249.7003
-17000 train 5.451448 (lr=4.0741e-06) (hash(x)=148929955)
-17100 val loss 5.4779
-17100 val perplexity 239.3535
-17100 train 5.500988 (lr=3.7945e-06) (hash(x)=147818677)
-17100 val loss 5.5206
-17100 val perplexity 249.7858
-17100 train 5.537355 (lr=4.0475e-06) (hash(x)=147818677)
-17200 val loss 5.4761
-17200 val perplexity 238.9149
-17200 train 5.405346 (lr=3.7750e-06) (hash(x)=147119446)
-17200 val loss 5.5186
-17200 val perplexity 249.2853
-17200 train 5.448709 (lr=4.0267e-06) (hash(x)=147119446)
-17300 val loss 5.4766
-17300 val perplexity 239.0285
-17300 train 5.538480 (lr=3.7611e-06) (hash(x)=150494142)
-17300 val loss 5.5197
-17300 val perplexity 249.5582
-17300 train 5.575846 (lr=4.0119e-06) (hash(x)=150494142)
-17400 val loss 5.4770
-17400 val perplexity 239.1375
-17400 train 5.496178 (lr=3.7528e-06) (hash(x)=155918058)
-17400 val loss 5.5196
-17400 val perplexity 249.5304
-17400 train 5.545844 (lr=4.0030e-06) (hash(x)=155918058)
-17499 val loss 5.4740
-17499 val perplexity 238.4194
-17499 val loss 5.5160
-17499 val perplexity 248.6281
+max_steps: 8750
+7100 val loss 5.8251
+7100 val perplexity 338.6792
+7100 train 5.700815 (lr=6.5080e-06) (hash(x)=148277836)
+0 val loss 11.2898
+0 val perplexity 80003.3281
+7200 val loss 5.8238
+7200 val perplexity 338.2640
+7200 train 5.831926 (lr=6.1648e-06) (hash(x)=168591659)
+0 train 11.289023 (lr=8.0000e-08) (hash(x)=150332693)
+7300 val loss 5.8168
+7300 val perplexity 335.9115
+7300 train 5.797405 (lr=5.8405e-06) (hash(x)=152764851)
+100 val loss 10.0512
+100 val perplexity 23184.4512
+100 train 10.007780 (lr=8.0800e-06) (hash(x)=149277319)
+7400 val loss 5.8128
+7400 val perplexity 334.5652
+7400 train 5.765549 (lr=5.5357e-06) (hash(x)=142920131)
+200 val loss 9.4244
+200 val perplexity 12386.6855
+200 train 9.491747 (lr=1.6080e-05) (hash(x)=155934075)
+7500 val loss 5.8108
+7500 val perplexity 333.8896
+7500 train 5.807702 (lr=5.2508e-06) (hash(x)=146954649)
+300 val loss 8.2609
+300 val perplexity 3869.7136
+300 train 8.296400 (lr=2.4080e-05) (hash(x)=165402628)
+7600 val loss 5.8105
+7600 val perplexity 333.8024
+7600 train 5.602143 (lr=4.9862e-06) (hash(x)=144529566)
+400 val loss 7.5974
+400 val perplexity 1992.9136
+400 train 7.401337 (lr=3.2080e-05) (hash(x)=139168939)
+7700 val loss 5.8047
+7700 val perplexity 331.8635
+7700 train 5.760702 (lr=4.7423e-06) (hash(x)=142669945)
+500 val loss 7.4160
+500 val perplexity 1662.4181
+500 train 7.336144 (lr=4.0000e-05) (hash(x)=147134338)
+7800 val loss 5.8011
+7800 val perplexity 330.6578
+7800 train 5.871948 (lr=4.5194e-06) (hash(x)=161563764)
+600 val loss 7.3080
+600 val perplexity 1492.2618
+600 train 7.216753 (lr=3.9987e-05) (hash(x)=159086459)
+7900 val loss 5.7989
+7900 val perplexity 329.9443
+7900 train 5.598620 (lr=4.3179e-06) (hash(x)=141516532)
+700 val loss 7.2178
+700 val perplexity 1363.4265
+700 train 7.253378 (lr=3.9948e-05) (hash(x)=152399099)
+8000 val loss 5.7998
+8000 val perplexity 330.2267
+8000 train 5.668744 (lr=4.1380e-06) (hash(x)=152418457)
+800 val loss 7.0971
+800 val perplexity 1208.4463
+800 train 7.100146 (lr=3.9883e-05) (hash(x)=156548843)
+8100 val loss 5.7940
+8100 val perplexity 328.3377
+8100 train 5.787144 (lr=3.9800e-06) (hash(x)=161109470)
+900 val loss 7.0009
+900 val perplexity 1097.5968
+900 train 6.970758 (lr=3.9792e-05) (hash(x)=157889911)
+8200 val loss 5.7904
+8200 val perplexity 327.1298
+8200 train 5.914674 (lr=3.8442e-06) (hash(x)=154408822)
+1000 val loss 6.9203
+1000 val perplexity 1012.6512
+1000 train 6.716169 (lr=3.9675e-05) (hash(x)=142943707)
+8300 val loss 5.7886
+8300 val perplexity 326.5483
+8300 train 5.672719 (lr=3.7307e-06) (hash(x)=146431691)
+1100 val loss 6.8508
+1100 val perplexity 944.6305
+1100 train 6.849274 (lr=3.9532e-05) (hash(x)=148938689)
+8400 val loss 5.7887
+8400 val perplexity 326.6004
+8400 train 5.564780 (lr=3.6397e-06) (hash(x)=150182527)
+1200 val loss 6.7516
+1200 val perplexity 855.4323
+1200 train 6.889379 (lr=3.9364e-05) (hash(x)=161962903)
+8500 val loss 5.7832
+8500 val perplexity 324.7987
+8500 train 5.846736 (lr=3.5713e-06) (hash(x)=154828709)
+1300 val loss 6.6709
+1300 val perplexity 789.0945
+1300 train 6.656214 (lr=3.9171e-05) (hash(x)=145979235)
+8600 val loss 5.7820
+8600 val perplexity 324.3983
+8600 train 5.691816 (lr=3.5257e-06) (hash(x)=154995428)
+1400 val loss 6.6137
+1400 val perplexity 745.2544
+1400 train 6.565486 (lr=3.8953e-05) (hash(x)=159157507)
+8700 val loss 5.7821
+8700 val perplexity 324.4330
+8700 train 5.676406 (lr=3.5029e-06) (hash(x)=142111043)
+1500 val loss 6.5521
+1500 val perplexity 700.7333
+1500 train 6.591178 (lr=3.8711e-05) (hash(x)=145943130)
+8749 val loss 5.7797
+8749 val perplexity 323.6483
+1600 val loss 6.4850
+1600 val perplexity 655.2203
+1600 train 6.287962 (lr=3.8444e-05) (hash(x)=143249173)

attention_kindselective_n_heads4_seed1342/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4554f9feef4cd1714de74f2d36dfb797afdd186752d5ae5a18a0c3ad58deb6d7
+size 92843394

attention_kindselective_n_heads4_seed1342/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86936929383c09f29442147958ddf75c30a2f329230a59348238ed2c4e7f9596
+size 179406214