Upload folder using huggingface_hub

attention_kindselective_n_heads8_seed1338/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads8_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 8, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 2e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "2e-5_10240_8_1338", "n_embd": 512}

attention_kindselective_n_heads8_seed1338/dataloader_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
+size 964

attention_kindselective_n_heads8_seed1338/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f03ed2ebf741f15e13c79e6cc1e9a19b308450d81cc3b4d8d0338c63d77ca59
+size 964

attention_kindselective_n_heads8_seed1338/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82590037fb2eecbec961f7967a8dd1b8d85515d31a252f66b92b8139858a8b7c
+size 964

attention_kindselective_n_heads8_seed1338/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c544303717d12355a69b8ffb1eb109434e4fdccfd5a61279b6e8ba2e870d6700
+size 964

attention_kindselective_n_heads8_seed1338/log2.txt

@@ -0,0 +1,1209 @@
+max_steps: 10000
+0 val loss 11.0412
+0 val perplexity 62392.4570
+0 val loss 11.0412
+0 val perplexity 62392.5195
+0 val loss 11.0412
+0 val perplexity 62392.4570
+0 val loss 11.0412
+0 val perplexity 62392.4570
+0 train 11.030708 (lr=2.5000e-07) (hash(x)=50671684)
+0 train 11.030660 (lr=3.5000e-07) (hash(x)=50671684)
+0 train 11.030660 (lr=1.0000e-07) (hash(x)=50671684)
+0 train 11.030662 (lr=1.5000e-07) (hash(x)=50671684)
+100 val loss 9.5150
+100 val perplexity 13562.2197
+100 train 9.475113 (lr=3.5350e-05) (hash(x)=52740221)
+100 val loss 9.8877
+100 val perplexity 19685.8281
+100 val loss 9.6106
+100 val perplexity 14921.5723
+100 train 9.853711 (lr=1.0100e-05) (hash(x)=52740221)
+100 val loss 9.7397
+100 val perplexity 16979.0996
+100 train 9.567675 (lr=2.5250e-05) (hash(x)=52740221)
+100 train 9.698705 (lr=1.5150e-05) (hash(x)=52740221)
+200 val loss 8.0793
+200 val perplexity 3227.0042
+200 train 8.077329 (lr=7.0000e-05) (hash(x)=49034180)
+200 val loss 9.1855
+200 val perplexity 9755.0391
+200 val loss 9.4434
+200 val perplexity 12623.9707
+200 train 9.460796 (lr=2.0000e-05) (hash(x)=49034180)
+200 train 9.197133 (lr=3.0000e-05) (hash(x)=49034180)
+200 val loss 8.5470
+200 val perplexity 5151.4980
+200 train 8.548553 (lr=5.0000e-05) (hash(x)=49034180)
+300 val loss 7.5602
+300 val perplexity 1920.1787
+300 train 7.826581 (lr=6.9984e-05) (hash(x)=63180688)
+300 val loss 8.8301
+300 val perplexity 6837.0103
+300 train 8.877916 (lr=1.9995e-05) (hash(x)=63180688)
+300 val loss 8.2489
+300 val perplexity 3823.5215
+300 train 8.357185 (lr=2.9993e-05) (hash(x)=63180688)
+300 val loss 7.8357
+300 val perplexity 2529.2393
+300 train 8.017459 (lr=4.9988e-05) (hash(x)=63180688)
+400 val loss 7.4376
+400 val perplexity 1698.6493
+400 train 7.438178 (lr=6.9935e-05) (hash(x)=50373500)
+400 val loss 8.2621
+400 val perplexity 3874.1113
+400 train 8.178881 (lr=1.9982e-05) (hash(x)=50373500)
+400 val loss 7.8110
+400 val perplexity 2467.4846
+400 train 7.777402 (lr=2.9972e-05) (hash(x)=50373500)
+400 val loss 7.6415
+400 val perplexity 2082.9680
+400 train 7.627953 (lr=4.9954e-05) (hash(x)=50373500)
+500 val loss 7.3646
+500 val perplexity 1579.0923
+500 train 7.238733 (lr=6.9854e-05) (hash(x)=44547422)
+500 val loss 7.9960
+500 val perplexity 2969.1821
+500 train 7.944740 (lr=1.9958e-05) (hash(x)=44547422)
+500 val loss 7.6391
+500 val perplexity 2077.9395
+500 train 7.550472 (lr=2.9938e-05) (hash(x)=44547422)
+500 val loss 7.5464
+500 val perplexity 1893.9757
+500 train 7.441219 (lr=4.9896e-05) (hash(x)=44547422)
+600 val loss 7.3356
+600 val perplexity 1533.9551
+600 train 7.176397 (lr=6.9741e-05) (hash(x)=47184699)
+600 val loss 7.8623
+600 val perplexity 2597.5964
+600 train 7.796846 (lr=1.9926e-05) (hash(x)=47184699)
+600 val loss 7.5444
+600 val perplexity 1890.1512
+600 train 7.428701 (lr=2.9889e-05) (hash(x)=47184699)
+600 val loss 7.4974
+600 val perplexity 1803.3756
+600 train 7.370130 (lr=4.9815e-05) (hash(x)=47184699)
+700 val loss 7.3017
+700 val perplexity 1482.8025
+700 train 7.180538 (lr=6.9596e-05) (hash(x)=51374582)
+700 val loss 7.7705
+700 val perplexity 2369.7173
+700 train 7.703903 (lr=1.9885e-05) (hash(x)=51374582)
+700 val loss 7.4869
+700 val perplexity 1784.5436
+700 train 7.378650 (lr=2.9827e-05) (hash(x)=51374582)
+700 val loss 7.4491
+700 val perplexity 1718.3898
+700 train 7.325911 (lr=4.9712e-05) (hash(x)=51374582)
+800 val loss 7.2514
+800 val perplexity 1410.0104
+800 train 6.968596 (lr=6.9419e-05) (hash(x)=46264805)
+800 val loss 7.6907
+800 val perplexity 2187.8003
+800 train 7.486768 (lr=1.9834e-05) (hash(x)=46264805)
+800 val loss 7.4399
+800 val perplexity 1702.6342
+800 train 7.190279 (lr=2.9751e-05) (hash(x)=46264805)
+800 val loss 7.4127
+800 val perplexity 1656.8792
+800 train 7.154434 (lr=4.9585e-05) (hash(x)=46264805)
+900 val loss 7.2094
+900 val perplexity 1352.0565
+900 train 7.378638 (lr=6.9210e-05) (hash(x)=61178712)
+900 val loss 7.6276
+900 val perplexity 2054.2126
+900 train 7.740030 (lr=1.9774e-05) (hash(x)=61178712)
+900 val loss 7.4194
+900 val perplexity 1668.0503
+900 train 7.569212 (lr=2.9662e-05) (hash(x)=61178712)
+900 val loss 7.3820
+900 val perplexity 1606.8147
+900 train 7.542928 (lr=4.9436e-05) (hash(x)=61178712)
+1000 val loss 7.1676
+1000 val perplexity 1296.7246
+1000 train 7.039244 (lr=6.8970e-05) (hash(x)=50886520)
+1000 val loss 7.5767
+1000 val perplexity 1952.1151
+1000 train 7.549294 (lr=1.9706e-05) (hash(x)=50886520)
+1000 val loss 7.3707
+1000 val perplexity 1588.7726
+1000 train 7.311100 (lr=2.9558e-05) (hash(x)=50886520)
+1000 val loss 7.3440
+1000 val perplexity 1546.8590
+1000 train 7.258212 (lr=4.9264e-05) (hash(x)=50886520)
+1100 val loss 7.1469
+1100 val perplexity 1270.1753
+1100 train 6.893501 (lr=6.8698e-05) (hash(x)=48600099)
+1100 val loss 7.5429
+1100 val perplexity 1887.3206
+1100 train 7.344742 (lr=1.9628e-05) (hash(x)=48600099)
+1100 val loss 7.3492
+1100 val perplexity 1554.8961
+1100 train 7.104578 (lr=2.9442e-05) (hash(x)=48600099)
+1100 val loss 7.3139
+1100 val perplexity 1501.0520
+1100 train 7.054069 (lr=4.9070e-05) (hash(x)=48600099)
+1200 val loss 7.1126
+1200 val perplexity 1227.3606
+1200 train 6.809518 (lr=6.8395e-05) (hash(x)=50146792)
+1200 val loss 7.5094
+1200 val perplexity 1825.0875
+1200 train 7.275321 (lr=1.9542e-05) (hash(x)=50146792)
+1200 val loss 7.3220
+1200 val perplexity 1513.1711
+1200 train 7.054025 (lr=2.9312e-05) (hash(x)=50146792)
+1200 val loss 7.2835
+1200 val perplexity 1456.0861
+1200 train 7.006647 (lr=4.8854e-05) (hash(x)=50146792)
+1300 val loss 7.1075
+1300 val perplexity 1221.0498
+1300 train 6.973897 (lr=6.8062e-05) (hash(x)=52617313)
+1300 val loss 7.4832
+1300 val perplexity 1777.9160
+1300 train 7.413532 (lr=1.9446e-05) (hash(x)=52617313)
+1300 val loss 7.2895
+1300 val perplexity 1464.8091
+1300 train 7.192160 (lr=2.9169e-05) (hash(x)=52617313)
+1300 val loss 7.2547
+1300 val perplexity 1414.8009
+1300 train 7.144293 (lr=4.8616e-05) (hash(x)=52617313)
+1400 val loss 7.0632
+1400 val perplexity 1168.1980
+1400 train 6.699810 (lr=6.7698e-05) (hash(x)=49794446)
+1400 val loss 7.4626
+1400 val perplexity 1741.7556
+1400 train 7.178441 (lr=1.9342e-05) (hash(x)=49794446)
+1400 val loss 7.2588
+1400 val perplexity 1420.5990
+1400 train 6.923851 (lr=2.9013e-05) (hash(x)=49794446)
+1400 val loss 7.2191
+1400 val perplexity 1365.2207
+1400 train 6.870090 (lr=4.8356e-05) (hash(x)=49794446)
+1500 val loss 7.0495
+1500 val perplexity 1152.2533
+1500 train 6.669672 (lr=6.7304e-05) (hash(x)=50766317)
+1500 val loss 7.4453
+1500 val perplexity 1711.8087
+1500 train 7.146240 (lr=1.9230e-05) (hash(x)=50766317)
+1500 val loss 7.2383
+1500 val perplexity 1391.7863
+1500 val loss 7.1959
+1500 val perplexity 1333.9630
+1500 train 6.877825 (lr=2.8845e-05) (hash(x)=50766317)
+1500 train 6.816164 (lr=4.8074e-05) (hash(x)=50766317)
+1600 val loss 7.0410
+1600 val perplexity 1142.4799
+1600 train 6.882990 (lr=6.6881e-05) (hash(x)=55551175)
+1600 val loss 7.4334
+1600 val perplexity 1691.6099
+1600 train 7.326027 (lr=1.9109e-05) (hash(x)=55551175)
+1600 val loss 7.2227
+1600 val perplexity 1370.1851
+1600 train 7.092454 (lr=2.8663e-05) (hash(x)=55551175)
+1600 val loss 7.1825
+1600 val perplexity 1316.2428
+1600 train 7.034845 (lr=4.7772e-05) (hash(x)=55551175)
+1700 val loss 6.9983
+1700 val perplexity 1094.7711
+1700 train 7.202261 (lr=6.6428e-05) (hash(x)=56717172)
+1700 val loss 7.4104
+1700 val perplexity 1653.1425
+1700 train 7.585999 (lr=1.8979e-05) (hash(x)=56717172)
+1700 val loss 7.1886
+1700 val perplexity 1324.2910
+1700 train 7.378125 (lr=2.8469e-05) (hash(x)=56717172)
+1700 val loss 7.1430
+1700 val perplexity 1265.2288
+1700 train 7.331745 (lr=4.7448e-05) (hash(x)=56717172)
+1800 val loss 6.9486
+1800 val perplexity 1041.6569
+1800 train 7.104018 (lr=6.5947e-05) (hash(x)=55376447)
+1800 val loss 7.3684
+1800 val perplexity 1585.1343
+1800 train 7.653419 (lr=1.8842e-05) (hash(x)=55376447)
+1800 val loss 7.1570
+1800 val perplexity 1283.0609
+1800 train 7.384525 (lr=2.8263e-05) (hash(x)=55376447)
+1800 val loss 7.1026
+1800 val perplexity 1215.0824
+1800 train 7.298141 (lr=4.7105e-05) (hash(x)=55376447)
+1900 val loss 6.9468
+1900 val perplexity 1039.7795
+1900 train 6.644609 (lr=6.5437e-05) (hash(x)=43810837)
+1900 val loss 7.3462
+1900 val perplexity 1550.2446
+1900 train 7.057503 (lr=1.8696e-05) (hash(x)=43810837)
+1900 val loss 7.1444
+1900 val perplexity 1266.9542
+1900 train 6.842966 (lr=2.8044e-05) (hash(x)=43810837)
+1900 val loss 7.1072
+1900 val perplexity 1220.6906
+1900 train 6.805607 (lr=4.6741e-05) (hash(x)=43810837)
+2000 val loss 6.9125
+2000 val perplexity 1004.7640
+2000 train 6.955251 (lr=6.4900e-05) (hash(x)=50881655)
+2000 val loss 7.3188
+2000 val perplexity 1508.4669
+2000 train 7.377332 (lr=1.8543e-05) (hash(x)=50881655)
+2000 val loss 7.1095
+2000 val perplexity 1223.4854
+2000 train 7.146984 (lr=2.7814e-05) (hash(x)=50881655)
+2000 val loss 7.0598
+2000 val perplexity 1164.1548
+2000 train 7.097054 (lr=4.6357e-05) (hash(x)=50881655)
+2100 val loss 6.8942
+2100 val perplexity 986.5238
+2100 train 6.770719 (lr=6.4335e-05) (hash(x)=49386015)
+2100 val loss 7.2984
+2100 val perplexity 1478.0065
+2100 train 7.180274 (lr=1.8382e-05) (hash(x)=49386015)
+2100 val loss 7.0916
+2100 val perplexity 1201.8446
+2100 train 6.963319 (lr=2.7572e-05) (hash(x)=49386015)
+2100 val loss 7.0394
+2100 val perplexity 1140.6602
+2100 train 6.919005 (lr=4.5954e-05) (hash(x)=49386015)
+2200 val loss 6.8933
+2200 val perplexity 985.6652
+2200 train 6.780797 (lr=6.3745e-05) (hash(x)=48572079)
+2200 val loss 7.2870
+2200 val perplexity 1461.1439
+2200 train 7.191564 (lr=1.8213e-05) (hash(x)=48572079)
+2200 val loss 7.0769
+2200 val perplexity 1184.3407
+2200 train 6.976396 (lr=2.7319e-05) (hash(x)=48572079)
+2200 val loss 7.0254
+2200 val perplexity 1124.8413
+2200 train 6.915151 (lr=4.5532e-05) (hash(x)=48572079)
+2300 val loss 6.8545
+2300 val perplexity 948.1169
+2300 train 6.901963 (lr=6.3128e-05) (hash(x)=54950719)
+2300 val loss 7.2620
+2300 val perplexity 1425.1230
+2300 train 7.301581 (lr=1.8036e-05) (hash(x)=54950719)
+2300 val loss 7.0468
+2300 val perplexity 1149.1204
+2300 train 7.070806 (lr=2.7055e-05) (hash(x)=54950719)
+2300 val loss 6.9928
+2300 val perplexity 1088.7826
+2300 train 7.020673 (lr=4.5091e-05) (hash(x)=54950719)
+2400 val loss 6.8428
+2400 val perplexity 937.0630
+2400 train 6.468015 (lr=6.2486e-05) (hash(x)=42190240)
+2400 val loss 7.2473
+2400 val perplexity 1404.2675
+2400 train 6.883503 (lr=1.7853e-05) (hash(x)=42190240)
+2400 val loss 7.0281
+2400 val perplexity 1127.8716
+2400 train 6.654352 (lr=2.6780e-05) (hash(x)=42190240)
+2400 val loss 6.9754
+2400 val perplexity 1069.9851
+2400 train 6.604122 (lr=4.4633e-05) (hash(x)=42190240)
+2500 val loss 6.8213
+2500 val perplexity 917.2199
+2500 train 6.914586 (lr=6.1819e-05) (hash(x)=45223539)
+2500 val loss 7.2317
+2500 val perplexity 1382.6361
+2500 train 7.320289 (lr=1.7663e-05) (hash(x)=45223539)
+2500 val loss 7.0148
+2500 val perplexity 1112.9597
+2500 val loss 6.9591
+2500 val perplexity 1052.7030
+2500 train 7.101954 (lr=2.6494e-05) (hash(x)=45223539)
+2500 train 7.038404 (lr=4.4156e-05) (hash(x)=45223539)
+2600 val loss 6.8081
+2600 val perplexity 905.1906
+2600 train 6.809896 (lr=6.1128e-05) (hash(x)=54037353)
+2600 val loss 7.2149
+2600 val perplexity 1359.4930
+2600 train 7.229406 (lr=1.7465e-05) (hash(x)=54037353)
+2600 val loss 7.0019
+2600 val perplexity 1098.6744
+2600 train 7.021422 (lr=2.6198e-05) (hash(x)=54037353)
+2600 val loss 6.9374
+2600 val perplexity 1030.1311
+2600 train 6.954820 (lr=4.3663e-05) (hash(x)=54037353)
+2700 val loss 6.7813
+2700 val perplexity 881.1724
+2700 train 7.265512 (lr=6.0414e-05) (hash(x)=59131616)
+2700 val loss 7.1999
+2700 val perplexity 1339.3392
+2700 train 7.691195 (lr=1.7261e-05) (hash(x)=59131616)
+2700 val loss 6.9835
+2700 val perplexity 1078.6578
+2700 train 7.498905 (lr=2.5892e-05) (hash(x)=59131616)
+2700 val loss 6.9108
+2700 val perplexity 1003.0886
+2700 train 7.397624 (lr=4.3153e-05) (hash(x)=59131616)
+2800 val loss 6.7650
+2800 val perplexity 866.9603
+2800 train 6.601818 (lr=5.9677e-05) (hash(x)=45882743)
+2800 val loss 7.1851
+2800 val perplexity 1319.6088
+2800 train 7.032475 (lr=1.7051e-05) (hash(x)=45882743)
+2800 val loss 6.9720
+2800 val perplexity 1066.3402
+2800 val loss 6.8969
+2800 val perplexity 989.2418
+2800 train 6.810520 (lr=2.5576e-05) (hash(x)=45882743)
+2800 train 6.740892 (lr=4.2627e-05) (hash(x)=45882743)
+2900 val loss 6.7591
+2900 val perplexity 861.8439
+2900 train 6.378328 (lr=5.8919e-05) (hash(x)=43758910)
+2900 val loss 7.1679
+2900 val perplexity 1297.1414
+2900 train 6.796428 (lr=1.6834e-05) (hash(x)=43758910)
+2900 val loss 6.8926
+2900 val perplexity 984.9496
+2900 train 6.507185 (lr=4.2085e-05) (hash(x)=43758910)
+2900 val loss 6.9604
+2900 val perplexity 1054.0331
+2900 train 6.588078 (lr=2.5251e-05) (hash(x)=43758910)
+3000 val loss 6.7508
+3000 val perplexity 854.7387
+3000 train 6.644414 (lr=5.8140e-05) (hash(x)=47965974)
+3000 val loss 7.1723
+3000 val perplexity 1302.8431
+3000 train 7.095241 (lr=1.6611e-05) (hash(x)=47965974)
+3000 val loss 6.8833
+3000 val perplexity 975.8807
+3000 train 6.776263 (lr=4.1529e-05) (hash(x)=47965974)
+3100 val loss 6.7373
+3100 val perplexity 843.3165
+3100 train 6.585240 (lr=5.7340e-05) (hash(x)=48205243)
+3000 val loss 6.9506
+3000 val perplexity 1043.7854
+3000 train 6.852409 (lr=2.4917e-05) (hash(x)=47965974)
+3100 val loss 7.1504
+3100 val perplexity 1274.6481
+3100 train 7.029405 (lr=1.6383e-05) (hash(x)=48205243)
+3200 val loss 6.7279
+3200 val perplexity 835.3999
+3200 train 6.822510 (lr=5.6522e-05) (hash(x)=54511383)
+3100 val loss 6.8696
+3100 val perplexity 962.5995
+3100 train 6.728429 (lr=4.0957e-05) (hash(x)=48205243)
+3100 val loss 6.9418
+3100 val perplexity 1034.6399
+3100 train 6.792998 (lr=2.4574e-05) (hash(x)=48205243)
+3200 val loss 7.1385
+3200 val perplexity 1259.5211
+3200 train 7.222794 (lr=1.6149e-05) (hash(x)=54511383)
+3300 val loss 6.7237
+3300 val perplexity 831.9065
+3300 train 6.682970 (lr=5.5684e-05) (hash(x)=54428388)
+3200 val loss 6.8627
+3200 val perplexity 955.9105
+3200 train 6.959624 (lr=4.0373e-05) (hash(x)=54511383)
+3200 val loss 6.9223
+3200 val perplexity 1014.6286
+3200 train 7.015221 (lr=2.4224e-05) (hash(x)=54511383)
+3300 val loss 7.1309
+3300 val perplexity 1249.9888
+3300 train 7.094438 (lr=1.5910e-05) (hash(x)=54428388)
+3400 val loss 6.7185
+3400 val perplexity 827.5817
+3400 train 6.686805 (lr=5.4829e-05) (hash(x)=48115990)
+3300 val loss 6.8569
+3300 val perplexity 950.3951
+3300 train 6.825317 (lr=3.9775e-05) (hash(x)=54428388)
+3300 val loss 6.9152
+3300 val perplexity 1007.5222
+3300 train 6.874907 (lr=2.3865e-05) (hash(x)=54428388)
+3400 val loss 7.1204
+3400 val perplexity 1236.9988
+3400 train 7.180803 (lr=1.5666e-05) (hash(x)=48115990)
+3500 val loss 6.7021
+3500 val perplexity 814.1271
+3500 train 6.281488 (lr=5.3958e-05) (hash(x)=41137345)
+3400 val loss 6.8489
+3400 val perplexity 942.8759
+3400 train 6.851781 (lr=3.9164e-05) (hash(x)=48115990)
+3400 val loss 6.9011
+3400 val perplexity 993.3831
+3400 train 6.906279 (lr=2.3498e-05) (hash(x)=48115990)
+3500 val loss 7.1075
+3500 val perplexity 1221.1249
+3500 train 6.737636 (lr=1.5416e-05) (hash(x)=41137345)
+3600 val loss 6.6806
+3600 val perplexity 796.7698
+3600 train 6.564756 (lr=5.3070e-05) (hash(x)=55186224)
+3500 val loss 6.8443
+3500 val perplexity 938.5548
+3500 train 6.456430 (lr=3.8541e-05) (hash(x)=41137345)
+3500 val loss 6.8925
+3500 val perplexity 984.8782
+3500 train 6.500405 (lr=2.3125e-05) (hash(x)=41137345)
+3600 val loss 7.0997
+3600 val perplexity 1211.5752
+3600 train 6.996099 (lr=1.5163e-05) (hash(x)=55186224)
+3700 val loss 6.6738
+3700 val perplexity 791.3663
+3700 train 6.579067 (lr=5.2167e-05) (hash(x)=54990049)
+3600 val loss 6.8360
+3600 val perplexity 930.7857
+3600 train 6.722075 (lr=3.7907e-05) (hash(x)=55186224)
+3600 val loss 6.8866
+3600 val perplexity 979.0352
+3600 train 6.769399 (lr=2.2744e-05) (hash(x)=55186224)
+3700 val loss 7.0944
+3700 val perplexity 1205.2024
+3700 train 6.992174 (lr=1.4905e-05) (hash(x)=54990049)
+3800 val loss 6.6671
+3800 val perplexity 786.1088
+3800 train 6.389765 (lr=5.1251e-05) (hash(x)=46288812)
+3700 val loss 6.8334
+3700 val perplexity 928.3057
+3700 train 6.733560 (lr=3.7262e-05) (hash(x)=54990049)
+3700 val loss 6.8801
+3700 val perplexity 972.7336
+3700 train 6.785053 (lr=2.2357e-05) (hash(x)=54990049)
+3800 val loss 7.0916
+3800 val perplexity 1201.8423
+3800 train 6.850882 (lr=1.4643e-05) (hash(x)=46288812)
+3900 val loss 6.6555
+3900 val perplexity 777.0841
+3900 train 6.301198 (lr=5.0321e-05) (hash(x)=45829773)
+3800 val loss 6.8309
+3800 val perplexity 925.9794
+3800 train 6.575776 (lr=3.6608e-05) (hash(x)=46288812)
+3800 val loss 6.8744
+3800 val perplexity 967.1729
+3800 train 6.619145 (lr=2.1965e-05) (hash(x)=46288812)
+3900 val loss 7.0761
+3900 val perplexity 1183.3990
+3900 train 6.760256 (lr=1.4377e-05) (hash(x)=45829773)
+4000 val loss 6.6570
+4000 val perplexity 778.1891
+4000 train 6.411054 (lr=4.9379e-05) (hash(x)=52499943)
+3900 val loss 6.8091
+3900 val perplexity 906.0413
+3900 val loss 6.8577
+3900 val perplexity 951.1495
+3900 train 6.470855 (lr=3.5944e-05) (hash(x)=45829773)
+3900 train 6.522695 (lr=2.1566e-05) (hash(x)=45829773)
+4000 val loss 7.0684
+4000 val perplexity 1174.2996
+4000 train 6.878099 (lr=1.4108e-05) (hash(x)=52499943)
+4100 val loss 6.6296
+4100 val perplexity 757.1706
+4100 train 6.538190 (lr=4.8426e-05) (hash(x)=48563796)
+4000 val loss 6.8547
+4000 val perplexity 948.2933
+4000 val loss 6.8092
+4000 val perplexity 906.1381
+4000 train 6.591424 (lr=3.5271e-05) (hash(x)=52499943)
+4000 train 6.649190 (lr=2.1162e-05) (hash(x)=52499943)
+4100 val loss 7.0474
+4100 val perplexity 1149.8827
+4100 train 6.979160 (lr=1.3836e-05) (hash(x)=48563796)
+4200 val loss 6.6076
+4200 val perplexity 740.7054
+4200 train 6.571188 (lr=4.7463e-05) (hash(x)=49165143)
+4100 val loss 6.7875
+4100 val perplexity 886.7177
+4100 train 6.698511 (lr=3.4590e-05) (hash(x)=48563796)
+4100 val loss 6.8290
+4100 val perplexity 924.2396
+4100 train 6.757933 (lr=2.0754e-05) (hash(x)=48563796)
+4200 val loss 7.0299
+4200 val perplexity 1129.8665
+4200 train 7.013832 (lr=1.3561e-05) (hash(x)=49165143)
+4300 val loss 6.5900
+4300 val perplexity 727.8115
+4300 train 6.669829 (lr=4.6490e-05) (hash(x)=50973176)
+4200 val loss 6.8093
+4200 val perplexity 906.2409
+4200 train 6.791944 (lr=2.0341e-05) (hash(x)=49165143)
+4200 val loss 6.7663
+4200 val perplexity 868.0909
+4200 train 6.749839 (lr=3.3902e-05) (hash(x)=49165143)
+4300 val loss 7.0162
+4300 val perplexity 1114.5115
+4300 train 7.075418 (lr=1.3283e-05) (hash(x)=50973176)
+4400 val loss 6.5760
+4400 val perplexity 717.6350
+4400 train 6.643890 (lr=4.5509e-05) (hash(x)=55275124)
+4300 val loss 6.7969
+4300 val perplexity 895.0617
+4300 train 6.871970 (lr=1.9924e-05) (hash(x)=50973176)
+4300 val loss 6.7452
+4300 val perplexity 849.9814
+4300 train 6.811849 (lr=3.3207e-05) (hash(x)=50973176)
+4400 val loss 7.0096
+4400 val perplexity 1107.1816
+4400 train 7.059990 (lr=1.3003e-05) (hash(x)=55275124)
+4500 val loss 6.5617
+4500 val perplexity 707.4891
+4500 train 6.930486 (lr=4.4521e-05) (hash(x)=58646505)
+4400 val loss 6.7828
+4400 val perplexity 882.5753
+4400 train 6.836937 (lr=1.9504e-05) (hash(x)=55275124)
+4400 val loss 6.7274
+4400 val perplexity 834.9734
+4500 val loss 6.9973
+4500 val perplexity 1093.6769
+4400 train 6.781178 (lr=3.2507e-05) (hash(x)=55275124)
+4500 train 7.363664 (lr=1.2720e-05) (hash(x)=58646505)
+4600 val loss 6.5481
+4600 val perplexity 697.8899
+4600 train 6.364978 (lr=4.3527e-05) (hash(x)=42554666)
+4500 val loss 6.7737
+4500 val perplexity 874.5845
+4500 train 7.154090 (lr=1.9081e-05) (hash(x)=58646505)
+4600 val loss 6.9881
+4600 val perplexity 1083.6766
+4600 train 6.796409 (lr=1.2436e-05) (hash(x)=42554666)
+4500 val loss 6.7220
+4500 val perplexity 830.4634
+4500 train 7.101217 (lr=3.1801e-05) (hash(x)=58646505)
+4700 val loss 6.5355
+4700 val perplexity 689.1530
+4700 train 6.470394 (lr=4.2528e-05) (hash(x)=47846764)
+4600 val loss 6.7566
+4600 val perplexity 859.7034
+4600 train 6.551023 (lr=1.8655e-05) (hash(x)=42554666)
+4700 val loss 6.9803
+4700 val perplexity 1075.2849
+4700 train 6.901748 (lr=1.2151e-05) (hash(x)=47846764)
+4600 val loss 6.7128
+4600 val perplexity 822.8683
+4600 train 6.511578 (lr=3.1091e-05) (hash(x)=42554666)
+4800 val loss 6.5178
+4800 val perplexity 677.0767
+4800 train 7.064293 (lr=4.1525e-05) (hash(x)=58239019)
+4700 val loss 6.7513
+4700 val perplexity 855.1525
+4800 val loss 6.9707
+4800 val perplexity 1064.9286
+4700 train 6.683542 (lr=1.8226e-05) (hash(x)=47846764)
+4800 train 7.519922 (lr=1.1864e-05) (hash(x)=58239019)
+4700 val loss 6.6994
+4700 val perplexity 811.9506
+4700 train 6.628958 (lr=3.0377e-05) (hash(x)=47846764)
+4900 val loss 6.5181
+4900 val perplexity 677.2886
+4900 train 6.572011 (lr=4.0518e-05) (hash(x)=50711220)
+4800 val loss 6.7374
+4800 val perplexity 843.3644
+4900 val loss 6.9609
+4900 val perplexity 1054.5725
+4800 train 7.297200 (lr=1.7796e-05) (hash(x)=58239019)
+4900 train 7.010784 (lr=1.1577e-05) (hash(x)=50711220)
+4800 val loss 6.6917
+4800 val perplexity 805.7076
+4800 train 7.249681 (lr=2.9661e-05) (hash(x)=58239019)
+5000 val loss 6.5144
+5000 val perplexity 674.8202
+5000 train 6.457778 (lr=3.9510e-05) (hash(x)=45994194)
+5000 val loss 6.9575
+5000 val perplexity 1050.9631
+4900 val loss 6.7300
+4900 val perplexity 837.1061
+4900 train 6.796998 (lr=1.7365e-05) (hash(x)=50711220)
+4900 val loss 6.6740
+4900 val perplexity 791.5267
+4900 train 6.739973 (lr=2.8942e-05) (hash(x)=50711220)
+5000 train 6.903171 (lr=1.1288e-05) (hash(x)=45994194)
+5100 val loss 6.4941
+5100 val perplexity 661.2037
+5100 train 6.372785 (lr=3.8500e-05) (hash(x)=48659050)
+5100 val loss 6.9444
+5100 val perplexity 1037.3356
+5100 train 6.808917 (lr=1.1000e-05) (hash(x)=48659050)
+5000 val loss 6.7196
+5000 val perplexity 828.4827
+5000 val loss 6.6801
+5000 val perplexity 796.3593
+5000 train 6.678499 (lr=1.6933e-05) (hash(x)=45994194)
+5000 train 6.628725 (lr=2.8221e-05) (hash(x)=45994194)
+5200 val loss 6.4883
+5200 val perplexity 657.3944
+5200 train 6.391860 (lr=3.7490e-05) (hash(x)=49369682)
+5200 val loss 6.9390
+5200 val perplexity 1031.7484
+5200 train 6.868454 (lr=1.0712e-05) (hash(x)=49369682)
+5100 val loss 6.7082
+5100 val perplexity 819.0988
+5100 val loss 6.6555
+5100 val perplexity 777.0222
+5100 train 6.584312 (lr=1.6500e-05) (hash(x)=48659050)
+5100 train 6.527529 (lr=2.7500e-05) (hash(x)=48659050)
+5300 val loss 6.4765
+5300 val perplexity 649.6841
+5300 train 6.899568 (lr=3.6482e-05) (hash(x)=57787700)
+5300 val loss 6.9323
+5300 val perplexity 1024.8837
+5300 train 7.331203 (lr=1.0423e-05) (hash(x)=57787700)
+5200 val loss 6.6987
+5200 val perplexity 811.3156
+5200 val loss 6.6475
+5200 val perplexity 770.8427
+5200 train 6.620082 (lr=1.6067e-05) (hash(x)=49369682)
+5200 train 6.554370 (lr=2.6779e-05) (hash(x)=49369682)
+5400 val loss 6.4795
+5400 val perplexity 651.6630
+5400 train 6.409245 (lr=3.5475e-05) (hash(x)=49365400)
+5400 val loss 6.9261
+5400 val perplexity 1018.4915
+5400 train 6.856396 (lr=1.0136e-05) (hash(x)=49365400)
+5300 val loss 6.6931
+5300 val perplexity 806.8045
+5300 train 7.125628 (lr=1.5635e-05) (hash(x)=57787700)
+5300 val loss 6.6388
+5300 val perplexity 764.1794
+5300 train 7.059060 (lr=2.6058e-05) (hash(x)=57787700)
+5500 val loss 6.4668
+5500 val perplexity 643.4189
+5500 train 6.418372 (lr=3.4472e-05) (hash(x)=48720412)
+5500 val loss 6.9221
+5500 val perplexity 1014.4897
+5500 train 6.863081 (lr=9.8491e-06) (hash(x)=48720412)
+5400 val loss 6.6870
+5400 val perplexity 801.8978
+5400 train 6.628658 (lr=1.5204e-05) (hash(x)=49365400)
+5400 val loss 6.6302
+5400 val perplexity 757.6622
+5400 train 6.565665 (lr=2.5339e-05) (hash(x)=49365400)
+5600 val loss 6.4579
+5600 val perplexity 637.7405
+5600 train 6.811317 (lr=3.3473e-05) (hash(x)=55784800)
+5600 val loss 6.9129
+5600 val perplexity 1005.1623
+5600 train 7.253220 (lr=9.5636e-06) (hash(x)=55784800)
+5500 val loss 6.6801
+5500 val perplexity 796.3778
+5500 train 6.631643 (lr=1.4774e-05) (hash(x)=48720412)
+5500 val loss 6.6239
+5500 val perplexity 752.8723
+5500 train 6.577877 (lr=2.4623e-05) (hash(x)=48720412)
+5700 val loss 6.4601
+5700 val perplexity 639.1117
+5700 train 6.290679 (lr=3.2479e-05) (hash(x)=50073634)
+5700 val loss 6.9127
+5700 val perplexity 1004.9950
+5700 train 6.758874 (lr=9.2796e-06) (hash(x)=50073634)
+5600 val loss 6.6729
+5600 val perplexity 790.6545
+5600 train 7.028577 (lr=1.4345e-05) (hash(x)=55784800)
+5600 val loss 6.6146
+5600 val perplexity 745.8879
+5600 train 6.962052 (lr=2.3909e-05) (hash(x)=55784800)
+5800 val loss 6.4550
+5800 val perplexity 635.8431
+5800 train 6.277244 (lr=3.1491e-05) (hash(x)=50170324)
+5800 val loss 6.9058
+5800 val perplexity 998.0084
+5800 train 6.738111 (lr=8.9973e-06) (hash(x)=50170324)
+5700 val loss 6.6696
+5700 val perplexity 788.0435
+5700 train 6.520940 (lr=1.3919e-05) (hash(x)=50073634)
+5700 val loss 6.6135
+5700 val perplexity 745.0647
+5700 train 6.450591 (lr=2.3199e-05) (hash(x)=50073634)
+5900 val loss 6.4499
+5900 val perplexity 632.6409
+5900 train 6.166365 (lr=3.0510e-05) (hash(x)=48410268)
+5900 val loss 6.9088
+5900 val perplexity 1001.0864
+5900 train 6.630468 (lr=8.7171e-06) (hash(x)=48410268)
+5800 val loss 6.6111
+5800 val perplexity 743.3010
+5800 val loss 6.6599
+5800 val perplexity 780.4451
+5800 train 6.442124 (lr=2.2493e-05) (hash(x)=50170324)
+5800 train 6.497536 (lr=1.3496e-05) (hash(x)=50170324)
+6000 val loss 6.4450
+6000 val perplexity 629.5153
+6000 train 6.299426 (lr=2.9537e-05) (hash(x)=49527342)
+6000 val loss 6.9027
+6000 val perplexity 994.9408
+6000 train 6.801613 (lr=8.4393e-06) (hash(x)=49527342)
+5900 val loss 6.6010
+5900 val perplexity 735.8242
+5900 train 6.316404 (lr=2.1793e-05) (hash(x)=48410268)
+5900 val loss 6.6584
+5900 val perplexity 779.3054
+5900 train 6.376041 (lr=1.3076e-05) (hash(x)=48410268)
+6100 val loss 6.4465
+6100 val perplexity 630.4760
+6100 train 6.241553 (lr=2.8574e-05) (hash(x)=49550294)
+6100 val loss 6.8991
+6100 val perplexity 991.4178
+6100 train 6.739030 (lr=8.1640e-06) (hash(x)=49550294)
+6000 val loss 6.6054
+6000 val perplexity 739.0561
+6000 train 6.494583 (lr=2.1098e-05) (hash(x)=49527342)
+6000 val loss 6.6552
+6000 val perplexity 776.8270
+6000 train 6.549714 (lr=1.2659e-05) (hash(x)=49527342)
+6200 val loss 6.4413
+6200 val perplexity 627.2357
+6200 train 5.957240 (lr=2.7621e-05) (hash(x)=42126106)
+6200 val loss 6.8939
+6200 val perplexity 986.2703
+6200 train 6.446277 (lr=7.8917e-06) (hash(x)=42126106)
+6100 val loss 6.5968
+6100 val perplexity 732.7717
+6100 train 6.419609 (lr=2.0410e-05) (hash(x)=49550294)
+6100 val loss 6.6495
+6100 val perplexity 772.4101
+6100 train 6.473972 (lr=1.2246e-05) (hash(x)=49550294)
+6300 val loss 6.4439
+6300 val perplexity 628.8303
+6300 train 6.190170 (lr=2.6679e-05) (hash(x)=49608772)
+6300 val loss 6.8934
+6300 val perplexity 985.7131
+6300 train 6.680425 (lr=7.6226e-06) (hash(x)=49608772)
+6200 val loss 6.5978
+6200 val perplexity 733.4502
+6200 train 6.131247 (lr=1.9729e-05) (hash(x)=42126106)
+6200 val loss 6.6458
+6200 val perplexity 769.5638
+6200 train 6.174097 (lr=1.1838e-05) (hash(x)=42126106)
+6400 val loss 6.4316
+6400 val perplexity 621.1544
+6400 train 5.968055 (lr=2.5749e-05) (hash(x)=52324417)
+6400 val loss 6.8855
+6400 val perplexity 977.9658
+6400 train 6.481584 (lr=7.3569e-06) (hash(x)=52324417)
+6300 val loss 6.5943
+6300 val perplexity 730.8967
+6300 train 6.353149 (lr=1.9056e-05) (hash(x)=49608772)
+6300 val loss 6.6441
+6300 val perplexity 768.2593
+6300 train 6.394776 (lr=1.1434e-05) (hash(x)=49608772)
+6500 val loss 6.4077
+6500 val perplexity 606.4824
+6500 train 6.457166 (lr=2.4833e-05) (hash(x)=46207215)
+6500 val loss 6.8695
+6500 val perplexity 962.4999
+6500 train 6.899956 (lr=7.0950e-06) (hash(x)=46207215)
+6400 val loss 6.5863
+6400 val perplexity 725.1245
+6400 train 6.154537 (lr=1.8392e-05) (hash(x)=52324417)
+6400 val loss 6.6324
+6400 val perplexity 759.3197
+6400 train 6.207419 (lr=1.1035e-05) (hash(x)=52324417)
+6600 val loss 6.3960
+6600 val perplexity 599.4461
+6600 train 6.364021 (lr=2.3930e-05) (hash(x)=49027014)
+6600 val loss 6.8598
+6600 val perplexity 953.2013
+6600 train 6.788251 (lr=6.8372e-06) (hash(x)=49027014)
+6500 val loss 6.5651
+6500 val perplexity 709.9162
+6500 train 6.600822 (lr=1.7738e-05) (hash(x)=46207215)
+6500 val loss 6.6128
+6500 val perplexity 744.5444
+6500 train 6.661718 (lr=1.0643e-05) (hash(x)=46207215)
+6700 val loss 6.3989
+6700 val perplexity 601.1656
+6700 train 6.432356 (lr=2.3042e-05) (hash(x)=46232513)
+6700 val loss 6.8536
+6700 val perplexity 947.2444
+6700 train 6.824396 (lr=6.5835e-06) (hash(x)=46232513)
+6600 val loss 6.5582
+6600 val perplexity 704.9709
+6600 train 6.505264 (lr=1.7093e-05) (hash(x)=49027014)
+6600 val loss 6.6036
+6600 val perplexity 737.7280
+6600 train 6.547662 (lr=1.0256e-05) (hash(x)=49027014)
+6800 val loss 6.3890
+6800 val perplexity 595.2584
+6800 train 6.318826 (lr=2.2171e-05) (hash(x)=47348403)
+6800 val loss 6.8536
+6800 val perplexity 947.2534
+6800 train 6.787578 (lr=6.3345e-06) (hash(x)=47348403)
+6700 val loss 6.5480
+6700 val perplexity 697.8230
+6700 train 6.583887 (lr=1.6459e-05) (hash(x)=46232513)
+6700 val loss 6.5991
+6700 val perplexity 734.4021
+6700 train 6.615988 (lr=9.8753e-06) (hash(x)=46232513)
+6900 val loss 6.3967
+6900 val perplexity 599.8644
+6900 train 6.417536 (lr=2.1316e-05) (hash(x)=49806647)
+6900 val loss 6.8555
+6900 val perplexity 949.0623
+6900 train 6.840402 (lr=6.0902e-06) (hash(x)=49806647)
+6800 val loss 6.5493
+6800 val perplexity 698.7370
+6800 train 6.484587 (lr=1.5836e-05) (hash(x)=47348403)
+7000 val loss 6.3745
+7000 val perplexity 586.6633
+6800 val loss 6.5951
+6800 val perplexity 731.4786
+7000 train 6.413634 (lr=2.0478e-05) (hash(x)=50893018)
+6800 train 6.525431 (lr=9.5017e-06) (hash(x)=47348403)
+7000 val loss 6.8389
+7000 val perplexity 933.5077
+7000 train 6.886159 (lr=5.8510e-06) (hash(x)=50893018)
+6900 val loss 6.5492
+6900 val perplexity 698.6997
+6900 train 6.569705 (lr=1.5225e-05) (hash(x)=49806647)
+7100 val loss 6.3662
+7100 val perplexity 581.8609
+7100 train 6.399204 (lr=1.9660e-05) (hash(x)=49157639)
+6900 val loss 6.5899
+6900 val perplexity 727.7289
+6900 train 6.611711 (lr=9.1353e-06) (hash(x)=49806647)
+7100 val loss 6.8357
+7100 val perplexity 930.4750
+7100 train 6.874208 (lr=5.6170e-06) (hash(x)=49157639)
+7000 val loss 6.5346
+7000 val perplexity 688.5408
+7000 train 6.584486 (lr=1.4627e-05) (hash(x)=50893018)
+7200 val loss 6.3606
+7200 val perplexity 578.5686
+7200 train 6.369837 (lr=1.8860e-05) (hash(x)=47014759)
+7000 val loss 6.5786
+7000 val perplexity 719.5305
+7000 train 6.634720 (lr=8.7764e-06) (hash(x)=50893018)
+7200 val loss 6.8313
+7200 val perplexity 926.3760
+7200 train 6.809377 (lr=5.3886e-06) (hash(x)=47014759)
+7100 val loss 6.5322
+7100 val perplexity 686.9103
+7300 val loss 6.3576
+7300 val perplexity 576.8370
+7100 train 6.576263 (lr=1.4043e-05) (hash(x)=49157639)
+7300 train 6.341197 (lr=1.8081e-05) (hash(x)=47325591)
+7100 val loss 6.5737
+7100 val perplexity 716.0148
+7100 train 6.610196 (lr=8.4255e-06) (hash(x)=49157639)
+7300 val loss 6.8313
+7300 val perplexity 926.4335
+7300 train 6.779392 (lr=5.1659e-06) (hash(x)=47325591)
+7400 val loss 6.3512
+7400 val perplexity 573.1561
+7400 train 6.235737 (lr=1.7323e-05) (hash(x)=49184604)
+7200 val loss 6.5260
+7200 val perplexity 682.6921
+7200 train 6.518712 (lr=1.3471e-05) (hash(x)=47014759)
+7200 val loss 6.5675
+7200 val perplexity 711.5921
+7200 train 6.546741 (lr=8.0829e-06) (hash(x)=47014759)
+7400 val loss 6.8224
+7400 val perplexity 918.1414
+7400 train 6.680563 (lr=4.9493e-06) (hash(x)=49184604)
+7500 val loss 6.3495
+7500 val perplexity 572.2331
+7300 val loss 6.5227
+7300 val perplexity 680.4317
+7300 train 6.489656 (lr=1.2915e-05) (hash(x)=47325591)
+7500 train 6.565956 (lr=1.6586e-05) (hash(x)=55053584)
+7300 val loss 6.5669
+7300 val perplexity 711.1389
+7300 train 6.538014 (lr=7.7489e-06) (hash(x)=47325591)
+7500 val loss 6.8192
+7500 val perplexity 915.2512
+7500 train 7.127323 (lr=4.7389e-06) (hash(x)=55053584)
+7400 val loss 6.5154
+7400 val perplexity 675.4557
+7400 train 6.394844 (lr=1.2373e-05) (hash(x)=49184604)
+7600 val loss 6.3460
+7600 val perplexity 570.1979
+7600 train 6.293826 (lr=1.5872e-05) (hash(x)=48693923)
+7400 val loss 6.5571
+7400 val perplexity 704.2495
+7400 train 6.427595 (lr=7.4239e-06) (hash(x)=49184604)
+7600 val loss 6.8172
+7600 val perplexity 913.3918
+7600 train 6.765959 (lr=4.5349e-06) (hash(x)=48693923)
+7500 val loss 6.5101
+7500 val perplexity 671.9246
+7500 train 6.810478 (lr=1.1847e-05) (hash(x)=55053584)
+7700 val loss 6.3417
+7700 val perplexity 567.7472
+7700 train 5.880659 (lr=1.5181e-05) (hash(x)=40952882)
+7500 val loss 6.5521
+7500 val perplexity 700.7015
+7500 train 6.888753 (lr=7.1083e-06) (hash(x)=55053584)
+7700 val loss 6.8143
+7700 val perplexity 910.7701
+7700 train 6.320709 (lr=4.3375e-06) (hash(x)=40952882)
+7800 val loss 6.3429
+7800 val perplexity 568.4157
+7800 train 6.443004 (lr=1.4514e-05) (hash(x)=52487845)
+7600 val loss 6.5085
+7600 val perplexity 670.8518
+7600 train 6.467136 (lr=1.1337e-05) (hash(x)=48693923)
+7600 val loss 6.5490
+7600 val perplexity 698.5761
+7600 train 6.507745 (lr=6.8023e-06) (hash(x)=48693923)
+7800 val loss 6.8130
+7800 val perplexity 909.5957
+7800 train 6.895899 (lr=4.1470e-06) (hash(x)=52487845)
+7900 val loss 6.3350
+7900 val perplexity 563.9832
+7900 train 6.403783 (lr=1.3872e-05) (hash(x)=50221547)
+7700 val loss 6.5029
+7700 val perplexity 667.0413
+7700 train 6.025047 (lr=1.0844e-05) (hash(x)=40952882)
+7700 val loss 6.5466
+7700 val perplexity 696.8989
+7700 train 6.057941 (lr=6.5062e-06) (hash(x)=40952882)
+7900 val loss 6.8090
+7900 val perplexity 905.9458
+7900 train 6.892159 (lr=3.9635e-06) (hash(x)=50221547)
+8000 val loss 6.3324
+8000 val perplexity 562.5143
+8000 train 6.561235 (lr=1.3255e-05) (hash(x)=62294204)
+7800 val loss 6.4995
+7800 val perplexity 664.7764
+7800 train 6.599380 (lr=1.0367e-05) (hash(x)=52487845)
+7800 val loss 6.5443
+7800 val perplexity 695.2772
+7800 train 6.642978 (lr=6.2205e-06) (hash(x)=52487845)
+8000 val loss 6.8080
+8000 val perplexity 905.0434
+8000 train 7.153604 (lr=3.7873e-06) (hash(x)=62294204)
+8100 val loss 6.3286
+8100 val perplexity 560.3528
+8100 train 5.986205 (lr=1.2665e-05) (hash(x)=44401967)
+7900 val loss 6.4989
+7900 val perplexity 664.4284
+7900 train 6.581850 (lr=9.9088e-06) (hash(x)=50221547)
+7900 val loss 6.5393
+7900 val perplexity 691.8358
+7900 train 6.626468 (lr=5.9453e-06) (hash(x)=50221547)
+8100 val loss 6.8051
+8100 val perplexity 902.4113
+8100 train 6.457790 (lr=3.6184e-06) (hash(x)=44401967)
+8200 val loss 6.3292
+8200 val perplexity 560.6861
+8200 train 6.249769 (lr=1.2100e-05) (hash(x)=52769095)
+8000 val loss 6.4917
+8000 val perplexity 659.6194
+8000 train 6.824268 (lr=9.4682e-06) (hash(x)=62294204)
+8000 val loss 6.5362
+8000 val perplexity 689.6366
+8000 train 6.911385 (lr=5.6809e-06) (hash(x)=62294204)
+8200 val loss 6.8045
+8200 val perplexity 901.8989
+8200 train 6.761432 (lr=3.4572e-06) (hash(x)=52769095)
+8300 val loss 6.3299
+8300 val perplexity 561.1138
+8300 train 6.275354 (lr=1.1563e-05) (hash(x)=56829883)
+8100 val loss 6.4893
+8100 val perplexity 658.0931
+8100 train 6.156446 (lr=9.0461e-06) (hash(x)=44401967)
+8100 val loss 6.5327
+8100 val perplexity 687.2546
+8100 train 6.194956 (lr=5.4277e-06) (hash(x)=44401967)
+8300 val loss 6.8066
+8300 val perplexity 903.8031
+8300 train 6.767237 (lr=3.3037e-06) (hash(x)=56829883)
+8400 val loss 6.3270
+8400 val perplexity 559.4752
+8400 train 6.257383 (lr=1.1053e-05) (hash(x)=52147375)
+8200 val loss 6.4897
+8200 val perplexity 658.3050
+8200 train 6.420258 (lr=8.6430e-06) (hash(x)=52769095)
+8200 val loss 6.5323
+8200 val perplexity 686.9604
+8200 train 6.474307 (lr=5.1858e-06) (hash(x)=52769095)
+8400 val loss 6.8053
+8400 val perplexity 902.6118
+8400 train 6.752232 (lr=3.1581e-06) (hash(x)=52147375)
+8500 val loss 6.3305
+8500 val perplexity 561.4124
+8500 train 6.627474 (lr=1.0572e-05) (hash(x)=60197820)
+8300 val loss 6.4888
+8300 val perplexity 657.7073
+8300 train 6.448828 (lr=8.2593e-06) (hash(x)=56829883)
+8300 val loss 6.5343
+8300 val perplexity 688.3612
+8300 train 6.490636 (lr=4.9556e-06) (hash(x)=56829883)
+8500 val loss 6.8045
+8500 val perplexity 901.8800
+8500 train 7.037055 (lr=3.0206e-06) (hash(x)=60197820)
+8600 val loss 6.3244
+8600 val perplexity 558.0261
+8600 train 6.074996 (lr=1.0119e-05) (hash(x)=49377068)
+8400 val loss 6.4885
+8400 val perplexity 657.5069
+8400 train 6.421007 (lr=7.8953e-06) (hash(x)=52147375)
+8400 val loss 6.5331
+8400 val perplexity 687.5152
+8400 train 6.478803 (lr=4.7372e-06) (hash(x)=52147375)
+8600 val loss 6.8019
+8600 val perplexity 899.5659
+8600 train 6.592875 (lr=2.8913e-06) (hash(x)=49377068)
+8700 val loss 6.3242
+8700 val perplexity 557.9348
+8700 train 6.308025 (lr=9.6960e-06) (hash(x)=51092724)
+8500 val loss 6.4867
+8500 val perplexity 656.3723
+8500 train 6.771042 (lr=7.5515e-06) (hash(x)=60197820)
+8500 val loss 6.5317
+8500 val perplexity 686.5867
+8500 train 6.808648 (lr=4.5309e-06) (hash(x)=60197820)
+8700 val loss 6.8009
+8700 val perplexity 898.6459
+8700 train 6.764141 (lr=2.7703e-06) (hash(x)=51092724)
+8800 val loss 6.3245
+8800 val perplexity 558.0521
+8800 train 6.319681 (lr=9.3021e-06) (hash(x)=48642928)
+8600 val loss 6.4817
+8600 val perplexity 653.0836
+8600 train 6.252263 (lr=7.2282e-06) (hash(x)=49377068)
+8600 val loss 6.5274
+8600 val perplexity 683.5885
+8600 train 6.298992 (lr=4.3369e-06) (hash(x)=49377068)
+8800 val loss 6.7992
+8800 val perplexity 897.1229
+8800 train 6.821721 (lr=2.6577e-06) (hash(x)=48642928)
+8900 val loss 6.3333
+8900 val perplexity 563.0059
+8900 train 6.536117 (lr=8.9382e-06) (hash(x)=55342246)
+8700 val loss 6.4842
+8700 val perplexity 654.7362
+8700 train 6.468521 (lr=6.9257e-06) (hash(x)=51092724)
+8700 val loss 6.5261
+8700 val perplexity 682.7126
+8700 train 6.495368 (lr=4.1554e-06) (hash(x)=51092724)
+8900 val loss 6.8031
+8900 val perplexity 900.6793
+8900 train 7.076955 (lr=2.5538e-06) (hash(x)=55342246)
+9000 val loss 6.3072
+9000 val perplexity 548.5292
+9000 train 6.311553 (lr=8.6047e-06) (hash(x)=48093368)
+8800 val loss 6.4768
+8800 val perplexity 649.8600
+8800 train 6.481946 (lr=6.6444e-06) (hash(x)=48642928)
+8800 val loss 6.5237
+8800 val perplexity 681.0806
+8800 train 6.529369 (lr=3.9866e-06) (hash(x)=48642928)
+9000 val loss 6.7872
+9000 val perplexity 886.3854
+9000 train 6.737963 (lr=2.4585e-06) (hash(x)=48093368)
+9100 val loss 6.3026
+9100 val perplexity 545.9979
+9100 train 6.379838 (lr=8.3020e-06) (hash(x)=48578183)
+8900 val loss 6.4853
+8900 val perplexity 655.4434
+8900 train 6.709653 (lr=6.3845e-06) (hash(x)=55342246)
+8900 val loss 6.5276
+8900 val perplexity 683.7887
+8900 train 6.777533 (lr=3.8307e-06) (hash(x)=55342246)
+9100 val loss 6.7850
+9100 val perplexity 884.5032
+9100 train 6.877484 (lr=2.3720e-06) (hash(x)=48578183)
+9200 val loss 6.2996
+9200 val perplexity 544.3685
+9200 train 6.467710 (lr=8.0302e-06) (hash(x)=50794720)
+9000 val loss 6.4627
+9000 val perplexity 640.7952
+9000 train 6.448120 (lr=6.1462e-06) (hash(x)=48093368)
+9000 val loss 6.5091
+9000 val perplexity 671.1916
+9000 train 6.476361 (lr=3.6877e-06) (hash(x)=48093368)
+9200 val loss 6.7823
+9200 val perplexity 882.1112
+9200 train 6.995069 (lr=2.2943e-06) (hash(x)=50794720)
+9300 val loss 6.2941
+9300 val perplexity 541.3758
+9300 train 6.044296 (lr=7.7898e-06) (hash(x)=46513190)
+9100 val loss 6.4606
+9100 val perplexity 639.4345
+9100 train 6.546059 (lr=5.9300e-06) (hash(x)=48578183)
+9100 val loss 6.5059
+9100 val perplexity 669.0497
+9100 train 6.587250 (lr=3.5580e-06) (hash(x)=48578183)
+9300 val loss 6.7809
+9300 val perplexity 880.8246
+9300 train 6.540437 (lr=2.2256e-06) (hash(x)=46513190)
+9400 val loss 6.2939
+9400 val perplexity 541.2728
+9400 train 5.963445 (lr=7.5809e-06) (hash(x)=43808238)
+9200 val loss 6.4558
+9200 val perplexity 636.3748
+9200 train 6.658762 (lr=5.7359e-06) (hash(x)=50794720)
+9200 val loss 6.5040
+9200 val perplexity 667.7790
+9200 train 6.728305 (lr=3.4415e-06) (hash(x)=50794720)
+9400 val loss 6.7788
+9400 val perplexity 879.0392
+9400 train 6.409406 (lr=2.1660e-06) (hash(x)=43808238)
+9500 val loss 6.2907
+9500 val perplexity 539.5046
+9500 train 6.018129 (lr=7.4038e-06) (hash(x)=45021888)
+9300 val loss 6.4518
+9300 val perplexity 633.8415
+9300 train 6.202276 (lr=5.5641e-06) (hash(x)=46513190)
+9300 val loss 6.5013
+9300 val perplexity 666.0020
+9300 train 6.249645 (lr=3.3385e-06) (hash(x)=46513190)
+9500 val loss 6.7771
+9500 val perplexity 877.5475
+9500 train 6.546412 (lr=2.1154e-06) (hash(x)=45021888)
+9600 val loss 6.2910
+9600 val perplexity 539.6711
+9600 train 6.393634 (lr=7.2586e-06) (hash(x)=56525570)
+9400 val loss 6.4526
+9400 val perplexity 634.3289
+9400 train 6.111716 (lr=5.4149e-06) (hash(x)=43808238)
+9400 val loss 6.5003
+9400 val perplexity 665.3675
+9400 train 6.157181 (lr=3.2490e-06) (hash(x)=43808238)
+9600 val loss 6.7750
+9600 val perplexity 875.6418
+9600 train 6.851060 (lr=2.0739e-06) (hash(x)=56525570)
+9700 val loss 6.2842
+9700 val perplexity 536.0264
+9700 train 6.466522 (lr=7.1456e-06) (hash(x)=52585913)
+9500 val loss 6.4483
+9500 val perplexity 631.6206
+9500 train 6.191110 (lr=5.2884e-06) (hash(x)=45021888)
+9500 val loss 6.4981
+9500 val perplexity 663.8555
+9500 train 6.242024 (lr=3.1730e-06) (hash(x)=45021888)
+9700 val loss 6.7736
+9700 val perplexity 874.4531
+9700 train 6.914565 (lr=2.0416e-06) (hash(x)=52585913)
+9800 val loss 6.2846
+9800 val perplexity 536.2417
+9800 train 6.464126 (lr=7.0647e-06) (hash(x)=52344698)
+9600 val loss 6.4494
+9600 val perplexity 632.3007
+9600 train 6.547867 (lr=5.1847e-06) (hash(x)=56525570)
+9600 val loss 6.4961
+9600 val perplexity 662.5447
+9600 train 6.603984 (lr=3.1108e-06) (hash(x)=56525570)
+9800 val loss 6.7723
+9800 val perplexity 873.3168
+9800 train 6.975519 (lr=2.0185e-06) (hash(x)=52344698)
+9900 val loss 6.2817
+9900 val perplexity 534.6906
+9900 train 6.284701 (lr=7.0162e-06) (hash(x)=51740945)
+9700 val loss 6.4440
+9700 val perplexity 628.9359
+9700 val loss 6.4940
+9700 val perplexity 661.1810
+9700 train 6.661100 (lr=3.0624e-06) (hash(x)=52585913)
+9700 train 6.610982 (lr=5.1040e-06) (hash(x)=52585913)
+9800 val loss 6.4918
+9800 val perplexity 659.7147
+9800 train 6.709685 (lr=3.0277e-06) (hash(x)=52344698)
+9900 val loss 6.7701
+9900 val perplexity 871.4020
+9900 train 6.791570 (lr=2.0046e-06) (hash(x)=51740945)
+9999 val loss 6.2791
+9999 val perplexity 533.3184
+9800 val loss 6.4419
+9800 val perplexity 627.5698
+9800 train 6.656963 (lr=5.0462e-06) (hash(x)=52344698)
+9900 val loss 6.4900
+9900 val perplexity 658.5361
+9900 train 6.512582 (lr=3.0069e-06) (hash(x)=51740945)
+9999 val loss 6.4887
+9999 val perplexity 657.6709
+9999 val loss 6.7688
+9999 val perplexity 870.2236
+9900 val loss 6.4399
+9900 val perplexity 626.3513
+9900 train 6.445999 (lr=5.0116e-06) (hash(x)=51740945)
+9999 val loss 6.4393
+9999 val perplexity 625.9971

attention_kindselective_n_heads8_seed1338/model_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0f421aaeba65c1ac7d8f57367d664ff7980380b67f84b76cf91bd1c70585212
+size 257976706

attention_kindselective_n_heads8_seed1338/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c4f9fc9b119cc1c40dd40402afd1d7d323f35ed99f346d00949f1f5bfc4ada0
+size 257976706

attention_kindselective_n_heads8_seed1338/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a56b7521c2f642fe74f79d77ad0872fad2baca63cfa6818f3aa12e176fbc01ae
+size 257976706

attention_kindselective_n_heads8_seed1338/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2438144af715e7e5d8e4af477a0526246dcaafb32968d2669293d131fc55497
+size 257976706

attention_kindselective_n_heads8_seed1338/optimizer_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2abe43e4b5e3c310d8acd2bab9e701733ffdc7fb74d3f2a93e525c2ff8db6f7e
+size 509672838

attention_kindselective_n_heads8_seed1338/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48a423a0d6c8d63c72833cdeedcfbb50adb632c8e5335799734d33897f804c57
+size 509672838

attention_kindselective_n_heads8_seed1338/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70054d22ec62aa54be8932a09eedbb337d629e7c18f1f1de827eaef7acb3375e
+size 509672838

attention_kindselective_n_heads8_seed1338/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edfb58324c9646bbe15807f28e2c49e576fa2581cd7d8fb52ea20593dad7c531
+size 509672838