Upload folder using huggingface_hub

attention_kindselective_n_heads8_seed1340/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads8_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 8, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 7e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "7e-5_10240_8_1340", "n_embd": 512}

attention_kindselective_n_heads8_seed1340/dataloader_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
+size 964

attention_kindselective_n_heads8_seed1340/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f03ed2ebf741f15e13c79e6cc1e9a19b308450d81cc3b4d8d0338c63d77ca59
+size 964

attention_kindselective_n_heads8_seed1340/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82590037fb2eecbec961f7967a8dd1b8d85515d31a252f66b92b8139858a8b7c
+size 964

attention_kindselective_n_heads8_seed1340/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c544303717d12355a69b8ffb1eb109434e4fdccfd5a61279b6e8ba2e870d6700
+size 964

attention_kindselective_n_heads8_seed1340/log2.txt

@@ -0,0 +1,1209 @@
+max_steps: 10000
+0 val loss 11.0744
+0 val perplexity 64500.1523
+0 val loss 11.0744
+0 val perplexity 64500.2734
+0 val loss 11.0744
+0 val perplexity 64501.0742
+0 val loss 11.0744
+0 val perplexity 64500.1523
+0 train 11.079541 (lr=3.5000e-07) (hash(x)=57791809)
+0 train 11.079541 (lr=1.5000e-07) (hash(x)=57791809)
+0 train 11.079541 (lr=2.5000e-07) (hash(x)=57791809)
+0 train 11.079541 (lr=1.0000e-07) (hash(x)=57791809)
+100 val loss 9.5254
+100 val perplexity 13704.0791
+100 train 9.545608 (lr=3.5350e-05) (hash(x)=48211824)
+100 val loss 9.9436
+100 val perplexity 20819.3242
+100 val loss 9.6354
+100 val perplexity 15296.7988
+100 train 9.972991 (lr=1.0100e-05) (hash(x)=48211824)
+100 train 9.657048 (lr=2.5250e-05) (hash(x)=48211824)
+100 val loss 9.7540
+100 val perplexity 17222.1250
+100 train 9.779591 (lr=1.5150e-05) (hash(x)=48211824)
+200 val loss 8.2580
+200 val perplexity 3858.4006
+200 train 8.204547 (lr=7.0000e-05) (hash(x)=50375849)
+200 val loss 9.4624
+200 val perplexity 12866.3037
+200 train 9.433879 (lr=2.0000e-05) (hash(x)=50375849)
+200 val loss 9.1285
+200 val perplexity 9214.0938
+200 val loss 8.8667
+200 val perplexity 7091.8809
+200 train 9.091208 (lr=3.0000e-05) (hash(x)=50375849)
+200 train 8.824692 (lr=5.0000e-05) (hash(x)=50375849)
+300 val loss 7.7658
+300 val perplexity 2358.4961
+300 train 8.038077 (lr=6.9984e-05) (hash(x)=57250808)
+300 val loss 8.8112
+300 val perplexity 6709.1323
+300 train 8.984029 (lr=1.9995e-05) (hash(x)=57250808)
+300 val loss 8.1672
+300 val perplexity 3523.3943
+300 train 8.388642 (lr=2.9993e-05) (hash(x)=57250808)
+300 val loss 7.9807
+300 val perplexity 2924.0942
+300 train 8.223899 (lr=4.9988e-05) (hash(x)=57250808)
+400 val loss 7.5800
+400 val perplexity 1958.6438
+400 train 8.198520 (lr=6.9935e-05) (hash(x)=62519858)
+400 val loss 8.2008
+400 val perplexity 3643.9265
+400 train 8.714503 (lr=1.9982e-05) (hash(x)=62519858)
+400 val loss 7.7380
+400 val perplexity 2293.7683
+400 train 8.321421 (lr=2.9972e-05) (hash(x)=62519858)
+400 val loss 7.6893
+400 val perplexity 2184.7803
+400 train 8.280112 (lr=4.9954e-05) (hash(x)=62519858)
+500 val loss 7.4864
+500 val perplexity 1783.6954
+500 train 7.365067 (lr=6.9854e-05) (hash(x)=47226806)
+500 val loss 7.9001
+500 val perplexity 2697.5540
+500 train 7.817379 (lr=1.9958e-05) (hash(x)=47226806)
+500 val loss 7.5787
+500 val perplexity 1956.1013
+500 train 7.463418 (lr=2.9938e-05) (hash(x)=47226806)
+500 val loss 7.5497
+500 val perplexity 1900.0834
+500 train 7.439267 (lr=4.9896e-05) (hash(x)=47226806)
+600 val loss 7.4098
+600 val perplexity 1652.0936
+600 train 7.449531 (lr=6.9741e-05) (hash(x)=51149322)
+600 val loss 7.7479
+600 val perplexity 2316.7852
+600 train 7.808668 (lr=1.9926e-05) (hash(x)=51149322)
+600 val loss 7.4978
+600 val perplexity 1804.1498
+600 train 7.543031 (lr=2.9889e-05) (hash(x)=51149322)
+600 val loss 7.4542
+600 val perplexity 1727.0232
+600 train 7.500797 (lr=4.9815e-05) (hash(x)=51149322)
+700 val loss 7.3662
+700 val perplexity 1581.5858
+700 train 7.328444 (lr=6.9596e-05) (hash(x)=51564551)
+700 val loss 7.6381
+700 val perplexity 2075.7202
+700 train 7.601674 (lr=1.9885e-05) (hash(x)=51564551)
+700 val loss 7.4472
+700 val perplexity 1715.0033
+700 train 7.412354 (lr=2.9827e-05) (hash(x)=51564551)
+700 val loss 7.4139
+700 val perplexity 1658.8911
+700 train 7.377543 (lr=4.9712e-05) (hash(x)=51564551)
+800 val loss 7.3339
+800 val perplexity 1531.3365
+800 train 7.086210 (lr=6.9419e-05) (hash(x)=45093459)
+800 val loss 7.5613
+800 val perplexity 1922.2756
+800 train 7.341402 (lr=1.9834e-05) (hash(x)=45093459)
+800 val loss 7.3970
+800 val perplexity 1631.0750
+800 train 7.159473 (lr=2.9751e-05) (hash(x)=45093459)
+800 val loss 7.3797
+800 val perplexity 1603.0342
+800 train 7.141194 (lr=4.9585e-05) (hash(x)=45093459)
+900 val loss 7.3133
+900 val perplexity 1500.0953
+900 train 7.530349 (lr=6.9210e-05) (hash(x)=54988361)
+900 val loss 7.5042
+900 val perplexity 1815.7388
+900 train 7.794142 (lr=1.9774e-05) (hash(x)=54988361)
+900 val loss 7.3566
+900 val perplexity 1566.5011
+900 train 7.629829 (lr=2.9662e-05) (hash(x)=54988361)
+900 val loss 7.3483
+900 val perplexity 1553.5599
+900 train 7.611835 (lr=4.9436e-05) (hash(x)=54988361)
+1000 val loss 7.3015
+1000 val perplexity 1482.5197
+1000 train 7.151457 (lr=6.8970e-05) (hash(x)=47588648)
+1000 val loss 7.4704
+1000 val perplexity 1755.3352
+1000 train 7.308885 (lr=1.9706e-05) (hash(x)=47588648)
+1000 val loss 7.3298
+1000 val perplexity 1525.1024
+1000 train 7.173532 (lr=2.9558e-05) (hash(x)=47588648)
+1000 val loss 7.3243
+1000 val perplexity 1516.7701
+1000 train 7.170599 (lr=4.9264e-05) (hash(x)=47588648)
+1100 val loss 7.2816
+1100 val perplexity 1453.3807
+1100 val loss 7.4328
+1100 val perplexity 1690.5720
+1100 train 6.891258 (lr=6.8698e-05) (hash(x)=37984588)
+1100 train 7.063292 (lr=1.9628e-05) (hash(x)=37984588)
+1100 val loss 7.2966
+1100 val perplexity 1475.2267
+1100 train 6.895032 (lr=2.9442e-05) (hash(x)=37984588)
+1100 val loss 7.2944
+1100 val perplexity 1472.0934
+1100 train 6.891774 (lr=4.9070e-05) (hash(x)=37984588)
+1200 val loss 7.2477
+1200 val perplexity 1404.8997
+1200 train 7.403861 (lr=6.8395e-05) (hash(x)=56333817)
+1200 val loss 7.4056
+1200 val perplexity 1645.1812
+1200 train 7.572041 (lr=1.9542e-05) (hash(x)=56333817)
+1200 val loss 7.2510
+1200 val perplexity 1409.5781
+1200 train 7.413085 (lr=2.9312e-05) (hash(x)=56333817)
+1200 val loss 7.2683
+1200 val perplexity 1434.0852
+1200 train 7.427673 (lr=4.8854e-05) (hash(x)=56333817)
+1300 val loss 7.2387
+1300 val perplexity 1392.2408
+1300 val loss 7.3904
+1300 val perplexity 1620.3455
+1300 train 7.393292 (lr=6.8062e-05) (hash(x)=53454056)
+1300 train 7.515656 (lr=1.9446e-05) (hash(x)=53454056)
+1300 val loss 7.2148
+1300 val perplexity 1359.3795
+1300 train 7.371761 (lr=2.9169e-05) (hash(x)=53454056)
+1300 val loss 7.2425
+1300 val perplexity 1397.6387
+1300 train 7.405399 (lr=4.8616e-05) (hash(x)=53454056)
+1400 val loss 7.2102
+1400 val perplexity 1353.1600
+1400 train 7.318605 (lr=6.7698e-05) (hash(x)=55284163)
+1400 val loss 7.3416
+1400 val perplexity 1543.2137
+1400 train 7.421918 (lr=1.9342e-05) (hash(x)=55284163)
+1400 val loss 7.1703
+1400 val perplexity 1300.1732
+1400 train 7.267991 (lr=2.9013e-05) (hash(x)=55284163)
+1400 val loss 7.2076
+1400 val perplexity 1349.6500
+1400 train 7.308177 (lr=4.8356e-05) (hash(x)=55284163)
+1500 val loss 7.1755
+1500 val perplexity 1307.0302
+1500 train 6.990077 (lr=6.7304e-05) (hash(x)=48162598)
+1500 val loss 7.3069
+1500 val perplexity 1490.5138
+1500 train 7.126318 (lr=1.9230e-05) (hash(x)=48162598)
+1500 val loss 7.1275
+1500 val perplexity 1245.7611
+1500 train 6.953938 (lr=2.8845e-05) (hash(x)=48162598)
+1500 val loss 7.1882
+1500 val perplexity 1323.7600
+1500 train 7.001610 (lr=4.8074e-05) (hash(x)=48162598)
+1600 val loss 7.1604
+1600 val perplexity 1287.4865
+1600 train 7.235291 (lr=6.6881e-05) (hash(x)=54214535)
+1600 val loss 7.2764
+1600 val perplexity 1445.7960
+1600 train 7.336366 (lr=1.9109e-05) (hash(x)=54214535)
+1600 val loss 7.0945
+1600 val perplexity 1205.3110
+1600 train 7.182061 (lr=2.8663e-05) (hash(x)=54214535)
+1600 val loss 7.1639
+1600 val perplexity 1291.9230
+1600 train 7.246048 (lr=4.7772e-05) (hash(x)=54214535)
+1700 val loss 7.1594
+1700 val perplexity 1286.1304
+1700 train 7.379553 (lr=6.6428e-05) (hash(x)=53525003)
+1700 val loss 7.2678
+1700 val perplexity 1433.3823
+1700 train 7.495866 (lr=1.8979e-05) (hash(x)=53525003)
+1700 val loss 7.0801
+1700 val perplexity 1188.0975
+1700 train 7.320861 (lr=2.8469e-05) (hash(x)=53525003)
+1700 val loss 7.1502
+1700 val perplexity 1274.3279
+1700 train 7.380573 (lr=4.7448e-05) (hash(x)=53525003)
+1800 val loss 7.1304
+1800 val perplexity 1249.4257
+1800 train 7.203484 (lr=6.5947e-05) (hash(x)=51848994)
+1800 val loss 7.2311
+1800 val perplexity 1381.7793
+1800 train 7.303027 (lr=1.8842e-05) (hash(x)=51848994)
+1800 val loss 7.0370
+1800 val perplexity 1137.9156
+1800 train 7.110418 (lr=2.8263e-05) (hash(x)=51848994)
+1800 val loss 7.1128
+1800 val perplexity 1227.6375
+1800 train 7.184612 (lr=4.7105e-05) (hash(x)=51848994)
+1900 val loss 7.1304
+1900 val perplexity 1249.3917
+1900 train 6.945359 (lr=6.5437e-05) (hash(x)=48405987)
+1900 val loss 7.2135
+1900 val perplexity 1357.6111
+1900 train 7.020858 (lr=1.8696e-05) (hash(x)=48405987)
+1900 val loss 7.0105
+1900 val perplexity 1108.2396
+1900 train 6.820072 (lr=2.8044e-05) (hash(x)=48405987)
+1900 val loss 7.0923
+1900 val perplexity 1202.6534
+1900 train 6.898511 (lr=4.6741e-05) (hash(x)=48405987)
+2000 val loss 7.0969
+2000 val perplexity 1208.1680
+2000 train 7.451231 (lr=6.4900e-05) (hash(x)=58592291)
+2000 val loss 7.1832
+2000 val perplexity 1317.0741
+2000 train 7.533998 (lr=1.8543e-05) (hash(x)=58592291)
+2000 val loss 6.9804
+2000 val perplexity 1075.3429
+2000 train 7.339841 (lr=2.7814e-05) (hash(x)=58592291)
+2000 val loss 7.0678
+2000 val perplexity 1173.6071
+2000 train 7.429214 (lr=4.6357e-05) (hash(x)=58592291)
+2100 val loss 7.0806
+2100 val perplexity 1188.7402
+2100 train 7.179558 (lr=6.4335e-05) (hash(x)=51167081)
+2100 val loss 7.1699
+2100 val perplexity 1299.7511
+2100 train 7.262040 (lr=1.8382e-05) (hash(x)=51167081)
+2100 val loss 6.9650
+2100 val perplexity 1058.9535
+2100 train 7.059167 (lr=2.7572e-05) (hash(x)=51167081)
+2100 val loss 7.0631
+2100 val perplexity 1168.0159
+2100 train 7.157566 (lr=4.5954e-05) (hash(x)=51167081)
+2200 val loss 7.0772
+2200 val perplexity 1184.5981
+2200 train 7.055668 (lr=6.3745e-05) (hash(x)=47994988)
+2200 val loss 7.1410
+2200 val perplexity 1262.7300
+2200 train 7.130829 (lr=1.8213e-05) (hash(x)=47994988)
+2200 val loss 6.9354
+2200 val perplexity 1028.0294
+2200 train 6.934694 (lr=2.7319e-05) (hash(x)=47994988)
+2200 val loss 7.0341
+2200 val perplexity 1134.6560
+2200 train 7.013680 (lr=4.5532e-05) (hash(x)=47994988)
+2300 val loss 7.0683
+2300 val perplexity 1174.1478
+2300 train 7.064332 (lr=6.3128e-05) (hash(x)=47377604)
+2300 val loss 7.1320
+2300 val perplexity 1251.4082
+2300 train 7.126850 (lr=1.8036e-05) (hash(x)=47377604)
+2300 val loss 6.9205
+2300 val perplexity 1012.7773
+2300 train 6.902592 (lr=2.7055e-05) (hash(x)=47377604)
+2400 val loss 7.0480
+2400 val perplexity 1150.5189
+2300 val loss 7.0176
+2300 val perplexity 1116.0878
+2400 train 7.102792 (lr=6.2486e-05) (hash(x)=53554323)
+2300 train 7.011597 (lr=4.5091e-05) (hash(x)=47377604)
+2400 val loss 7.1077
+2400 val perplexity 1221.3223
+2400 train 7.168054 (lr=1.7853e-05) (hash(x)=53554323)
+2400 val loss 6.8895
+2400 val perplexity 981.9048
+2400 train 6.956583 (lr=2.6780e-05) (hash(x)=53554323)
+2500 val loss 7.0545
+2500 val perplexity 1158.1093
+2400 val loss 6.9913
+2400 val perplexity 1087.1801
+2500 val loss 7.0940
+2500 val perplexity 1204.7324
+2500 train 7.091203 (lr=6.1819e-05) (hash(x)=50780417)
+2400 train 7.054268 (lr=4.4633e-05) (hash(x)=53554323)
+2500 train 7.134183 (lr=1.7663e-05) (hash(x)=50780417)
+2500 val loss 6.8720
+2500 val perplexity 964.8711
+2500 train 6.929877 (lr=2.6494e-05) (hash(x)=50780417)
+2600 val loss 7.0427
+2600 val perplexity 1144.4280
+2600 train 6.935129 (lr=6.1128e-05) (hash(x)=46453562)
+2500 val loss 6.9796
+2500 val perplexity 1074.4991
+2600 val loss 7.0705
+2600 val perplexity 1176.6801
+2600 train 6.983191 (lr=1.7465e-05) (hash(x)=46453562)
+2500 train 7.024747 (lr=4.4156e-05) (hash(x)=50780417)
+2600 val loss 6.8481
+2600 val perplexity 942.0485
+2600 train 6.759700 (lr=2.6198e-05) (hash(x)=46453562)
+2700 val loss 7.0361
+2700 val perplexity 1136.9296
+2700 train 7.019847 (lr=6.0414e-05) (hash(x)=54404221)
+2700 val loss 7.0556
+2700 val perplexity 1159.3474
+2700 train 7.057378 (lr=1.7261e-05) (hash(x)=54404221)
+2600 val loss 6.9612
+2600 val perplexity 1054.9165
+2600 train 6.864810 (lr=4.3663e-05) (hash(x)=46453562)
+2700 val loss 6.8362
+2700 val perplexity 930.9543
+2700 train 6.829805 (lr=2.5892e-05) (hash(x)=54404221)
+2800 val loss 7.0333
+2800 val perplexity 1133.7198
+2800 train 7.761267 (lr=5.9677e-05) (hash(x)=59318895)
+2800 val loss 7.0442
+2800 val perplexity 1146.1642
+2800 train 7.818988 (lr=1.7051e-05) (hash(x)=59318895)
+2700 val loss 6.9554
+2700 val perplexity 1048.7799
+2700 train 6.948943 (lr=4.3153e-05) (hash(x)=54404221)
+2800 val loss 6.8236
+2800 val perplexity 919.3036
+2800 train 7.550114 (lr=2.5576e-05) (hash(x)=59318895)
+2900 val loss 7.0409
+2900 val perplexity 1142.4042
+2900 train 7.038341 (lr=5.8919e-05) (hash(x)=47845760)
+2900 val loss 7.0287
+2900 val perplexity 1128.5403
+2900 train 7.030903 (lr=1.6834e-05) (hash(x)=47845760)
+2800 val loss 6.9471
+2800 val perplexity 1040.0840
+2800 train 7.687397 (lr=4.2627e-05) (hash(x)=59318895)
+2900 val loss 6.8010
+2900 val perplexity 898.7311
+2900 train 6.806674 (lr=2.5251e-05) (hash(x)=47845760)
+3000 val loss 7.0295
+3000 val perplexity 1129.4146
+3000 train 6.731463 (lr=5.8140e-05) (hash(x)=44336167)
+3000 val loss 7.0128
+3000 val perplexity 1110.7865
+3000 train 6.712865 (lr=1.6611e-05) (hash(x)=44336167)
+2900 val loss 6.9376
+2900 val perplexity 1030.3340
+2900 train 6.940581 (lr=4.2085e-05) (hash(x)=47845760)
+3000 val loss 6.7838
+3000 val perplexity 883.4333
+3000 train 6.473313 (lr=2.4917e-05) (hash(x)=44336167)
+3100 val loss 7.0272
+3100 val perplexity 1126.9003
+3100 train 7.329674 (lr=5.7340e-05) (hash(x)=44479330)
+3100 val loss 6.9967
+3100 val perplexity 1093.0607
+3100 train 7.308694 (lr=1.6383e-05) (hash(x)=44479330)
+3000 val loss 6.9218
+3000 val perplexity 1014.1106
+3000 train 6.614216 (lr=4.1529e-05) (hash(x)=44336167)
+3100 val loss 6.7648
+3100 val perplexity 866.7640
+3100 train 7.090666 (lr=2.4574e-05) (hash(x)=44479330)
+3200 val loss 7.0308
+3200 val perplexity 1130.9462
+3200 train 7.100553 (lr=5.6522e-05) (hash(x)=54593096)
+3200 val loss 6.9944
+3200 val perplexity 1090.5176
+3200 train 7.079770 (lr=1.6149e-05) (hash(x)=54593096)
+3100 val loss 6.9163
+3100 val perplexity 1008.5403
+3100 train 7.235619 (lr=4.0957e-05) (hash(x)=44479330)
+3200 val loss 6.7609
+3200 val perplexity 863.3954
+3200 train 6.855031 (lr=2.4224e-05) (hash(x)=54593096)
+3300 val loss 7.0069
+3300 val perplexity 1104.2101
+3300 train 6.964014 (lr=5.5684e-05) (hash(x)=45347643)
+3300 val loss 6.9772
+3300 val perplexity 1071.8771
+3300 train 6.932001 (lr=1.5910e-05) (hash(x)=45347643)
+3200 val loss 6.9159
+3200 val perplexity 1008.1403
+3200 train 7.010983 (lr=4.0373e-05) (hash(x)=54593096)
+3300 val loss 6.7374
+3300 val perplexity 843.3978
+3300 train 6.732250 (lr=2.3865e-05) (hash(x)=45347643)
+3400 val loss 6.9972
+3400 val perplexity 1093.6138
+3400 train 7.143245 (lr=5.4829e-05) (hash(x)=47797247)
+3400 val loss 6.9505
+3400 val perplexity 1043.6790
+3400 train 7.096212 (lr=1.5666e-05) (hash(x)=47797247)
+3300 val loss 6.9194
+3300 val perplexity 1011.7419
+3300 train 6.877916 (lr=3.9775e-05) (hash(x)=45347643)
+3400 val loss 6.7233
+3400 val perplexity 831.5302
+3400 train 6.864102 (lr=2.3498e-05) (hash(x)=47797247)
+3500 val loss 6.9898
+3500 val perplexity 1085.4670
+3500 train 6.855062 (lr=5.3958e-05) (hash(x)=46115683)
+3500 val loss 6.9364
+3500 val perplexity 1029.0530
+3500 train 6.807493 (lr=1.5416e-05) (hash(x)=46115683)
+3400 val loss 6.8847
+3400 val perplexity 977.1854
+3400 train 7.030772 (lr=3.9164e-05) (hash(x)=47797247)
+3500 val loss 6.6976
+3500 val perplexity 810.4366
+3500 train 6.536535 (lr=2.3125e-05) (hash(x)=46115683)
+3600 val loss 6.9711
+3600 val perplexity 1065.3461
+3600 train 6.831712 (lr=5.3070e-05) (hash(x)=44502074)
+3600 val loss 6.9254
+3600 val perplexity 1017.8438
+3600 train 6.797766 (lr=1.5163e-05) (hash(x)=44502074)
+3500 val loss 6.8773
+3500 val perplexity 969.9569
+3500 train 6.728076 (lr=3.8541e-05) (hash(x)=46115683)
+3600 val loss 6.6856
+3600 val perplexity 800.8084
+3600 train 6.566401 (lr=2.2744e-05) (hash(x)=44502074)
+3700 val loss 6.9689
+3700 val perplexity 1063.0332
+3700 train 7.124398 (lr=5.2167e-05) (hash(x)=55388443)
+3700 val loss 6.9106
+3700 val perplexity 1002.8619
+3700 train 7.083196 (lr=1.4905e-05) (hash(x)=55388443)
+3700 val loss 6.6702
+3700 val perplexity 788.5517
+3600 val loss 6.8693
+3600 val perplexity 962.2554
+3700 train 6.845844 (lr=2.2357e-05) (hash(x)=55388443)
+3600 train 6.741370 (lr=3.7907e-05) (hash(x)=44502074)
+3800 val loss 6.9651
+3800 val perplexity 1058.9833
+3800 train 6.742047 (lr=5.1251e-05) (hash(x)=43790341)
+3800 val loss 6.8986
+3800 val perplexity 990.8743
+3800 train 6.670383 (lr=1.4643e-05) (hash(x)=43790341)
+3800 val loss 6.6571
+3800 val perplexity 778.3168
+3800 train 6.436588 (lr=2.1965e-05) (hash(x)=43790341)
+3700 val loss 6.8565
+3700 val perplexity 950.0204
+3700 train 7.029319 (lr=3.7262e-05) (hash(x)=55388443)
+3900 val loss 6.9636
+3900 val perplexity 1057.3928
+3900 train 6.931797 (lr=5.0321e-05) (hash(x)=50013318)
+3900 val loss 6.8866
+3900 val perplexity 979.1108
+3900 train 6.861619 (lr=1.4377e-05) (hash(x)=50013318)
+3900 val loss 6.6434
+3900 val perplexity 767.6965
+3900 train 6.618001 (lr=2.1566e-05) (hash(x)=50013318)
+3800 val loss 6.8465
+3800 val perplexity 940.5781
+3800 train 6.626799 (lr=3.6608e-05) (hash(x)=43790341)
+4000 val loss 6.9586
+4000 val perplexity 1052.1274
+4000 train 7.005599 (lr=4.9379e-05) (hash(x)=51704787)
+4000 val loss 6.8761
+4000 val perplexity 968.8410
+4000 train 6.948847 (lr=1.4108e-05) (hash(x)=51704787)
+4000 val loss 6.6273
+4000 val perplexity 755.4662
+4000 train 6.686940 (lr=2.1162e-05) (hash(x)=51704787)
+3900 val loss 6.8405
+3900 val perplexity 934.9680
+3900 train 6.808554 (lr=3.5944e-05) (hash(x)=50013318)
+4100 val loss 6.9637
+4100 val perplexity 1057.5693
+4100 train 7.027716 (lr=4.8426e-05) (hash(x)=50821964)
+4100 val loss 6.8649
+4100 val perplexity 958.0593
+4100 train 6.937582 (lr=1.3836e-05) (hash(x)=50821964)
+4100 val loss 6.6167
+4100 val perplexity 747.4631
+4100 train 6.681018 (lr=2.0754e-05) (hash(x)=50821964)
+4000 val loss 6.8263
+4000 val perplexity 921.7486
+4000 train 6.867013 (lr=3.5271e-05) (hash(x)=51704787)
+4200 val loss 6.9647
+4200 val perplexity 1058.6071
+4200 train 6.991130 (lr=4.7463e-05) (hash(x)=49675080)
+4200 val loss 6.8555
+4200 val perplexity 949.0610
+4200 train 6.890470 (lr=1.3561e-05) (hash(x)=49675080)
+4200 val loss 6.6082
+4200 val perplexity 741.1528
+4200 train 6.649536 (lr=2.0341e-05) (hash(x)=49675080)
+4100 val loss 6.8227
+4100 val perplexity 918.4326
+4100 train 6.889338 (lr=3.4590e-05) (hash(x)=50821964)
+4300 val loss 6.9660
+4300 val perplexity 1059.9851
+4300 train 6.611563 (lr=4.6490e-05) (hash(x)=43239281)
+4300 val loss 6.8436
+4300 val perplexity 937.8506
+4300 train 6.500687 (lr=1.3283e-05) (hash(x)=43239281)
+4300 val loss 6.5957
+4300 val perplexity 731.9608
+4300 train 6.246088 (lr=1.9924e-05) (hash(x)=43239281)
+4200 val loss 6.8298
+4200 val perplexity 925.0024
+4200 train 6.867212 (lr=3.3902e-05) (hash(x)=49675080)
+4400 val loss 6.9531
+4400 val perplexity 1046.4331
+4400 train 6.618497 (lr=4.5509e-05) (hash(x)=45076737)
+4400 val loss 6.8358
+4400 val perplexity 930.5940
+4400 train 6.493437 (lr=1.3003e-05) (hash(x)=45076737)
+4400 val loss 6.5910
+4400 val perplexity 728.4740
+4400 train 6.258480 (lr=1.9504e-05) (hash(x)=45076737)
+4500 val loss 6.9524
+4500 val perplexity 1045.6770
+4500 train 7.078568 (lr=4.4521e-05) (hash(x)=57930262)
+4300 val loss 6.8005
+4300 val perplexity 898.3245
+4300 train 6.453156 (lr=3.3207e-05) (hash(x)=43239281)
+4500 val loss 6.8233
+4500 val perplexity 918.9745
+4500 train 6.945100 (lr=1.2720e-05) (hash(x)=57930262)
+4500 val loss 6.5777
+4500 val perplexity 718.9064
+4500 train 6.710505 (lr=1.9081e-05) (hash(x)=57930262)
+4600 val loss 6.9370
+4600 val perplexity 1029.6769
+4600 train 6.716589 (lr=4.3527e-05) (hash(x)=46721614)
+4400 val loss 6.8052
+4400 val perplexity 902.5490
+4600 val loss 6.8141
+4600 val perplexity 910.5617
+4400 train 6.455774 (lr=3.2507e-05) (hash(x)=45076737)
+4600 train 6.585222 (lr=1.2436e-05) (hash(x)=46721614)
+4600 val loss 6.5683
+4600 val perplexity 712.1240
+4600 train 6.316262 (lr=1.8655e-05) (hash(x)=46721614)
+4700 val loss 6.9392
+4700 val perplexity 1031.9817
+4700 train 6.750459 (lr=4.2528e-05) (hash(x)=49837920)
+4700 val loss 6.8054
+4700 val perplexity 902.6893
+4700 train 6.615937 (lr=1.2151e-05) (hash(x)=49837920)
+4500 val loss 6.7860
+4500 val perplexity 885.3331
+4500 train 6.911989 (lr=3.1801e-05) (hash(x)=57930262)
+4700 val loss 6.5557
+4700 val perplexity 703.2458
+4700 train 6.345678 (lr=1.8226e-05) (hash(x)=49837920)
+4800 val loss 6.9232
+4800 val perplexity 1015.5730
+4800 train 7.062541 (lr=4.1525e-05) (hash(x)=48380045)
+4800 val loss 6.7953
+4800 val perplexity 893.6684
+4800 train 6.939090 (lr=1.1864e-05) (hash(x)=48380045)
+4600 val loss 6.7746
+4600 val perplexity 875.3221
+4600 train 6.539246 (lr=3.1091e-05) (hash(x)=46721614)
+4800 val loss 6.5463
+4800 val perplexity 696.6906
+4800 train 6.705057 (lr=1.7796e-05) (hash(x)=48380045)
+4900 val loss 6.9166
+4900 val perplexity 1008.8375
+4900 train 6.791525 (lr=4.0518e-05) (hash(x)=44202577)
+4900 val loss 6.7833
+4900 val perplexity 882.9680
+4900 train 6.638532 (lr=1.1577e-05) (hash(x)=44202577)
+4700 val loss 6.7656
+4700 val perplexity 867.4582
+4700 train 6.562875 (lr=3.0377e-05) (hash(x)=49837920)
+4900 val loss 6.5368
+4900 val perplexity 690.0550
+4900 train 6.406009 (lr=1.7365e-05) (hash(x)=44202577)
+5000 val loss 6.9087
+5000 val perplexity 1000.9222
+5000 train 6.899564 (lr=3.9510e-05) (hash(x)=52038024)
+5000 val loss 6.7696
+5000 val perplexity 870.9617
+5000 train 6.774422 (lr=1.1288e-05) (hash(x)=52038024)
+4800 val loss 6.7573
+4800 val perplexity 860.2833
+4800 train 6.900402 (lr=2.9661e-05) (hash(x)=48380045)
+5000 val loss 6.5254
+5000 val perplexity 682.2469
+5000 train 6.508739 (lr=1.6933e-05) (hash(x)=52038024)
+5100 val loss 6.8995
+5100 val perplexity 991.7739
+5100 train 7.089410 (lr=3.8500e-05) (hash(x)=53700038)
+5100 val loss 6.7566
+5100 val perplexity 859.7440
+5100 train 6.936323 (lr=1.1000e-05) (hash(x)=53700038)
+4900 val loss 6.7479
+4900 val perplexity 852.2854
+4900 train 6.608902 (lr=2.8942e-05) (hash(x)=44202577)
+5200 val loss 6.8979
+5200 val perplexity 990.1672
+5200 train 6.912906 (lr=3.7490e-05) (hash(x)=48137625)
+5100 val loss 6.5085
+5100 val perplexity 670.8521
+5100 train 6.672117 (lr=1.6500e-05) (hash(x)=53700038)
+5200 val loss 6.7483
+5200 val perplexity 852.6248
+5200 train 6.789814 (lr=1.0712e-05) (hash(x)=48137625)
+5000 val loss 6.7348
+5000 val perplexity 841.1663
+5000 train 6.733591 (lr=2.8221e-05) (hash(x)=52038024)
+5300 val loss 6.8939
+5300 val perplexity 986.2891
+5300 train 6.749971 (lr=3.6482e-05) (hash(x)=43161573)
+5200 val loss 6.5073
+5200 val perplexity 670.0002
+5200 train 6.552045 (lr=1.6067e-05) (hash(x)=48137625)
+5300 val loss 6.7398
+5300 val perplexity 845.4045
+5300 train 6.591602 (lr=1.0423e-05) (hash(x)=43161573)
+5100 val loss 6.7271
+5100 val perplexity 834.7480
+5100 train 6.914661 (lr=2.7500e-05) (hash(x)=53700038)
+5400 val loss 6.8924
+5400 val perplexity 984.7928
+5400 train 7.057383 (lr=3.5475e-05) (hash(x)=56673322)
+5300 val loss 6.4953
+5300 val perplexity 662.0350
+5300 train 6.333315 (lr=1.5635e-05) (hash(x)=43161573)
+5400 val loss 6.7355
+5400 val perplexity 841.7444
+5400 train 6.927864 (lr=1.0136e-05) (hash(x)=56673322)
+5200 val loss 6.7212
+5200 val perplexity 829.8277
+5200 train 6.751077 (lr=2.6779e-05) (hash(x)=48137625)
+5500 val loss 6.8680
+5500 val perplexity 961.0191
+5500 train 7.047554 (lr=3.4472e-05) (hash(x)=53468295)
+5400 val loss 6.4899
+5400 val perplexity 658.4774
+5500 val loss 6.7250
+5500 val perplexity 832.9433
+5400 train 6.677568 (lr=1.5204e-05) (hash(x)=56673322)
+5500 train 6.919577 (lr=9.8491e-06) (hash(x)=53468295)
+5300 val loss 6.7096
+5300 val perplexity 820.2217
+5300 train 6.554457 (lr=2.6058e-05) (hash(x)=43161573)
+5600 val loss 6.8605
+5600 val perplexity 953.8215
+5600 train 7.188080 (lr=3.3473e-05) (hash(x)=59287280)
+5600 val loss 6.7138
+5600 val perplexity 823.6735
+5500 val loss 6.4886
+5500 val perplexity 657.5712
+5600 train 7.046098 (lr=9.5636e-06) (hash(x)=59287280)
+5500 train 6.689147 (lr=1.4774e-05) (hash(x)=53468295)
+5400 val loss 6.7112
+5400 val perplexity 821.5338
+5400 train 6.880655 (lr=2.5339e-05) (hash(x)=56673322)
+5700 val loss 6.8700
+5700 val perplexity 962.9370
+5700 train 7.115635 (lr=3.2479e-05) (hash(x)=57575806)
+5700 val loss 6.7076
+5700 val perplexity 818.6053
+5700 train 6.936852 (lr=9.2796e-06) (hash(x)=57575806)
+5600 val loss 6.4805
+5600 val perplexity 652.3239
+5600 train 6.805542 (lr=1.4345e-05) (hash(x)=59287280)
+5500 val loss 6.7051
+5500 val perplexity 816.5854
+5500 train 6.890529 (lr=2.4623e-05) (hash(x)=53468295)
+5800 val loss 6.8754
+5800 val perplexity 968.1921
+5800 train 6.828178 (lr=3.1491e-05) (hash(x)=46897279)
+5800 val loss 6.7025
+5800 val perplexity 814.4427
+5800 train 6.697758 (lr=8.9973e-06) (hash(x)=46897279)
+5700 val loss 6.4664
+5700 val perplexity 643.1498
+5700 train 6.672063 (lr=1.3919e-05) (hash(x)=57575806)
+5600 val loss 6.6962
+5600 val perplexity 809.3145
+5600 train 7.011116 (lr=2.3909e-05) (hash(x)=59287280)
+5900 val loss 6.8609
+5900 val perplexity 954.2668
+5900 train 6.717267 (lr=3.0510e-05) (hash(x)=47565679)
+5900 val loss 6.6951
+5900 val perplexity 808.4362
+5900 train 6.575296 (lr=8.7171e-06) (hash(x)=47565679)
+5800 val loss 6.4659
+5800 val perplexity 642.8671
+5800 train 6.516923 (lr=1.3496e-05) (hash(x)=46897279)
+5700 val loss 6.6804
+5700 val perplexity 796.6323
+5700 train 6.919859 (lr=2.3199e-05) (hash(x)=57575806)
+6000 val loss 6.8543
+6000 val perplexity 947.9298
+6000 train 6.732675 (lr=2.9537e-05) (hash(x)=51590090)
+6000 val loss 6.6898
+6000 val perplexity 804.1218
+6000 train 6.565755 (lr=8.4393e-06) (hash(x)=51590090)
+5900 val loss 6.4598
+5900 val perplexity 638.9581
+5900 train 6.347005 (lr=1.3076e-05) (hash(x)=47565679)
+5800 val loss 6.6835
+5800 val perplexity 799.0714
+5800 train 6.678881 (lr=2.2493e-05) (hash(x)=46897279)
+6100 val loss 6.8477
+6100 val perplexity 941.7269
+6100 train 7.248047 (lr=2.8574e-05) (hash(x)=59732271)
+6100 val loss 6.6780
+6100 val perplexity 794.7079
+6100 train 7.095581 (lr=8.1640e-06) (hash(x)=59732271)
+6000 val loss 6.4558
+6000 val perplexity 636.3523
+6000 train 6.310637 (lr=1.2659e-05) (hash(x)=51590090)
+5900 val loss 6.6804
+5900 val perplexity 796.6262
+5900 train 6.560131 (lr=2.1793e-05) (hash(x)=47565679)
+6200 val loss 6.8392
+6200 val perplexity 933.7579
+6200 train 6.832141 (lr=2.7621e-05) (hash(x)=46394422)
+6200 val loss 6.6687
+6200 val perplexity 787.3771
+6200 train 6.698349 (lr=7.8917e-06) (hash(x)=46394422)
+6100 val loss 6.4453
+6100 val perplexity 629.7390
+6100 train 6.882823 (lr=1.2246e-05) (hash(x)=59732271)
+6000 val loss 6.6699
+6000 val perplexity 788.3404
+6000 train 6.557931 (lr=2.1098e-05) (hash(x)=51590090)
+6300 val loss 6.8440
+6300 val perplexity 938.2667
+6300 train 6.859726 (lr=2.6679e-05) (hash(x)=53748145)
+6300 val loss 6.6657
+6300 val perplexity 785.0262
+6300 train 6.699683 (lr=7.6226e-06) (hash(x)=53748145)
+6200 val loss 6.4406
+6200 val perplexity 626.7773
+6200 train 6.547752 (lr=1.1838e-05) (hash(x)=46394422)
+6100 val loss 6.6590
+6100 val perplexity 779.7674
+6100 train 7.079908 (lr=2.0410e-05) (hash(x)=59732271)
+6400 val loss 6.8349
+6400 val perplexity 929.7654
+6400 train 6.713243 (lr=2.5749e-05) (hash(x)=46054751)
+6400 val loss 6.6582
+6400 val perplexity 779.1374
+6400 train 6.540910 (lr=7.3569e-06) (hash(x)=46054751)
+6300 val loss 6.4352
+6300 val perplexity 623.3842
+6300 train 6.474613 (lr=1.1434e-05) (hash(x)=53748145)
+6500 val loss 6.8288
+6500 val perplexity 924.0612
+6500 train 7.062670 (lr=2.4833e-05) (hash(x)=51816809)
+6200 val loss 6.6508
+6200 val perplexity 773.4122
+6200 train 6.688356 (lr=1.9729e-05) (hash(x)=46394422)
+6500 val loss 6.6497
+6500 val perplexity 772.5416
+6500 train 6.918588 (lr=7.0950e-06) (hash(x)=51816809)
+6400 val loss 6.4275
+6400 val perplexity 618.6315
+6400 train 6.307833 (lr=1.1035e-05) (hash(x)=46054751)
+6600 val loss 6.8306
+6600 val perplexity 925.7701
+6600 train 6.681698 (lr=2.3930e-05) (hash(x)=52453336)
+6300 val loss 6.6429
+6300 val perplexity 767.3305
+6300 train 6.688078 (lr=1.9056e-05) (hash(x)=53748145)
+6600 val loss 6.6472
+6600 val perplexity 770.6442
+6600 train 6.543104 (lr=6.8372e-06) (hash(x)=52453336)
+6500 val loss 6.4223
+6500 val perplexity 615.4333
+6500 train 6.724624 (lr=1.0643e-05) (hash(x)=51816809)
+6700 val loss 6.8288
+6700 val perplexity 924.0959
+6700 train 6.853996 (lr=2.3042e-05) (hash(x)=49108775)
+6400 val loss 6.6365
+6400 val perplexity 762.4250
+6400 train 6.516543 (lr=1.8392e-05) (hash(x)=46054751)
+6700 val loss 6.6395
+6700 val perplexity 764.6962
+6700 train 6.668485 (lr=6.5835e-06) (hash(x)=49108775)
+6600 val loss 6.4189
+6600 val perplexity 613.3281
+6600 train 6.276807 (lr=1.0256e-05) (hash(x)=52453336)
+6800 val loss 6.8176
+6800 val perplexity 913.8017
+6800 train 6.734222 (lr=2.2171e-05) (hash(x)=46745396)
+6500 val loss 6.6280
+6500 val perplexity 755.9351
+6500 train 6.887875 (lr=1.7738e-05) (hash(x)=51816809)
+6800 val loss 6.6352
+6800 val perplexity 761.4408
+6800 train 6.550888 (lr=6.3345e-06) (hash(x)=46745396)
+6700 val loss 6.4147
+6700 val perplexity 610.7313
+6700 train 6.440950 (lr=9.8753e-06) (hash(x)=49108775)
+6900 val loss 6.8119
+6900 val perplexity 908.6169
+6900 train 6.969987 (lr=2.1316e-05) (hash(x)=46534986)
+6900 val loss 6.6270
+6900 val perplexity 755.2350
+6900 train 6.828880 (lr=6.0902e-06) (hash(x)=46534986)
+6600 val loss 6.6247
+6600 val perplexity 753.4476
+6600 train 6.471210 (lr=1.7093e-05) (hash(x)=52453336)
+6800 val loss 6.4107
+6800 val perplexity 608.3489
+6800 train 6.298498 (lr=9.5017e-06) (hash(x)=46745396)
+7000 val loss 6.8048
+7000 val perplexity 902.1484
+7000 train 7.197929 (lr=2.0478e-05) (hash(x)=49317888)
+7000 val loss 6.6200
+7000 val perplexity 749.9446
+7000 train 6.991396 (lr=5.8510e-06) (hash(x)=49317888)
+6900 val loss 6.4029
+6900 val perplexity 603.6034
+6700 val loss 6.6157
+6700 val perplexity 746.7203
+6900 train 6.632865 (lr=9.1353e-06) (hash(x)=46534986)
+6700 train 6.645592 (lr=1.6459e-05) (hash(x)=49108775)
+7100 val loss 6.8015
+7100 val perplexity 899.2177
+7100 train 6.811685 (lr=1.9660e-05) (hash(x)=50360484)
+7100 val loss 6.6150
+7100 val perplexity 746.1974
+7100 train 6.616625 (lr=5.6170e-06) (hash(x)=50360484)
+7000 val loss 6.3969
+7000 val perplexity 599.9835
+7000 train 6.782572 (lr=8.7764e-06) (hash(x)=49317888)
+6800 val loss 6.6173
+6800 val perplexity 747.9319
+6800 train 6.525100 (lr=1.5836e-05) (hash(x)=46745396)
+7200 val loss 6.7989
+7200 val perplexity 896.8526
+7200 train 6.673613 (lr=1.8860e-05) (hash(x)=49515094)
+7200 val loss 6.6101
+7200 val perplexity 742.5444
+7200 train 6.475906 (lr=5.3886e-06) (hash(x)=49515094)
+7100 val loss 6.3945
+7100 val perplexity 598.5322
+7100 train 6.410477 (lr=8.4255e-06) (hash(x)=50360484)
+6900 val loss 6.6010
+6900 val perplexity 735.8467
+6900 train 6.792136 (lr=1.5225e-05) (hash(x)=46534986)
+7300 val loss 6.7962
+7300 val perplexity 894.4124
+7300 train 6.783378 (lr=1.8081e-05) (hash(x)=51546861)
+7300 val loss 6.6118
+7300 val perplexity 743.8251
+7300 train 6.599928 (lr=5.1659e-06) (hash(x)=51546861)
+7200 val loss 6.3870
+7200 val perplexity 594.0425
+7200 train 6.228770 (lr=8.0829e-06) (hash(x)=49515094)
+7000 val loss 6.5920
+7000 val perplexity 729.2160
+7000 train 6.981744 (lr=1.4627e-05) (hash(x)=49317888)
+7400 val loss 6.7896
+7400 val perplexity 888.5211
+7400 train 6.785089 (lr=1.7323e-05) (hash(x)=48320948)
+7400 val loss 6.6037
+7400 val perplexity 737.7843
+7400 train 6.612427 (lr=4.9493e-06) (hash(x)=48320948)
+7300 val loss 6.3924
+7300 val perplexity 597.2842
+7300 train 6.382062 (lr=7.7489e-06) (hash(x)=51546861)
+7100 val loss 6.5873
+7100 val perplexity 725.8042
+7100 train 6.607678 (lr=1.4043e-05) (hash(x)=50360484)
+7500 val loss 6.7864
+7500 val perplexity 885.7453
+7500 train 6.726026 (lr=1.6586e-05) (hash(x)=40167457)
+7500 val loss 6.6045
+7500 val perplexity 738.4333
+7500 train 6.547146 (lr=4.7389e-06) (hash(x)=40167457)
+7400 val loss 6.3855
+7400 val perplexity 593.1888
+7400 train 6.382369 (lr=7.4239e-06) (hash(x)=48320948)
+7200 val loss 6.5815
+7200 val perplexity 721.6158
+7200 train 6.439557 (lr=1.3471e-05) (hash(x)=49515094)
+7600 val loss 6.7853
+7600 val perplexity 884.7520
+7600 train 6.721845 (lr=1.5872e-05) (hash(x)=49942165)
+7600 val loss 6.5947
+7600 val perplexity 731.2087
+7600 train 6.535936 (lr=4.5349e-06) (hash(x)=49942165)
+7500 val loss 6.3889
+7500 val perplexity 595.2056
+7500 train 6.335147 (lr=7.1083e-06) (hash(x)=40167457)
+7300 val loss 6.5827
+7300 val perplexity 722.4566
+7300 train 6.567333 (lr=1.2915e-05) (hash(x)=51546861)
+7700 val loss 6.7805
+7700 val perplexity 880.5395
+7700 train 6.506702 (lr=1.5181e-05) (hash(x)=48853311)
+7700 val loss 6.5914
+7700 val perplexity 728.8103
+7700 train 6.309133 (lr=4.3375e-06) (hash(x)=48853311)
+7600 val loss 6.3745
+7600 val perplexity 586.7179
+7600 train 6.329422 (lr=6.8023e-06) (hash(x)=49942165)
+7400 val loss 6.5734
+7400 val perplexity 715.8018
+7400 train 6.580335 (lr=1.2373e-05) (hash(x)=48320948)
+7800 val loss 6.7783
+7800 val perplexity 878.5657
+7800 train 6.677202 (lr=1.4514e-05) (hash(x)=48510117)
+7800 val loss 6.5876
+7800 val perplexity 726.0670
+7800 train 6.485681 (lr=4.1470e-06) (hash(x)=48510117)
+7700 val loss 6.3719
+7700 val perplexity 585.1765
+7700 train 6.077825 (lr=6.5062e-06) (hash(x)=48853311)
+7500 val loss 6.5774
+7500 val perplexity 718.6973
+7500 train 6.505441 (lr=1.1847e-05) (hash(x)=40167457)
+7900 val loss 6.7788
+7900 val perplexity 879.0137
+7900 train 6.777487 (lr=1.3872e-05) (hash(x)=48339781)
+7900 val loss 6.5867
+7900 val perplexity 725.3773
+7900 train 6.586193 (lr=3.9635e-06) (hash(x)=48339781)
+7800 val loss 6.3706
+7800 val perplexity 584.4119
+7800 train 6.258163 (lr=6.2205e-06) (hash(x)=48510117)
+7600 val loss 6.5672
+7600 val perplexity 711.3448
+7600 train 6.511395 (lr=1.1337e-05) (hash(x)=49942165)
+8000 val loss 6.7733
+8000 val perplexity 874.1851
+8000 train 6.894838 (lr=1.3255e-05) (hash(x)=54927320)
+8000 val loss 6.5813
+8000 val perplexity 721.4551
+8000 train 6.694423 (lr=3.7873e-06) (hash(x)=54927320)
+7900 val loss 6.3693
+7900 val perplexity 583.6351
+7900 train 6.374780 (lr=5.9453e-06) (hash(x)=48339781)
+7700 val loss 6.5615
+7700 val perplexity 707.3499
+7700 train 6.275876 (lr=1.0844e-05) (hash(x)=48853311)
+8100 val loss 6.7714
+8100 val perplexity 872.5671
+8100 train 6.479921 (lr=1.2665e-05) (hash(x)=46461786)
+8100 val loss 6.5788
+8100 val perplexity 719.7039
+8100 train 6.297609 (lr=3.6184e-06) (hash(x)=46461786)
+8000 val loss 6.3640
+8000 val perplexity 580.5896
+8000 train 6.444406 (lr=5.6809e-06) (hash(x)=54927320)
+7800 val loss 6.5596
+7800 val perplexity 705.9629
+7800 train 6.453285 (lr=1.0367e-05) (hash(x)=48510117)
+8200 val loss 6.7708
+8200 val perplexity 872.0380
+8200 train 6.699470 (lr=1.2100e-05) (hash(x)=51536260)
+8200 val loss 6.5759
+8200 val perplexity 717.6141
+8200 train 6.503042 (lr=3.4572e-06) (hash(x)=51536260)
+8100 val loss 6.3618
+8100 val perplexity 579.2977
+8100 train 6.076931 (lr=5.4277e-06) (hash(x)=46461786)
+7900 val loss 6.5584
+7900 val perplexity 705.1194
+7900 train 6.559155 (lr=9.9088e-06) (hash(x)=48339781)
+8300 val loss 6.7694
+8300 val perplexity 870.7827
+8300 train 6.520802 (lr=1.1563e-05) (hash(x)=44770722)
+8300 val loss 6.5741
+8300 val perplexity 716.2737
+8300 train 6.306182 (lr=3.3037e-06) (hash(x)=44770722)
+8200 val loss 6.3588
+8200 val perplexity 577.5776
+8200 train 6.273705 (lr=5.1858e-06) (hash(x)=51536260)
+8400 val loss 6.7649
+8400 val perplexity 866.8718
+8400 train 6.764847 (lr=1.1053e-05) (hash(x)=50104957)
+8000 val loss 6.5552
+8000 val perplexity 702.9102
+8000 train 6.663426 (lr=9.4682e-06) (hash(x)=54927320)
+8400 val loss 6.5708
+8400 val perplexity 713.9655
+8400 train 6.575215 (lr=3.1581e-06) (hash(x)=50104957)
+8300 val loss 6.3586
+8300 val perplexity 577.4501
+8300 train 6.086116 (lr=4.9556e-06) (hash(x)=44770722)
+8500 val loss 6.7612
+8500 val perplexity 863.7002
+8500 train 6.898350 (lr=1.0572e-05) (hash(x)=50132971)
+8100 val loss 6.5496
+8100 val perplexity 698.9459
+8100 train 6.264082 (lr=9.0461e-06) (hash(x)=46461786)
+8500 val loss 6.5681
+8500 val perplexity 711.9845
+8500 train 6.696193 (lr=3.0206e-06) (hash(x)=50132971)
+8400 val loss 6.3557
+8400 val perplexity 575.7655
+8400 train 6.362148 (lr=4.7372e-06) (hash(x)=50104957)
+8600 val loss 6.7565
+8600 val perplexity 859.6239
+8600 train 6.720913 (lr=1.0119e-05) (hash(x)=52193699)
+8200 val loss 6.5469
+8200 val perplexity 697.1107
+8200 train 6.480287 (lr=8.6430e-06) (hash(x)=51536260)
+8600 val loss 6.5642
+8600 val perplexity 709.2699
+8600 train 6.535054 (lr=2.8913e-06) (hash(x)=52193699)
+8500 val loss 6.3527
+8500 val perplexity 574.0456
+8500 train 6.457326 (lr=4.5309e-06) (hash(x)=50132971)
+8700 val loss 6.7556
+8700 val perplexity 858.8803
+8700 train 6.753688 (lr=9.6960e-06) (hash(x)=47902319)
+8300 val loss 6.5480
+8300 val perplexity 697.8151
+8300 train 6.279721 (lr=8.2593e-06) (hash(x)=44770722)
+8700 val loss 6.5619
+8700 val perplexity 707.6201
+8700 train 6.558250 (lr=2.7703e-06) (hash(x)=47902319)
+8600 val loss 6.3497
+8600 val perplexity 572.3204
+8600 train 6.325282 (lr=4.3369e-06) (hash(x)=52193699)
+8800 val loss 6.7519
+8800 val perplexity 855.7248
+8800 train 7.023129 (lr=9.3021e-06) (hash(x)=54904230)
+8400 val loss 6.5447
+8400 val perplexity 695.5720
+8400 train 6.552617 (lr=7.8953e-06) (hash(x)=50104957)
+8800 val loss 6.5575
+8800 val perplexity 704.4779
+8800 train 6.865329 (lr=2.6577e-06) (hash(x)=54904230)
+8700 val loss 6.3472
+8700 val perplexity 570.8699
+8700 train 6.359556 (lr=4.1554e-06) (hash(x)=47902319)
+8900 val loss 6.7474
+8900 val perplexity 851.8157
+8900 train 6.624123 (lr=8.9382e-06) (hash(x)=46311615)
+8500 val loss 6.5417
+8500 val perplexity 693.4591
+8500 train 6.659644 (lr=7.5515e-06) (hash(x)=50132971)
+8900 val loss 6.5562
+8900 val perplexity 703.5956
+8900 train 6.433506 (lr=2.5538e-06) (hash(x)=46311615)
+8800 val loss 6.3426
+8800 val perplexity 568.2990
+8800 train 6.660896 (lr=3.9866e-06) (hash(x)=54904230)
+9000 val loss 6.7487
+9000 val perplexity 852.9575
+9000 train 6.639927 (lr=8.6047e-06) (hash(x)=48535188)
+8600 val loss 6.5351
+8600 val perplexity 688.9059
+8600 train 6.505647 (lr=7.2282e-06) (hash(x)=52193699)
+9000 val loss 6.5569
+9000 val perplexity 704.0800
+9000 train 6.438684 (lr=2.4585e-06) (hash(x)=48535188)
+8900 val loss 6.3420
+8900 val perplexity 567.9246
+8900 train 6.216720 (lr=3.8307e-06) (hash(x)=46311615)
+9100 val loss 6.7458
+9100 val perplexity 850.4387
+9100 train 6.777208 (lr=8.3020e-06) (hash(x)=51757372)
+8700 val loss 6.5346
+8700 val perplexity 688.5260
+8700 train 6.549554 (lr=6.9257e-06) (hash(x)=47902319)
+9100 val loss 6.5527
+9100 val perplexity 701.1675
+9100 train 6.597699 (lr=2.3720e-06) (hash(x)=51757372)
+9000 val loss 6.3423
+9000 val perplexity 568.1190
+9000 train 6.216728 (lr=3.6877e-06) (hash(x)=48535188)
+9200 val loss 6.7434
+9200 val perplexity 848.4285
+9200 train 6.608101 (lr=8.0302e-06) (hash(x)=51131708)
+9200 val loss 6.5506
+9200 val perplexity 699.6799
+9200 train 6.395917 (lr=2.2943e-06) (hash(x)=51131708)
+8800 val loss 6.5294
+8800 val perplexity 684.9847
+8800 train 6.834278 (lr=6.6444e-06) (hash(x)=54904230)
+9100 val loss 6.3364
+9100 val perplexity 564.7474
+9100 train 6.369995 (lr=3.5580e-06) (hash(x)=51757372)
+9300 val loss 6.7413
+9300 val perplexity 846.6523
+9300 train 6.670128 (lr=7.7898e-06) (hash(x)=44784276)
+9300 val loss 6.5488
+9300 val perplexity 698.4379
+9300 train 6.492771 (lr=2.2256e-06) (hash(x)=44784276)
+8900 val loss 6.5279
+8900 val perplexity 683.9315
+8900 train 6.411209 (lr=6.3845e-06) (hash(x)=46311615)
+9200 val loss 6.3365
+9200 val perplexity 564.8207
+9200 train 6.164150 (lr=3.4415e-06) (hash(x)=51131708)
+9400 val loss 6.7413
+9400 val perplexity 846.6563
+9400 train 6.825796 (lr=7.5809e-06) (hash(x)=51981169)
+9400 val loss 6.5481
+9400 val perplexity 697.9249
+9400 train 6.641167 (lr=2.1660e-06) (hash(x)=51981169)
+9000 val loss 6.5335
+9000 val perplexity 687.7910
+9000 train 6.425806 (lr=6.1462e-06) (hash(x)=48535188)
+9300 val loss 6.3350
+9300 val perplexity 563.9686
+9300 train 6.284737 (lr=3.3385e-06) (hash(x)=44784276)
+9500 val loss 6.7437
+9500 val perplexity 848.6773
+9500 train 6.667314 (lr=7.4038e-06) (hash(x)=47232936)
+9500 val loss 6.5462
+9500 val perplexity 696.6165
+9500 train 6.484810 (lr=2.1154e-06) (hash(x)=47232936)
+9100 val loss 6.5242
+9100 val perplexity 681.4025
+9100 train 6.559541 (lr=5.9300e-06) (hash(x)=51757372)
+9400 val loss 6.3345
+9400 val perplexity 563.6874
+9400 train 6.435622 (lr=3.2490e-06) (hash(x)=51981169)
+9600 val loss 6.7411
+9600 val perplexity 846.4912
+9600 train 6.770916 (lr=7.2586e-06) (hash(x)=53800450)
+9600 val loss 6.5455
+9600 val perplexity 696.0746
+9600 train 6.573491 (lr=2.0739e-06) (hash(x)=53800450)
+9200 val loss 6.5233
+9200 val perplexity 680.8189
+9200 train 6.369421 (lr=5.7359e-06) (hash(x)=51131708)
+9500 val loss 6.3333
+9500 val perplexity 563.0075
+9500 train 6.274373 (lr=3.1730e-06) (hash(x)=47232936)
+9700 val loss 6.7405
+9700 val perplexity 845.9715
+9700 train 6.864671 (lr=7.1456e-06) (hash(x)=55768123)
+9700 val loss 6.5442
+9700 val perplexity 695.2151
+9700 train 6.679401 (lr=2.0416e-06) (hash(x)=55768123)
+9300 val loss 6.5223
+9300 val perplexity 680.1148
+9300 train 6.468675 (lr=5.5641e-06) (hash(x)=44784276)
+9600 val loss 6.3314
+9600 val perplexity 561.9553
+9600 train 6.342242 (lr=3.1108e-06) (hash(x)=53800450)
+9800 val loss 6.7390
+9800 val perplexity 844.7344
+9800 train 6.674843 (lr=7.0647e-06) (hash(x)=47745177)
+9800 val loss 6.5437
+9800 val perplexity 694.8572
+9800 train 6.458412 (lr=2.0185e-06) (hash(x)=47745177)
+9400 val loss 6.5186
+9400 val perplexity 677.6028
+9400 train 6.622711 (lr=5.4149e-06) (hash(x)=51981169)
+9700 val loss 6.3310
+9700 val perplexity 561.7244
+9700 train 6.463716 (lr=3.0624e-06) (hash(x)=55768123)
+9900 val loss 6.7386
+9900 val perplexity 844.3973
+9900 train 6.993019 (lr=7.0162e-06) (hash(x)=56592246)
+9900 val loss 6.5427
+9900 val perplexity 694.1486
+9900 train 6.819444 (lr=2.0046e-06) (hash(x)=56592246)
+9500 val loss 6.5166
+9500 val perplexity 676.2491
+9500 train 6.458758 (lr=5.2884e-06) (hash(x)=47232936)
+9999 val loss 6.7382
+9999 val perplexity 844.0145
+9800 val loss 6.3319
+9800 val perplexity 562.2196
+9800 train 6.251424 (lr=3.0277e-06) (hash(x)=47745177)
+9999 val loss 6.5435
+9999 val perplexity 694.7287
+9600 val loss 6.5156
+9600 val perplexity 675.5681
+9600 train 6.540743 (lr=5.1847e-06) (hash(x)=53800450)
+9900 val loss 6.3312
+9900 val perplexity 561.8361
+9900 train 6.614131 (lr=3.0069e-06) (hash(x)=56592246)
+9999 val loss 6.3315
+9999 val perplexity 562.0116
+9700 val loss 6.5144
+9700 val perplexity 674.7742
+9700 train 6.652847 (lr=5.1040e-06) (hash(x)=55768123)
+9800 val loss 6.5141
+9800 val perplexity 674.5625
+9800 train 6.437059 (lr=5.0462e-06) (hash(x)=47745177)
+9900 val loss 6.5129
+9900 val perplexity 673.7457
+9900 train 6.774715 (lr=5.0116e-06) (hash(x)=56592246)
+9999 val loss 6.5129
+9999 val perplexity 673.7534

attention_kindselective_n_heads8_seed1340/model_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:546196196f4e135fa42acf649a491e59d4e404903294df6008574f8fcb529fcc
+size 257976706

attention_kindselective_n_heads8_seed1340/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e0d6f68d89b38b064adf1c87e26bc401ffd6c6c1f9e6dc751711747053f53c2
+size 257976706

attention_kindselective_n_heads8_seed1340/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5836d089233e1744dd72efd8c6b025464fc61c5cbee97c1610b275dd0c748e5
+size 257976706

attention_kindselective_n_heads8_seed1340/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:803e7f1d779f4eaa96237848c494b11c7ea9c19ab7ee06466b575e3a7025fff8
+size 257976706

attention_kindselective_n_heads8_seed1340/optimizer_02500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47e117f9e735979d69333522e6bf577f699bf362355432d8a8d55ee77a8e102a
+size 509672838

attention_kindselective_n_heads8_seed1340/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dc9e23aa59ecd4e79f1695ce1f36b3fbb68e7d1c9daabc7bb7ab239cb5ec48f
+size 509672838

attention_kindselective_n_heads8_seed1340/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7b908f20aa5a501633bcacffab720c0cbaea5aa7a8a0dfc45110899b03c5827
+size 509672838

attention_kindselective_n_heads8_seed1340/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3f6b50490bbd4ce73aeef1748bd4d1d083b3dd237165be68b2a0cc0f109a719
+size 509672838