Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_10240_4_1341", "n_embd": 256}

attention_kindselective_n_heads4_seed1341/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
+oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
 size 964

attention_kindselective_n_heads4_seed1341/dataloader_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+oid sha256:7f03ed2ebf741f15e13c79e6cc1e9a19b308450d81cc3b4d8d0338c63d77ca59
 size 964

attention_kindselective_n_heads4_seed1341/dataloader_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+oid sha256:82590037fb2eecbec961f7967a8dd1b8d85515d31a252f66b92b8139858a8b7c
 size 964

attention_kindselective_n_heads4_seed1341/dataloader_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+oid sha256:c544303717d12355a69b8ffb1eb109434e4fdccfd5a61279b6e8ba2e870d6700
 size 964

attention_kindselective_n_heads4_seed1341/log2.txt

@@ -1,303 +1,605 @@
 max_steps: 10000
-0 val loss 11.3107
-0 val perplexity 81693.5234
-0 train 11.308768 (lr=1.5000e-07) (hash(x)=145079536)
-100 val loss 9.9810
-100 val perplexity 21610.8398
-100 train 9.920817 (lr=1.5150e-05) (hash(x)=154745873)
-200 val loss 9.0327
-200 val perplexity 8372.2393
-200 train 9.070545 (lr=3.0000e-05) (hash(x)=155800595)
-300 val loss 7.9078
-300 val perplexity 2718.5388
-300 train 7.873547 (lr=2.9993e-05) (hash(x)=148595389)
-400 val loss 7.6379
-400 val perplexity 2075.3203
-400 train 7.546945 (lr=2.9972e-05) (hash(x)=145606733)
-500 val loss 7.5460
-500 val perplexity 1893.0764
-500 train 7.381058 (lr=2.9938e-05) (hash(x)=138221231)
-600 val loss 7.4750
-600 val perplexity 1763.4865
-600 train 7.483062 (lr=2.9889e-05) (hash(x)=150367139)
-700 val loss 7.4357
-700 val perplexity 1695.4303
-700 train 7.589180 (lr=2.9827e-05) (hash(x)=155579314)
-800 val loss 7.3914
-800 val perplexity 1621.9232
-800 train 7.467270 (lr=2.9751e-05) (hash(x)=155747374)
-900 val loss 7.3559
-900 val perplexity 1565.4408
-900 train 7.563746 (lr=2.9662e-05) (hash(x)=159334575)
-1000 val loss 7.3269
-1000 val perplexity 1520.6451
-1000 train 7.201617 (lr=2.9558e-05) (hash(x)=140604760)
-1100 val loss 7.3036
-1100 val perplexity 1485.6582
-1100 train 7.141465 (lr=2.9442e-05) (hash(x)=146208052)
-1200 val loss 7.2811
-1200 val perplexity 1452.6318
-1200 train 7.186431 (lr=2.9312e-05) (hash(x)=148404734)
-1300 val loss 7.2576
-1300 val perplexity 1418.9017
-1300 train 7.193643 (lr=2.9169e-05) (hash(x)=155681970)
-1400 val loss 7.2337
-1400 val perplexity 1385.4006
-1400 train 7.187166 (lr=2.9013e-05) (hash(x)=148115934)
-1500 val loss 7.2029
-1500 val perplexity 1343.2867
-1500 train 7.354657 (lr=2.8845e-05) (hash(x)=157074034)
-1600 val loss 7.1849
-1600 val perplexity 1319.3363
-1600 train 6.942118 (lr=2.8663e-05) (hash(x)=137464699)
-1700 val loss 7.1570
-1700 val perplexity 1283.0671
-1700 train 7.334529 (lr=2.8469e-05) (hash(x)=166955614)
-1800 val loss 7.1327
-1800 val perplexity 1252.2875
-1800 train 6.962445 (lr=2.8263e-05) (hash(x)=143886042)
-1900 val loss 7.1017
-1900 val perplexity 1214.0776
-1900 train 7.764103 (lr=2.8044e-05) (hash(x)=193610391)
-2000 val loss 7.0833
-2000 val perplexity 1191.8937
-2000 train 7.255627 (lr=2.7814e-05) (hash(x)=163799796)
-2100 val loss 7.0436
-2100 val perplexity 1145.5090
-2100 train 7.025841 (lr=2.7572e-05) (hash(x)=154107339)
-2200 val loss 7.0151
-2200 val perplexity 1113.3079
-2200 train 7.011511 (lr=2.7319e-05) (hash(x)=144592844)
-2300 val loss 6.9857
-2300 val perplexity 1081.1161
-2300 train 7.070469 (lr=2.7055e-05) (hash(x)=154535861)
-2400 val loss 6.9636
-2400 val perplexity 1057.4553
-2400 train 7.703825 (lr=2.6780e-05) (hash(x)=204706354)
-2500 val loss 6.9441
-2500 val perplexity 1036.9701
-2500 train 6.739681 (lr=2.6494e-05) (hash(x)=143783202)
-2600 val loss 6.9068
-2600 val perplexity 999.0374
-2600 train 6.801236 (lr=2.6198e-05) (hash(x)=150862210)
-2700 val loss 6.8760
-2700 val perplexity 968.7800
-2700 train 6.937062 (lr=2.5892e-05) (hash(x)=150753426)
-2800 val loss 6.8560
-2800 val perplexity 949.5924
-2800 train 6.852540 (lr=2.5576e-05) (hash(x)=147766811)
-2900 val loss 6.8234
-2900 val perplexity 919.1029
-2900 train 6.759596 (lr=2.5251e-05) (hash(x)=151177814)
-3000 val loss 6.8052
-3000 val perplexity 902.5270
-3000 train 6.647370 (lr=2.4917e-05) (hash(x)=135925327)
-3100 val loss 6.7907
-3100 val perplexity 889.5784
-3100 train 6.448046 (lr=2.4574e-05) (hash(x)=124842353)
-3200 val loss 6.7714
-3200 val perplexity 872.5413
-3200 train 6.888403 (lr=2.4224e-05) (hash(x)=160440642)
-3300 val loss 6.7501
-3300 val perplexity 854.1573
-3300 train 6.648814 (lr=2.3865e-05) (hash(x)=143017131)
-3400 val loss 6.7283
-3400 val perplexity 835.7366
-3400 train 7.011107 (lr=2.3498e-05) (hash(x)=148143474)
-3500 val loss 6.7111
-3500 val perplexity 821.4731
-3500 train 6.533078 (lr=2.3125e-05) (hash(x)=148732639)
-3600 val loss 6.6867
-3600 val perplexity 801.6344
-3600 train 6.676949 (lr=2.2744e-05) (hash(x)=146333803)
-3700 val loss 6.6650
-3700 val perplexity 784.4567
-3700 train 6.462936 (lr=2.2357e-05) (hash(x)=143611445)
-3800 val loss 6.6509
-3800 val perplexity 773.4550
-3800 train 6.446225 (lr=2.1965e-05) (hash(x)=141272115)
-3900 val loss 6.6264
-3900 val perplexity 754.7770
-3900 train 6.550550 (lr=2.1566e-05) (hash(x)=149161848)
-4000 val loss 6.6091
-4000 val perplexity 741.8365
-4000 train 6.691909 (lr=2.1162e-05) (hash(x)=152036955)
-4100 val loss 6.5838
-4100 val perplexity 723.3132
-4100 train 6.641014 (lr=2.0754e-05) (hash(x)=153260968)
-4200 val loss 6.5629
-4200 val perplexity 708.3047
-4200 train 6.442538 (lr=2.0341e-05) (hash(x)=147629478)
-4300 val loss 6.5478
-4300 val perplexity 697.7399
-4300 train 6.550487 (lr=1.9924e-05) (hash(x)=161701080)
-4400 val loss 6.5317
-4400 val perplexity 686.5733
-4400 train 6.570666 (lr=1.9504e-05) (hash(x)=155290524)
-4500 val loss 6.5223
-4500 val perplexity 680.1618
-4500 train 6.317447 (lr=1.9081e-05) (hash(x)=144726593)
-4600 val loss 6.5042
-4600 val perplexity 667.9535
-4600 train 6.444550 (lr=1.8655e-05) (hash(x)=149517202)
-4700 val loss 6.4828
-4700 val perplexity 653.8280
-4700 train 6.483470 (lr=1.8226e-05) (hash(x)=139403008)
-4800 val loss 6.4693
-4800 val perplexity 645.0618
-4800 train 6.548851 (lr=1.7796e-05) (hash(x)=153387238)
-4900 val loss 6.4597
-4900 val perplexity 638.8582
-4900 train 6.207753 (lr=1.7365e-05) (hash(x)=139881569)
-5000 val loss 6.4438
-5000 val perplexity 628.7875
-5000 train 6.428659 (lr=1.6933e-05) (hash(x)=152602580)
-5100 val loss 6.4318
-5100 val perplexity 621.3099
-5100 train 6.479546 (lr=1.6500e-05) (hash(x)=150598460)
-5200 val loss 6.4224
-5200 val perplexity 615.5011
-5200 train 6.313307 (lr=1.6067e-05) (hash(x)=147660496)
-5300 val loss 6.4107
-5300 val perplexity 608.3204
-5300 train 6.151624 (lr=1.5635e-05) (hash(x)=147693848)
-5400 val loss 6.4053
-5400 val perplexity 605.0364
-5400 train 6.148688 (lr=1.5204e-05) (hash(x)=136048145)
-5500 val loss 6.3898
-5500 val perplexity 595.7108
-5500 train 6.384548 (lr=1.4774e-05) (hash(x)=140313123)
-5600 val loss 6.3793
-5600 val perplexity 589.5198
-5600 train 6.400584 (lr=1.4345e-05) (hash(x)=156997993)
-5700 val loss 6.3661
-5700 val perplexity 581.7758
-5700 train 6.370698 (lr=1.3919e-05) (hash(x)=143513880)
-5800 val loss 6.3591
-5800 val perplexity 577.7076
-5800 train 6.495447 (lr=1.3496e-05) (hash(x)=153736628)
-5900 val loss 6.3509
-5900 val perplexity 573.0050
-5900 train 6.522499 (lr=1.3076e-05) (hash(x)=166385417)
-6000 val loss 6.3421
-6000 val perplexity 567.9657
-6000 train 6.563224 (lr=1.2659e-05) (hash(x)=153216517)
-6100 val loss 6.3358
-6100 val perplexity 564.4168
-6100 train 6.212199 (lr=1.2246e-05) (hash(x)=145387545)
-6200 val loss 6.3286
-6200 val perplexity 560.3442
-6200 train 6.448741 (lr=1.1838e-05) (hash(x)=156306460)
-6300 val loss 6.3156
-6300 val perplexity 553.1222
-6300 train 6.230059 (lr=1.1434e-05) (hash(x)=148610447)
-6400 val loss 6.3071
-6400 val perplexity 548.4486
-6400 train 6.526285 (lr=1.1035e-05) (hash(x)=158343535)
-6500 val loss 6.3001
-6500 val perplexity 544.6276
-6500 train 6.452993 (lr=1.0643e-05) (hash(x)=157742557)
-6600 val loss 6.2955
-6600 val perplexity 542.1091
-6600 train 6.213193 (lr=1.0256e-05) (hash(x)=150457308)
-6700 val loss 6.2904
-6700 val perplexity 539.3461
-6700 train 6.166604 (lr=9.8753e-06) (hash(x)=142426205)
-6800 val loss 6.2867
-6800 val perplexity 537.3521
-6800 train 6.090304 (lr=9.5017e-06) (hash(x)=151522525)
-6900 val loss 6.2809
-6900 val perplexity 534.2869
-6900 train 6.224025 (lr=9.1353e-06) (hash(x)=152029630)
-7000 val loss 6.2692
-7000 val perplexity 528.0710
-7000 train 6.124932 (lr=8.7764e-06) (hash(x)=142978954)
-7100 val loss 6.2643
-7100 val perplexity 525.4924
-7100 train 6.144693 (lr=8.4255e-06) (hash(x)=148064359)
-7200 val loss 6.2599
-7200 val perplexity 523.1679
-7200 train 6.171076 (lr=8.0829e-06) (hash(x)=155771845)
-7300 val loss 6.2554
-7300 val perplexity 520.8312
-7300 train 6.050687 (lr=7.7489e-06) (hash(x)=141639844)
-7400 val loss 6.2522
-7400 val perplexity 519.1713
-7400 train 6.302989 (lr=7.4239e-06) (hash(x)=153466021)
-7500 val loss 6.2444
-7500 val perplexity 515.1309
-7500 train 6.255723 (lr=7.1083e-06) (hash(x)=148843587)
-7600 val loss 6.2382
-7600 val perplexity 511.9514
-7600 train 6.392756 (lr=6.8023e-06) (hash(x)=152053933)
-7700 val loss 6.2335
-7700 val perplexity 509.5445
-7700 train 6.063930 (lr=6.5062e-06) (hash(x)=141564003)
-7800 val loss 6.2306
-7800 val perplexity 508.0515
-7800 train 6.104179 (lr=6.2205e-06) (hash(x)=148617843)
-7900 val loss 6.2282
-7900 val perplexity 506.8590
-7900 train 6.183061 (lr=5.9453e-06) (hash(x)=146417668)
-8000 val loss 6.2244
-8000 val perplexity 504.9313
-8000 train 6.031130 (lr=5.6809e-06) (hash(x)=150165488)
-8100 val loss 6.2213
-8100 val perplexity 503.3661
-8100 train 6.198036 (lr=5.4277e-06) (hash(x)=144818855)
-8200 val loss 6.2150
-8200 val perplexity 500.2049
-8200 train 6.276789 (lr=5.1858e-06) (hash(x)=143074954)
-8300 val loss 6.2112
-8300 val perplexity 498.3220
-8300 train 6.106345 (lr=4.9556e-06) (hash(x)=145172494)
-8400 val loss 6.2099
-8400 val perplexity 497.6450
-8400 train 6.234631 (lr=4.7372e-06) (hash(x)=152284393)
-8500 val loss 6.2073
-8500 val perplexity 496.3461
-8500 train 6.067934 (lr=4.5309e-06) (hash(x)=147044705)
-8600 val loss 6.2057
-8600 val perplexity 495.5492
-8600 train 6.101382 (lr=4.3369e-06) (hash(x)=147762608)
-8700 val loss 6.2016
-8700 val perplexity 493.5237
-8700 train 6.190577 (lr=4.1554e-06) (hash(x)=141994567)
-8800 val loss 6.1973
-8800 val perplexity 491.4120
-8800 train 6.262373 (lr=3.9866e-06) (hash(x)=152552896)
-8900 val loss 6.1948
-8900 val perplexity 490.2008
-8900 train 6.026112 (lr=3.8307e-06) (hash(x)=139714454)
-9000 val loss 6.1927
-9000 val perplexity 489.1827
-9000 train 6.210496 (lr=3.6877e-06) (hash(x)=152274419)
-9100 val loss 6.1915
-9100 val perplexity 488.5645
-9100 train 6.273561 (lr=3.5580e-06) (hash(x)=160536885)
-9200 val loss 6.1912
-9200 val perplexity 488.4222
-9200 train 6.001785 (lr=3.4415e-06) (hash(x)=138386899)
-9300 val loss 6.1875
-9300 val perplexity 486.6331
-9300 train 6.231057 (lr=3.3385e-06) (hash(x)=149957457)
-9400 val loss 6.1838
-9400 val perplexity 484.8300
-9400 train 6.253336 (lr=3.2490e-06) (hash(x)=161671548)
-9500 val loss 6.1822
-9500 val perplexity 484.0442
-9500 train 6.334615 (lr=3.1730e-06) (hash(x)=171857400)
-9600 val loss 6.1806
-9600 val perplexity 483.2801
-9600 train 5.907129 (lr=3.1108e-06) (hash(x)=135378471)
-9700 val loss 6.1796
-9700 val perplexity 482.8083
-9700 train 6.100302 (lr=3.0624e-06) (hash(x)=149514591)
-9800 val loss 6.1767
-9800 val perplexity 481.3884
-9800 train 6.151616 (lr=3.0277e-06) (hash(x)=152679072)
-9900 val loss 6.1744
-9900 val perplexity 480.2796
-9900 train 6.098384 (lr=3.0069e-06) (hash(x)=150415193)
-9999 val loss 6.1729
-9999 val perplexity 479.5658
+0 val loss 11.3128
+0 val perplexity 81862.0547
+0 val loss 11.3128
+0 val perplexity 81862.0547
+0 train 11.310279 (lr=3.5000e-07) (hash(x)=47078120)
+0 train 11.310279 (lr=2.5000e-07) (hash(x)=47078120)
+100 val loss 9.7617
+100 val perplexity 17356.8340
+100 train 9.694991 (lr=3.5350e-05) (hash(x)=43429388)
+100 val loss 9.8512
+100 val perplexity 18981.9902
+100 train 9.787387 (lr=2.5250e-05) (hash(x)=43429388)
+200 val loss 8.2894
+200 val perplexity 3981.2551
+200 train 8.452739 (lr=7.0000e-05) (hash(x)=52929681)
+200 val loss 8.4601
+200 val perplexity 4722.3330
+200 train 8.603586 (lr=5.0000e-05) (hash(x)=52929681)
+300 val loss 7.7492
+300 val perplexity 2319.6724
+300 train 7.759899 (lr=6.9984e-05) (hash(x)=49930367)
+300 val loss 7.7631
+300 val perplexity 2352.1753
+300 train 7.778935 (lr=4.9988e-05) (hash(x)=49930367)
+400 val loss 7.6106
+400 val perplexity 2019.5231
+400 train 7.412497 (lr=6.9935e-05) (hash(x)=48542946)
+400 val loss 7.6169
+400 val perplexity 2032.2474
+400 train 7.425069 (lr=4.9954e-05) (hash(x)=48542946)
+500 val loss 7.5685
+500 val perplexity 1936.2578
+500 train 7.781235 (lr=6.9854e-05) (hash(x)=55286048)
+500 val loss 7.5577
+500 val perplexity 1915.3969
+500 train 7.776724 (lr=4.9896e-05) (hash(x)=55286048)
+600 val loss 7.5280
+600 val perplexity 1859.3456
+600 train 7.784773 (lr=6.9741e-05) (hash(x)=51577760)
+600 val loss 7.5079
+600 val perplexity 1822.3160
+600 train 7.754216 (lr=4.9815e-05) (hash(x)=51577760)
+700 val loss 7.4630
+700 val perplexity 1742.3579
+700 val loss 7.4881
+700 val perplexity 1786.6348
+700 train 7.642385 (lr=6.9596e-05) (hash(x)=57433471)
+700 train 7.625514 (lr=4.9712e-05) (hash(x)=57433471)
+800 val loss 7.4173
+800 val perplexity 1664.5129
+800 train 7.335365 (lr=4.9585e-05) (hash(x)=49799291)
+800 val loss 7.4641
+800 val perplexity 1744.2440
+800 train 7.393112 (lr=6.9419e-05) (hash(x)=49799291)
+900 val loss 7.3836
+900 val perplexity 1609.2952
+900 train 7.390461 (lr=4.9436e-05) (hash(x)=49502839)
+900 val loss 7.4507
+900 val perplexity 1721.0458
+900 train 7.472989 (lr=6.9210e-05) (hash(x)=49502839)
+1000 val loss 7.3367
+1000 val perplexity 1535.6757
+1000 train 7.610759 (lr=4.9264e-05) (hash(x)=51142904)
+1000 val loss 7.4245
+1000 val perplexity 1676.5538
+1000 train 7.708606 (lr=6.8970e-05) (hash(x)=51142904)
+1100 val loss 7.3006
+1100 val perplexity 1481.1467
+1100 train 7.366391 (lr=4.9070e-05) (hash(x)=52751086)
+1100 val loss 7.4098
+1100 val perplexity 1652.0471
+1100 train 7.481995 (lr=6.8698e-05) (hash(x)=52751086)
+1200 val loss 7.2841
+1200 val perplexity 1456.9362
+1200 train 7.263017 (lr=4.8854e-05) (hash(x)=51538621)
+1200 val loss 7.4052
+1200 val perplexity 1644.5803
+1200 train 7.392343 (lr=6.8395e-05) (hash(x)=51538621)
+1300 val loss 7.2626
+1300 val perplexity 1425.9829
+1300 train 7.337047 (lr=4.8616e-05) (hash(x)=52034040)
+1300 val loss 7.3926
+1300 val perplexity 1623.9207
+1300 train 7.473956 (lr=6.8062e-05) (hash(x)=52034040)
+1400 val loss 7.2474
+1400 val perplexity 1404.4630
+1400 train 7.300510 (lr=4.8356e-05) (hash(x)=50640105)
+1400 val loss 7.3499
+1400 val perplexity 1556.0629
+1400 train 7.395884 (lr=6.7698e-05) (hash(x)=50640105)
+1500 val loss 7.2309
+1500 val perplexity 1381.5065
+1500 train 7.167299 (lr=4.8074e-05) (hash(x)=49016270)
+1500 val loss 7.3171
+1500 val perplexity 1505.8344
+1500 train 7.244659 (lr=6.7304e-05) (hash(x)=49016270)
+1600 val loss 7.2159
+1600 val perplexity 1360.8336
+1600 train 6.991704 (lr=4.7772e-05) (hash(x)=46100488)
+1600 val loss 7.2733
+1600 val perplexity 1441.3094
+1600 train 7.050428 (lr=6.6881e-05) (hash(x)=46100488)
+1700 val loss 7.2009
+1700 val perplexity 1340.6190
+1700 train 7.251199 (lr=4.7448e-05) (hash(x)=49185350)
+1700 val loss 7.2556
+1700 val perplexity 1416.0603
+1700 train 7.300930 (lr=6.6428e-05) (hash(x)=49185350)
+1800 val loss 7.1965
+1800 val perplexity 1334.7596
+1800 train 7.101544 (lr=4.7105e-05) (hash(x)=48024574)
+1800 val loss 7.2196
+1800 val perplexity 1365.9220
+1800 train 7.116072 (lr=6.5947e-05) (hash(x)=48024574)
+1900 val loss 7.1782
+1900 val perplexity 1310.5806
+1900 train 7.008852 (lr=4.6741e-05) (hash(x)=45823189)
+1900 val loss 7.2347
+1900 val perplexity 1386.6748
+1900 train 7.042696 (lr=6.5437e-05) (hash(x)=45823189)
+2000 val loss 7.1713
+2000 val perplexity 1301.4883
+2000 train 6.995096 (lr=4.6357e-05) (hash(x)=45703932)
+2000 val loss 7.1963
+2000 val perplexity 1334.4949
+2000 train 7.022106 (lr=6.4900e-05) (hash(x)=45703932)
+2100 val loss 7.1582
+2100 val perplexity 1284.6196
+2100 train 7.814315 (lr=4.5954e-05) (hash(x)=58570170)
+2100 val loss 7.1742
+2100 val perplexity 1305.3441
+2100 train 7.821298 (lr=6.4335e-05) (hash(x)=58570170)
+2200 val loss 7.1705
+2200 val perplexity 1300.4889
+2200 train 7.209018 (lr=4.5532e-05) (hash(x)=55262880)
+2200 val loss 7.1913
+2200 val perplexity 1327.8857
+2200 train 7.220510 (lr=6.3745e-05) (hash(x)=55262880)
+2300 val loss 7.1378
+2300 val perplexity 1258.6638
+2300 train 6.894910 (lr=4.5091e-05) (hash(x)=46415497)
+2300 val loss 7.1616
+2300 val perplexity 1288.9939
+2300 train 6.941603 (lr=6.3128e-05) (hash(x)=46415497)
+2400 val loss 7.1111
+2400 val perplexity 1225.4788
+2400 train 7.024413 (lr=4.4633e-05) (hash(x)=49272278)
+2400 val loss 7.1233
+2400 val perplexity 1240.5813
+2400 train 7.048873 (lr=6.2486e-05) (hash(x)=49272278)
+2500 val loss 7.0761
+2500 val perplexity 1183.3629
+2500 train 6.943182 (lr=4.4156e-05) (hash(x)=48390803)
+2500 val loss 7.1063
+2500 val perplexity 1219.6753
+2500 train 6.968485 (lr=6.1819e-05) (hash(x)=48390803)
+2600 val loss 7.0582
+2600 val perplexity 1162.3698
+2600 train 6.934459 (lr=4.3663e-05) (hash(x)=47450116)
+2600 val loss 7.0812
+2600 val perplexity 1189.4194
+2600 train 6.962214 (lr=6.1128e-05) (hash(x)=47450116)
+2700 val loss 7.0514
+2700 val perplexity 1154.4388
+2700 train 7.127511 (lr=4.3153e-05) (hash(x)=52681152)
+2700 val loss 7.0574
+2700 val perplexity 1161.4263
+2700 train 7.132858 (lr=6.0414e-05) (hash(x)=52681152)
+2800 val loss 7.0329
+2800 val perplexity 1133.3129
+2800 train 7.101951 (lr=4.2627e-05) (hash(x)=50664094)
+2800 val loss 7.0451
+2800 val perplexity 1147.2659
+2800 train 7.097102 (lr=5.9677e-05) (hash(x)=50664094)
+2900 val loss 7.0183
+2900 val perplexity 1116.9050
+2900 train 6.814544 (lr=4.2085e-05) (hash(x)=47067144)
+2900 val loss 7.0468
+2900 val perplexity 1149.1554
+2900 train 6.833675 (lr=5.8919e-05) (hash(x)=47067144)
+3000 val loss 7.0048
+3000 val perplexity 1101.9557
+3000 train 6.823463 (lr=4.1529e-05) (hash(x)=45015009)
+3000 val loss 7.0270
+3000 val perplexity 1126.6256
+3000 train 6.849481 (lr=5.8140e-05) (hash(x)=45015009)
+3100 val loss 6.9903
+3100 val perplexity 1086.0919
+3100 train 6.819701 (lr=4.0957e-05) (hash(x)=45245896)
+3100 val loss 7.0127
+3100 val perplexity 1110.6350
+3100 train 6.829636 (lr=5.7340e-05) (hash(x)=45245896)
+3200 val loss 6.9845
+3200 val perplexity 1079.7936
+3200 train 6.933752 (lr=4.0373e-05) (hash(x)=49995942)
+3200 val loss 6.9907
+3200 val perplexity 1086.5137
+3200 train 6.932501 (lr=5.6522e-05) (hash(x)=49995942)
+3300 val loss 6.9702
+3300 val perplexity 1064.4474
+3300 train 6.866320 (lr=3.9775e-05) (hash(x)=52311504)
+3300 val loss 6.9877
+3300 val perplexity 1083.1827
+3300 train 6.867157 (lr=5.5684e-05) (hash(x)=52311504)
+3400 val loss 6.9502
+3400 val perplexity 1043.3903
+3400 train 6.946955 (lr=3.9164e-05) (hash(x)=44332917)
+3400 val loss 6.9558
+3400 val perplexity 1049.1891
+3400 train 6.959603 (lr=5.4829e-05) (hash(x)=44332917)
+3500 val loss 6.9310
+3500 val perplexity 1023.5314
+3500 train 7.043299 (lr=3.8541e-05) (hash(x)=56517159)
+3500 val loss 6.9587
+3500 val perplexity 1052.2147
+3500 train 7.056464 (lr=5.3958e-05) (hash(x)=56517159)
+3600 val loss 6.9298
+3600 val perplexity 1022.2837
+3600 train 6.814017 (lr=3.7907e-05) (hash(x)=50720920)
+3600 val loss 6.9627
+3600 val perplexity 1056.4498
+3600 train 6.823014 (lr=5.3070e-05) (hash(x)=50720920)
+3700 val loss 6.9230
+3700 val perplexity 1015.3115
+3700 train 7.408505 (lr=3.7262e-05) (hash(x)=62727701)
+3700 val loss 6.9457
+3700 val perplexity 1038.6671
+3700 train 7.405176 (lr=5.2167e-05) (hash(x)=62727701)
+3800 val loss 6.8997
+3800 val perplexity 991.9767
+3800 train 6.744309 (lr=3.6608e-05) (hash(x)=54772539)
+3800 val loss 6.9512
+3800 val perplexity 1044.3679
+3800 train 6.802071 (lr=5.1251e-05) (hash(x)=54772539)
+3900 val loss 6.8893
+3900 val perplexity 981.6801
+3900 train 6.907066 (lr=3.5944e-05) (hash(x)=52274485)
+3900 val loss 6.9282
+3900 val perplexity 1020.6476
+3900 train 6.943563 (lr=5.0321e-05) (hash(x)=52274485)
+4000 val loss 6.8797
+4000 val perplexity 972.3515
+4000 train 6.587507 (lr=3.5271e-05) (hash(x)=50118307)
+4000 val loss 6.9188
+4000 val perplexity 1011.1333
+4000 train 6.632916 (lr=4.9379e-05) (hash(x)=50118307)
+4100 val loss 6.8683
+4100 val perplexity 961.2698
+4100 train 6.477684 (lr=3.4590e-05) (hash(x)=42771647)
+4100 val loss 6.9173
+4100 val perplexity 1009.5603
+4100 train 6.526671 (lr=4.8426e-05) (hash(x)=42771647)
+4200 val loss 6.8648
+4200 val perplexity 957.9387
+4200 train 6.946112 (lr=3.3902e-05) (hash(x)=51748836)
+4200 val loss 6.8992
+4200 val perplexity 991.4320
+4200 train 6.988651 (lr=4.7463e-05) (hash(x)=51748836)
+4300 val loss 6.8309
+4300 val perplexity 926.0364
+4300 train 6.852984 (lr=3.3207e-05) (hash(x)=49021280)
+4300 val loss 6.8634
+4300 val perplexity 956.5743
+4300 train 6.886794 (lr=4.6490e-05) (hash(x)=49021280)
+4400 val loss 6.8110
+4400 val perplexity 907.7654
+4400 train 6.871642 (lr=3.2507e-05) (hash(x)=55200309)
+4400 val loss 6.8500
+4400 val perplexity 943.8713
+4400 train 6.919098 (lr=4.5509e-05) (hash(x)=55200309)
+4500 val loss 6.8002
+4500 val perplexity 898.0311
+4500 train 6.855990 (lr=3.1801e-05) (hash(x)=52085049)
+4500 val loss 6.8419
+4500 val perplexity 936.2292
+4500 train 6.906362 (lr=4.4521e-05) (hash(x)=52085049)
+4600 val loss 6.7947
+4600 val perplexity 893.1308
+4600 train 6.786499 (lr=3.1091e-05) (hash(x)=48935595)
+4600 val loss 6.8137
+4600 val perplexity 910.2144
+4600 train 6.811741 (lr=4.3527e-05) (hash(x)=48935595)
+4700 val loss 6.7692
+4700 val perplexity 870.6316
+4700 train 6.986565 (lr=3.0377e-05) (hash(x)=49182380)
+4700 val loss 6.7991
+4700 val perplexity 897.0356
+4700 train 7.015434 (lr=4.2528e-05) (hash(x)=49182380)
+4800 val loss 6.7546
+4800 val perplexity 857.9871
+4800 train 6.557003 (lr=2.9661e-05) (hash(x)=43941929)
+4800 val loss 6.7950
+4800 val perplexity 893.3301
+4800 train 6.608094 (lr=4.1525e-05) (hash(x)=43941929)
+4900 val loss 6.7453
+4900 val perplexity 850.0325
+4900 train 6.908017 (lr=2.8942e-05) (hash(x)=51852773)
+4900 val loss 6.7802
+4900 val perplexity 880.2544
+4900 train 6.932560 (lr=4.0518e-05) (hash(x)=51852773)
+5000 val loss 6.7367
+5000 val perplexity 842.7851
+5000 train 6.356645 (lr=2.8221e-05) (hash(x)=40509616)
+5000 val loss 6.7714
+5000 val perplexity 872.4963
+5000 train 6.396081 (lr=3.9510e-05) (hash(x)=40509616)
+5100 val loss 6.7270
+5100 val perplexity 834.6163
+5100 train 6.991583 (lr=2.7500e-05) (hash(x)=57585369)
+5100 val loss 6.7416
+5100 val perplexity 846.9026
+5100 train 6.995601 (lr=3.8500e-05) (hash(x)=57585369)
+5200 val loss 6.7101
+5200 val perplexity 820.6826
+5200 train 6.710048 (lr=2.6779e-05) (hash(x)=51042313)
+5200 val loss 6.7433
+5200 val perplexity 848.3735
+5200 train 6.749459 (lr=3.7490e-05) (hash(x)=51042313)
+5300 val loss 6.6991
+5300 val perplexity 811.6769
+5300 train 6.843317 (lr=2.6058e-05) (hash(x)=52001684)
+5300 val loss 6.7278
+5300 val perplexity 835.3138
+5300 train 6.872353 (lr=3.6482e-05) (hash(x)=52001684)
+5400 val loss 6.6844
+5400 val perplexity 799.8459
+5400 train 6.608623 (lr=2.5339e-05) (hash(x)=48831647)
+5400 val loss 6.7170
+5400 val perplexity 826.3600
+5400 train 6.639812 (lr=3.5475e-05) (hash(x)=48831647)
+5500 val loss 6.6733
+5500 val perplexity 790.9973
+5500 train 7.073305 (lr=2.4623e-05) (hash(x)=50192069)
+5500 val loss 6.7281
+5500 val perplexity 835.5864
+5500 train 7.133408 (lr=3.4472e-05) (hash(x)=50192069)
+5600 val loss 6.6626
+5600 val perplexity 782.5651
+5600 train 6.471318 (lr=2.3909e-05) (hash(x)=47208852)
+5600 val loss 6.7018
+5600 val perplexity 813.8849
+5600 train 6.518745 (lr=3.3473e-05) (hash(x)=47208852)
+5700 val loss 6.6626
+5700 val perplexity 782.5812
+5700 train 6.235447 (lr=2.3199e-05) (hash(x)=44061694)
+5700 val loss 6.7001
+5700 val perplexity 812.4940
+5700 train 6.281430 (lr=3.2479e-05) (hash(x)=44061694)
+5800 val loss 6.6482
+5800 val perplexity 771.4292
+5800 train 7.028475 (lr=2.2493e-05) (hash(x)=56513279)
+5800 val loss 6.6854
+5800 val perplexity 800.6347
+5800 train 7.046527 (lr=3.1491e-05) (hash(x)=56513279)
+5900 val loss 6.6390
+5900 val perplexity 764.3682
+5900 train 6.778142 (lr=2.1793e-05) (hash(x)=50412818)
+5900 val loss 6.6802
+5900 val perplexity 796.4485
+5900 train 6.831610 (lr=3.0510e-05) (hash(x)=50412818)
+6000 val loss 6.6357
+6000 val perplexity 761.7829
+6000 train 6.446944 (lr=2.1098e-05) (hash(x)=47159634)
+6000 val loss 6.6713
+6000 val perplexity 789.4302
+6000 train 6.488885 (lr=2.9537e-05) (hash(x)=47159634)
+6100 val loss 6.6175
+6100 val perplexity 748.0796
+6100 train 6.715433 (lr=2.0410e-05) (hash(x)=54312795)
+6100 val loss 6.6555
+6100 val perplexity 777.0111
+6100 train 6.751032 (lr=2.8574e-05) (hash(x)=54312795)
+6200 val loss 6.6087
+6200 val perplexity 741.5112
+6200 train 6.761733 (lr=1.9729e-05) (hash(x)=54187587)
+6200 val loss 6.6418
+6200 val perplexity 766.5114
+6200 train 6.792206 (lr=2.7621e-05) (hash(x)=54187587)
+6300 val loss 6.6008
+6300 val perplexity 735.6593
+6300 train 6.716049 (lr=1.9056e-05) (hash(x)=53620387)
+6300 val loss 6.6270
+6300 val perplexity 755.2397
+6300 train 6.738632 (lr=2.6679e-05) (hash(x)=53620387)
+6400 val loss 6.5969
+6400 val perplexity 732.8492
+6400 train 6.540919 (lr=1.8392e-05) (hash(x)=48761774)
+6400 val loss 6.6193
+6400 val perplexity 749.4073
+6400 train 6.558865 (lr=2.5749e-05) (hash(x)=48761774)
+6500 val loss 6.5836
+6500 val perplexity 723.1132
+6500 train 6.806363 (lr=1.7738e-05) (hash(x)=56690281)
+6500 val loss 6.6142
+6500 val perplexity 745.6394
+6500 train 6.849979 (lr=2.4833e-05) (hash(x)=56690281)
+6600 val loss 6.5759
+6600 val perplexity 717.5851
+6600 train 6.373225 (lr=1.7093e-05) (hash(x)=42985269)
+6600 val loss 6.6067
+6600 val perplexity 740.0142
+6600 train 6.412946 (lr=2.3930e-05) (hash(x)=42985269)
+6700 val loss 6.5883
+6700 val perplexity 726.5438
+6700 train 6.632374 (lr=1.6459e-05) (hash(x)=53315447)
+6700 val loss 6.6465
+6700 val perplexity 770.0811
+6700 train 6.687102 (lr=2.3042e-05) (hash(x)=53315447)
+6800 val loss 6.5659
+6800 val perplexity 710.4279
+6800 train 7.029819 (lr=1.5836e-05) (hash(x)=61577166)
+6800 val loss 6.5924
+6800 val perplexity 729.5439
+6800 train 7.053003 (lr=2.2171e-05) (hash(x)=61577166)
+6900 val loss 6.5586
+6900 val perplexity 705.2489
+6900 train 6.690564 (lr=1.5225e-05) (hash(x)=54641005)
+6900 val loss 6.5893
+6900 val perplexity 727.2960
+6900 train 6.726195 (lr=2.1316e-05) (hash(x)=54641005)
+7000 val loss 6.5510
+7000 val perplexity 699.9125
+7000 train 7.094835 (lr=1.4627e-05) (hash(x)=60579512)
+7000 val loss 6.5950
+7000 val perplexity 731.3928
+7000 train 7.124228 (lr=2.0478e-05) (hash(x)=60579512)
+7100 val loss 6.5470
+7100 val perplexity 697.1821
+7100 train 6.442610 (lr=1.4043e-05) (hash(x)=53151549)
+7100 val loss 6.5818
+7100 val perplexity 721.8381
+7100 train 6.473633 (lr=1.9660e-05) (hash(x)=53151549)
+7200 val loss 6.5441
+7200 val perplexity 695.1605
+7200 train 7.482133 (lr=1.3471e-05) (hash(x)=71842455)
+7200 val loss 6.5744
+7200 val perplexity 716.4940
+7200 train 7.518528 (lr=1.8860e-05) (hash(x)=71842455)
+7300 val loss 6.5423
+7300 val perplexity 693.9010
+7300 train 6.313259 (lr=1.2915e-05) (hash(x)=44516452)
+7300 val loss 6.5735
+7300 val perplexity 715.8937
+7300 train 6.335499 (lr=1.8081e-05) (hash(x)=44516452)
+7400 val loss 6.5372
+7400 val perplexity 690.3647
+7400 train 6.185552 (lr=1.2373e-05) (hash(x)=42667710)
+7400 val loss 6.5690
+7400 val perplexity 712.6825
+7400 train 6.217746 (lr=1.7323e-05) (hash(x)=42667710)
+7500 val loss 6.5379
+7500 val perplexity 690.8422
+7500 train 6.279439 (lr=1.1847e-05) (hash(x)=47050797)
+7500 val loss 6.5736
+7500 val perplexity 715.9745
+7500 train 6.322675 (lr=1.6586e-05) (hash(x)=47050797)
+7600 val loss 6.5328
+7600 val perplexity 687.3523
+7600 train 6.446810 (lr=1.1337e-05) (hash(x)=49785056)
+7600 val loss 6.5663
+7600 val perplexity 710.7661
+7600 train 6.485307 (lr=1.5872e-05) (hash(x)=49785056)
+7700 val loss 6.5258
+7700 val perplexity 682.5475
+7700 train 6.347061 (lr=1.0844e-05) (hash(x)=53232030)
+7700 val loss 6.5539
+7700 val perplexity 701.9633
+7700 train 6.363538 (lr=1.5181e-05) (hash(x)=53232030)
+7800 val loss 6.5232
+7800 val perplexity 680.7534
+7800 train 6.337240 (lr=1.0367e-05) (hash(x)=48049749)
+7800 val loss 6.5506
+7800 val perplexity 699.6725
+7800 train 6.387172 (lr=1.4514e-05) (hash(x)=48049749)
+7900 val loss 6.5147
+7900 val perplexity 675.0194
+7900 train 6.333706 (lr=9.9088e-06) (hash(x)=44768513)
+7900 val loss 6.5412
+7900 val perplexity 693.1060
+7900 train 6.345912 (lr=1.3872e-05) (hash(x)=44768513)
+8000 val loss 6.5126
+8000 val perplexity 673.5455
+8000 train 6.364021 (lr=9.4682e-06) (hash(x)=46228039)
+8000 val loss 6.5334
+8000 val perplexity 687.7585
+8000 train 6.399539 (lr=1.3255e-05) (hash(x)=46228039)
+8100 val loss 6.5056
+8100 val perplexity 668.8973
+8100 train 6.890257 (lr=9.0461e-06) (hash(x)=60017091)
+8100 val loss 6.5337
+8100 val perplexity 687.9074
+8100 train 6.910211 (lr=1.2665e-05) (hash(x)=60017091)
+8200 val loss 6.5021
+8200 val perplexity 666.5466
+8200 train 6.503376 (lr=8.6430e-06) (hash(x)=49910198)
+8200 val loss 6.5272
+8200 val perplexity 683.4747
+8200 train 6.535112 (lr=1.2100e-05) (hash(x)=49910198)
+8300 val loss 6.4962
+8300 val perplexity 662.6019
+8300 train 6.844116 (lr=8.2593e-06) (hash(x)=57919055)
+8300 val loss 6.5207
+8300 val perplexity 679.0652
+8300 train 6.860990 (lr=1.1563e-05) (hash(x)=57919055)
+8400 val loss 6.4953
+8400 val perplexity 662.0015
+8400 train 6.590104 (lr=7.8953e-06) (hash(x)=49694964)
+8400 val loss 6.5190
+8400 val perplexity 677.8727
+8400 train 6.614913 (lr=1.1053e-05) (hash(x)=49694964)
+8500 val loss 6.4913
+8500 val perplexity 659.3549
+8500 train 6.505794 (lr=7.5515e-06) (hash(x)=53762585)
+8500 val loss 6.5127
+8500 val perplexity 673.6345
+8500 train 6.534412 (lr=1.0572e-05) (hash(x)=53762585)
+8600 val loss 6.4897
+8600 val perplexity 658.3483
+8600 train 6.516029 (lr=7.2282e-06) (hash(x)=51166973)
+8600 val loss 6.5104
+8600 val perplexity 672.0836
+8600 train 6.537435 (lr=1.0119e-05) (hash(x)=51166973)
+8700 val loss 6.5081
+8700 val perplexity 670.5390
+8700 train 6.594219 (lr=9.6960e-06) (hash(x)=53968049)
+8700 val loss 6.4891
+8700 val perplexity 657.9178
+8700 train 6.569641 (lr=6.9257e-06) (hash(x)=53968049)
+8800 val loss 6.5047
+8800 val perplexity 668.3008
+8800 train 6.594789 (lr=9.3021e-06) (hash(x)=59231056)
+8800 val loss 6.4841
+8800 val perplexity 654.6766
+8800 train 6.569866 (lr=6.6444e-06) (hash(x)=59231056)
+8900 val loss 6.5031
+8900 val perplexity 667.2048
+8900 train 6.406780 (lr=8.9382e-06) (hash(x)=50488048)
+8900 val loss 6.4835
+8900 val perplexity 654.2831
+9000 val loss 6.5006
+9000 val perplexity 665.5312
+8900 train 6.380212 (lr=6.3845e-06) (hash(x)=50488048)
+9000 train 6.212546 (lr=8.6047e-06) (hash(x)=44492956)
+9100 val loss 6.5057
+9100 val perplexity 668.9218
+9100 train 6.543286 (lr=8.3020e-06) (hash(x)=51134989)
+9000 val loss 6.4798
+9000 val perplexity 651.8441
+9000 train 6.196507 (lr=6.1462e-06) (hash(x)=44492956)
+9200 val loss 6.4988
+9200 val perplexity 664.3596
+9200 train 6.313716 (lr=8.0302e-06) (hash(x)=48636056)
+9100 val loss 6.4836
+9100 val perplexity 654.3164
+9100 train 6.517267 (lr=5.9300e-06) (hash(x)=51134989)
+9300 val loss 6.4975
+9300 val perplexity 663.4982
+9300 train 6.441718 (lr=7.7898e-06) (hash(x)=50200551)
+9200 val loss 6.4788
+9200 val perplexity 651.1703
+9200 train 6.282190 (lr=5.7359e-06) (hash(x)=48636056)
+9400 val loss 6.4984
+9400 val perplexity 664.0980
+9400 train 6.325517 (lr=7.5809e-06) (hash(x)=48057228)
+9300 val loss 6.4773
+9300 val perplexity 650.2233
+9300 train 6.417827 (lr=5.5641e-06) (hash(x)=50200551)
+9500 val loss 6.4931
+9500 val perplexity 660.5939
+9500 train 6.270067 (lr=7.4038e-06) (hash(x)=48125171)
+9400 val loss 6.4772
+9400 val perplexity 650.1170
+9400 train 6.301267 (lr=5.4149e-06) (hash(x)=48057228)
+9600 val loss 6.4923
+9600 val perplexity 660.0577
+9600 train 6.415076 (lr=7.2586e-06) (hash(x)=53375853)
+9500 val loss 6.4735
+9500 val perplexity 647.7742
+9500 train 6.245581 (lr=5.2884e-06) (hash(x)=48125171)
+9700 val loss 6.4883
+9700 val perplexity 657.3944
+9700 train 7.296200 (lr=7.1456e-06) (hash(x)=53924631)
+9600 val loss 6.4741
+9600 val perplexity 648.1082
+9600 train 6.401201 (lr=5.1847e-06) (hash(x)=53375853)
+9800 val loss 6.4832
+9800 val perplexity 654.0672
+9800 train 6.527805 (lr=7.0647e-06) (hash(x)=48895047)
+9700 val loss 6.4699
+9700 val perplexity 645.3882
+9700 train 7.271974 (lr=5.1040e-06) (hash(x)=53924631)
+9900 val loss 6.4799
+9900 val perplexity 651.8783
+9900 train 6.321083 (lr=7.0162e-06) (hash(x)=44269923)
+9800 val loss 6.4660
+9800 val perplexity 642.9058
+9800 train 6.506448 (lr=5.0462e-06) (hash(x)=48895047)
+9999 val loss 6.4860
+9999 val perplexity 655.9136
+9900 val loss 6.4629
+9900 val perplexity 640.8951
+9900 train 6.308586 (lr=5.0116e-06) (hash(x)=44269923)
+9999 val loss 6.4654
+9999 val perplexity 642.5481

attention_kindselective_n_heads4_seed1341/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4dd19d55eccbca4cc0d4abab192709a4e04037b423dcf01f739eec9d2889807
+oid sha256:49dcb2bbcd59a985e2fea57fad06afedf1f64d61094d1dc8f6cfc83c6fdf6cd1
 size 92843394

attention_kindselective_n_heads4_seed1341/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:897c2fca90a8a86414cec7ef9f9dba951412bdc1d1b3736048c368ad5d6d206d
+oid sha256:231a8f89882888110c171b221d63ff6e1cb161b75abe119498640b4cf4f64a3d
 size 92843394

attention_kindselective_n_heads4_seed1341/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53d429564946189dea5d288e8a03b4bfd8eb188459cbc1d8e724ccb87f538f77
+oid sha256:8c22e2287221c2854c890def3ad5a680c15132745c807ad9edbd776085af63c0
 size 92843394

attention_kindselective_n_heads4_seed1341/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7002c239e79b53ed1ff51816d5a0819c177cd02442ad4f91edd01037ed04dc90
+oid sha256:ae8e7546121eb2fcbccb16e950a8578382d106d75e43bf389c1df910a26b45cf
 size 92843394

attention_kindselective_n_heads4_seed1341/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31d88e061c99a7f36e0d25bb9be9cf00fb2ca8080cc9da28f9c5e50fa85dfaa4
+oid sha256:0e25a8af3279e7d30f2fa3cec153d0be60a339227e6c25055bedb248f25844a2
 size 179406214

attention_kindselective_n_heads4_seed1341/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45a23b5438972806ac3ff4f774bbdaca4f8dd88b515c93a9b89cd11d0b848d99
+oid sha256:0b29711ebb29c7f449c48556bab06813e77398c99c72a393938e1c4250ad3ba6
 size 179406214

attention_kindselective_n_heads4_seed1341/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd8888fc9d1ae8d4db6810cdff83bfd3f1d60b67f0b6a4ebc53db99906b4ba9c
+oid sha256:17523d4ab8ec9adf3994022ccee185ada0f5971a9a3fbd0cd6986ee2cf6e6613
 size 179406214

attention_kindselective_n_heads4_seed1341/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:26c6af83136a26760446afc9ba36e45eaa02cd5ef16fbb40c5a070b2857403be
+oid sha256:3d57bc15456b0e24b344a28176223bc2bcd78aa80ee88708afb702ad14b20aa6
 size 179406214