Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_3/attention_kindselective_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_3", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1339", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_2_1339", "n_embd": 128}

attention_kindselective_n_heads2_seed1339/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.7467
 0 val perplexity 126331.5391
-0 train 11.740535 (lr=7.5000e-07) (hash(x)=150724848)
-100 val loss 9.4365
-100 val perplexity 12538.0400
-100 train 9.480801 (lr=7.5750e-05) (hash(x)=149217061)
-200 val loss 7.6992
-200 val perplexity 2206.4814
-200 train 7.687409 (lr=1.5000e-04) (hash(x)=149910534)
-300 val loss 7.6357
-300 val perplexity 2070.8967
-300 train 7.504811 (lr=1.4997e-04) (hash(x)=142185643)
-400 val loss 7.6326
-400 val perplexity 2064.4209
-400 train 7.661612 (lr=1.4986e-04) (hash(x)=148123706)
-500 val loss 7.6077
-500 val perplexity 2013.7218
-500 train 7.464101 (lr=1.4969e-04) (hash(x)=149952383)
-600 val loss 7.6349
-600 val perplexity 2069.1643
-600 train 7.453125 (lr=1.4945e-04) (hash(x)=146678221)
-700 val loss 7.5944
-700 val perplexity 1987.0646
-700 train 7.617968 (lr=1.4913e-04) (hash(x)=156180736)
-800 val loss 7.5767
-800 val perplexity 1952.2250
-800 train 7.557334 (lr=1.4876e-04) (hash(x)=151700982)
-900 val loss 7.5475
-900 val perplexity 1895.9889
-900 train 7.363899 (lr=1.4831e-04) (hash(x)=147288467)
-1000 val loss 7.5335
-1000 val perplexity 1869.6943
-1000 train 7.592796 (lr=1.4779e-04) (hash(x)=156182087)
-1100 val loss 7.5065
-1100 val perplexity 1819.7588
-1100 train 7.376440 (lr=1.4721e-04) (hash(x)=147861550)
-1200 val loss 7.4818
-1200 val perplexity 1775.3508
-1200 train 7.343384 (lr=1.4656e-04) (hash(x)=149318660)
-1300 val loss 7.4929
-1300 val perplexity 1795.2754
-1300 train 7.324528 (lr=1.4585e-04) (hash(x)=151631103)
-1400 val loss 7.4838
-1400 val perplexity 1779.0669
-1400 train 7.515356 (lr=1.4507e-04) (hash(x)=150482428)
-1500 val loss 7.4626
-1500 val perplexity 1741.7150
-1500 train 7.344044 (lr=1.4422e-04) (hash(x)=144458617)
-1600 val loss 7.3651
-1600 val perplexity 1579.8989
-1600 train 7.253219 (lr=1.4332e-04) (hash(x)=143268605)
-1700 val loss 7.3347
-1700 val perplexity 1532.5645
-1700 train 7.271320 (lr=1.4235e-04) (hash(x)=150817011)
-1800 val loss 7.3513
-1800 val perplexity 1558.1730
-1800 train 7.360166 (lr=1.4131e-04) (hash(x)=152322423)
-1900 val loss 7.2976
-1900 val perplexity 1476.7399
-1900 train 7.127239 (lr=1.4022e-04) (hash(x)=143574126)
-2000 val loss 7.3026
-2000 val perplexity 1484.1338
-2000 train 7.090799 (lr=1.3907e-04) (hash(x)=147904298)
-2100 val loss 7.2743
-2100 val perplexity 1442.7886
-2100 train 7.011063 (lr=1.3786e-04) (hash(x)=144719400)
-2200 val loss 7.2155
-2200 val perplexity 1360.3833
-2200 train 7.452095 (lr=1.3660e-04) (hash(x)=154343147)
-2300 val loss 7.1752
-2300 val perplexity 1306.6619
-2300 train 7.215823 (lr=1.3527e-04) (hash(x)=154372635)
-2400 val loss 7.1679
-2400 val perplexity 1297.0833
-2400 train 7.051396 (lr=1.3390e-04) (hash(x)=141843115)
-2500 val loss 7.1449
-2500 val perplexity 1267.6400
-2500 train 7.062555 (lr=1.3247e-04) (hash(x)=149417679)
-2600 val loss 7.1157
-2600 val perplexity 1231.1683
-2600 train 7.002046 (lr=1.3099e-04) (hash(x)=145279030)
-2700 val loss 7.1132
-2700 val perplexity 1228.1211
-2700 train 7.009352 (lr=1.2946e-04) (hash(x)=146363911)
-2800 val loss 7.1103
-2800 val perplexity 1224.4899
-2800 train 7.086044 (lr=1.2788e-04) (hash(x)=152507639)
-2900 val loss 7.1079
-2900 val perplexity 1221.6112
-2900 train 6.923614 (lr=1.2626e-04) (hash(x)=139296191)
-3000 val loss 7.1002
-3000 val perplexity 1212.2461
-3000 train 7.111192 (lr=1.2459e-04) (hash(x)=148473774)
-3100 val loss 7.0391
-3100 val perplexity 1140.3469
-3100 train 7.136829 (lr=1.2287e-04) (hash(x)=154130527)
-3200 val loss 6.9963
-3200 val perplexity 1092.5740
-3200 train 6.936981 (lr=1.2112e-04) (hash(x)=151117002)
-3300 val loss 7.0011
-3300 val perplexity 1097.8837
-3300 train 7.113845 (lr=1.1932e-04) (hash(x)=151035383)
-3400 val loss 6.9671
-3400 val perplexity 1061.1619
-3400 train 7.074393 (lr=1.1749e-04) (hash(x)=150614749)
-3500 val loss 6.9428
-3500 val perplexity 1035.6863
-3500 train 6.973127 (lr=1.1562e-04) (hash(x)=151431391)
-3600 val loss 6.9251
-3600 val perplexity 1017.4580
-3600 train 6.882280 (lr=1.1372e-04) (hash(x)=151140797)
-3700 val loss 6.9142
-3700 val perplexity 1006.4932
-3700 train 7.091825 (lr=1.1179e-04) (hash(x)=154608340)
-3800 val loss 6.8872
-3800 val perplexity 979.6236
-3800 train 6.868873 (lr=1.0982e-04) (hash(x)=169641309)
-3900 val loss 6.8553
-3900 val perplexity 948.8849
-3900 train 6.805549 (lr=1.0783e-04) (hash(x)=150592479)
-4000 val loss 6.8401
-4000 val perplexity 934.5740
-4000 train 6.819444 (lr=1.0581e-04) (hash(x)=155495317)
-4100 val loss 6.8482
-4100 val perplexity 942.1501
-4100 train 7.033608 (lr=1.0377e-04) (hash(x)=170323970)
-4200 val loss 6.8270
-4200 val perplexity 922.4595
-4200 train 6.649766 (lr=1.0171e-04) (hash(x)=140854360)
-4300 val loss 6.8013
-4300 val perplexity 899.0114
-4300 train 6.802259 (lr=9.9622e-05) (hash(x)=148926644)
-4400 val loss 6.7754
-4400 val perplexity 875.9948
-4400 train 6.811431 (lr=9.7520e-05) (hash(x)=149997871)
-4500 val loss 6.7574
-4500 val perplexity 860.4335
-4500 train 6.723810 (lr=9.5403e-05) (hash(x)=149219289)
-4600 val loss 6.7387
-4600 val perplexity 844.4553
-4600 train 6.559234 (lr=9.3273e-05) (hash(x)=139836197)
-4700 val loss 6.7162
-4700 val perplexity 825.6404
-4700 train 6.670391 (lr=9.1132e-05) (hash(x)=150722883)
-4800 val loss 6.7031
-4800 val perplexity 814.8918
-4800 train 6.871788 (lr=8.8982e-05) (hash(x)=166864648)
-4900 val loss 6.6805
-4900 val perplexity 796.7543
-4900 train 6.777838 (lr=8.6825e-05) (hash(x)=157746743)
-5000 val loss 6.6579
-5000 val perplexity 778.9282
-5000 train 6.616292 (lr=8.4663e-05) (hash(x)=153032257)
-5100 val loss 6.6439
-5100 val perplexity 768.1103
-5100 train 6.713952 (lr=8.2500e-05) (hash(x)=151862371)
-5200 val loss 6.6381
-5200 val perplexity 763.6665
-5200 train 6.914250 (lr=8.0337e-05) (hash(x)=168862995)
-5300 val loss 6.6273
-5300 val perplexity 755.4161
-5300 train 6.587457 (lr=7.8175e-05) (hash(x)=148243567)
-5400 val loss 6.6006
-5400 val perplexity 735.5362
-5400 train 6.533762 (lr=7.6018e-05) (hash(x)=145381742)
-5500 val loss 6.5832
-5500 val perplexity 722.8611
-5500 train 6.719530 (lr=7.3868e-05) (hash(x)=155602174)
-5600 val loss 6.5691
-5600 val perplexity 712.7457
-5600 train 6.656135 (lr=7.1727e-05) (hash(x)=160637672)
-5700 val loss 6.5469
-5700 val perplexity 697.0565
-5700 train 6.475524 (lr=6.9597e-05) (hash(x)=152101131)
-5800 val loss 6.5437
-5800 val perplexity 694.8675
-5800 train 6.523787 (lr=6.7480e-05) (hash(x)=152327089)
-5900 val loss 6.5318
-5900 val perplexity 686.6349
-5900 train 6.474981 (lr=6.5378e-05) (hash(x)=151094543)
-6000 val loss 6.5252
-6000 val perplexity 682.1018
-6000 train 6.609112 (lr=6.3294e-05) (hash(x)=160233315)
-6100 val loss 6.5045
-6100 val perplexity 668.1599
-6100 train 6.587837 (lr=6.1230e-05) (hash(x)=158295337)
-6200 val loss 6.4956
-6200 val perplexity 662.2494
-6200 train 6.458873 (lr=5.9188e-05) (hash(x)=151269817)
-6300 val loss 6.4957
-6300 val perplexity 662.2895
-6300 train 6.622801 (lr=5.7169e-05) (hash(x)=156912756)
-6400 val loss 6.4735
-6400 val perplexity 647.7662
-6400 train 6.468271 (lr=5.5177e-05) (hash(x)=125779637)
-6500 val loss 6.4767
-6500 val perplexity 649.8300
-6500 train 6.383432 (lr=5.3213e-05) (hash(x)=148157661)
-6600 val loss 6.4666
-6600 val perplexity 643.2645
-6600 train 6.497171 (lr=5.1279e-05) (hash(x)=152820049)
-6700 val loss 6.4568
-6700 val perplexity 637.0014
-6700 train 6.417848 (lr=4.9377e-05) (hash(x)=155818717)
-6800 val loss 6.4443
-6800 val perplexity 629.1221
-6800 train 6.465776 (lr=4.7509e-05) (hash(x)=158833881)
-6900 val loss 6.4343
-6900 val perplexity 622.8764
-6900 train 6.397180 (lr=4.5676e-05) (hash(x)=164984909)
-7000 val loss 6.4265
-7000 val perplexity 617.9967
-7000 train 6.319517 (lr=4.3882e-05) (hash(x)=151172493)
-7100 val loss 6.4253
-7100 val perplexity 617.2631
-7100 train 6.444489 (lr=4.2128e-05) (hash(x)=154023633)
-7200 val loss 6.4220
-7200 val perplexity 615.2341
-7200 train 6.309972 (lr=4.0414e-05) (hash(x)=139351250)
-7300 val loss 6.4162
-7300 val perplexity 611.6689
-7300 train 6.344171 (lr=3.8745e-05) (hash(x)=158719458)
-7400 val loss 6.3990
-7400 val perplexity 601.2144
-7400 train 6.475732 (lr=3.7120e-05) (hash(x)=162250810)
-7500 val loss 6.3922
-7500 val perplexity 597.1456
-7500 train 6.276504 (lr=3.5541e-05) (hash(x)=142827557)
-7600 val loss 6.3901
-7600 val perplexity 595.9185
-7600 train 6.312879 (lr=3.4011e-05) (hash(x)=147830801)
-7700 val loss 6.3845
-7700 val perplexity 592.5839
-7700 train 6.500862 (lr=3.2531e-05) (hash(x)=165233314)
-7800 val loss 6.3826
-7800 val perplexity 591.4781
-7800 train 6.380383 (lr=3.1102e-05) (hash(x)=160654717)
-7900 val loss 6.3789
-7900 val perplexity 589.2634
-7900 train 6.249607 (lr=2.9726e-05) (hash(x)=140712586)
-8000 val loss 6.3717
-8000 val perplexity 585.0411
-8000 train 6.513039 (lr=2.8405e-05) (hash(x)=150309061)
-8100 val loss 6.3597
-8100 val perplexity 578.0466
-8100 train 6.361465 (lr=2.7138e-05) (hash(x)=158805466)
-8200 val loss 6.3576
-8200 val perplexity 576.8763
-8200 train 6.229097 (lr=2.5929e-05) (hash(x)=142814078)
-8300 val loss 6.3545
-8300 val perplexity 575.0615
-8300 train 6.310745 (lr=2.4778e-05) (hash(x)=148340417)
-8400 val loss 6.3521
-8400 val perplexity 573.7128
-8400 train 6.309781 (lr=2.3686e-05) (hash(x)=157132639)
-8500 val loss 6.3498
-8500 val perplexity 572.3974
-8500 train 6.310510 (lr=2.2655e-05) (hash(x)=152994240)
-8600 val loss 6.3441
-8600 val perplexity 569.1282
-8600 train 6.290955 (lr=2.1685e-05) (hash(x)=152210154)
-8700 val loss 6.3343
-8700 val perplexity 563.5939
-8700 train 6.391054 (lr=2.0777e-05) (hash(x)=149631247)
-8800 val loss 6.3324
-8800 val perplexity 562.4875
-8800 train 6.204415 (lr=1.9933e-05) (hash(x)=145617784)
-8900 val loss 6.3294
-8900 val perplexity 560.8342
-8900 train 6.412525 (lr=1.9153e-05) (hash(x)=155984970)
-9000 val loss 6.3313
-9000 val perplexity 561.8929
-9000 train 6.085052 (lr=1.8439e-05) (hash(x)=143433013)
-9100 val loss 6.3293
-9100 val perplexity 560.7858
-9100 train 6.209786 (lr=1.7790e-05) (hash(x)=152507533)
-9200 val loss 6.3209
-9200 val perplexity 556.0552
-9200 train 6.429985 (lr=1.7208e-05) (hash(x)=162400650)
-9300 val loss 6.3179
-9300 val perplexity 554.4227
-9300 train 6.355889 (lr=1.6692e-05) (hash(x)=148419068)
-9400 val loss 6.3182
-9400 val perplexity 554.5536
-9400 train 6.285955 (lr=1.6245e-05) (hash(x)=145025169)
-9500 val loss 6.3124
-9500 val perplexity 551.3729
-9500 train 6.280643 (lr=1.5865e-05) (hash(x)=145104235)
-9600 val loss 6.3172
-9600 val perplexity 554.0310
-9600 train 6.251455 (lr=1.5554e-05) (hash(x)=150499797)
-9700 val loss 6.3145
-9700 val perplexity 552.5436
-9700 train 6.136825 (lr=1.5312e-05) (hash(x)=139690022)
-9800 val loss 6.3089
-9800 val perplexity 549.4202
-9800 train 6.298676 (lr=1.5139e-05) (hash(x)=145645279)
-9900 val loss 6.3025
-9900 val perplexity 545.9294
-9900 train 6.201912 (lr=1.5035e-05) (hash(x)=150103432)
-9999 val loss 6.3028
-9999 val perplexity 546.0888
+0 train 11.740535 (lr=2.5000e-07) (hash(x)=150724848)
+100 val loss 10.4025
+100 val perplexity 32942.0703
+100 train 10.435096 (lr=2.5250e-05) (hash(x)=149217061)
+200 val loss 8.6411
+200 val perplexity 5659.7891
+200 train 8.616951 (lr=5.0000e-05) (hash(x)=149910534)
+300 val loss 7.7488
+300 val perplexity 2318.7954
+300 train 7.632362 (lr=4.9988e-05) (hash(x)=142185643)
+400 val loss 7.6018
+400 val perplexity 2001.8092
+400 train 7.636534 (lr=4.9954e-05) (hash(x)=148123706)
+500 val loss 7.5696
+500 val perplexity 1938.4028
+500 train 7.433709 (lr=4.9896e-05) (hash(x)=149952383)
+600 val loss 7.5472
+600 val perplexity 1895.3499
+600 train 7.349388 (lr=4.9815e-05) (hash(x)=146678221)
+700 val loss 7.5005
+700 val perplexity 1808.9504
+700 train 7.523438 (lr=4.9712e-05) (hash(x)=156180736)
+800 val loss 7.4568
+800 val perplexity 1731.6691
+800 train 7.442268 (lr=4.9585e-05) (hash(x)=151700982)
+900 val loss 7.4146
+900 val perplexity 1660.1090
+900 train 7.219617 (lr=4.9436e-05) (hash(x)=147288467)
+1000 val loss 7.3795
+1000 val perplexity 1602.7788
+1000 train 7.424051 (lr=4.9264e-05) (hash(x)=156182087)
+1100 val loss 7.3479
+1100 val perplexity 1552.8918
+1100 train 7.208760 (lr=4.9070e-05) (hash(x)=147861550)
+1200 val loss 7.3147
+1200 val perplexity 1502.2614
+1200 train 7.178025 (lr=4.8854e-05) (hash(x)=149318660)
+1300 val loss 7.3030
+1300 val perplexity 1484.6936
+1300 train 7.139358 (lr=4.8616e-05) (hash(x)=151631103)
+1400 val loss 7.2869
+1400 val perplexity 1461.0060
+1400 train 7.325649 (lr=4.8356e-05) (hash(x)=150482428)
+1500 val loss 7.2215
+1500 val perplexity 1368.5865
+1500 train 7.062772 (lr=4.8074e-05) (hash(x)=144458617)
+1600 val loss 7.1778
+1600 val perplexity 1310.0883
+1600 train 7.060380 (lr=4.7772e-05) (hash(x)=143268605)
+1700 val loss 7.1428
+1700 val perplexity 1264.9622
+1700 train 7.076792 (lr=4.7448e-05) (hash(x)=150817011)
+1800 val loss 7.1078
+1800 val perplexity 1221.4225
+1800 train 7.102525 (lr=4.7105e-05) (hash(x)=152322423)
+1900 val loss 7.0664
+1900 val perplexity 1171.8892
+1900 train 6.872022 (lr=4.6741e-05) (hash(x)=143574126)
+2000 val loss 7.0288
+2000 val perplexity 1128.6533
+2000 train 6.792527 (lr=4.6357e-05) (hash(x)=147904298)
+2100 val loss 6.9943
+2100 val perplexity 1090.4375
+2100 train 6.740319 (lr=4.5954e-05) (hash(x)=144719400)
+2200 val loss 6.9503
+2200 val perplexity 1043.4559
+2200 train 7.198068 (lr=4.5532e-05) (hash(x)=154343147)
+2300 val loss 6.8955
+2300 val perplexity 987.8442
+2300 train 6.939746 (lr=4.5091e-05) (hash(x)=154372635)
+2400 val loss 6.8531
+2400 val perplexity 946.8014
+2400 train 6.698762 (lr=4.4633e-05) (hash(x)=141843115)
+2500 val loss 6.8142
+2500 val perplexity 910.7258
+2500 train 6.709142 (lr=4.4156e-05) (hash(x)=149417679)
+2600 val loss 6.7830
+2600 val perplexity 882.6696
+2600 train 6.663291 (lr=4.3663e-05) (hash(x)=145279030)
+2700 val loss 6.7474
+2700 val perplexity 851.8775
+2700 train 6.656663 (lr=4.3153e-05) (hash(x)=146363911)
+2800 val loss 6.7253
+2800 val perplexity 833.2512
+2800 train 6.681330 (lr=4.2627e-05) (hash(x)=152507639)
+2900 val loss 6.6957
+2900 val perplexity 808.9321
+2900 train 6.486669 (lr=4.2085e-05) (hash(x)=139296191)
+3000 val loss 6.6649
+3000 val perplexity 784.4163
+3000 train 6.682261 (lr=4.1529e-05) (hash(x)=148473774)
+3100 val loss 6.6050
+3100 val perplexity 738.7633
+3100 train 6.694517 (lr=4.0957e-05) (hash(x)=154130527)
+3200 val loss 6.5747
+3200 val perplexity 716.7247
+3200 train 6.522137 (lr=4.0373e-05) (hash(x)=151117002)
+3300 val loss 6.5659
+3300 val perplexity 710.4384
+3300 train 6.724684 (lr=3.9775e-05) (hash(x)=151035383)
+3400 val loss 6.5302
+3400 val perplexity 685.5111
+3400 train 6.655223 (lr=3.9164e-05) (hash(x)=150614749)
+3500 val loss 6.5049
+3500 val perplexity 668.4155
+3500 train 6.523757 (lr=3.8541e-05) (hash(x)=151431391)
+3600 val loss 6.4827
+3600 val perplexity 653.7342
+3600 train 6.420113 (lr=3.7907e-05) (hash(x)=151140797)
+3700 val loss 6.4595
+3700 val perplexity 638.7455
+3700 train 6.653959 (lr=3.7262e-05) (hash(x)=154608340)
+3800 val loss 6.4378
+3800 val perplexity 625.0529
+3800 train 6.450654 (lr=3.6608e-05) (hash(x)=169641309)
+3900 val loss 6.4161
+3900 val perplexity 611.6103
+3900 train 6.381898 (lr=3.5944e-05) (hash(x)=150592479)
+4000 val loss 6.3932
+4000 val perplexity 597.7495
+4000 train 6.379524 (lr=3.5271e-05) (hash(x)=155495317)
+4100 val loss 6.3845
+4100 val perplexity 592.6045
+4100 train 6.582943 (lr=3.4590e-05) (hash(x)=170323970)
+4200 val loss 6.3710
+4200 val perplexity 584.6144
+4200 train 6.205431 (lr=3.3902e-05) (hash(x)=140854360)
+4300 val loss 6.3471
+4300 val perplexity 570.8184
+4300 train 6.348463 (lr=3.3207e-05) (hash(x)=148926644)
+4400 val loss 6.3300
+4400 val perplexity 561.1761
+4400 train 6.365700 (lr=3.2507e-05) (hash(x)=149997871)
+4500 val loss 6.3119
+4500 val perplexity 551.0798
+4500 train 6.272457 (lr=3.1801e-05) (hash(x)=149219289)
+4600 val loss 6.3005
+4600 val perplexity 544.8549
+4600 train 6.106970 (lr=3.1091e-05) (hash(x)=139836197)
+4700 val loss 6.2902
+4700 val perplexity 539.2803
+4700 train 6.239379 (lr=3.0377e-05) (hash(x)=150722883)
+4800 val loss 6.2791
+4800 val perplexity 533.3046
+4800 train 6.481926 (lr=2.9661e-05) (hash(x)=166864648)
+4900 val loss 6.2589
+4900 val perplexity 522.6470
+4900 train 6.376144 (lr=2.8942e-05) (hash(x)=157746743)
+5000 val loss 6.2425
+5000 val perplexity 514.1351
+5000 train 6.196634 (lr=2.8221e-05) (hash(x)=153032257)
+5100 val loss 6.2367
+5100 val perplexity 511.1816
+5100 train 6.311671 (lr=2.7500e-05) (hash(x)=151862371)
+5200 val loss 6.2239
+5200 val perplexity 504.6796
+5200 train 6.545998 (lr=2.6779e-05) (hash(x)=168862995)
+5300 val loss 6.2230
+5300 val perplexity 504.2054
+5300 train 6.190140 (lr=2.6058e-05) (hash(x)=148243567)
+5400 val loss 6.2093
+5400 val perplexity 497.3521
+5400 train 6.134599 (lr=2.5339e-05) (hash(x)=145381742)
+5500 val loss 6.2002
+5500 val perplexity 492.8568
+5500 train 6.354118 (lr=2.4623e-05) (hash(x)=155602174)
+5600 val loss 6.1799
+5600 val perplexity 482.9541
+5600 train 6.256996 (lr=2.3909e-05) (hash(x)=160637672)
+5700 val loss 6.1744
+5700 val perplexity 480.3039
+5700 train 6.102977 (lr=2.3199e-05) (hash(x)=152101131)
+5800 val loss 6.1643
+5800 val perplexity 475.4475
+5800 train 6.129929 (lr=2.2493e-05) (hash(x)=152327089)
+5900 val loss 6.1617
+5900 val perplexity 474.2258
+5900 train 6.115116 (lr=2.1793e-05) (hash(x)=151094543)
+6000 val loss 6.1547
+6000 val perplexity 470.9344
+6000 train 6.266313 (lr=2.1098e-05) (hash(x)=160233315)
+6100 val loss 6.1427
+6100 val perplexity 465.3227
+6100 train 6.246418 (lr=2.0410e-05) (hash(x)=158295337)
+6200 val loss 6.1317
+6200 val perplexity 460.2089
+6200 train 6.082817 (lr=1.9729e-05) (hash(x)=151269817)
+6300 val loss 6.1250
+6300 val perplexity 457.1342
+6300 train 6.272613 (lr=1.9056e-05) (hash(x)=156912756)
+6400 val loss 6.1183
+6400 val perplexity 454.1139
+6400 train 6.069808 (lr=1.8392e-05) (hash(x)=125779637)
+6500 val loss 6.1209
+6500 val perplexity 455.2741
+6500 train 6.025056 (lr=1.7738e-05) (hash(x)=148157661)
+6600 val loss 6.1137
+6600 val perplexity 451.9875
+6600 train 6.132632 (lr=1.7093e-05) (hash(x)=152820049)
+6700 val loss 6.1073
+6700 val perplexity 449.1447
+6700 train 6.042549 (lr=1.6459e-05) (hash(x)=155818717)
+6800 val loss 6.0981
+6800 val perplexity 445.0075
+6800 train 6.135156 (lr=1.5836e-05) (hash(x)=158833881)
+6900 val loss 6.0922
+6900 val perplexity 442.4055
+6900 train 6.018761 (lr=1.5225e-05) (hash(x)=164984909)
+7000 val loss 6.0899
+7000 val perplexity 441.3766
+7000 train 5.979160 (lr=1.4627e-05) (hash(x)=151172493)
+7100 val loss 6.0860
+7100 val perplexity 439.6446
+7100 train 6.136322 (lr=1.4043e-05) (hash(x)=154023633)
+7200 val loss 6.0815
+7200 val perplexity 437.6871
+7200 train 5.934602 (lr=1.3471e-05) (hash(x)=139351250)
+7300 val loss 6.0764
+7300 val perplexity 435.4692
+7300 train 6.026505 (lr=1.2915e-05) (hash(x)=158719458)
+7400 val loss 6.0684
+7400 val perplexity 431.9809
+7400 train 6.137913 (lr=1.2373e-05) (hash(x)=162250810)
+7500 val loss 6.0626
+7500 val perplexity 429.5063
+7500 train 5.942214 (lr=1.1847e-05) (hash(x)=142827557)
+7600 val loss 6.0585
+7600 val perplexity 427.7453
+7600 train 5.955554 (lr=1.1337e-05) (hash(x)=147830801)
+7700 val loss 6.0555
+7700 val perplexity 426.4401
+7700 train 6.161001 (lr=1.0844e-05) (hash(x)=165233314)
+7800 val loss 6.0556
+7800 val perplexity 426.4869
+7800 train 6.048510 (lr=1.0367e-05) (hash(x)=160654717)
+7900 val loss 6.0524
+7900 val perplexity 425.1184
+7900 train 5.907644 (lr=9.9088e-06) (hash(x)=140712586)
+8000 val loss 6.0453
+8000 val perplexity 422.1395
+8000 train 6.212321 (lr=9.4682e-06) (hash(x)=150309061)
+8100 val loss 6.0427
+8100 val perplexity 421.0160
+8100 train 6.045451 (lr=9.0461e-06) (hash(x)=158805466)
+8200 val loss 6.0367
+8200 val perplexity 418.5128
+8200 train 5.912158 (lr=8.6430e-06) (hash(x)=142814078)
+8300 val loss 6.0354
+8300 val perplexity 417.9462
+8300 train 5.992675 (lr=8.2593e-06) (hash(x)=148340417)
+8400 val loss 6.0360
+8400 val perplexity 418.2068
+8400 train 5.979285 (lr=7.8953e-06) (hash(x)=157132639)
+8500 val loss 6.0357
+8500 val perplexity 418.0764
+8500 train 5.972235 (lr=7.5515e-06) (hash(x)=152994240)
+8600 val loss 6.0275
+8600 val perplexity 414.6911
+8600 train 5.985933 (lr=7.2282e-06) (hash(x)=152210154)
+8700 val loss 6.0254
+8700 val perplexity 413.8256
+8700 train 6.090410 (lr=6.9257e-06) (hash(x)=149631247)
+8800 val loss 6.0220
+8800 val perplexity 412.4035
+8800 train 5.890935 (lr=6.6444e-06) (hash(x)=145617784)
+8900 val loss 6.0217
+8900 val perplexity 412.2670
+8900 train 6.111032 (lr=6.3845e-06) (hash(x)=155984970)
+9000 val loss 6.0222
+9000 val perplexity 412.4674
+9000 train 5.757221 (lr=6.1462e-06) (hash(x)=143433013)
+9100 val loss 6.0210
+9100 val perplexity 411.9792
+9100 train 5.914127 (lr=5.9300e-06) (hash(x)=152507533)
+9200 val loss 6.0152
+9200 val perplexity 409.5916
+9200 train 6.130023 (lr=5.7359e-06) (hash(x)=162400650)
+9300 val loss 6.0124
+9300 val perplexity 408.4705
+9300 train 6.066665 (lr=5.5641e-06) (hash(x)=148419068)
+9400 val loss 6.0114
+9400 val perplexity 408.0426
+9400 train 5.982956 (lr=5.4149e-06) (hash(x)=145025169)
+9500 val loss 6.0094
+9500 val perplexity 407.2299
+9500 train 5.996061 (lr=5.2884e-06) (hash(x)=145104235)
+9600 val loss 6.0103
+9600 val perplexity 407.6214
+9600 train 5.956517 (lr=5.1847e-06) (hash(x)=150499797)
+9700 val loss 6.0096
+9700 val perplexity 407.3307
+9700 train 5.820603 (lr=5.1040e-06) (hash(x)=139690022)
+9800 val loss 6.0035
+9800 val perplexity 404.8292
+9800 train 5.991593 (lr=5.0462e-06) (hash(x)=145645279)
+9900 val loss 6.0030
+9900 val perplexity 404.6236
+9900 train 5.909470 (lr=5.0116e-06) (hash(x)=150103432)
+9999 val loss 6.0007
+9999 val perplexity 403.7148

attention_kindselective_n_heads2_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb1443cb2223a3202b6db20d87fc5c06325ae65ec82e291e8b6a510e727775c1
+oid sha256:26725685fb50827dc83ff9f82278f8ba42e9d22b96701aec4a728d4a8dbd0f9c
 size 38587970

attention_kindselective_n_heads2_seed1339/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96a1976960dfa6e5f5541b027d8b1326f833d19d32c42d74bed201967fd5a0b3
+oid sha256:21ec684a166123440416fba30806e36722f229e82332d1d22abc1a74df7a99a9
 size 38587970

attention_kindselective_n_heads2_seed1339/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c9e4fe573fb5956acb8a1b31c6ee7bbff7b8ed970c371503fa40301f3a0651a
+oid sha256:5986d3501491e985e8e00c8458eb498d763d7f9a78ece6147050485e100bcdb5
 size 38587970

attention_kindselective_n_heads2_seed1339/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:172ad7c2469fd0b7aa14daef98d1cb52b19cc8e90bd5095af623e814bb489078
+oid sha256:a8ee4d4d9b92e53fb7fed22798ddd855f33919f39d84d10bc1b01b00ea2240bf
 size 38587970

attention_kindselective_n_heads2_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fff3493189e40fabdd0eb47a94f5ab767303f05484bcae36576e6fc71ba90972
+oid sha256:034ef43e482023b6bd9f0964b417fbdd8203f1186153ac6987211b9695b65496
 size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae20372cd86ef92af86587c9501cc81592410a5df6ce190e3560e17a9616f28e
+oid sha256:d8857e209bbc4d1f53aa2895a05bd2b6a1c84e646a62b371d514bc1acb38a4cb
 size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:efa3f6adbdeaac2e96b04670e11f4081965c41854ca90eb5be8a00fb3cbbe147
+oid sha256:3b56dc8c39a61d3f18c12d3b72d372b1ab49fdbe6bcc2e8bfbf9bc6140bb0953
 size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54de0acaec2b5b040fdf1208efb00cdf6fb5b9b0a6ce755aad2926c9707faa27
+oid sha256:ab36f6184851e8f0b703753114d767fbac91c51fd4f93a77d98854f772b45afa
 size 70895430