Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads2_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_10240_2_1341", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_6/attention_kindselective_n_heads2_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_6", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.0001, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "10e-5_10240_2_1341", "n_embd": 128}

attention_kindselective_n_heads2_seed1341/log2.txt

@@ -1,591 +1,3 @@
 max_steps: 10000
 0 val loss 11.7712
 0 val perplexity 129466.1094
-0 val loss 11.7712
-0 val perplexity 129466.1094
-0 train 11.770625 (lr=3.5000e-07) (hash(x)=47078120)
-0 train 11.770625 (lr=2.5000e-07) (hash(x)=47078120)
-100 val loss 10.0727
-100 val perplexity 23688.1914
-100 train 10.071457 (lr=3.5350e-05) (hash(x)=43429388)
-100 val loss 10.2257
-100 val perplexity 27604.7012
-100 train 10.230239 (lr=2.5250e-05) (hash(x)=43429388)
-200 val loss 8.5407
-200 val perplexity 5118.7700
-200 train 8.701613 (lr=7.0000e-05) (hash(x)=52929681)
-200 val loss 9.0275
-200 val perplexity 8328.7500
-200 train 9.171297 (lr=5.0000e-05) (hash(x)=52929681)
-300 val loss 7.7996
-300 val perplexity 2439.5366
-300 train 7.821887 (lr=6.9984e-05) (hash(x)=49930367)
-300 val loss 7.9612
-300 val perplexity 2867.4238
-300 train 7.983008 (lr=4.9988e-05) (hash(x)=49930367)
-400 val loss 7.6933
-400 val perplexity 2193.5842
-400 train 7.520776 (lr=6.9935e-05) (hash(x)=48542946)
-400 val loss 7.7024
-400 val perplexity 2213.6982
-400 train 7.530426 (lr=4.9954e-05) (hash(x)=48542946)
-500 val loss 7.6582
-500 val perplexity 2117.9321
-500 train 7.850955 (lr=4.9896e-05) (hash(x)=55286048)
-500 val loss 7.6649
-500 val perplexity 2132.2778
-500 train 7.856366 (lr=6.9854e-05) (hash(x)=55286048)
-600 val loss 7.6322
-600 val perplexity 2063.6858
-600 train 7.885755 (lr=4.9815e-05) (hash(x)=51577760)
-600 val loss 7.6414
-600 val perplexity 2082.6948
-600 train 7.883400 (lr=6.9741e-05) (hash(x)=51577760)
-700 val loss 7.5973
-700 val perplexity 1992.7426
-700 train 7.745926 (lr=4.9712e-05) (hash(x)=57433471)
-700 val loss 7.6175
-700 val perplexity 2033.5338
-700 train 7.751812 (lr=6.9596e-05) (hash(x)=57433471)
-800 val loss 7.5873
-800 val perplexity 1972.9855
-800 train 7.505336 (lr=4.9585e-05) (hash(x)=49799291)
-800 val loss 7.5945
-800 val perplexity 1987.2598
-800 train 7.511701 (lr=6.9419e-05) (hash(x)=49799291)
-900 val loss 7.5716
-900 val perplexity 1942.1638
-900 train 7.573707 (lr=4.9436e-05) (hash(x)=49502839)
-900 val loss 7.5740
-900 val perplexity 1946.9084
-900 train 7.579421 (lr=6.9210e-05) (hash(x)=49502839)
-1000 val loss 7.5416
-1000 val perplexity 1884.8977
-1000 train 7.802229 (lr=4.9264e-05) (hash(x)=51142904)
-1000 val loss 7.5218
-1000 val perplexity 1847.9504
-1000 train 7.800779 (lr=6.8970e-05) (hash(x)=51142904)
-1100 val loss 7.5367
-1100 val perplexity 1875.6093
-1100 train 7.609015 (lr=4.9070e-05) (hash(x)=52751086)
-1100 val loss 7.5060
-1100 val perplexity 1818.8705
-1100 train 7.595724 (lr=6.8698e-05) (hash(x)=52751086)
-1200 val loss 7.5167
-1200 val perplexity 1838.4198
-1200 train 7.513431 (lr=4.8854e-05) (hash(x)=51538621)
-1200 val loss 7.4812
-1200 val perplexity 1774.2981
-1200 train 7.480718 (lr=6.8395e-05) (hash(x)=51538621)
-1300 val loss 7.5203
-1300 val perplexity 1845.1267
-1300 train 7.599769 (lr=4.8616e-05) (hash(x)=52034040)
-1300 val loss 7.4708
-1300 val perplexity 1755.9396
-1300 train 7.550740 (lr=6.8062e-05) (hash(x)=52034040)
-1400 val loss 7.4884
-1400 val perplexity 1787.2448
-1400 train 7.515759 (lr=4.8356e-05) (hash(x)=50640105)
-1400 val loss 7.4255
-1400 val perplexity 1678.2040
-1400 train 7.458371 (lr=6.7698e-05) (hash(x)=50640105)
-1500 val loss 7.4658
-1500 val perplexity 1747.2191
-1500 train 7.407789 (lr=4.8074e-05) (hash(x)=49016270)
-1500 val loss 7.3886
-1500 val perplexity 1617.4746
-1500 train 7.317589 (lr=6.7304e-05) (hash(x)=49016270)
-1600 val loss 7.4415
-1600 val perplexity 1705.3562
-1600 train 7.206252 (lr=4.7772e-05) (hash(x)=46100488)
-1700 val loss 7.4194
-1700 val perplexity 1668.0033
-1700 train 7.459988 (lr=4.7448e-05) (hash(x)=49185350)
-1600 val loss 7.3380
-1600 val perplexity 1537.6078
-1600 train 7.104100 (lr=6.6881e-05) (hash(x)=46100488)
-1800 val loss 7.3992
-1800 val perplexity 1634.6200
-1800 train 7.278754 (lr=4.7105e-05) (hash(x)=48024574)
-1700 val loss 7.2826
-1700 val perplexity 1454.7106
-1700 train 7.309137 (lr=6.6428e-05) (hash(x)=49185350)
-1900 val loss 7.3710
-1900 val perplexity 1589.1908
-1900 train 7.176445 (lr=4.6741e-05) (hash(x)=45823189)
-1800 val loss 7.2495
-1800 val perplexity 1407.4564
-1800 train 7.130040 (lr=6.5947e-05) (hash(x)=48024574)
-2000 val loss 7.3510
-2000 val perplexity 1557.7837
-2000 train 7.191787 (lr=4.6357e-05) (hash(x)=45703932)
-1900 val loss 7.2065
-1900 val perplexity 1348.1270
-1900 train 7.000680 (lr=6.5437e-05) (hash(x)=45823189)
-2100 val loss 7.3243
-2100 val perplexity 1516.6754
-2100 train 7.937493 (lr=4.5954e-05) (hash(x)=58570170)
-2000 val loss 7.1571
-2000 val perplexity 1283.1888
-2000 train 6.968543 (lr=6.4900e-05) (hash(x)=45703932)
-2200 val loss 7.3308
-2200 val perplexity 1526.6041
-2200 train 7.352339 (lr=4.5532e-05) (hash(x)=55262880)
-2100 val loss 7.1143
-2100 val perplexity 1229.3721
-2100 train 7.765666 (lr=6.4335e-05) (hash(x)=58570170)
-2300 val loss 7.2808
-2300 val perplexity 1452.1927
-2300 train 7.059850 (lr=4.5091e-05) (hash(x)=46415497)
-2200 val loss 7.0929
-2200 val perplexity 1203.3556
-2200 train 7.112502 (lr=6.3745e-05) (hash(x)=55262880)
-2400 val loss 7.2422
-2400 val perplexity 1397.1962
-2400 train 7.170742 (lr=4.4633e-05) (hash(x)=49272278)
-2300 val loss 7.0295
-2300 val perplexity 1129.4420
-2300 train 6.759797 (lr=6.3128e-05) (hash(x)=46415497)
-2500 val loss 7.2227
-2500 val perplexity 1370.1537
-2500 train 7.073672 (lr=4.4156e-05) (hash(x)=48390803)
-2400 val loss 6.9712
-2400 val perplexity 1065.5042
-2400 train 6.889137 (lr=6.2486e-05) (hash(x)=49272278)
-2600 val loss 7.1874
-2600 val perplexity 1322.6218
-2600 train 7.069160 (lr=4.3663e-05) (hash(x)=47450116)
-2500 val loss 6.9361
-2500 val perplexity 1028.7850
-2500 train 6.783605 (lr=6.1819e-05) (hash(x)=48390803)
-2700 val loss 7.1763
-2700 val perplexity 1308.1176
-2700 train 7.254964 (lr=4.3153e-05) (hash(x)=52681152)
-2800 val loss 7.1497
-2800 val perplexity 1273.7507
-2800 train 7.206081 (lr=4.2627e-05) (hash(x)=50664094)
-2600 val loss 6.8966
-2600 val perplexity 988.8942
-2600 train 6.776588 (lr=6.1128e-05) (hash(x)=47450116)
-2900 val loss 7.1360
-2900 val perplexity 1256.4001
-2900 train 6.937007 (lr=4.2085e-05) (hash(x)=47067144)
-2700 val loss 6.8688
-2700 val perplexity 961.7508
-2700 train 6.956961 (lr=6.0414e-05) (hash(x)=52681152)
-3000 val loss 7.1024
-3000 val perplexity 1214.9301
-3000 train 6.913496 (lr=4.1529e-05) (hash(x)=45015009)
-2800 val loss 6.8370
-2800 val perplexity 931.7004
-2800 train 6.900253 (lr=5.9677e-05) (hash(x)=50664094)
-3100 val loss 7.0774
-3100 val perplexity 1184.8473
-3100 train 6.905035 (lr=4.0957e-05) (hash(x)=45245896)
-2900 val loss 6.8098
-2900 val perplexity 906.7277
-2900 train 6.608460 (lr=5.8919e-05) (hash(x)=47067144)
-3200 val loss 7.0524
-3200 val perplexity 1155.6483
-3200 train 7.008028 (lr=4.0373e-05) (hash(x)=49995942)
-3000 val loss 6.7790
-3000 val perplexity 879.1960
-3000 train 6.598187 (lr=5.8140e-05) (hash(x)=45015009)
-3300 val loss 7.0304
-3300 val perplexity 1130.5018
-3300 train 6.907940 (lr=3.9775e-05) (hash(x)=52311504)
-3100 val loss 6.7659
-3100 val perplexity 867.7515
-3100 train 6.592752 (lr=5.7340e-05) (hash(x)=45245896)
-3400 val loss 6.9985
-3400 val perplexity 1094.9454
-3400 train 7.000693 (lr=3.9164e-05) (hash(x)=44332917)
-3200 val loss 6.7531
-3200 val perplexity 856.6802
-3200 train 6.716655 (lr=5.6522e-05) (hash(x)=49995942)
-3500 val loss 6.9684
-3500 val perplexity 1062.4990
-3500 train 7.082510 (lr=3.8541e-05) (hash(x)=56517159)
-3300 val loss 6.7407
-3300 val perplexity 846.1188
-3300 train 6.605199 (lr=5.5684e-05) (hash(x)=52311504)
-3600 val loss 6.9526
-3600 val perplexity 1045.8395
-3600 train 6.833664 (lr=3.7907e-05) (hash(x)=50720920)
-3400 val loss 6.7160
-3400 val perplexity 825.5479
-3400 train 6.696031 (lr=5.4829e-05) (hash(x)=44332917)
-3700 val loss 6.9372
-3700 val perplexity 1029.9263
-3700 train 7.426872 (lr=3.7262e-05) (hash(x)=62727701)
-3500 val loss 6.7101
-3500 val perplexity 820.6426
-3500 train 6.807840 (lr=5.3958e-05) (hash(x)=56517159)
-3800 val loss 6.9073
-3800 val perplexity 999.5339
-3800 train 6.755322 (lr=3.6608e-05) (hash(x)=54772539)
-3600 val loss 6.6921
-3600 val perplexity 806.0220
-3600 train 6.541726 (lr=5.3070e-05) (hash(x)=50720920)
-3900 val loss 6.8920
-3900 val perplexity 984.4092
-3900 train 6.885997 (lr=3.5944e-05) (hash(x)=52274485)
-3700 val loss 6.6747
-3700 val perplexity 792.1251
-4000 val loss 6.8666
-4000 val perplexity 959.7191
-3700 train 7.175059 (lr=5.2167e-05) (hash(x)=62727701)
-4000 train 6.580018 (lr=3.5271e-05) (hash(x)=50118307)
-4100 val loss 6.8453
-4100 val perplexity 939.4526
-4100 train 6.425397 (lr=3.4590e-05) (hash(x)=42771647)
-3800 val loss 6.6577
-3800 val perplexity 778.7675
-3800 train 6.517581 (lr=5.1251e-05) (hash(x)=54772539)
-4200 val loss 6.8241
-4200 val perplexity 919.7592
-4200 train 6.917285 (lr=3.3902e-05) (hash(x)=51748836)
-3900 val loss 6.6442
-3900 val perplexity 768.3301
-3900 train 6.636859 (lr=5.0321e-05) (hash(x)=52274485)
-4300 val loss 6.8008
-4300 val perplexity 898.5447
-4300 train 6.814617 (lr=3.3207e-05) (hash(x)=49021280)
-4000 val loss 6.6322
-4000 val perplexity 759.1318
-4000 train 6.323574 (lr=4.9379e-05) (hash(x)=50118307)
-4400 val loss 6.7751
-4400 val perplexity 875.7759
-4400 train 6.851520 (lr=3.2507e-05) (hash(x)=55200309)
-4100 val loss 6.6264
-4100 val perplexity 754.7734
-4100 train 6.207099 (lr=4.8426e-05) (hash(x)=42771647)
-4500 val loss 6.7518
-4500 val perplexity 855.5616
-4500 train 6.813970 (lr=3.1801e-05) (hash(x)=52085049)
-4200 val loss 6.6130
-4200 val perplexity 744.7159
-4200 train 6.700623 (lr=4.7463e-05) (hash(x)=51748836)
-4600 val loss 6.7426
-4600 val perplexity 847.7891
-4600 train 6.735619 (lr=3.1091e-05) (hash(x)=48935595)
-4300 val loss 6.5854
-4300 val perplexity 724.4294
-4300 train 6.597935 (lr=4.6490e-05) (hash(x)=49021280)
-4700 val loss 6.7275
-4700 val perplexity 835.0339
-4700 train 6.922513 (lr=3.0377e-05) (hash(x)=49182380)
-4400 val loss 6.5741
-4400 val perplexity 716.2935
-4400 train 6.621936 (lr=4.5509e-05) (hash(x)=55200309)
-4800 val loss 6.7063
-4800 val perplexity 817.5575
-4800 train 6.516437 (lr=2.9661e-05) (hash(x)=43941929)
-4500 val loss 6.5614
-4500 val perplexity 707.2348
-4500 train 6.612576 (lr=4.4521e-05) (hash(x)=52085049)
-4900 val loss 6.6963
-4900 val perplexity 809.4009
-4900 train 6.859163 (lr=2.8942e-05) (hash(x)=51852773)
-4600 val loss 6.5465
-4600 val perplexity 696.8232
-4600 train 6.553600 (lr=4.3527e-05) (hash(x)=48935595)
-5000 val loss 6.6846
-5000 val perplexity 800.0298
-5000 train 6.301474 (lr=2.8221e-05) (hash(x)=40509616)
-4700 val loss 6.5361
-4700 val perplexity 689.5744
-4700 train 6.776329 (lr=4.2528e-05) (hash(x)=49182380)
-5100 val loss 6.6730
-5100 val perplexity 790.7288
-5100 train 6.920803 (lr=2.7500e-05) (hash(x)=57585369)
-4800 val loss 6.5219
-4800 val perplexity 679.8577
-4800 train 6.354631 (lr=4.1525e-05) (hash(x)=43941929)
-5200 val loss 6.6732
-5200 val perplexity 790.8970
-5200 train 6.662302 (lr=2.6779e-05) (hash(x)=51042313)
-5300 val loss 6.6516
-5300 val perplexity 774.0305
-5300 train 6.798846 (lr=2.6058e-05) (hash(x)=52001684)
-4900 val loss 6.5276
-4900 val perplexity 683.7730
-4900 train 6.678216 (lr=4.0518e-05) (hash(x)=51852773)
-5400 val loss 6.6401
-5400 val perplexity 765.1759
-5400 train 6.575788 (lr=2.5339e-05) (hash(x)=48831647)
-5000 val loss 6.5124
-5000 val perplexity 673.4216
-5000 train 6.107291 (lr=3.9510e-05) (hash(x)=40509616)
-5500 val loss 6.6363
-5500 val perplexity 762.2680
-5500 train 7.035235 (lr=2.4623e-05) (hash(x)=50192069)
-5100 val loss 6.5035
-5100 val perplexity 667.4753
-5100 train 6.713647 (lr=3.8500e-05) (hash(x)=57585369)
-5600 val loss 6.6367
-5600 val perplexity 762.5941
-5600 train 6.465759 (lr=2.3909e-05) (hash(x)=47208852)
-5200 val loss 6.4972
-5200 val perplexity 663.2990
-5200 train 6.471662 (lr=3.7490e-05) (hash(x)=51042313)
-5700 val loss 6.6367
-5700 val perplexity 762.5676
-5700 train 6.232710 (lr=2.3199e-05) (hash(x)=44061694)
-5300 val loss 6.4865
-5300 val perplexity 656.2524
-5300 train 6.634539 (lr=3.6482e-05) (hash(x)=52001684)
-5800 val loss 6.6225
-5800 val perplexity 751.8287
-5800 train 7.010595 (lr=2.2493e-05) (hash(x)=56513279)
-5400 val loss 6.4770
-5400 val perplexity 650.0268
-5400 train 6.397245 (lr=3.5475e-05) (hash(x)=48831647)
-5900 val loss 6.6089
-5900 val perplexity 741.6647
-5900 train 6.761731 (lr=2.1793e-05) (hash(x)=50412818)
-5500 val loss 6.4741
-5500 val perplexity 648.1672
-5500 train 6.909862 (lr=3.4472e-05) (hash(x)=50192069)
-6000 val loss 6.6003
-6000 val perplexity 735.2872
-6000 train 6.414360 (lr=2.1098e-05) (hash(x)=47159634)
-5600 val loss 6.4683
-5600 val perplexity 644.4017
-5600 train 6.314424 (lr=3.3473e-05) (hash(x)=47208852)
-6100 val loss 6.5934
-6100 val perplexity 730.2899
-6100 train 6.695201 (lr=2.0410e-05) (hash(x)=54312795)
-5700 val loss 6.4732
-5700 val perplexity 647.5825
-5700 train 6.043439 (lr=3.2479e-05) (hash(x)=44061694)
-6200 val loss 6.5796
-6200 val perplexity 720.2858
-6200 train 6.707597 (lr=1.9729e-05) (hash(x)=54187587)
-5800 val loss 6.4643
-5800 val perplexity 641.8052
-5800 train 6.857479 (lr=3.1491e-05) (hash(x)=56513279)
-6300 val loss 6.5726
-6300 val perplexity 715.2604
-6300 train 6.670837 (lr=1.9056e-05) (hash(x)=53620387)
-6400 val loss 6.5608
-6400 val perplexity 706.8127
-6400 train 6.493462 (lr=1.8392e-05) (hash(x)=48761774)
-5900 val loss 6.4570
-5900 val perplexity 637.1591
-5900 train 6.608946 (lr=3.0510e-05) (hash(x)=50412818)
-6500 val loss 6.5542
-6500 val perplexity 702.1666
-6500 train 6.777394 (lr=1.7738e-05) (hash(x)=56690281)
-6000 val loss 6.4471
-6000 val perplexity 630.8537
-6000 train 6.271699 (lr=2.9537e-05) (hash(x)=47159634)
-6600 val loss 6.5486
-6600 val perplexity 698.2621
-6600 train 6.347186 (lr=1.7093e-05) (hash(x)=42985269)
-6100 val loss 6.4398
-6100 val perplexity 626.2506
-6100 train 6.526561 (lr=2.8574e-05) (hash(x)=54312795)
-6700 val loss 6.5654
-6700 val perplexity 710.0743
-6700 train 6.615873 (lr=1.6459e-05) (hash(x)=53315447)
-6200 val loss 6.4319
-6200 val perplexity 621.3448
-6200 train 6.558974 (lr=2.7621e-05) (hash(x)=54187587)
-6800 val loss 6.5371
-6800 val perplexity 690.3044
-6800 train 7.013281 (lr=1.5836e-05) (hash(x)=61577166)
-6300 val loss 6.4216
-6300 val perplexity 615.0062
-6300 train 6.523766 (lr=2.6679e-05) (hash(x)=53620387)
-6900 val loss 6.5317
-6900 val perplexity 686.5308
-6900 train 6.668251 (lr=1.5225e-05) (hash(x)=54641005)
-6400 val loss 6.4139
-6400 val perplexity 610.2687
-6400 train 6.355747 (lr=2.5749e-05) (hash(x)=48761774)
-7000 val loss 6.5290
-7000 val perplexity 684.6970
-7000 train 7.082861 (lr=1.4627e-05) (hash(x)=60579512)
-6500 val loss 6.4031
-6500 val perplexity 603.6854
-6500 train 6.621194 (lr=2.4833e-05) (hash(x)=56690281)
-7100 val loss 6.5275
-7100 val perplexity 683.7108
-7100 train 6.407035 (lr=1.4043e-05) (hash(x)=53151549)
-6600 val loss 6.4005
-6600 val perplexity 602.1273
-6600 train 6.214245 (lr=2.3930e-05) (hash(x)=42985269)
-7200 val loss 6.5212
-7200 val perplexity 679.3991
-7200 train 7.457515 (lr=1.3471e-05) (hash(x)=71842455)
-6700 val loss 6.4037
-6700 val perplexity 604.0975
-6700 train 6.470212 (lr=2.3042e-05) (hash(x)=53315447)
-7300 val loss 6.5188
-7300 val perplexity 677.7803
-7300 train 6.291944 (lr=1.2915e-05) (hash(x)=44516452)
-6800 val loss 6.3889
-6800 val perplexity 595.2116
-6800 train 6.838380 (lr=2.2171e-05) (hash(x)=61577166)
-7400 val loss 6.5163
-7400 val perplexity 676.0811
-7400 train 6.169219 (lr=1.2373e-05) (hash(x)=42667710)
-6900 val loss 6.3828
-6900 val perplexity 591.6084
-6900 train 6.541791 (lr=2.1316e-05) (hash(x)=54641005)
-7500 val loss 6.5125
-7500 val perplexity 673.5272
-7500 train 6.262584 (lr=1.1847e-05) (hash(x)=47050797)
-7600 val loss 6.5128
-7600 val perplexity 673.7106
-7000 val loss 6.3797
-7000 val perplexity 589.7284
-7600 train 6.422743 (lr=1.1337e-05) (hash(x)=49785056)
-7000 train 6.912982 (lr=2.0478e-05) (hash(x)=60579512)
-7700 val loss 6.5022
-7700 val perplexity 666.5777
-7700 train 6.310266 (lr=1.0844e-05) (hash(x)=53232030)
-7100 val loss 6.3786
-7100 val perplexity 589.1092
-7100 train 6.265176 (lr=1.9660e-05) (hash(x)=53151549)
-7800 val loss 6.4957
-7800 val perplexity 662.2895
-7800 train 6.329096 (lr=1.0367e-05) (hash(x)=48049749)
-7200 val loss 6.3746
-7200 val perplexity 586.7686
-7200 train 7.320697 (lr=1.8860e-05) (hash(x)=71842455)
-7900 val loss 6.4906
-7900 val perplexity 658.9155
-7900 train 6.314969 (lr=9.9088e-06) (hash(x)=44768513)
-7300 val loss 6.3713
-7300 val perplexity 584.7982
-7300 train 6.152587 (lr=1.8081e-05) (hash(x)=44516452)
-8000 val loss 6.4841
-8000 val perplexity 654.6609
-8000 train 6.349458 (lr=9.4682e-06) (hash(x)=46228039)
-7400 val loss 6.3714
-7400 val perplexity 584.8604
-7400 train 6.004220 (lr=1.7323e-05) (hash(x)=42667710)
-8100 val loss 6.4850
-8100 val perplexity 655.2425
-8100 train 6.849713 (lr=9.0461e-06) (hash(x)=60017091)
-7500 val loss 6.3709
-7500 val perplexity 584.6019
-7500 train 6.115476 (lr=1.6586e-05) (hash(x)=47050797)
-8200 val loss 6.4761
-8200 val perplexity 649.4072
-8200 train 6.484280 (lr=8.6430e-06) (hash(x)=49910198)
-7600 val loss 6.3649
-7600 val perplexity 581.0688
-7600 train 6.270297 (lr=1.5872e-05) (hash(x)=49785056)
-8300 val loss 6.4709
-8300 val perplexity 646.0813
-8300 train 6.789174 (lr=8.2593e-06) (hash(x)=57919055)
-7700 val loss 6.3568
-7700 val perplexity 576.3995
-7700 train 6.166145 (lr=1.5181e-05) (hash(x)=53232030)
-8400 val loss 6.4732
-8400 val perplexity 647.5664
-8400 train 6.565013 (lr=7.8953e-06) (hash(x)=49694964)
-7800 val loss 6.3557
-7800 val perplexity 575.7751
-7800 train 6.184354 (lr=1.4514e-05) (hash(x)=48049749)
-8500 val loss 6.4678
-8500 val perplexity 644.0825
-8500 train 6.496820 (lr=7.5515e-06) (hash(x)=53762585)
-7900 val loss 6.3477
-7900 val perplexity 571.2045
-7900 train 6.176773 (lr=1.3872e-05) (hash(x)=44768513)
-8600 val loss 6.4652
-8600 val perplexity 642.3760
-8600 train 6.497360 (lr=7.2282e-06) (hash(x)=51166973)
-8000 val loss 6.3411
-8000 val perplexity 567.4194
-8000 train 6.197558 (lr=1.3255e-05) (hash(x)=46228039)
-8700 val loss 6.4624
-8700 val perplexity 640.6073
-8700 train 6.541679 (lr=6.9257e-06) (hash(x)=53968049)
-8100 val loss 6.3394
-8100 val perplexity 566.4333
-8100 train 6.685860 (lr=1.2665e-05) (hash(x)=60017091)
-8800 val loss 6.4577
-8800 val perplexity 637.6141
-8800 train 6.545799 (lr=6.6444e-06) (hash(x)=59231056)
-8200 val loss 6.3328
-8200 val perplexity 562.7375
-8200 train 6.369884 (lr=1.2100e-05) (hash(x)=49910198)
-8900 val loss 6.4566
-8900 val perplexity 636.8945
-8900 train 6.358093 (lr=6.3845e-06) (hash(x)=50488048)
-8300 val loss 6.3289
-8300 val perplexity 560.5214
-8300 train 6.634696 (lr=1.1563e-05) (hash(x)=57919055)
-9000 val loss 6.4566
-9000 val perplexity 636.9036
-9000 train 6.185684 (lr=6.1462e-06) (hash(x)=44492956)
-8400 val loss 6.3304
-8400 val perplexity 561.3581
-8400 train 6.415321 (lr=1.1053e-05) (hash(x)=49694964)
-9100 val loss 6.4624
-9100 val perplexity 640.5786
-9100 train 6.489594 (lr=5.9300e-06) (hash(x)=51134989)
-8500 val loss 6.3274
-8500 val perplexity 559.7225
-8500 train 6.364176 (lr=1.0572e-05) (hash(x)=53762585)
-9200 val loss 6.4543
-9200 val perplexity 635.4424
-9200 train 6.258815 (lr=5.7359e-06) (hash(x)=48636056)
-9300 val loss 6.4538
-9300 val perplexity 635.1309
-9300 train 6.410451 (lr=5.5641e-06) (hash(x)=50200551)
-8600 val loss 6.3260
-8600 val perplexity 558.9059
-8600 train 6.361839 (lr=1.0119e-05) (hash(x)=51166973)
-9400 val loss 6.4542
-9400 val perplexity 635.3648
-9400 train 6.281915 (lr=5.4149e-06) (hash(x)=48057228)
-8700 val loss 6.3224
-8700 val perplexity 556.8817
-8700 train 6.396859 (lr=9.6960e-06) (hash(x)=53968049)
-9500 val loss 6.4501
-9500 val perplexity 632.7894
-9500 train 6.230661 (lr=5.2884e-06) (hash(x)=48125171)
-8800 val loss 6.3167
-8800 val perplexity 553.7402
-8800 train 6.420463 (lr=9.3021e-06) (hash(x)=59231056)
-9600 val loss 6.4492
-9600 val perplexity 632.1690
-9600 train 6.376224 (lr=5.1847e-06) (hash(x)=53375853)
-8900 val loss 6.3177
-8900 val perplexity 554.2752
-8900 train 6.200024 (lr=8.9382e-06) (hash(x)=50488048)
-9700 val loss 6.4451
-9700 val perplexity 629.6221
-9700 train 7.265676 (lr=5.1040e-06) (hash(x)=53924631)
-9000 val loss 6.3171
-9000 val perplexity 553.9613
-9000 train 6.043607 (lr=8.6047e-06) (hash(x)=44492956)
-9800 val loss 6.4416
-9800 val perplexity 627.3945
-9800 train 6.482964 (lr=5.0462e-06) (hash(x)=48895047)
-9100 val loss 6.3195
-9100 val perplexity 555.2841
-9100 train 6.320802 (lr=8.3020e-06) (hash(x)=51134989)
-9900 val loss 6.4396
-9900 val perplexity 626.1697
-9900 train 6.285259 (lr=5.0116e-06) (hash(x)=44269923)
-9200 val loss 6.3156
-9200 val perplexity 553.1354
-9200 train 6.110624 (lr=8.0302e-06) (hash(x)=48636056)
-9999 val loss 6.4422
-9999 val perplexity 627.8090
-9300 val loss 6.3171
-9300 val perplexity 553.9850
-9300 train 6.267875 (lr=7.7898e-06) (hash(x)=50200551)
-9400 val loss 6.3169
-9400 val perplexity 553.8564
-9400 train 6.124255 (lr=7.5809e-06) (hash(x)=48057228)
-9500 val loss 6.3137
-9500 val perplexity 552.0832
-9500 train 6.094636 (lr=7.4038e-06) (hash(x)=48125171)

attention_kindselective_n_heads2_seed1341/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0927621769e86865fd11c7c2ac120514613ed33d559219ce56792a9196b3a0e8
+oid sha256:8a9c526ea9748d4265486e8e5cf4f2e92786d25f160ec94c33eaf1d85b3fb394
 size 38587970

attention_kindselective_n_heads2_seed1341/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8cade60da08fc83cbddf22af2ab6080ef30ec634d051725fd7c91acbd24a7a43
+oid sha256:1134e0ac55901d13e7ee3307ec72f3f8085485b36581ad3855f9598d346b1fd4
 size 70895430