Upload folder using huggingface_hub

attention_kindself_n_heads2_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1339", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_2_1339", "n_embd": 128}

attention_kindself_n_heads2_seed1339/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.7353
 0 val perplexity 124899.7500
-0 train 11.734357 (lr=7.5000e-07) (hash(x)=150724848)
-100 val loss 9.7906
-100 val perplexity 17864.8477
-100 train 9.831616 (lr=7.5750e-05) (hash(x)=149217061)
-200 val loss 7.9135
-200 val perplexity 2733.8181
-200 train 7.903269 (lr=1.5000e-04) (hash(x)=149910534)
-300 val loss 7.6195
-300 val perplexity 2037.6289
-300 train 7.489441 (lr=1.4997e-04) (hash(x)=142185643)
-400 val loss 7.5327
-400 val perplexity 1868.0796
-400 train 7.563246 (lr=1.4986e-04) (hash(x)=148123706)
-500 val loss 7.4229
-500 val perplexity 1673.8356
-500 train 7.254730 (lr=1.4969e-04) (hash(x)=149952383)
-600 val loss 7.3229
-600 val perplexity 1514.5962
-600 train 7.112319 (lr=1.4945e-04) (hash(x)=146678221)
-700 val loss 7.2195
-700 val perplexity 1365.8119
-700 train 7.256211 (lr=1.4913e-04) (hash(x)=156180736)
-800 val loss 7.1607
-800 val perplexity 1287.7596
-800 train 7.143748 (lr=1.4876e-04) (hash(x)=151700982)
-900 val loss 7.1306
-900 val perplexity 1249.6790
-900 train 6.930069 (lr=1.4831e-04) (hash(x)=147288467)
-1000 val loss 7.1058
-1000 val perplexity 1218.9619
-1000 train 7.177827 (lr=1.4779e-04) (hash(x)=156182087)
-1100 val loss 7.0983
-1100 val perplexity 1209.8513
-1100 train 6.947954 (lr=1.4721e-04) (hash(x)=147861550)
-1200 val loss 7.0880
-1200 val perplexity 1197.5005
-1200 train 6.959338 (lr=1.4656e-04) (hash(x)=149318660)
-1300 val loss 7.1193
-1300 val perplexity 1235.6134
-1300 train 6.947792 (lr=1.4585e-04) (hash(x)=151631103)
-1400 val loss 7.1081
-1400 val perplexity 1221.8605
-1400 train 7.125467 (lr=1.4507e-04) (hash(x)=150482428)
-1500 val loss 7.0467
-1500 val perplexity 1149.0096
-1500 train 6.902546 (lr=1.4422e-04) (hash(x)=144458617)
-1600 val loss 7.0057
-1600 val perplexity 1102.9209
-1600 train 6.918149 (lr=1.4332e-04) (hash(x)=143268605)
-1700 val loss 6.9909
-1700 val perplexity 1086.6871
-1700 train 6.951296 (lr=1.4235e-04) (hash(x)=150817011)
-1800 val loss 6.9886
-1800 val perplexity 1084.1615
-1800 train 6.995378 (lr=1.4131e-04) (hash(x)=152322423)
-1900 val loss 6.9769
-1900 val perplexity 1071.6195
-1900 train 6.790772 (lr=1.4022e-04) (hash(x)=143574126)
-2000 val loss 6.9831
-2000 val perplexity 1078.2130
-2000 train 6.761238 (lr=1.3907e-04) (hash(x)=147904298)
-2100 val loss 6.9866
-2100 val perplexity 1082.0898
-2100 train 6.752719 (lr=1.3786e-04) (hash(x)=144719400)
-2200 val loss 6.9856
-2200 val perplexity 1081.0084
-2200 train 7.212669 (lr=1.3660e-04) (hash(x)=154343147)
-2300 val loss 6.9550
-2300 val perplexity 1048.3564
-2300 train 7.002470 (lr=1.3527e-04) (hash(x)=154372635)
-2400 val loss 6.9724
-2400 val perplexity 1066.8245
-2400 train 6.826124 (lr=1.3390e-04) (hash(x)=141843115)
-2500 val loss 6.9567
-2500 val perplexity 1050.2083
-2500 train 6.861791 (lr=1.3247e-04) (hash(x)=149417679)
-2600 val loss 6.9538
-2600 val perplexity 1047.0795
-2600 train 6.827319 (lr=1.3099e-04) (hash(x)=145279030)
-2700 val loss 6.9492
-2700 val perplexity 1042.3495
-2700 train 6.853619 (lr=1.2946e-04) (hash(x)=146363911)
-2800 val loss 6.9600
-2800 val perplexity 1053.5844
-2800 train 6.925730 (lr=1.2788e-04) (hash(x)=152507639)
-2900 val loss 6.9717
-2900 val perplexity 1066.0504
-2900 train 6.777905 (lr=1.2626e-04) (hash(x)=139296191)
-3000 val loss 6.9777
-3000 val perplexity 1072.5018
-3000 train 6.983221 (lr=1.2459e-04) (hash(x)=148473774)
-3100 val loss 6.9501
-3100 val perplexity 1043.2361
-3100 train 7.037889 (lr=1.2287e-04) (hash(x)=154130527)
-3200 val loss 6.9663
-3200 val perplexity 1060.2772
-3200 train 6.903945 (lr=1.2112e-04) (hash(x)=151117002)
-3300 val loss 6.9904
-3300 val perplexity 1086.1484
-3300 train 7.110773 (lr=1.1932e-04) (hash(x)=151035383)
-3400 val loss 6.9912
-3400 val perplexity 1087.0308
-3400 train 7.093910 (lr=1.1749e-04) (hash(x)=150614749)
-3500 val loss 6.9915
-3500 val perplexity 1087.3533
-3500 train 7.043681 (lr=1.1562e-04) (hash(x)=151431391)
-3600 val loss 7.0193
-3600 val perplexity 1118.0074
-3600 train 6.978808 (lr=1.1372e-04) (hash(x)=151140797)
-3700 val loss 6.9775
-3700 val perplexity 1072.2477
-3700 train 7.151230 (lr=1.1179e-04) (hash(x)=154608340)
-3800 val loss 6.9801
-3800 val perplexity 1075.0132
-3800 train 6.973941 (lr=1.0982e-04) (hash(x)=169641309)
-3900 val loss 6.9675
-3900 val perplexity 1061.6168
-3900 train 6.916593 (lr=1.0783e-04) (hash(x)=150592479)
-4000 val loss 6.9695
-4000 val perplexity 1063.6959
-4000 train 6.955076 (lr=1.0581e-04) (hash(x)=155495317)
-4100 val loss 6.9653
-4100 val perplexity 1059.2281
-4100 train 7.145298 (lr=1.0377e-04) (hash(x)=170323970)
-4200 val loss 6.9714
-4200 val perplexity 1065.6687
-4200 train 6.779059 (lr=1.0171e-04) (hash(x)=140854360)
-4300 val loss 6.9555
-4300 val perplexity 1048.8595
-4300 train 6.944369 (lr=9.9622e-05) (hash(x)=148926644)
-4400 val loss 6.9527
-4400 val perplexity 1045.9672
-4400 train 6.997404 (lr=9.7520e-05) (hash(x)=149997871)
-4500 val loss 6.9458
-4500 val perplexity 1038.7681
-4500 train 6.908257 (lr=9.5403e-05) (hash(x)=149219289)
-4600 val loss 6.9330
-4600 val perplexity 1025.5349
-4600 train 6.739017 (lr=9.3273e-05) (hash(x)=139836197)
-4700 val loss 6.9361
-4700 val perplexity 1028.7106
-4700 train 6.893769 (lr=9.1132e-05) (hash(x)=150722883)
-4800 val loss 6.9405
-4800 val perplexity 1033.2831
-4800 train 7.139781 (lr=8.8982e-05) (hash(x)=166864648)
-4900 val loss 6.9288
-4900 val perplexity 1021.2498
-4900 train 7.030507 (lr=8.6825e-05) (hash(x)=157746743)
-5000 val loss 6.9255
-5000 val perplexity 1017.8948
-5000 train 6.881103 (lr=8.4663e-05) (hash(x)=153032257)
-5100 val loss 6.9227
-5100 val perplexity 1015.0491
-5100 train 6.985375 (lr=8.2500e-05) (hash(x)=151862371)
-5200 val loss 6.9293
-5200 val perplexity 1021.7968
-5200 train 7.186607 (lr=8.0337e-05) (hash(x)=168862995)
-5300 val loss 6.9301
-5300 val perplexity 1022.5923
-5300 train 6.893943 (lr=7.8175e-05) (hash(x)=148243567)
-5400 val loss 6.9281
-5400 val perplexity 1020.5435
-5400 train 6.853803 (lr=7.6018e-05) (hash(x)=145381742)
-5500 val loss 6.9387
-5500 val perplexity 1031.3790
-5500 train 7.071217 (lr=7.3868e-05) (hash(x)=155602174)
-5600 val loss 6.9308
-5600 val perplexity 1023.2815
-5600 train 7.025819 (lr=7.1727e-05) (hash(x)=160637672)
-5700 val loss 6.9346
-5700 val perplexity 1027.1803
-5700 train 6.879603 (lr=6.9597e-05) (hash(x)=152101131)
-5800 val loss 6.9140
-5800 val perplexity 1006.3104
-5800 train 6.884543 (lr=6.7480e-05) (hash(x)=152327089)
-5900 val loss 6.9178
-5900 val perplexity 1010.0659
-5900 train 6.851806 (lr=6.5378e-05) (hash(x)=151094543)
-6000 val loss 6.9214
-6000 val perplexity 1013.7050
-6000 train 7.002866 (lr=6.3294e-05) (hash(x)=160233315)
-6100 val loss 6.9204
-6100 val perplexity 1012.6880
-6100 train 7.003745 (lr=6.1230e-05) (hash(x)=158295337)
-6200 val loss 6.9196
-6200 val perplexity 1011.9175
-6200 train 6.873456 (lr=5.9188e-05) (hash(x)=151269817)
-6300 val loss 6.9151
-6300 val perplexity 1007.4050
-6300 train 6.991065 (lr=5.7169e-05) (hash(x)=156912756)
-6400 val loss 6.9218
-6400 val perplexity 1014.1149
-6400 train 6.839902 (lr=5.5177e-05) (hash(x)=125779637)
-6500 val loss 6.9173
-6500 val perplexity 1009.5565
-6500 train 6.820065 (lr=5.3213e-05) (hash(x)=148157661)
-6600 val loss 6.9251
-6600 val perplexity 1017.4473
-6600 train 6.981798 (lr=5.1279e-05) (hash(x)=152820049)
-6700 val loss 6.9197
-6700 val perplexity 1011.9740
-6700 train 6.877532 (lr=4.9377e-05) (hash(x)=155818717)
-6800 val loss 6.9144
-6800 val perplexity 1006.6727
-6800 train 6.965318 (lr=4.7509e-05) (hash(x)=158833881)
-6900 val loss 6.9193
-6900 val perplexity 1011.6271
-6900 train 6.940590 (lr=4.5676e-05) (hash(x)=164984909)
-7000 val loss 6.9143
-7000 val perplexity 1006.5340
-7000 train 6.805004 (lr=4.3882e-05) (hash(x)=151172493)
-7100 val loss 6.9159
-7100 val perplexity 1008.1864
-7100 train 6.888276 (lr=4.2128e-05) (hash(x)=154023633)
-7200 val loss 6.9209
-7200 val perplexity 1013.2715
-7200 train 6.802105 (lr=4.0414e-05) (hash(x)=139351250)
-7300 val loss 6.9159
-7300 val perplexity 1008.2215
-7300 train 6.877492 (lr=3.8745e-05) (hash(x)=158719458)
-7400 val loss 6.9136
-7400 val perplexity 1005.8325
-7400 train 7.032726 (lr=3.7120e-05) (hash(x)=162250810)
-7500 val loss 6.9135
-7500 val perplexity 1005.7472
-7500 train 6.775838 (lr=3.5541e-05) (hash(x)=142827557)
-7600 val loss 6.9058
-7600 val perplexity 998.0494
-7600 train 6.817025 (lr=3.4011e-05) (hash(x)=147830801)
-7700 val loss 6.9035
-7700 val perplexity 995.7875
-7700 train 7.017764 (lr=3.2531e-05) (hash(x)=165233314)
-7800 val loss 6.9087
-7800 val perplexity 1000.9251
-7800 train 6.919121 (lr=3.1102e-05) (hash(x)=160654717)
-7900 val loss 6.9144
-7900 val perplexity 1006.6353
-7900 train 6.797275 (lr=2.9726e-05) (hash(x)=140712586)
-8000 val loss 6.9056
-8000 val perplexity 997.8057
-8000 train 7.014383 (lr=2.8405e-05) (hash(x)=150309061)
-8100 val loss 6.9044
-8100 val perplexity 996.6664
-8100 train 6.935093 (lr=2.7138e-05) (hash(x)=158805466)
-8200 val loss 6.9134
-8200 val perplexity 1005.6608
-8200 train 6.762922 (lr=2.5929e-05) (hash(x)=142814078)
-8300 val loss 6.9123
-8300 val perplexity 1004.5633
-8300 train 6.855933 (lr=2.4778e-05) (hash(x)=148340417)
-8400 val loss 6.9173
-8400 val perplexity 1009.5709
-8400 train 6.869319 (lr=2.3686e-05) (hash(x)=157132639)
-8500 val loss 6.9167
-8500 val perplexity 1008.9866
-8500 train 6.875008 (lr=2.2655e-05) (hash(x)=152994240)
-8600 val loss 6.9130
-8600 val perplexity 1005.2601
-8600 train 6.867662 (lr=2.1685e-05) (hash(x)=152210154)
-8700 val loss 6.9102
-8700 val perplexity 1002.4277
-8700 train 6.945439 (lr=2.0777e-05) (hash(x)=149631247)
-8800 val loss 6.9110
-8800 val perplexity 1003.2904
-8800 train 6.752805 (lr=1.9933e-05) (hash(x)=145617784)
-8900 val loss 6.9149
-8900 val perplexity 1007.1778
-8900 train 7.021801 (lr=1.9153e-05) (hash(x)=155984970)
-9000 val loss 6.9155
-9000 val perplexity 1007.7278
-9000 train 6.684093 (lr=1.8439e-05) (hash(x)=143433013)
-9100 val loss 6.9143
-9100 val perplexity 1006.5181
-9100 train 6.815769 (lr=1.7790e-05) (hash(x)=152507533)
-9200 val loss 6.9159
-9200 val perplexity 1008.1912
-9200 train 7.023916 (lr=1.7208e-05) (hash(x)=162400650)
-9300 val loss 6.9134
-9300 val perplexity 1005.6873
-9300 train 6.943089 (lr=1.6692e-05) (hash(x)=148419068)
-9400 val loss 6.9130
-9400 val perplexity 1005.2514
-9400 train 6.881789 (lr=1.6245e-05) (hash(x)=145025169)
-9500 val loss 6.9133
-9500 val perplexity 1005.6000
-9500 train 6.864194 (lr=1.5865e-05) (hash(x)=145104235)
-9600 val loss 6.9163
-9600 val perplexity 1008.5455
-9600 train 6.835734 (lr=1.5554e-05) (hash(x)=150499797)
-9700 val loss 6.9179
-9700 val perplexity 1010.2282
-9700 train 6.737312 (lr=1.5312e-05) (hash(x)=139690022)
-9800 val loss 6.9151
-9800 val perplexity 1007.3925
-9800 train 6.893308 (lr=1.5139e-05) (hash(x)=145645279)
-9900 val loss 6.9154
-9900 val perplexity 1007.6418
-9900 train 6.841991 (lr=1.5035e-05) (hash(x)=150103432)
-9999 val loss 6.9172
-9999 val perplexity 1009.4544
+0 train 11.734357 (lr=2.5000e-07) (hash(x)=150724848)
+100 val loss 10.4651
+100 val perplexity 35068.8242
+100 train 10.497286 (lr=2.5250e-05) (hash(x)=149217061)
+200 val loss 9.4784
+200 val perplexity 13074.8525
+200 train 9.459652 (lr=5.0000e-05) (hash(x)=149910534)
+300 val loss 8.4916
+300 val perplexity 4873.5464
+300 train 8.435646 (lr=4.9988e-05) (hash(x)=142185643)
+400 val loss 7.9347
+400 val perplexity 2792.5461
+400 train 7.970343 (lr=4.9954e-05) (hash(x)=148123706)
+500 val loss 7.7107
+500 val perplexity 2232.1318
+500 train 7.594043 (lr=4.9896e-05) (hash(x)=149952383)
+600 val loss 7.6270
+600 val perplexity 2052.8398
+600 train 7.453488 (lr=4.9815e-05) (hash(x)=146678221)
+700 val loss 7.5568
+700 val perplexity 1913.7664
+700 train 7.582264 (lr=4.9712e-05) (hash(x)=156180736)
+800 val loss 7.5122
+800 val perplexity 1830.2686
+800 train 7.493997 (lr=4.9585e-05) (hash(x)=151700982)
+900 val loss 7.4732
+900 val perplexity 1760.2922
+900 train 7.285657 (lr=4.9436e-05) (hash(x)=147288467)
+1000 val loss 7.4353
+1000 val perplexity 1694.8337
+1000 train 7.500896 (lr=4.9264e-05) (hash(x)=156182087)
+1100 val loss 7.3997
+1100 val perplexity 1635.4713
+1100 train 7.265240 (lr=4.9070e-05) (hash(x)=147861550)
+1200 val loss 7.3593
+1200 val perplexity 1570.7631
+1200 train 7.223550 (lr=4.8854e-05) (hash(x)=149318660)
+1300 val loss 7.3372
+1300 val perplexity 1536.3575
+1300 train 7.182025 (lr=4.8616e-05) (hash(x)=151631103)
+1400 val loss 7.3177
+1400 val perplexity 1506.7767
+1400 train 7.353926 (lr=4.8356e-05) (hash(x)=150482428)
+1500 val loss 7.2556
+1500 val perplexity 1416.0056
+1500 train 7.098469 (lr=4.8074e-05) (hash(x)=144458617)
+1600 val loss 7.2306
+1600 val perplexity 1381.0382
+1600 train 7.117816 (lr=4.7772e-05) (hash(x)=143268605)
+1700 val loss 7.2024
+1700 val perplexity 1342.6124
+1700 train 7.140574 (lr=4.7448e-05) (hash(x)=150817011)
+1800 val loss 7.1735
+1800 val perplexity 1304.3784
+1800 train 7.182352 (lr=4.7105e-05) (hash(x)=152322423)
+1900 val loss 7.1587
+1900 val perplexity 1285.2250
+1900 train 6.990263 (lr=4.6741e-05) (hash(x)=143574126)
+2000 val loss 7.1590
+2000 val perplexity 1285.6001
+2000 train 6.933070 (lr=4.6357e-05) (hash(x)=147904298)
+2100 val loss 7.1400
+2100 val perplexity 1261.4673
+2100 train 6.915145 (lr=4.5954e-05) (hash(x)=144719400)
+2200 val loss 7.1296
+2200 val perplexity 1248.3776
+2200 train 7.368258 (lr=4.5532e-05) (hash(x)=154343147)
+2300 val loss 7.1069
+2300 val perplexity 1220.2990
+2300 train 7.155266 (lr=4.5091e-05) (hash(x)=154372635)
+2400 val loss 7.1004
+2400 val perplexity 1212.4403
+2400 train 6.952942 (lr=4.4633e-05) (hash(x)=141843115)
+2500 val loss 7.0758
+2500 val perplexity 1182.9872
+2500 train 6.977271 (lr=4.4156e-05) (hash(x)=149417679)
+2600 val loss 7.0675
+2600 val perplexity 1173.1707
+2600 train 6.943630 (lr=4.3663e-05) (hash(x)=145279030)
+2700 val loss 7.0585
+2700 val perplexity 1162.6952
+2700 train 6.957924 (lr=4.3153e-05) (hash(x)=146363911)
+2800 val loss 7.0487
+2800 val perplexity 1151.3960
+2800 train 7.027252 (lr=4.2627e-05) (hash(x)=152507639)
+2900 val loss 7.0534
+2900 val perplexity 1156.7444
+2900 train 6.867245 (lr=4.2085e-05) (hash(x)=139296191)
+3000 val loss 7.0461
+3000 val perplexity 1148.3628
+3000 train 7.069539 (lr=4.1529e-05) (hash(x)=148473774)
+3100 val loss 7.0199
+3100 val perplexity 1118.6302
+3100 train 7.116752 (lr=4.0957e-05) (hash(x)=154130527)
+3200 val loss 7.0328
+3200 val perplexity 1133.1583
+3200 train 6.965783 (lr=4.0373e-05) (hash(x)=151117002)
+3300 val loss 7.0206
+3300 val perplexity 1119.4948
+3300 train 7.142177 (lr=3.9775e-05) (hash(x)=151035383)
+3400 val loss 7.0035
+3400 val perplexity 1100.5216
+3400 train 7.106104 (lr=3.9164e-05) (hash(x)=150614749)
+3500 val loss 6.9936
+3500 val perplexity 1089.6199
+3500 train 7.052362 (lr=3.8541e-05) (hash(x)=151431391)
+3600 val loss 6.9841
+3600 val perplexity 1079.3010
+3600 train 6.946033 (lr=3.7907e-05) (hash(x)=151140797)
+3700 val loss 6.9774
+3700 val perplexity 1072.1720
+3700 train 7.211062 (lr=3.7262e-05) (hash(x)=154608340)
+3800 val loss 6.9552
+3800 val perplexity 1048.5380
+3800 train 6.954602 (lr=3.6608e-05) (hash(x)=169641309)
+3900 val loss 6.9448
+3900 val perplexity 1037.7443
+3900 train 6.901346 (lr=3.5944e-05) (hash(x)=150592479)
+4000 val loss 6.9374
+4000 val perplexity 1030.1282
+4000 train 6.914086 (lr=3.5271e-05) (hash(x)=155495317)
+4100 val loss 6.9406
+4100 val perplexity 1033.4181
+4100 train 7.135403 (lr=3.4590e-05) (hash(x)=170323970)
+4200 val loss 6.9227
+4200 val perplexity 1015.0854
+4200 train 6.735570 (lr=3.3902e-05) (hash(x)=140854360)
+4300 val loss 6.9131
+4300 val perplexity 1005.3391
+4300 train 6.906529 (lr=3.3207e-05) (hash(x)=148926644)
+4400 val loss 6.9095
+4400 val perplexity 1001.7760
+4400 train 6.958065 (lr=3.2507e-05) (hash(x)=149997871)
+4500 val loss 6.8977
+4500 val perplexity 989.9657
+4500 train 6.864591 (lr=3.1801e-05) (hash(x)=149219289)
+4600 val loss 6.8935
+4600 val perplexity 985.8222
+4600 train 6.694082 (lr=3.1091e-05) (hash(x)=139836197)
+4700 val loss 6.9032
+4700 val perplexity 995.4861
+4700 train 6.865824 (lr=3.0377e-05) (hash(x)=150722883)
+4800 val loss 6.8859
+4800 val perplexity 978.4028
+4800 train 7.063042 (lr=2.9661e-05) (hash(x)=166864648)
+4900 val loss 6.8857
+4900 val perplexity 978.1995
+4900 train 7.000870 (lr=2.8942e-05) (hash(x)=157746743)
+5000 val loss 6.8581
+5000 val perplexity 951.5841
+5000 train 6.818891 (lr=2.8221e-05) (hash(x)=153032257)
+5100 val loss 6.8565
+5100 val perplexity 950.0789
+5100 train 6.935815 (lr=2.7500e-05) (hash(x)=151862371)
+5200 val loss 6.8392
+5200 val perplexity 933.7040
+5200 train 7.130702 (lr=2.6779e-05) (hash(x)=168862995)
+5300 val loss 6.8410
+5300 val perplexity 935.4055
+5300 train 6.816749 (lr=2.6058e-05) (hash(x)=148243567)
+5400 val loss 6.8288
+5400 val perplexity 924.0881
+5400 train 6.752869 (lr=2.5339e-05) (hash(x)=145381742)
+5500 val loss 6.8278
+5500 val perplexity 923.1636
+5500 train 6.989860 (lr=2.4623e-05) (hash(x)=155602174)
+5600 val loss 6.8086
+5600 val perplexity 905.5921
+5600 train 6.898433 (lr=2.3909e-05) (hash(x)=160637672)
+5700 val loss 6.8047
+5700 val perplexity 902.1183
+5700 train 6.749375 (lr=2.3199e-05) (hash(x)=152101131)
+5800 val loss 6.7908
+5800 val perplexity 889.6378
+5800 train 6.761727 (lr=2.2493e-05) (hash(x)=152327089)
+5900 val loss 6.7963
+5900 val perplexity 894.5493
+5900 train 6.732069 (lr=2.1793e-05) (hash(x)=151094543)
+6000 val loss 6.7812
+6000 val perplexity 881.1279
+6000 train 6.880424 (lr=2.1098e-05) (hash(x)=160233315)
+6100 val loss 6.7784
+6100 val perplexity 878.6605
+6100 train 6.872136 (lr=2.0410e-05) (hash(x)=158295337)
+6200 val loss 6.7672
+6200 val perplexity 868.8906
+6200 train 6.724010 (lr=1.9729e-05) (hash(x)=151269817)
+6300 val loss 6.7602
+6300 val perplexity 862.8197
+6300 train 6.847940 (lr=1.9056e-05) (hash(x)=156912756)
+6400 val loss 6.7576
+6400 val perplexity 860.5812
+6400 train 6.714630 (lr=1.8392e-05) (hash(x)=125779637)
+6500 val loss 6.7565
+6500 val perplexity 859.6633
+6500 train 6.662803 (lr=1.7738e-05) (hash(x)=148157661)
+6600 val loss 6.7478
+6600 val perplexity 852.1619
+6600 train 6.795863 (lr=1.7093e-05) (hash(x)=152820049)
+6700 val loss 6.7412
+6700 val perplexity 846.5380
+6700 train 6.698226 (lr=1.6459e-05) (hash(x)=155818717)
+6800 val loss 6.7321
+6800 val perplexity 838.9412
+6800 train 6.793108 (lr=1.5836e-05) (hash(x)=158833881)
+6900 val loss 6.7339
+6900 val perplexity 840.3877
+6900 train 6.756989 (lr=1.5225e-05) (hash(x)=164984909)
+7000 val loss 6.7327
+7000 val perplexity 839.4137
+7000 train 6.607749 (lr=1.4627e-05) (hash(x)=151172493)
+7100 val loss 6.7451
+7100 val perplexity 849.8951
+7100 train 6.723945 (lr=1.4043e-05) (hash(x)=154023633)
+7200 val loss 6.7121
+7200 val perplexity 822.2639
+7200 train 6.588626 (lr=1.3471e-05) (hash(x)=139351250)
+7300 val loss 6.7147
+7300 val perplexity 824.3981
+7300 train 6.681081 (lr=1.2915e-05) (hash(x)=158719458)
+7400 val loss 6.7026
+7400 val perplexity 814.5391
+7400 train 6.810424 (lr=1.2373e-05) (hash(x)=162250810)
+7500 val loss 6.6964
+7500 val perplexity 809.4514
+7500 train 6.549837 (lr=1.1847e-05) (hash(x)=142827557)
+7600 val loss 6.6928
+7600 val perplexity 806.6149
+7600 train 6.593294 (lr=1.1337e-05) (hash(x)=147830801)
+7700 val loss 6.6912
+7700 val perplexity 805.2506
+7700 train 6.814297 (lr=1.0844e-05) (hash(x)=165233314)
+7800 val loss 6.6926
+7800 val perplexity 806.3911
+7800 train 6.703671 (lr=1.0367e-05) (hash(x)=160654717)
+7900 val loss 6.6848
+7900 val perplexity 800.1847
+7900 train 6.564507 (lr=9.9088e-06) (hash(x)=140712586)
+8000 val loss 6.6777
+8000 val perplexity 794.4768
+8000 train 6.790144 (lr=9.4682e-06) (hash(x)=150309061)
+8100 val loss 6.6745
+8100 val perplexity 791.9514
+8100 train 6.723730 (lr=9.0461e-06) (hash(x)=158805466)
+8200 val loss 6.6719
+8200 val perplexity 789.8681
+8200 train 6.532796 (lr=8.6430e-06) (hash(x)=142814078)
+8300 val loss 6.6696
+8300 val perplexity 788.1194
+8300 train 6.625739 (lr=8.2593e-06) (hash(x)=148340417)
+8400 val loss 6.6704
+8400 val perplexity 788.7062
+8400 train 6.617031 (lr=7.8953e-06) (hash(x)=157132639)
+8500 val loss 6.6660
+8500 val perplexity 785.2834
+8500 train 6.630250 (lr=7.5515e-06) (hash(x)=152994240)
+8600 val loss 6.6639
+8600 val perplexity 783.5849
+8600 train 6.615349 (lr=7.2282e-06) (hash(x)=152210154)
+8700 val loss 6.6610
+8700 val perplexity 781.3484
+8700 train 6.691710 (lr=6.9257e-06) (hash(x)=149631247)
+8800 val loss 6.6558
+8800 val perplexity 777.3157
+8800 train 6.500901 (lr=6.6444e-06) (hash(x)=145617784)
+8900 val loss 6.6552
+8900 val perplexity 776.7936
+8900 train 6.751377 (lr=6.3845e-06) (hash(x)=155984970)
+9000 val loss 6.6549
+9000 val perplexity 776.6133
+9000 train 6.419650 (lr=6.1462e-06) (hash(x)=143433013)
+9100 val loss 6.6529
+9100 val perplexity 775.0507
+9100 train 6.539464 (lr=5.9300e-06) (hash(x)=152507533)
+9200 val loss 6.6483
+9200 val perplexity 771.4741
+9200 train 6.775714 (lr=5.7359e-06) (hash(x)=162400650)
+9300 val loss 6.6472
+9300 val perplexity 770.6464
+9300 train 6.674854 (lr=5.5641e-06) (hash(x)=148419068)
+9400 val loss 6.6443
+9400 val perplexity 768.3872
+9400 train 6.611318 (lr=5.4149e-06) (hash(x)=145025169)
+9500 val loss 6.6483
+9500 val perplexity 771.4730
+9500 train 6.606385 (lr=5.2884e-06) (hash(x)=145104235)
+9600 val loss 6.6447
+9600 val perplexity 768.6980
+9600 train 6.568452 (lr=5.1847e-06) (hash(x)=150499797)
+9700 val loss 6.6428
+9700 val perplexity 767.2230
+9700 train 6.452896 (lr=5.1040e-06) (hash(x)=139690022)
+9800 val loss 6.6368
+9800 val perplexity 762.6857
+9800 train 6.622273 (lr=5.0462e-06) (hash(x)=145645279)
+9900 val loss 6.6362
+9900 val perplexity 762.1913
+9900 train 6.553835 (lr=5.0116e-06) (hash(x)=150103432)
+9999 val loss 6.6367
+9999 val perplexity 762.5792

attention_kindself_n_heads2_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:495bb269f966567bc0c4a1b6dde4d76b4247ecc9dfec0e58226d786a6e26ac33
+oid sha256:909f58a8dfeabb72fcfeeb417429e4030abaefb3b9d9a048431bae84dcc17b19
 size 38587970

attention_kindself_n_heads2_seed1339/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e900cb79898a34aa43dd12ccdc5f8dfc07be2d9553d9f5d9654bc5b5ccf8c9e
+oid sha256:d12694eddb4e563c6596b3d065e6ef9d19e9fa14751a0eb96d7d909d24680519
 size 38587970

attention_kindself_n_heads2_seed1339/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac73275343867ef02ea1c89fd2ba612301e671b0f500dfa3700943771dc8a46a
+oid sha256:cef669e7987742ce62ef9b50a232227f031532358a62492016abf4601b5539f3
 size 38587970

attention_kindself_n_heads2_seed1339/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19ec0d3cd9470a4f858151feaa09d711eeb7cc132ce08ba0241e132618d6c6dd
+oid sha256:9e4ddc957ba2752164b50a97bdd83eea77afbc9706b30fb6d2eae3af5a68f8aa
 size 38587970

attention_kindself_n_heads2_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b2d2fdc0f24530cf7016c528e4cbf7ef4bc0d58a4d286fba5b245550a7a695b
+oid sha256:4e44bda42a029f004f0038fd1ffc3360799ab524869dd634d9021cd088889e57
 size 70895430

attention_kindself_n_heads2_seed1339/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:baadc04fd132ee555b189b92f65de978cec8429a93489fe00dde18b7475e5dbd
+oid sha256:803d6c0490c84281eb9ea042594da3b72013eb563c17462d39878058f7fa52d4
 size 70895430

attention_kindself_n_heads2_seed1339/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:14fc0781ccbae4541e4719aa83d114c7a31e7d6f49828e08ef431d61fb0ee967
+oid sha256:ae1e38720c9bc3eba40bf0e63ef1247c07917a53981537b21fff8e4b9d837a06
 size 70895430

attention_kindself_n_heads2_seed1339/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1684693033f6813206bfef9fd35016f237558a9cd9aab8773340f08ea4aea567
+oid sha256:8b08722ea4238195b07a11c3e1371d2b9556eedf60434203bc268f4d6818d3d6
 size 70895430