Upload folder using huggingface_hub

attention_kindself_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1339", "n_embd": 256}

attention_kindself_n_heads4_seed1339/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.2796
 0 val perplexity 79186.1172
-0 train 11.291462 (lr=2.5000e-07) (hash(x)=150724848)
-100 val loss 9.9021
-100 val perplexity 19971.4043
-100 train 9.951036 (lr=2.5250e-05) (hash(x)=149217061)
-200 val loss 9.0426
-200 val perplexity 8455.4004
-200 train 9.061214 (lr=5.0000e-05) (hash(x)=149910534)
-300 val loss 8.2609
-300 val perplexity 3869.5623
-300 train 8.200714 (lr=4.9988e-05) (hash(x)=142185643)
-400 val loss 7.8750
-400 val perplexity 2630.7476
-400 train 7.933581 (lr=4.9954e-05) (hash(x)=148123706)
-500 val loss 7.6928
-500 val perplexity 2192.4968
-500 train 7.587177 (lr=4.9896e-05) (hash(x)=149952383)
-600 val loss 7.5998
-600 val perplexity 1997.7612
-600 train 7.429729 (lr=4.9815e-05) (hash(x)=146678221)
-700 val loss 7.5332
-700 val perplexity 1869.1604
-700 train 7.564937 (lr=4.9712e-05) (hash(x)=156180736)
-800 val loss 7.4854
-800 val perplexity 1781.7887
-800 train 7.475163 (lr=4.9585e-05) (hash(x)=151700982)
-900 val loss 7.4627
-900 val perplexity 1741.8279
-900 train 7.287769 (lr=4.9436e-05) (hash(x)=147288467)
-1000 val loss 7.4293
-1000 val perplexity 1684.6075
-1000 train 7.493608 (lr=4.9264e-05) (hash(x)=156182087)
-1100 val loss 7.4115
-1100 val perplexity 1654.9202
-1100 train 7.291558 (lr=4.9070e-05) (hash(x)=147861550)
-1200 val loss 7.3934
-1200 val perplexity 1625.1987
-1200 train 7.271595 (lr=4.8854e-05) (hash(x)=149318660)
-1300 val loss 7.3770
-1300 val perplexity 1598.8241
-1300 train 7.238466 (lr=4.8616e-05) (hash(x)=151631103)
-1400 val loss 7.3561
-1400 val perplexity 1565.7140
-1400 train 7.398652 (lr=4.8356e-05) (hash(x)=150482428)
-1500 val loss 7.3213
-1500 val perplexity 1512.1592
-1500 train 7.179976 (lr=4.8074e-05) (hash(x)=144458617)
-1600 val loss 7.2966
-1600 val perplexity 1475.3420
-1600 train 7.183173 (lr=4.7772e-05) (hash(x)=143268605)
-1700 val loss 7.2711
-1700 val perplexity 1438.1329
-1700 train 7.207503 (lr=4.7448e-05) (hash(x)=150817011)
-1800 val loss 7.2608
-1800 val perplexity 1423.3289
-1800 train 7.258028 (lr=4.7105e-05) (hash(x)=152322423)
-1900 val loss 7.2405
-1900 val perplexity 1394.7865
-1900 train 7.084047 (lr=4.6741e-05) (hash(x)=143574126)
-2000 val loss 7.2436
-2000 val perplexity 1399.0969
-2000 train 7.032498 (lr=4.6357e-05) (hash(x)=147904298)
-2100 val loss 7.2320
-2100 val perplexity 1382.9723
-2100 train 7.020701 (lr=4.5954e-05) (hash(x)=144719400)
-2200 val loss 7.2250
-2200 val perplexity 1373.3562
-2200 train 7.456707 (lr=4.5532e-05) (hash(x)=154343147)
-2300 val loss 7.2107
-2300 val perplexity 1353.8384
-2300 train 7.265342 (lr=4.5091e-05) (hash(x)=154372635)
-2400 val loss 7.2090
-2400 val perplexity 1351.5879
-2400 train 7.050181 (lr=4.4633e-05) (hash(x)=141843115)
-2500 val loss 7.2026
-2500 val perplexity 1342.9486
-2500 train 7.107991 (lr=4.4156e-05) (hash(x)=149417679)
-2600 val loss 7.1948
-2600 val perplexity 1332.4684
-2600 train 7.049023 (lr=4.3663e-05) (hash(x)=145279030)
-2700 val loss 7.1873
-2700 val perplexity 1322.5468
-2700 train 7.072419 (lr=4.3153e-05) (hash(x)=146363911)
-2800 val loss 7.2067
-2800 val perplexity 1348.4304
-2800 train 7.189149 (lr=4.2627e-05) (hash(x)=152507639)
-2900 val loss 7.1827
-2900 val perplexity 1316.4399
-2900 train 6.992226 (lr=4.2085e-05) (hash(x)=139296191)
-3000 val loss 7.1794
-3000 val perplexity 1312.0787
-3000 train 7.195726 (lr=4.1529e-05) (hash(x)=148473774)
-3100 val loss 7.1690
-3100 val perplexity 1298.5116
-3100 train 7.270919 (lr=4.0957e-05) (hash(x)=154130527)
-3200 val loss 7.1663
-3200 val perplexity 1295.0099
-3200 train 7.096659 (lr=4.0373e-05) (hash(x)=151117002)
-3300 val loss 7.1644
-3300 val perplexity 1292.6143
-3300 train 7.261252 (lr=3.9775e-05) (hash(x)=151035383)
-3400 val loss 7.1602
-3400 val perplexity 1287.1372
-3400 train 7.248283 (lr=3.9164e-05) (hash(x)=150614749)
-3500 val loss 7.1526
-3500 val perplexity 1277.3953
-3500 train 7.218451 (lr=3.8541e-05) (hash(x)=151431391)
-3600 val loss 7.1520
-3600 val perplexity 1276.6804
-3600 train 7.085993 (lr=3.7907e-05) (hash(x)=151140797)
-3700 val loss 7.1450
-3700 val perplexity 1267.8009
-3700 train 7.379134 (lr=3.7262e-05) (hash(x)=154608340)
-3800 val loss 7.1462
-3800 val perplexity 1269.2732
-3800 train 7.149647 (lr=3.6608e-05) (hash(x)=169641309)
-3900 val loss 7.1452
-3900 val perplexity 1267.9592
-3900 train 7.087150 (lr=3.5944e-05) (hash(x)=150592479)
-4000 val loss 7.1453
-4000 val perplexity 1268.1244
-4000 train 7.118044 (lr=3.5271e-05) (hash(x)=155495317)
-4100 val loss 7.1474
-4100 val perplexity 1270.8580
-4100 train 7.334249 (lr=3.4590e-05) (hash(x)=170323970)
-4200 val loss 7.1551
-4200 val perplexity 1280.6582
-4200 train 6.968613 (lr=3.3902e-05) (hash(x)=140854360)
-4300 val loss 7.1448
-4300 val perplexity 1267.5548
-4300 train 7.138808 (lr=3.3207e-05) (hash(x)=148926644)
-4400 val loss 7.1432
-4400 val perplexity 1265.4563
-4400 train 7.182815 (lr=3.2507e-05) (hash(x)=149997871)
-4500 val loss 7.1313
-4500 val perplexity 1250.4640
-4500 train 7.091198 (lr=3.1801e-05) (hash(x)=149219289)
-4600 val loss 7.1326
-4600 val perplexity 1252.0677
-4600 train 6.935356 (lr=3.1091e-05) (hash(x)=139836197)
-4700 val loss 7.1366
-4700 val perplexity 1257.1642
-4700 train 7.099030 (lr=3.0377e-05) (hash(x)=150722883)
-4800 val loss 7.1298
-4800 val perplexity 1248.6515
-4800 train 7.321390 (lr=2.9661e-05) (hash(x)=166864648)
-4900 val loss 7.1284
-4900 val perplexity 1246.8326
-4900 train 7.231492 (lr=2.8942e-05) (hash(x)=157746743)
-5000 val loss 7.1262
-5000 val perplexity 1244.1138
-5000 train 7.084176 (lr=2.8221e-05) (hash(x)=153032257)
-5100 val loss 7.1329
-5100 val perplexity 1252.4446
-5100 train 7.217553 (lr=2.7500e-05) (hash(x)=151862371)
-5200 val loss 7.1284
-5200 val perplexity 1246.9265
-5200 train 7.419171 (lr=2.6779e-05) (hash(x)=168862995)
-5300 val loss 7.1254
-5300 val perplexity 1243.1229
-5300 train 7.094783 (lr=2.6058e-05) (hash(x)=148243567)
-5400 val loss 7.1269
-5400 val perplexity 1244.9808
-5400 train 7.061582 (lr=2.5339e-05) (hash(x)=145381742)
-5500 val loss 7.1190
-5500 val perplexity 1235.1892
-5500 train 7.284770 (lr=2.4623e-05) (hash(x)=155602174)
-5600 val loss 7.1170
-5600 val perplexity 1232.7450
-5600 train 7.213219 (lr=2.3909e-05) (hash(x)=160637672)
-5700 val loss 7.1150
-5700 val perplexity 1230.3080
-5700 train 7.074363 (lr=2.3199e-05) (hash(x)=152101131)
-5800 val loss 7.1187
-5800 val perplexity 1234.8164
-5800 train 7.082228 (lr=2.2493e-05) (hash(x)=152327089)
-5900 val loss 7.1184
-5900 val perplexity 1234.5280
-5900 train 7.041556 (lr=2.1793e-05) (hash(x)=151094543)
-6000 val loss 7.1203
-6000 val perplexity 1236.8035
-6000 train 7.216980 (lr=2.1098e-05) (hash(x)=160233315)
-6100 val loss 7.1131
-6100 val perplexity 1227.9484
-6100 train 7.200726 (lr=2.0410e-05) (hash(x)=158295337)
-6200 val loss 7.1109
-6200 val perplexity 1225.2181
-6200 train 7.066869 (lr=1.9729e-05) (hash(x)=151269817)
-6300 val loss 7.1129
-6300 val perplexity 1227.7365
-6300 train 7.188467 (lr=1.9056e-05) (hash(x)=156912756)
-6400 val loss 7.1081
-6400 val perplexity 1221.8523
-6400 train 7.044916 (lr=1.8392e-05) (hash(x)=125779637)
-6500 val loss 7.1168
-6500 val perplexity 1232.4882
-6500 train 7.028541 (lr=1.7738e-05) (hash(x)=148157661)
-6600 val loss 7.1085
-6600 val perplexity 1222.3086
-6600 train 7.161902 (lr=1.7093e-05) (hash(x)=152820049)
-6700 val loss 7.1078
-6700 val perplexity 1221.4958
-6700 train 7.072041 (lr=1.6459e-05) (hash(x)=155818717)
-6800 val loss 7.0971
-6800 val perplexity 1208.4872
-6800 train 7.173000 (lr=1.5836e-05) (hash(x)=158833881)
-6900 val loss 7.0994
-6900 val perplexity 1211.2847
-6900 train 7.156277 (lr=1.5225e-05) (hash(x)=164984909)
-7000 val loss 7.0956
-7000 val perplexity 1206.6462
-7000 train 6.990550 (lr=1.4627e-05) (hash(x)=151172493)
-7100 val loss 7.0936
-7100 val perplexity 1204.1770
-7100 train 7.043107 (lr=1.4043e-05) (hash(x)=154023633)
-7200 val loss 7.0892
-7200 val perplexity 1198.9299
-7200 train 6.971313 (lr=1.3471e-05) (hash(x)=139351250)
-7300 val loss 7.0875
-7300 val perplexity 1196.9484
-7300 train 7.047305 (lr=1.2915e-05) (hash(x)=158719458)
-7400 val loss 7.0860
-7400 val perplexity 1195.0596
-7400 train 7.210918 (lr=1.2373e-05) (hash(x)=162250810)
-7500 val loss 7.0863
-7500 val perplexity 1195.4512
-7500 train 6.927986 (lr=1.1847e-05) (hash(x)=142827557)
-7600 val loss 7.0807
-7600 val perplexity 1188.7521
-7600 train 7.012818 (lr=1.1337e-05) (hash(x)=147830801)
-7700 val loss 7.0813
-7700 val perplexity 1189.5392
-7700 train 7.220162 (lr=1.0844e-05) (hash(x)=165233314)
-7800 val loss 7.0805
-7800 val perplexity 1188.5917
-7800 train 7.086352 (lr=1.0367e-05) (hash(x)=160654717)
-7900 val loss 7.0822
-7900 val perplexity 1190.5776
-7900 train 6.959390 (lr=9.9088e-06) (hash(x)=140712586)
-8000 val loss 7.0740
-8000 val perplexity 1180.8348
-8000 train 7.150942 (lr=9.4682e-06) (hash(x)=150309061)
-8100 val loss 7.0724
-8100 val perplexity 1178.9276
-8100 train 7.109391 (lr=9.0461e-06) (hash(x)=158805466)
-8200 val loss 7.0690
-8200 val perplexity 1174.9840
-8200 train 6.911159 (lr=8.6430e-06) (hash(x)=142814078)
-8300 val loss 7.0671
-8300 val perplexity 1172.7701
-8300 train 7.025698 (lr=8.2593e-06) (hash(x)=148340417)
-8400 val loss 7.0684
-8400 val perplexity 1174.2312
-8400 train 7.020465 (lr=7.8953e-06) (hash(x)=157132639)
-8500 val loss 7.0645
-8500 val perplexity 1169.6957
-8500 train 7.028442 (lr=7.5515e-06) (hash(x)=152994240)
-8600 val loss 7.0610
-8600 val perplexity 1165.5823
-8600 train 6.998075 (lr=7.2282e-06) (hash(x)=152210154)
-8700 val loss 7.0611
-8700 val perplexity 1165.7090
-8700 train 7.082347 (lr=6.9257e-06) (hash(x)=149631247)
-8800 val loss 7.0584
-8800 val perplexity 1162.5870
-8800 train 6.921595 (lr=6.6444e-06) (hash(x)=145617784)
-8900 val loss 7.0569
-8900 val perplexity 1160.7957
-8900 train 7.155268 (lr=6.3845e-06) (hash(x)=155984970)
-9000 val loss 7.0565
-9000 val perplexity 1160.4248
-9000 train 6.833085 (lr=6.1462e-06) (hash(x)=143433013)
-9100 val loss 7.0541
-9100 val perplexity 1157.6333
-9100 train 6.952363 (lr=5.9300e-06) (hash(x)=152507533)
-9200 val loss 7.0541
-9200 val perplexity 1157.5908
-9200 train 7.204437 (lr=5.7359e-06) (hash(x)=162400650)
-9300 val loss 7.0508
-9300 val perplexity 1153.7626
-9300 train 7.087835 (lr=5.5641e-06) (hash(x)=148419068)
-9400 val loss 7.0555
-9400 val perplexity 1159.2208
-9400 train 7.023118 (lr=5.4149e-06) (hash(x)=145025169)
-9500 val loss 7.0498
-9500 val perplexity 1152.5929
-9500 train 7.014523 (lr=5.2884e-06) (hash(x)=145104235)
-9600 val loss 7.0501
-9600 val perplexity 1152.9315
-9600 train 6.962839 (lr=5.1847e-06) (hash(x)=150499797)
-9700 val loss 7.0489
-9700 val perplexity 1151.5975
-9700 train 6.860659 (lr=5.1040e-06) (hash(x)=139690022)
-9800 val loss 7.0456
-9800 val perplexity 1147.8142
-9800 train 7.011253 (lr=5.0462e-06) (hash(x)=145645279)
-9900 val loss 7.0449
-9900 val perplexity 1146.9382
-9900 train 6.967855 (lr=5.0116e-06) (hash(x)=150103432)
-9999 val loss 7.0456
-9999 val perplexity 1147.8422
+0 train 11.291462 (lr=1.5000e-07) (hash(x)=150724848)
+100 val loss 9.9918
+100 val perplexity 21846.0312
+100 train 10.038823 (lr=1.5150e-05) (hash(x)=149217061)
+200 val loss 9.5847
+200 val perplexity 14540.3027
+200 train 9.604377 (lr=3.0000e-05) (hash(x)=149910534)
+300 val loss 8.8958
+300 val perplexity 7301.2344
+300 train 8.877235 (lr=2.9993e-05) (hash(x)=142185643)
+400 val loss 8.5006
+400 val perplexity 4917.8960
+400 train 8.568892 (lr=2.9972e-05) (hash(x)=148123706)
+500 val loss 8.2573
+500 val perplexity 3855.7300
+500 train 8.206617 (lr=2.9938e-05) (hash(x)=149952383)
+600 val loss 8.0601
+600 val perplexity 3165.6990
+600 train 7.956430 (lr=2.9889e-05) (hash(x)=146678221)
+700 val loss 7.9130
+700 val perplexity 2732.6284
+700 train 7.938250 (lr=2.9827e-05) (hash(x)=156180736)
+800 val loss 7.8032
+800 val perplexity 2448.4763
+800 train 7.794213 (lr=2.9751e-05) (hash(x)=151700982)
+900 val loss 7.7195
+900 val perplexity 2251.8474
+900 train 7.560434 (lr=2.9662e-05) (hash(x)=147288467)
+1000 val loss 7.6481
+1000 val perplexity 2096.5630
+1000 train 7.714798 (lr=2.9558e-05) (hash(x)=156182087)
+1100 val loss 7.5998
+1100 val perplexity 1997.8328
+1100 train 7.507259 (lr=2.9442e-05) (hash(x)=147861550)
+1200 val loss 7.5669
+1200 val perplexity 1933.1093
+1200 train 7.469774 (lr=2.9312e-05) (hash(x)=149318660)
+1300 val loss 7.5431
+1300 val perplexity 1887.5906
+1300 train 7.424637 (lr=2.9169e-05) (hash(x)=151631103)
+1400 val loss 7.5175
+1400 val perplexity 1840.0282
+1400 train 7.561960 (lr=2.9013e-05) (hash(x)=150482428)
+1500 val loss 7.4891
+1500 val perplexity 1788.4921
+1500 train 7.353311 (lr=2.8845e-05) (hash(x)=144458617)
+1600 val loss 7.4656
+1600 val perplexity 1746.9250
+1600 train 7.346329 (lr=2.8663e-05) (hash(x)=143268605)
+1700 val loss 7.4516
+1700 val perplexity 1722.6821
+1700 train 7.391551 (lr=2.8469e-05) (hash(x)=150817011)
+1800 val loss 7.4371
+1800 val perplexity 1697.8566
+1800 train 7.436510 (lr=2.8263e-05) (hash(x)=152322423)
+1900 val loss 7.4158
+1900 val perplexity 1662.0186
+1900 train 7.271878 (lr=2.8044e-05) (hash(x)=143574126)
+2000 val loss 7.4040
+2000 val perplexity 1642.6132
+2000 train 7.207763 (lr=2.7814e-05) (hash(x)=147904298)
+2100 val loss 7.3957
+2100 val perplexity 1629.0114
+2100 train 7.188506 (lr=2.7572e-05) (hash(x)=144719400)
+2200 val loss 7.3787
+2200 val perplexity 1601.5267
+2200 train 7.612090 (lr=2.7319e-05) (hash(x)=154343147)
+2300 val loss 7.3581
+2300 val perplexity 1568.9238
+2300 train 7.419537 (lr=2.7055e-05) (hash(x)=154372635)
+2400 val loss 7.3439
+2400 val perplexity 1546.6827
+2400 train 7.196864 (lr=2.6780e-05) (hash(x)=141843115)
+2500 val loss 7.3282
+2500 val perplexity 1522.5789
+2500 train 7.241656 (lr=2.6494e-05) (hash(x)=149417679)
+2600 val loss 7.3167
+2600 val perplexity 1505.1832
+2600 train 7.178639 (lr=2.6198e-05) (hash(x)=145279030)
+2700 val loss 7.3108
+2700 val perplexity 1496.4324
+2700 train 7.197565 (lr=2.5892e-05) (hash(x)=146363911)
+2800 val loss 7.3025
+2800 val perplexity 1483.9929
+2800 train 7.289475 (lr=2.5576e-05) (hash(x)=152507639)
+2900 val loss 7.2917
+2900 val perplexity 1468.0970
+2900 train 7.100662 (lr=2.5251e-05) (hash(x)=139296191)
+3000 val loss 7.2872
+3000 val perplexity 1461.4930
+3000 train 7.309252 (lr=2.4917e-05) (hash(x)=148473774)
+3100 val loss 7.2726
+3100 val perplexity 1440.2191
+3100 train 7.380096 (lr=2.4574e-05) (hash(x)=154130527)
+3200 val loss 7.2612
+3200 val perplexity 1424.0056
+3200 train 7.188822 (lr=2.4224e-05) (hash(x)=151117002)
+3300 val loss 7.2666
+3300 val perplexity 1431.6910
+3300 train 7.359443 (lr=2.3865e-05) (hash(x)=151035383)
+3400 val loss 7.2466
+3400 val perplexity 1403.3323
+3400 train 7.328700 (lr=2.3498e-05) (hash(x)=150614749)
+3500 val loss 7.2381
+3500 val perplexity 1391.4047
+3500 train 7.322695 (lr=2.3125e-05) (hash(x)=151431391)
+3600 val loss 7.2330
+3600 val perplexity 1384.3889
+3600 train 7.175946 (lr=2.2744e-05) (hash(x)=151140797)
+3700 val loss 7.2279
+3700 val perplexity 1377.3540
+3700 train 7.464332 (lr=2.2357e-05) (hash(x)=154608340)
+3800 val loss 7.2198
+3800 val perplexity 1366.2314
+3800 train 7.235229 (lr=2.1965e-05) (hash(x)=169641309)
+3900 val loss 7.2179
+3900 val perplexity 1363.5585
+3900 train 7.153865 (lr=2.1566e-05) (hash(x)=150592479)
+4000 val loss 7.2032
+4000 val perplexity 1343.7205
+4000 train 7.170217 (lr=2.1162e-05) (hash(x)=155495317)
+4100 val loss 7.2041
+4100 val perplexity 1344.9698
+4100 train 7.401001 (lr=2.0754e-05) (hash(x)=170323970)
+4200 val loss 7.1967
+4200 val perplexity 1334.9862
+4200 train 7.002602 (lr=2.0341e-05) (hash(x)=140854360)
+4300 val loss 7.1884
+4300 val perplexity 1323.9248
+4300 train 7.182519 (lr=1.9924e-05) (hash(x)=148926644)
+4400 val loss 7.1824
+4400 val perplexity 1316.0244
+4400 train 7.222973 (lr=1.9504e-05) (hash(x)=149997871)
+4500 val loss 7.1755
+4500 val perplexity 1306.9958
+4500 train 7.133025 (lr=1.9081e-05) (hash(x)=149219289)
+4600 val loss 7.1712
+4600 val perplexity 1301.4684
+4600 train 6.976065 (lr=1.8655e-05) (hash(x)=139836197)
+4700 val loss 7.1694
+4700 val perplexity 1299.0063
+4700 train 7.121901 (lr=1.8226e-05) (hash(x)=150722883)
+4800 val loss 7.1631
+4800 val perplexity 1290.8459
+4800 train 7.362895 (lr=1.7796e-05) (hash(x)=166864648)
+4900 val loss 7.1576
+4900 val perplexity 1283.8247
+4900 train 7.257826 (lr=1.7365e-05) (hash(x)=157746743)
+5000 val loss 7.1490
+5000 val perplexity 1272.8685
+5000 train 7.109448 (lr=1.6933e-05) (hash(x)=153032257)
+5100 val loss 7.1450
+5100 val perplexity 1267.7059
+5100 train 7.227070 (lr=1.6500e-05) (hash(x)=151862371)
+5200 val loss 7.1397
+5200 val perplexity 1261.0674
+5200 train 7.448361 (lr=1.6067e-05) (hash(x)=168862995)
+5300 val loss 7.1392
+5300 val perplexity 1260.4553
+5300 train 7.111847 (lr=1.5635e-05) (hash(x)=148243567)
+5400 val loss 7.1311
+5400 val perplexity 1250.2697
+5400 train 7.058176 (lr=1.5204e-05) (hash(x)=145381742)
+5500 val loss 7.1242
+5500 val perplexity 1241.7004
+5500 train 7.294051 (lr=1.4774e-05) (hash(x)=155602174)
+5600 val loss 7.1206
+5600 val perplexity 1237.1556
+5600 train 7.213657 (lr=1.4345e-05) (hash(x)=160637672)
+5700 val loss 7.1158
+5700 val perplexity 1231.3098
+5700 train 7.075980 (lr=1.3919e-05) (hash(x)=152101131)
+5800 val loss 7.1094
+5800 val perplexity 1223.4661
+5800 train 7.070355 (lr=1.3496e-05) (hash(x)=152327089)
+5900 val loss 7.1091
+5900 val perplexity 1223.0812
+5900 train 7.031635 (lr=1.3076e-05) (hash(x)=151094543)
+6000 val loss 7.1074
+6000 val perplexity 1221.0055
+6000 train 7.214882 (lr=1.2659e-05) (hash(x)=160233315)
+6100 val loss 7.0985
+6100 val perplexity 1210.0942
+6100 train 7.191561 (lr=1.2246e-05) (hash(x)=158295337)
+6200 val loss 7.0950
+6200 val perplexity 1205.8663
+6200 train 7.057549 (lr=1.1838e-05) (hash(x)=151269817)
+6300 val loss 7.0902
+6300 val perplexity 1200.1987
+6300 train 7.170448 (lr=1.1434e-05) (hash(x)=156912756)
+6400 val loss 7.0858
+6400 val perplexity 1194.8966
+6400 train 7.018799 (lr=1.1035e-05) (hash(x)=125779637)
+6500 val loss 7.0847
+6500 val perplexity 1193.5840
+6500 train 6.995352 (lr=1.0643e-05) (hash(x)=148157661)
+6600 val loss 7.0803
+6600 val perplexity 1188.2891
+6600 train 7.138786 (lr=1.0256e-05) (hash(x)=152820049)
+6700 val loss 7.0884
+6700 val perplexity 1198.0219
+6700 train 7.040927 (lr=9.8753e-06) (hash(x)=155818717)
+6800 val loss 7.0709
+6800 val perplexity 1177.1682
+6800 train 7.152566 (lr=9.5017e-06) (hash(x)=158833881)
+6900 val loss 7.0689
+6900 val perplexity 1174.8759
+6900 train 7.135961 (lr=9.1353e-06) (hash(x)=164984909)
+7000 val loss 7.0647
+7000 val perplexity 1169.9740
+7000 train 6.953945 (lr=8.7764e-06) (hash(x)=151172493)
+7100 val loss 7.0671
+7100 val perplexity 1172.7673
+7100 train 7.018477 (lr=8.4255e-06) (hash(x)=154023633)
+7200 val loss 7.0604
+7200 val perplexity 1164.9432
+7200 train 6.937146 (lr=8.0829e-06) (hash(x)=139351250)
+7300 val loss 7.0561
+7300 val perplexity 1159.9192
+7300 train 7.017184 (lr=7.7489e-06) (hash(x)=158719458)
+7400 val loss 7.0536
+7400 val perplexity 1157.0306
+7400 train 7.180736 (lr=7.4239e-06) (hash(x)=162250810)
+7500 val loss 7.0488
+7500 val perplexity 1151.5200
+7500 train 6.898161 (lr=7.1083e-06) (hash(x)=142827557)
+7600 val loss 7.0486
+7600 val perplexity 1151.2247
+7600 train 6.983459 (lr=6.8023e-06) (hash(x)=147830801)
+7700 val loss 7.0437
+7700 val perplexity 1145.5664
+7700 train 7.181130 (lr=6.5062e-06) (hash(x)=165233314)
+7800 val loss 7.0433
+7800 val perplexity 1145.1989
+7800 train 7.049610 (lr=6.2205e-06) (hash(x)=160654717)
+7900 val loss 7.0392
+7900 val perplexity 1140.5269
+7900 train 6.912971 (lr=5.9453e-06) (hash(x)=140712586)
+8000 val loss 7.0357
+8000 val perplexity 1136.4613
+8000 train 7.120284 (lr=5.6809e-06) (hash(x)=150309061)
+8100 val loss 7.0332
+8100 val perplexity 1133.6506
+8100 train 7.065866 (lr=5.4277e-06) (hash(x)=158805466)
+8200 val loss 7.0320
+8200 val perplexity 1132.2493
+8200 train 6.870091 (lr=5.1858e-06) (hash(x)=142814078)
+8300 val loss 7.0298
+8300 val perplexity 1129.7921
+8300 train 6.989686 (lr=4.9556e-06) (hash(x)=148340417)
+8400 val loss 7.0281
+8400 val perplexity 1127.9388
+8400 train 6.972344 (lr=4.7372e-06) (hash(x)=157132639)
+8500 val loss 7.0278
+8500 val perplexity 1127.4937
+8500 train 6.992925 (lr=4.5309e-06) (hash(x)=152994240)
+8600 val loss 7.0222
+8600 val perplexity 1121.2107
+8600 train 6.958762 (lr=4.3369e-06) (hash(x)=152210154)
+8700 val loss 7.0204
+8700 val perplexity 1119.1915
+8700 train 7.041991 (lr=4.1554e-06) (hash(x)=149631247)
+8800 val loss 7.0189
+8800 val perplexity 1117.5597
+8800 train 6.874503 (lr=3.9866e-06) (hash(x)=145617784)
+8900 val loss 7.0180
+8900 val perplexity 1116.5854
+8900 train 7.119063 (lr=3.8307e-06) (hash(x)=155984970)
+9000 val loss 7.0170
+9000 val perplexity 1115.4611
+9000 train 6.785057 (lr=3.6877e-06) (hash(x)=143433013)
+9100 val loss 7.0163
+9100 val perplexity 1114.6815
+9100 train 6.915003 (lr=3.5580e-06) (hash(x)=152507533)
+9200 val loss 7.0131
+9200 val perplexity 1111.1212
+9200 train 7.167346 (lr=3.4415e-06) (hash(x)=162400650)
+9300 val loss 7.0121
+9300 val perplexity 1109.9335
+9300 train 7.052631 (lr=3.3385e-06) (hash(x)=148419068)
+9400 val loss 7.0098
+9400 val perplexity 1107.4303
+9400 train 6.974667 (lr=3.2490e-06) (hash(x)=145025169)
+9500 val loss 7.0086
+9500 val perplexity 1106.1521
+9500 train 6.972902 (lr=3.1730e-06) (hash(x)=145104235)
+9600 val loss 7.0083
+9600 val perplexity 1105.7798
+9600 train 6.918948 (lr=3.1108e-06) (hash(x)=150499797)
+9700 val loss 7.0080
+9700 val perplexity 1105.4318
+9700 train 6.817394 (lr=3.0624e-06) (hash(x)=139690022)
+9800 val loss 7.0058
+9800 val perplexity 1102.9635
+9800 train 6.966627 (lr=3.0277e-06) (hash(x)=145645279)
+9900 val loss 7.0024
+9900 val perplexity 1099.2639
+9900 train 6.922573 (lr=3.0069e-06) (hash(x)=150103432)
+9999 val loss 7.0017
+9999 val perplexity 1098.4680

attention_kindself_n_heads4_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a874ade60cf3c0753973366d6e7d2b341ee44b01a89abdf29183b09378667ed0
+oid sha256:2bb06dd7c949a8d452155d92fc165c1973e274c0af8046c24a82cb4e25b49b59
 size 92843394

attention_kindself_n_heads4_seed1339/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:efc27109d173b6b4765cceff5fb4a95018ffd213c0cce94727a474679efce861
+oid sha256:49d5414002b7a6b6664c77a1dcf46ed6b25eac0f61d99c1536856806e9fd9b1c
 size 92843394

attention_kindself_n_heads4_seed1339/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7fb37a20573e497ce0e47b0220aad7a6a21093349418d5f6759a537c004f0dd4
+oid sha256:9d9a73bd4bdd2bfb475830f35e21349cfa4dd0d145f4d745fb8ecad38aa16732
 size 92843394

attention_kindself_n_heads4_seed1339/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5aba8d6fd761f326db45b1851d53b9054e3b5e0e1311bb3f97daecec7952bafa
+oid sha256:b1e681f7c4293d979b28cd541b80501adbb6523ae7de5a28f84ca394d84d6fe4
 size 92843394

attention_kindself_n_heads4_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de51f73152f1ec589219a7082771385aca728c448cf76ed0dc302da5eef79d35
+oid sha256:f2b71934ee25d4aa81a082fd78c32db0a1f209de4ee5a40d3c6bc1ee2da5535c
 size 179406214

attention_kindself_n_heads4_seed1339/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a56dfec2a17c48c414efd19d1a8ab1e4f0643dcb99d3ef53c5c78f4747dc5b9
+oid sha256:8475ae035d80c48a634cb7fe99edb33dd8a97495f68a2ad8dc8f92b81e0ad74d
 size 179406214

attention_kindself_n_heads4_seed1339/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dbbb78f04809887baa6eeed69c48402feb12dc5dd09cae80923992f23b48a3f4
+oid sha256:a629ffd3886ee531eb7ca7cd04d282132bfbd4d238a1314a2e7db7af4667e52a
 size 179406214

attention_kindself_n_heads4_seed1339/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f95950c9e251e960a028614648249dddff2253ad04c3767de54f848a62d0d3b
+oid sha256:4ef131c227c41a4cd55b825c2c174bd26c605af67da52a491e14fff6d8f6f9a4
 size 179406214