Upload folder using huggingface_hub

attention_kindself_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_4_1339", "n_embd": 256}

attention_kindself_n_heads4_seed1339/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.2796
 0 val perplexity 79186.1172
-0 train 11.291462 (lr=7.5000e-07) (hash(x)=150724848)
-100 val loss 9.3316
-100 val perplexity 11289.2031
-100 train 9.386396 (lr=7.5750e-05) (hash(x)=149217061)
-200 val loss 7.7580
-200 val perplexity 2340.2415
-200 train 7.766219 (lr=1.5000e-04) (hash(x)=149910534)
-300 val loss 7.5387
-300 val perplexity 1879.3298
-300 train 7.404105 (lr=1.4997e-04) (hash(x)=142185643)
-400 val loss 7.3849
-400 val perplexity 1611.4607
-400 train 7.409924 (lr=1.4986e-04) (hash(x)=148123706)
-500 val loss 7.2848
-500 val perplexity 1457.9835
-500 train 7.114595 (lr=1.4969e-04) (hash(x)=149952383)
-600 val loss 7.2275
-600 val perplexity 1376.8268
-600 train 7.018831 (lr=1.4945e-04) (hash(x)=146678221)
-700 val loss 7.1586
-700 val perplexity 1285.1116
-700 train 7.201500 (lr=1.4913e-04) (hash(x)=156180736)
-800 val loss 7.1339
-800 val perplexity 1253.7417
-800 train 7.115716 (lr=1.4876e-04) (hash(x)=151700982)
-900 val loss 7.0960
-900 val perplexity 1207.1274
-900 train 6.877567 (lr=1.4831e-04) (hash(x)=147288467)
-1000 val loss 7.0511
-1000 val perplexity 1154.1218
-1000 train 7.116826 (lr=1.4779e-04) (hash(x)=156182087)
-1100 val loss 7.0467
-1100 val perplexity 1149.0145
-1100 train 6.910080 (lr=1.4721e-04) (hash(x)=147861550)
-1200 val loss 7.0220
-1200 val perplexity 1121.0123
-1200 train 6.883329 (lr=1.4656e-04) (hash(x)=149318660)
-1300 val loss 7.0377
-1300 val perplexity 1138.7271
-1300 train 6.882957 (lr=1.4585e-04) (hash(x)=151631103)
-1400 val loss 7.0220
-1400 val perplexity 1121.0166
-1400 train 7.034495 (lr=1.4507e-04) (hash(x)=150482428)
-1500 val loss 6.9852
-1500 val perplexity 1080.5740
-1500 train 6.833119 (lr=1.4422e-04) (hash(x)=144458617)
-1600 val loss 6.9992
-1600 val perplexity 1095.8010
-1600 train 6.893284 (lr=1.4332e-04) (hash(x)=143268605)
-1700 val loss 6.9816
-1700 val perplexity 1076.6697
-1700 train 6.925849 (lr=1.4235e-04) (hash(x)=150817011)
-1800 val loss 6.9799
-1800 val perplexity 1074.7789
-1800 train 6.975879 (lr=1.4131e-04) (hash(x)=152322423)
-1900 val loss 6.9624
-1900 val perplexity 1056.1371
-1900 train 6.774517 (lr=1.4022e-04) (hash(x)=143574126)
-2000 val loss 6.9909
-2000 val perplexity 1086.6913
-2000 train 6.782327 (lr=1.3907e-04) (hash(x)=147904298)
-2100 val loss 6.9824
-2100 val perplexity 1077.4812
-2100 train 6.763468 (lr=1.3786e-04) (hash(x)=144719400)
-2200 val loss 6.9828
-2200 val perplexity 1077.9159
-2200 train 7.221006 (lr=1.3660e-04) (hash(x)=154343147)
-2300 val loss 6.9937
-2300 val perplexity 1089.7175
-2300 train 7.024708 (lr=1.3527e-04) (hash(x)=154372635)
-2400 val loss 6.9764
-2400 val perplexity 1071.0907
-2400 train 6.818256 (lr=1.3390e-04) (hash(x)=141843115)
-2500 val loss 6.9814
-2500 val perplexity 1076.4762
-2500 train 6.891120 (lr=1.3247e-04) (hash(x)=149417679)
-2600 val loss 6.9879
-2600 val perplexity 1083.4302
-2600 train 6.861284 (lr=1.3099e-04) (hash(x)=145279030)
-2700 val loss 7.0230
-2700 val perplexity 1122.1450
-2700 train 6.928087 (lr=1.2946e-04) (hash(x)=146363911)
-2800 val loss 7.0102
-2800 val perplexity 1107.8602
-2800 train 6.996984 (lr=1.2788e-04) (hash(x)=152507639)
-2900 val loss 7.0251
-2900 val perplexity 1124.5314
-2900 train 6.853853 (lr=1.2626e-04) (hash(x)=139296191)
-3000 val loss 7.0815
-3000 val perplexity 1189.8097
-3000 train 7.084819 (lr=1.2459e-04) (hash(x)=148473774)
-3100 val loss 7.0437
-3100 val perplexity 1145.5751
-3100 train 7.125661 (lr=1.2287e-04) (hash(x)=154130527)
-3200 val loss 7.0794
-3200 val perplexity 1187.2012
-3200 train 7.033405 (lr=1.2112e-04) (hash(x)=151117002)
-3300 val loss 7.0905
-3300 val perplexity 1200.5089
-3300 train 7.205884 (lr=1.1932e-04) (hash(x)=151035383)
-3400 val loss 7.0632
-3400 val perplexity 1168.2102
-3400 train 7.170041 (lr=1.1749e-04) (hash(x)=150614749)
-3500 val loss 7.0934
-3500 val perplexity 1204.0105
-3500 train 7.124344 (lr=1.1562e-04) (hash(x)=151431391)
-3600 val loss 7.1095
-3600 val perplexity 1223.4929
-3600 train 7.066493 (lr=1.1372e-04) (hash(x)=151140797)
-3700 val loss 7.1210
-3700 val perplexity 1237.6566
-3700 train 7.303615 (lr=1.1179e-04) (hash(x)=154608340)
-3800 val loss 7.1782
-3800 val perplexity 1310.5081
-3800 train 7.108573 (lr=1.0982e-04) (hash(x)=169641309)
-3900 val loss 7.1296
-3900 val perplexity 1248.3175
-3900 train 7.071731 (lr=1.0783e-04) (hash(x)=150592479)
-4000 val loss 7.1371
-4000 val perplexity 1257.7638
-4000 train 7.126672 (lr=1.0581e-04) (hash(x)=155495317)
-4100 val loss 7.1221
-4100 val perplexity 1239.0602
-4100 train 7.290395 (lr=1.0377e-04) (hash(x)=170323970)
-4200 val loss 7.1540
-4200 val perplexity 1279.2599
-4200 train 6.982704 (lr=1.0171e-04) (hash(x)=140854360)
-4300 val loss 7.1362
-4300 val perplexity 1256.6296
-4300 train 7.135914 (lr=9.9622e-05) (hash(x)=148926644)
-4400 val loss 7.1230
-4400 val perplexity 1240.2017
-4400 train 7.164248 (lr=9.7520e-05) (hash(x)=149997871)
-4500 val loss 7.1416
-4500 val perplexity 1263.4196
-4500 train 7.111341 (lr=9.5403e-05) (hash(x)=149219289)
-4600 val loss 7.1663
-4600 val perplexity 1295.0135
-4600 train 6.972812 (lr=9.3273e-05) (hash(x)=139836197)
-4700 val loss 7.1523
-4700 val perplexity 1277.0450
-4700 train 7.122142 (lr=9.1132e-05) (hash(x)=150722883)
-4800 val loss 7.1510
-4800 val perplexity 1275.3193
-4800 train 7.313559 (lr=8.8982e-05) (hash(x)=166864648)
-4900 val loss 7.1670
-4900 val perplexity 1295.9025
-4900 train 7.252068 (lr=8.6825e-05) (hash(x)=157746743)
-5000 val loss 7.1829
-5000 val perplexity 1316.6577
-5000 train 7.156727 (lr=8.4663e-05) (hash(x)=153032257)
-5100 val loss 7.1774
-5100 val perplexity 1309.5005
-5100 train 7.255030 (lr=8.2500e-05) (hash(x)=151862371)
-5200 val loss 7.1758
-5200 val perplexity 1307.4421
-5200 train 7.425144 (lr=8.0337e-05) (hash(x)=168862995)
-5300 val loss 7.1714
-5300 val perplexity 1301.7012
-5300 train 7.149970 (lr=7.8175e-05) (hash(x)=148243567)
-5400 val loss 7.1838
-5400 val perplexity 1317.8958
-5400 train 7.125934 (lr=7.6018e-05) (hash(x)=145381742)
-5500 val loss 7.2053
-5500 val perplexity 1346.5536
-5500 train 7.331578 (lr=7.3868e-05) (hash(x)=155602174)
-5600 val loss 7.1589
-5600 val perplexity 1285.4670
-5600 train 7.253695 (lr=7.1727e-05) (hash(x)=160637672)
-5700 val loss 7.1751
-5700 val perplexity 1306.5392
-5700 train 7.153136 (lr=6.9597e-05) (hash(x)=152101131)
-5800 val loss 7.1774
-5800 val perplexity 1309.5411
-5800 train 7.159649 (lr=6.7480e-05) (hash(x)=152327089)
-5900 val loss 7.1646
-5900 val perplexity 1292.8016
-5900 train 7.118360 (lr=6.5378e-05) (hash(x)=151094543)
-6000 val loss 7.1787
-6000 val perplexity 1311.2225
-6000 train 7.286202 (lr=6.3294e-05) (hash(x)=160233315)
-6100 val loss 7.1632
-6100 val perplexity 1290.9937
-6100 train 7.260243 (lr=6.1230e-05) (hash(x)=158295337)
-6200 val loss 7.1683
-6200 val perplexity 1297.5900
-6200 train 7.161770 (lr=5.9188e-05) (hash(x)=151269817)
-6300 val loss 7.1598
-6300 val perplexity 1286.6083
-6300 train 7.258369 (lr=5.7169e-05) (hash(x)=156912756)
-6400 val loss 7.1634
-6400 val perplexity 1291.2910
-6400 train 7.076042 (lr=5.5177e-05) (hash(x)=125779637)
-6500 val loss 7.1931
-6500 val perplexity 1330.2091
-6500 train 7.120435 (lr=5.3213e-05) (hash(x)=148157661)
-6600 val loss 7.1875
-6600 val perplexity 1322.8066
-6600 train 7.224742 (lr=5.1279e-05) (hash(x)=152820049)
-6700 val loss 7.1680
-6700 val perplexity 1297.1934
-6700 train 7.127453 (lr=4.9377e-05) (hash(x)=155818717)
-6800 val loss 7.1549
-6800 val perplexity 1280.3846
-6800 train 7.204819 (lr=4.7509e-05) (hash(x)=158833881)
-6900 val loss 7.1392
-6900 val perplexity 1260.3671
-6900 train 7.168673 (lr=4.5676e-05) (hash(x)=164984909)
-7000 val loss 7.1415
-7000 val perplexity 1263.3184
-7000 train 7.052586 (lr=4.3882e-05) (hash(x)=151172493)
-7100 val loss 7.1452
-7100 val perplexity 1268.0675
-7100 train 7.115786 (lr=4.2128e-05) (hash(x)=154023633)
-7200 val loss 7.1357
-7200 val perplexity 1256.0227
-7200 train 7.033777 (lr=4.0414e-05) (hash(x)=139351250)
-7300 val loss 7.1326
-7300 val perplexity 1252.1747
-7300 train 7.107747 (lr=3.8745e-05) (hash(x)=158719458)
-7400 val loss 7.1271
-7400 val perplexity 1245.2498
-7400 train 7.238004 (lr=3.7120e-05) (hash(x)=162250810)
-7500 val loss 7.1292
-7500 val perplexity 1247.8699
-7500 train 7.009450 (lr=3.5541e-05) (hash(x)=142827557)
-7600 val loss 7.1253
-7600 val perplexity 1243.0115
-7600 train 7.086011 (lr=3.4011e-05) (hash(x)=147830801)
-7700 val loss 7.1211
-7700 val perplexity 1237.7563
-7700 train 7.250870 (lr=3.2531e-05) (hash(x)=165233314)
-7800 val loss 7.1121
-7800 val perplexity 1226.6871
-7800 train 7.139935 (lr=3.1102e-05) (hash(x)=160654717)
-7900 val loss 7.1114
-7900 val perplexity 1225.8656
-7900 train 6.983757 (lr=2.9726e-05) (hash(x)=140712586)
-8000 val loss 7.1024
-8000 val perplexity 1214.8588
-8000 train 7.194554 (lr=2.8405e-05) (hash(x)=150309061)
-8100 val loss 7.1089
-8100 val perplexity 1222.7528
-8100 train 7.132548 (lr=2.7138e-05) (hash(x)=158805466)
-8200 val loss 7.1218
-8200 val perplexity 1238.6584
-8200 train 7.008895 (lr=2.5929e-05) (hash(x)=142814078)
-8300 val loss 7.1024
-8300 val perplexity 1214.8744
-8300 train 7.062787 (lr=2.4778e-05) (hash(x)=148340417)
-8400 val loss 7.1228
-8400 val perplexity 1239.8574
-8400 train 7.071526 (lr=2.3686e-05) (hash(x)=157132639)
-8500 val loss 7.1088
-8500 val perplexity 1222.6788
-8500 train 7.080158 (lr=2.2655e-05) (hash(x)=152994240)
-8600 val loss 7.0931
-8600 val perplexity 1203.6345
-8600 train 7.056999 (lr=2.1685e-05) (hash(x)=152210154)
-8700 val loss 7.0910
-8700 val perplexity 1201.0780
-8700 train 7.111240 (lr=2.0777e-05) (hash(x)=149631247)
-8800 val loss 7.0898
-8800 val perplexity 1199.6757
-8800 train 6.979906 (lr=1.9933e-05) (hash(x)=145617784)
-8900 val loss 7.0862
-8900 val perplexity 1195.3822
-8900 train 7.186379 (lr=1.9153e-05) (hash(x)=155984970)
-9000 val loss 7.0871
-9000 val perplexity 1196.4508
-9000 train 6.891949 (lr=1.8439e-05) (hash(x)=143433013)
-9100 val loss 7.0887
-9100 val perplexity 1198.3727
-9100 train 7.001738 (lr=1.7790e-05) (hash(x)=152507533)
-9200 val loss 7.0833
-9200 val perplexity 1191.8619
-9200 train 7.180696 (lr=1.7208e-05) (hash(x)=162400650)
-9300 val loss 7.0836
-9300 val perplexity 1192.3075
-9300 train 7.095459 (lr=1.6692e-05) (hash(x)=148419068)
-9400 val loss 7.0818
-9400 val perplexity 1190.0691
-9400 train 7.052043 (lr=1.6245e-05) (hash(x)=145025169)
-9500 val loss 7.0796
-9500 val perplexity 1187.4729
-9500 train 7.045411 (lr=1.5865e-05) (hash(x)=145104235)
-9600 val loss 7.0753
-9600 val perplexity 1182.3990
-9600 train 7.012111 (lr=1.5554e-05) (hash(x)=150499797)
-9700 val loss 7.0802
-9700 val perplexity 1188.2228
-9700 train 6.903148 (lr=1.5312e-05) (hash(x)=139690022)
-9800 val loss 7.0721
-9800 val perplexity 1178.6555
-9800 train 7.046154 (lr=1.5139e-05) (hash(x)=145645279)
-9900 val loss 7.0681
-9900 val perplexity 1173.9249
-9900 train 7.007585 (lr=1.5035e-05) (hash(x)=150103432)
-9999 val loss 7.0719
-9999 val perplexity 1178.4133
+0 train 11.291462 (lr=2.5000e-07) (hash(x)=150724848)
+100 val loss 9.9021
+100 val perplexity 19971.4043
+100 train 9.951036 (lr=2.5250e-05) (hash(x)=149217061)
+200 val loss 9.0426
+200 val perplexity 8455.4004
+200 train 9.061214 (lr=5.0000e-05) (hash(x)=149910534)
+300 val loss 8.2609
+300 val perplexity 3869.5623
+300 train 8.200714 (lr=4.9988e-05) (hash(x)=142185643)
+400 val loss 7.8750
+400 val perplexity 2630.7476
+400 train 7.933581 (lr=4.9954e-05) (hash(x)=148123706)
+500 val loss 7.6928
+500 val perplexity 2192.4968
+500 train 7.587177 (lr=4.9896e-05) (hash(x)=149952383)
+600 val loss 7.5998
+600 val perplexity 1997.7612
+600 train 7.429729 (lr=4.9815e-05) (hash(x)=146678221)
+700 val loss 7.5332
+700 val perplexity 1869.1604
+700 train 7.564937 (lr=4.9712e-05) (hash(x)=156180736)
+800 val loss 7.4854
+800 val perplexity 1781.7887
+800 train 7.475163 (lr=4.9585e-05) (hash(x)=151700982)
+900 val loss 7.4627
+900 val perplexity 1741.8279
+900 train 7.287769 (lr=4.9436e-05) (hash(x)=147288467)
+1000 val loss 7.4293
+1000 val perplexity 1684.6075
+1000 train 7.493608 (lr=4.9264e-05) (hash(x)=156182087)
+1100 val loss 7.4115
+1100 val perplexity 1654.9202
+1100 train 7.291558 (lr=4.9070e-05) (hash(x)=147861550)
+1200 val loss 7.3934
+1200 val perplexity 1625.1987
+1200 train 7.271595 (lr=4.8854e-05) (hash(x)=149318660)
+1300 val loss 7.3770
+1300 val perplexity 1598.8241
+1300 train 7.238466 (lr=4.8616e-05) (hash(x)=151631103)
+1400 val loss 7.3561
+1400 val perplexity 1565.7140
+1400 train 7.398652 (lr=4.8356e-05) (hash(x)=150482428)
+1500 val loss 7.3213
+1500 val perplexity 1512.1592
+1500 train 7.179976 (lr=4.8074e-05) (hash(x)=144458617)
+1600 val loss 7.2966
+1600 val perplexity 1475.3420
+1600 train 7.183173 (lr=4.7772e-05) (hash(x)=143268605)
+1700 val loss 7.2711
+1700 val perplexity 1438.1329
+1700 train 7.207503 (lr=4.7448e-05) (hash(x)=150817011)
+1800 val loss 7.2608
+1800 val perplexity 1423.3289
+1800 train 7.258028 (lr=4.7105e-05) (hash(x)=152322423)
+1900 val loss 7.2405
+1900 val perplexity 1394.7865
+1900 train 7.084047 (lr=4.6741e-05) (hash(x)=143574126)
+2000 val loss 7.2436
+2000 val perplexity 1399.0969
+2000 train 7.032498 (lr=4.6357e-05) (hash(x)=147904298)
+2100 val loss 7.2320
+2100 val perplexity 1382.9723
+2100 train 7.020701 (lr=4.5954e-05) (hash(x)=144719400)
+2200 val loss 7.2250
+2200 val perplexity 1373.3562
+2200 train 7.456707 (lr=4.5532e-05) (hash(x)=154343147)
+2300 val loss 7.2107
+2300 val perplexity 1353.8384
+2300 train 7.265342 (lr=4.5091e-05) (hash(x)=154372635)
+2400 val loss 7.2090
+2400 val perplexity 1351.5879
+2400 train 7.050181 (lr=4.4633e-05) (hash(x)=141843115)
+2500 val loss 7.2026
+2500 val perplexity 1342.9486
+2500 train 7.107991 (lr=4.4156e-05) (hash(x)=149417679)
+2600 val loss 7.1948
+2600 val perplexity 1332.4684
+2600 train 7.049023 (lr=4.3663e-05) (hash(x)=145279030)
+2700 val loss 7.1873
+2700 val perplexity 1322.5468
+2700 train 7.072419 (lr=4.3153e-05) (hash(x)=146363911)
+2800 val loss 7.2067
+2800 val perplexity 1348.4304
+2800 train 7.189149 (lr=4.2627e-05) (hash(x)=152507639)
+2900 val loss 7.1827
+2900 val perplexity 1316.4399
+2900 train 6.992226 (lr=4.2085e-05) (hash(x)=139296191)
+3000 val loss 7.1794
+3000 val perplexity 1312.0787
+3000 train 7.195726 (lr=4.1529e-05) (hash(x)=148473774)
+3100 val loss 7.1690
+3100 val perplexity 1298.5116
+3100 train 7.270919 (lr=4.0957e-05) (hash(x)=154130527)
+3200 val loss 7.1663
+3200 val perplexity 1295.0099
+3200 train 7.096659 (lr=4.0373e-05) (hash(x)=151117002)
+3300 val loss 7.1644
+3300 val perplexity 1292.6143
+3300 train 7.261252 (lr=3.9775e-05) (hash(x)=151035383)
+3400 val loss 7.1602
+3400 val perplexity 1287.1372
+3400 train 7.248283 (lr=3.9164e-05) (hash(x)=150614749)
+3500 val loss 7.1526
+3500 val perplexity 1277.3953
+3500 train 7.218451 (lr=3.8541e-05) (hash(x)=151431391)
+3600 val loss 7.1520
+3600 val perplexity 1276.6804
+3600 train 7.085993 (lr=3.7907e-05) (hash(x)=151140797)
+3700 val loss 7.1450
+3700 val perplexity 1267.8009
+3700 train 7.379134 (lr=3.7262e-05) (hash(x)=154608340)
+3800 val loss 7.1462
+3800 val perplexity 1269.2732
+3800 train 7.149647 (lr=3.6608e-05) (hash(x)=169641309)
+3900 val loss 7.1452
+3900 val perplexity 1267.9592
+3900 train 7.087150 (lr=3.5944e-05) (hash(x)=150592479)
+4000 val loss 7.1453
+4000 val perplexity 1268.1244
+4000 train 7.118044 (lr=3.5271e-05) (hash(x)=155495317)
+4100 val loss 7.1474
+4100 val perplexity 1270.8580
+4100 train 7.334249 (lr=3.4590e-05) (hash(x)=170323970)
+4200 val loss 7.1551
+4200 val perplexity 1280.6582
+4200 train 6.968613 (lr=3.3902e-05) (hash(x)=140854360)
+4300 val loss 7.1448
+4300 val perplexity 1267.5548
+4300 train 7.138808 (lr=3.3207e-05) (hash(x)=148926644)
+4400 val loss 7.1432
+4400 val perplexity 1265.4563
+4400 train 7.182815 (lr=3.2507e-05) (hash(x)=149997871)
+4500 val loss 7.1313
+4500 val perplexity 1250.4640
+4500 train 7.091198 (lr=3.1801e-05) (hash(x)=149219289)
+4600 val loss 7.1326
+4600 val perplexity 1252.0677
+4600 train 6.935356 (lr=3.1091e-05) (hash(x)=139836197)
+4700 val loss 7.1366
+4700 val perplexity 1257.1642
+4700 train 7.099030 (lr=3.0377e-05) (hash(x)=150722883)
+4800 val loss 7.1298
+4800 val perplexity 1248.6515
+4800 train 7.321390 (lr=2.9661e-05) (hash(x)=166864648)
+4900 val loss 7.1284
+4900 val perplexity 1246.8326
+4900 train 7.231492 (lr=2.8942e-05) (hash(x)=157746743)
+5000 val loss 7.1262
+5000 val perplexity 1244.1138
+5000 train 7.084176 (lr=2.8221e-05) (hash(x)=153032257)
+5100 val loss 7.1329
+5100 val perplexity 1252.4446
+5100 train 7.217553 (lr=2.7500e-05) (hash(x)=151862371)
+5200 val loss 7.1284
+5200 val perplexity 1246.9265
+5200 train 7.419171 (lr=2.6779e-05) (hash(x)=168862995)
+5300 val loss 7.1254
+5300 val perplexity 1243.1229
+5300 train 7.094783 (lr=2.6058e-05) (hash(x)=148243567)
+5400 val loss 7.1269
+5400 val perplexity 1244.9808
+5400 train 7.061582 (lr=2.5339e-05) (hash(x)=145381742)
+5500 val loss 7.1190
+5500 val perplexity 1235.1892
+5500 train 7.284770 (lr=2.4623e-05) (hash(x)=155602174)
+5600 val loss 7.1170
+5600 val perplexity 1232.7450
+5600 train 7.213219 (lr=2.3909e-05) (hash(x)=160637672)
+5700 val loss 7.1150
+5700 val perplexity 1230.3080
+5700 train 7.074363 (lr=2.3199e-05) (hash(x)=152101131)
+5800 val loss 7.1187
+5800 val perplexity 1234.8164
+5800 train 7.082228 (lr=2.2493e-05) (hash(x)=152327089)
+5900 val loss 7.1184
+5900 val perplexity 1234.5280
+5900 train 7.041556 (lr=2.1793e-05) (hash(x)=151094543)
+6000 val loss 7.1203
+6000 val perplexity 1236.8035
+6000 train 7.216980 (lr=2.1098e-05) (hash(x)=160233315)
+6100 val loss 7.1131
+6100 val perplexity 1227.9484
+6100 train 7.200726 (lr=2.0410e-05) (hash(x)=158295337)
+6200 val loss 7.1109
+6200 val perplexity 1225.2181
+6200 train 7.066869 (lr=1.9729e-05) (hash(x)=151269817)
+6300 val loss 7.1129
+6300 val perplexity 1227.7365
+6300 train 7.188467 (lr=1.9056e-05) (hash(x)=156912756)
+6400 val loss 7.1081
+6400 val perplexity 1221.8523
+6400 train 7.044916 (lr=1.8392e-05) (hash(x)=125779637)
+6500 val loss 7.1168
+6500 val perplexity 1232.4882
+6500 train 7.028541 (lr=1.7738e-05) (hash(x)=148157661)
+6600 val loss 7.1085
+6600 val perplexity 1222.3086
+6600 train 7.161902 (lr=1.7093e-05) (hash(x)=152820049)
+6700 val loss 7.1078
+6700 val perplexity 1221.4958
+6700 train 7.072041 (lr=1.6459e-05) (hash(x)=155818717)
+6800 val loss 7.0971
+6800 val perplexity 1208.4872
+6800 train 7.173000 (lr=1.5836e-05) (hash(x)=158833881)
+6900 val loss 7.0994
+6900 val perplexity 1211.2847
+6900 train 7.156277 (lr=1.5225e-05) (hash(x)=164984909)
+7000 val loss 7.0956
+7000 val perplexity 1206.6462
+7000 train 6.990550 (lr=1.4627e-05) (hash(x)=151172493)
+7100 val loss 7.0936
+7100 val perplexity 1204.1770
+7100 train 7.043107 (lr=1.4043e-05) (hash(x)=154023633)
+7200 val loss 7.0892
+7200 val perplexity 1198.9299
+7200 train 6.971313 (lr=1.3471e-05) (hash(x)=139351250)
+7300 val loss 7.0875
+7300 val perplexity 1196.9484
+7300 train 7.047305 (lr=1.2915e-05) (hash(x)=158719458)
+7400 val loss 7.0860
+7400 val perplexity 1195.0596
+7400 train 7.210918 (lr=1.2373e-05) (hash(x)=162250810)
+7500 val loss 7.0863
+7500 val perplexity 1195.4512
+7500 train 6.927986 (lr=1.1847e-05) (hash(x)=142827557)
+7600 val loss 7.0807
+7600 val perplexity 1188.7521
+7600 train 7.012818 (lr=1.1337e-05) (hash(x)=147830801)
+7700 val loss 7.0813
+7700 val perplexity 1189.5392
+7700 train 7.220162 (lr=1.0844e-05) (hash(x)=165233314)
+7800 val loss 7.0805
+7800 val perplexity 1188.5917
+7800 train 7.086352 (lr=1.0367e-05) (hash(x)=160654717)
+7900 val loss 7.0822
+7900 val perplexity 1190.5776
+7900 train 6.959390 (lr=9.9088e-06) (hash(x)=140712586)
+8000 val loss 7.0740
+8000 val perplexity 1180.8348
+8000 train 7.150942 (lr=9.4682e-06) (hash(x)=150309061)
+8100 val loss 7.0724
+8100 val perplexity 1178.9276
+8100 train 7.109391 (lr=9.0461e-06) (hash(x)=158805466)
+8200 val loss 7.0690
+8200 val perplexity 1174.9840
+8200 train 6.911159 (lr=8.6430e-06) (hash(x)=142814078)
+8300 val loss 7.0671
+8300 val perplexity 1172.7701
+8300 train 7.025698 (lr=8.2593e-06) (hash(x)=148340417)
+8400 val loss 7.0684
+8400 val perplexity 1174.2312
+8400 train 7.020465 (lr=7.8953e-06) (hash(x)=157132639)
+8500 val loss 7.0645
+8500 val perplexity 1169.6957
+8500 train 7.028442 (lr=7.5515e-06) (hash(x)=152994240)
+8600 val loss 7.0610
+8600 val perplexity 1165.5823
+8600 train 6.998075 (lr=7.2282e-06) (hash(x)=152210154)
+8700 val loss 7.0611
+8700 val perplexity 1165.7090
+8700 train 7.082347 (lr=6.9257e-06) (hash(x)=149631247)
+8800 val loss 7.0584
+8800 val perplexity 1162.5870
+8800 train 6.921595 (lr=6.6444e-06) (hash(x)=145617784)
+8900 val loss 7.0569
+8900 val perplexity 1160.7957
+8900 train 7.155268 (lr=6.3845e-06) (hash(x)=155984970)
+9000 val loss 7.0565
+9000 val perplexity 1160.4248
+9000 train 6.833085 (lr=6.1462e-06) (hash(x)=143433013)
+9100 val loss 7.0541
+9100 val perplexity 1157.6333
+9100 train 6.952363 (lr=5.9300e-06) (hash(x)=152507533)
+9200 val loss 7.0541
+9200 val perplexity 1157.5908
+9200 train 7.204437 (lr=5.7359e-06) (hash(x)=162400650)
+9300 val loss 7.0508
+9300 val perplexity 1153.7626
+9300 train 7.087835 (lr=5.5641e-06) (hash(x)=148419068)
+9400 val loss 7.0555
+9400 val perplexity 1159.2208
+9400 train 7.023118 (lr=5.4149e-06) (hash(x)=145025169)
+9500 val loss 7.0498
+9500 val perplexity 1152.5929
+9500 train 7.014523 (lr=5.2884e-06) (hash(x)=145104235)
+9600 val loss 7.0501
+9600 val perplexity 1152.9315
+9600 train 6.962839 (lr=5.1847e-06) (hash(x)=150499797)
+9700 val loss 7.0489
+9700 val perplexity 1151.5975
+9700 train 6.860659 (lr=5.1040e-06) (hash(x)=139690022)
+9800 val loss 7.0456
+9800 val perplexity 1147.8142
+9800 train 7.011253 (lr=5.0462e-06) (hash(x)=145645279)
+9900 val loss 7.0449
+9900 val perplexity 1146.9382
+9900 train 6.967855 (lr=5.0116e-06) (hash(x)=150103432)
+9999 val loss 7.0456
+9999 val perplexity 1147.8422

attention_kindself_n_heads4_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ba11bae78a2338db851c51b584bdbb28e92c38db347f007eb25550ca636a5f2
+oid sha256:a874ade60cf3c0753973366d6e7d2b341ee44b01a89abdf29183b09378667ed0
 size 92843394

attention_kindself_n_heads4_seed1339/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e60a13c58b3b32a653aff34dc26a7de2f820000fb6a45b551f251151a5e24f1c
+oid sha256:efc27109d173b6b4765cceff5fb4a95018ffd213c0cce94727a474679efce861
 size 92843394

attention_kindself_n_heads4_seed1339/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9569b576688f5ed68359c15b1252b68c654b9916fd3e04372fd2a8c76fc8ce3b
+oid sha256:7fb37a20573e497ce0e47b0220aad7a6a21093349418d5f6759a537c004f0dd4
 size 92843394

attention_kindself_n_heads4_seed1339/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6e86c352cf79dc3191b93412d0de0c33ee277ff7d480cf26f37ebb1eac08792
+oid sha256:5aba8d6fd761f326db45b1851d53b9054e3b5e0e1311bb3f97daecec7952bafa
 size 92843394

attention_kindself_n_heads4_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c5e423b3da9278fb38f4bb6d9d37eaa0dce092cbc9b59be54e83223283d8bc77
+oid sha256:de51f73152f1ec589219a7082771385aca728c448cf76ed0dc302da5eef79d35
 size 179406214

attention_kindself_n_heads4_seed1339/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c19594c4ee6043c95ad102a7ff571eda191ed6f3aeb274b9417977c9a2242d44
+oid sha256:5a56dfec2a17c48c414efd19d1a8ab1e4f0643dcb99d3ef53c5c78f4747dc5b9
 size 179406214

attention_kindself_n_heads4_seed1339/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be4742da3573b50a7ab496bebc83225e31d357a635d1f6fc9507877675197280
+oid sha256:dbbb78f04809887baa6eeed69c48402feb12dc5dd09cae80923992f23b48a3f4
 size 179406214

attention_kindself_n_heads4_seed1339/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c2bac616ceb7bc96fb3c0c662e9f3bffd6b79eb542cfae6fa36a9638b3c32075
+oid sha256:2f95950c9e251e960a028614648249dddff2253ad04c3767de54f848a62d0d3b
 size 179406214