Upload folder using huggingface_hub

attention_kindself_n_heads2_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_3/attention_kindself_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_3", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1339", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1339", "n_embd": 128}

attention_kindself_n_heads2_seed1339/log2.txt

@@ -2,302 +2,302 @@ max_steps: 10000
 0 val loss 11.7353
 0 val perplexity 124899.7500
 0 train 11.734357 (lr=7.5000e-07) (hash(x)=150724848)
-100 val loss 9.7726
-100 val perplexity 17545.4785
-100 train 9.813877 (lr=7.5750e-05) (hash(x)=149217061)
-200 val loss 8.0307
-200 val perplexity 3073.7659
-200 train 8.014332 (lr=1.5000e-04) (hash(x)=149910534)
-300 val loss 7.6553
-300 val perplexity 2111.8521
-300 train 7.533482 (lr=1.4997e-04) (hash(x)=142185643)
-400 val loss 7.5761
-400 val perplexity 1950.9631
-400 train 7.601968 (lr=1.4986e-04) (hash(x)=148123706)
-500 val loss 7.4882
-500 val perplexity 1786.7975
-500 train 7.325721 (lr=1.4969e-04) (hash(x)=149952383)
-600 val loss 7.4041
-600 val perplexity 1642.7119
-600 train 7.203185 (lr=1.4945e-04) (hash(x)=146678221)
-700 val loss 7.3024
-700 val perplexity 1483.8464
-700 train 7.340023 (lr=1.4913e-04) (hash(x)=156180736)
-800 val loss 7.2771
-800 val perplexity 1446.7216
-800 train 7.261787 (lr=1.4876e-04) (hash(x)=151700982)
-900 val loss 7.2097
-900 val perplexity 1352.5447
-900 train 6.998355 (lr=1.4831e-04) (hash(x)=147288467)
-1000 val loss 7.1825
-1000 val perplexity 1316.2076
-1000 train 7.251370 (lr=1.4779e-04) (hash(x)=156182087)
-1100 val loss 7.1756
-1100 val perplexity 1307.2065
-1100 train 7.030709 (lr=1.4721e-04) (hash(x)=147861550)
-1200 val loss 7.1420
-1200 val perplexity 1263.9703
-1200 train 6.998680 (lr=1.4656e-04) (hash(x)=149318660)
-1300 val loss 7.1447
-1300 val perplexity 1267.4147
-1300 train 6.989931 (lr=1.4585e-04) (hash(x)=151631103)
-1400 val loss 7.1348
-1400 val perplexity 1254.8351
-1400 train 7.153018 (lr=1.4507e-04) (hash(x)=150482428)
-1500 val loss 7.1031
-1500 val perplexity 1215.7200
-1500 train 6.968789 (lr=1.4422e-04) (hash(x)=144458617)
-1600 val loss 7.0542
-1600 val perplexity 1157.7150
-1600 train 6.957231 (lr=1.4332e-04) (hash(x)=143268605)
-1700 val loss 7.0536
-1700 val perplexity 1157.0317
-1700 train 6.998997 (lr=1.4235e-04) (hash(x)=150817011)
-1800 val loss 7.0532
-1800 val perplexity 1156.5905
-1800 train 7.054293 (lr=1.4131e-04) (hash(x)=152322423)
-1900 val loss 7.0219
-1900 val perplexity 1120.9279
-1900 train 6.839796 (lr=1.4022e-04) (hash(x)=143574126)
-2000 val loss 7.0374
-2000 val perplexity 1138.4637
-2000 train 6.824109 (lr=1.3907e-04) (hash(x)=147904298)
-2100 val loss 7.0565
-2100 val perplexity 1160.4254
-2100 train 6.824200 (lr=1.3786e-04) (hash(x)=144719400)
-2200 val loss 7.0406
-2200 val perplexity 1142.0664
-2200 train 7.274655 (lr=1.3660e-04) (hash(x)=154343147)
-2300 val loss 7.0282
-2300 val perplexity 1128.0421
-2300 train 7.071688 (lr=1.3527e-04) (hash(x)=154372635)
-2400 val loss 7.0393
-2400 val perplexity 1140.5922
-2400 train 6.903203 (lr=1.3390e-04) (hash(x)=141843115)
-2500 val loss 7.0292
-2500 val perplexity 1129.1168
-2500 train 6.928387 (lr=1.3247e-04) (hash(x)=149417679)
-2600 val loss 7.0380
-2600 val perplexity 1139.1198
-2600 train 6.913923 (lr=1.3099e-04) (hash(x)=145279030)
-2700 val loss 7.0337
-2700 val perplexity 1134.2703
-2700 train 6.934816 (lr=1.2946e-04) (hash(x)=146363911)
-2800 val loss 7.0472
-2800 val perplexity 1149.6542
-2800 train 7.011040 (lr=1.2788e-04) (hash(x)=152507639)
-2900 val loss 7.0594
-2900 val perplexity 1163.7246
-2900 train 6.877071 (lr=1.2626e-04) (hash(x)=139296191)
-3000 val loss 7.0639
-3000 val perplexity 1169.0483
-3000 train 7.075006 (lr=1.2459e-04) (hash(x)=148473774)
-3100 val loss 7.0261
-3100 val perplexity 1125.5903
-3100 train 7.102339 (lr=1.2287e-04) (hash(x)=154130527)
-3200 val loss 7.0256
-3200 val perplexity 1125.0581
-3200 train 6.960464 (lr=1.2112e-04) (hash(x)=151117002)
-3300 val loss 7.0234
-3300 val perplexity 1122.5508
-3300 train 7.141053 (lr=1.1932e-04) (hash(x)=151035383)
-3400 val loss 7.0216
-3400 val perplexity 1120.6094
-3400 train 7.131077 (lr=1.1749e-04) (hash(x)=150614749)
-3500 val loss 7.0022
-3500 val perplexity 1099.0701
-3500 train 7.045189 (lr=1.1562e-04) (hash(x)=151431391)
-3600 val loss 7.0122
-3600 val perplexity 1110.0785
-3600 train 6.969791 (lr=1.1372e-04) (hash(x)=151140797)
-3700 val loss 7.0076
-3700 val perplexity 1104.9702
-3700 train 7.197464 (lr=1.1179e-04) (hash(x)=154608340)
-3800 val loss 7.0241
-3800 val perplexity 1123.4337
-3800 train 7.019115 (lr=1.0982e-04) (hash(x)=169641309)
-3900 val loss 7.0132
-3900 val perplexity 1111.1599
-3900 train 6.957850 (lr=1.0783e-04) (hash(x)=150592479)
-4000 val loss 6.9882
-4000 val perplexity 1083.7604
-4000 train 6.972362 (lr=1.0581e-04) (hash(x)=155495317)
-4100 val loss 6.9885
-4100 val perplexity 1084.1361
-4100 train 7.163860 (lr=1.0377e-04) (hash(x)=170323970)
-4200 val loss 6.9760
-4200 val perplexity 1070.6144
-4200 train 6.787168 (lr=1.0171e-04) (hash(x)=140854360)
-4300 val loss 6.9719
-4300 val perplexity 1066.2782
-4300 train 6.963425 (lr=9.9622e-05) (hash(x)=148926644)
-4400 val loss 6.9518
-4400 val perplexity 1044.9806
-4400 train 6.990151 (lr=9.7520e-05) (hash(x)=149997871)
-4500 val loss 6.9349
-4500 val perplexity 1027.5555
-4500 train 6.909260 (lr=9.5403e-05) (hash(x)=149219289)
-4600 val loss 6.9528
-4600 val perplexity 1046.0680
-4600 train 6.752494 (lr=9.3273e-05) (hash(x)=139836197)
-4700 val loss 6.9502
-4700 val perplexity 1043.3162
-4700 train 6.911660 (lr=9.1132e-05) (hash(x)=150722883)
-4800 val loss 6.9306
-4800 val perplexity 1023.1166
-4800 train 7.115446 (lr=8.8982e-05) (hash(x)=166864648)
-4900 val loss 6.9272
-4900 val perplexity 1019.5853
-4900 train 7.025172 (lr=8.6825e-05) (hash(x)=157746743)
-5000 val loss 6.9264
-5000 val perplexity 1018.8072
-5000 train 6.886820 (lr=8.4663e-05) (hash(x)=153032257)
-5100 val loss 6.9071
-5100 val perplexity 999.3232
-5100 train 6.971920 (lr=8.2500e-05) (hash(x)=151862371)
-5200 val loss 6.9168
-5200 val perplexity 1009.1252
-5200 train 7.182480 (lr=8.0337e-05) (hash(x)=168862995)
-5300 val loss 6.9219
-5300 val perplexity 1014.2078
-5300 train 6.877322 (lr=7.8175e-05) (hash(x)=148243567)
-5400 val loss 6.9206
-5400 val perplexity 1012.9241
-5400 train 6.864289 (lr=7.6018e-05) (hash(x)=145381742)
-5500 val loss 6.9204
-5500 val perplexity 1012.7020
-5500 train 7.048699 (lr=7.3868e-05) (hash(x)=155602174)
-5600 val loss 6.9294
-5600 val perplexity 1021.8431
-5600 train 7.007695 (lr=7.1727e-05) (hash(x)=160637672)
-5700 val loss 6.9148
-5700 val perplexity 1007.0419
-5700 train 6.867439 (lr=6.9597e-05) (hash(x)=152101131)
-5800 val loss 6.9122
-5800 val perplexity 1004.4901
-5800 train 6.883580 (lr=6.7480e-05) (hash(x)=152327089)
-5900 val loss 6.9251
-5900 val perplexity 1017.4454
-5900 train 6.865512 (lr=6.5378e-05) (hash(x)=151094543)
-6000 val loss 6.9199
-6000 val perplexity 1012.2385
-6000 train 6.998046 (lr=6.3294e-05) (hash(x)=160233315)
-6100 val loss 6.9092
-6100 val perplexity 1001.4684
-6100 train 6.989634 (lr=6.1230e-05) (hash(x)=158295337)
-6200 val loss 6.9051
-6200 val perplexity 997.3576
-6200 train 6.864357 (lr=5.9188e-05) (hash(x)=151269817)
-6300 val loss 6.9087
-6300 val perplexity 1000.9571
-6300 train 6.993814 (lr=5.7169e-05) (hash(x)=156912756)
-6400 val loss 6.9069
-6400 val perplexity 999.1902
-6400 train 6.829133 (lr=5.5177e-05) (hash(x)=125779637)
-6500 val loss 6.9165
-6500 val perplexity 1008.7678
-6500 train 6.815157 (lr=5.3213e-05) (hash(x)=148157661)
-6600 val loss 6.9218
-6600 val perplexity 1014.1851
-6600 train 6.967198 (lr=5.1279e-05) (hash(x)=152820049)
-6700 val loss 6.9065
-6700 val perplexity 998.7644
-6700 train 6.869339 (lr=4.9377e-05) (hash(x)=155818717)
-6800 val loss 6.9026
-6800 val perplexity 994.8915
-6800 train 6.959887 (lr=4.7509e-05) (hash(x)=158833881)
-6900 val loss 6.8969
-6900 val perplexity 989.2281
-6900 train 6.941722 (lr=4.5676e-05) (hash(x)=164984909)
-7000 val loss 6.8960
-7000 val perplexity 988.3021
-7000 train 6.765996 (lr=4.3882e-05) (hash(x)=151172493)
-7100 val loss 6.8978
-7100 val perplexity 990.1153
-7100 train 6.869718 (lr=4.2128e-05) (hash(x)=154023633)
-7200 val loss 6.9033
-7200 val perplexity 995.5060
-7200 train 6.771121 (lr=4.0414e-05) (hash(x)=139351250)
-7300 val loss 6.8981
-7300 val perplexity 990.3571
-7300 train 6.861828 (lr=3.8745e-05) (hash(x)=158719458)
-7400 val loss 6.8909
-7400 val perplexity 983.3278
-7400 train 7.006841 (lr=3.7120e-05) (hash(x)=162250810)
-7500 val loss 6.8890
-7500 val perplexity 981.3918
-7500 train 6.747966 (lr=3.5541e-05) (hash(x)=142827557)
-7600 val loss 6.8887
-7600 val perplexity 981.1364
-7600 train 6.809592 (lr=3.4011e-05) (hash(x)=147830801)
-7700 val loss 6.8842
-7700 val perplexity 976.6763
-7700 train 6.998799 (lr=3.2531e-05) (hash(x)=165233314)
-7800 val loss 6.8781
-7800 val perplexity 970.7328
-7800 train 6.888107 (lr=3.1102e-05) (hash(x)=160654717)
-7900 val loss 6.8828
-7900 val perplexity 975.3890
-7900 train 6.760226 (lr=2.9726e-05) (hash(x)=140712586)
-8000 val loss 6.8813
-8000 val perplexity 973.8698
-8000 train 6.973397 (lr=2.8405e-05) (hash(x)=150309061)
-8100 val loss 6.8759
-8100 val perplexity 968.6202
-8100 train 6.917377 (lr=2.7138e-05) (hash(x)=158805466)
-8200 val loss 6.8768
-8200 val perplexity 969.5352
-8200 train 6.721768 (lr=2.5929e-05) (hash(x)=142814078)
-8300 val loss 6.8779
-8300 val perplexity 970.5875
-8300 train 6.822116 (lr=2.4778e-05) (hash(x)=148340417)
-8400 val loss 6.8837
-8400 val perplexity 976.2219
-8400 train 6.829408 (lr=2.3686e-05) (hash(x)=157132639)
-8500 val loss 6.8814
-8500 val perplexity 973.9942
-8500 train 6.841990 (lr=2.2655e-05) (hash(x)=152994240)
-8600 val loss 6.8780
-8600 val perplexity 970.6819
-8600 train 6.828690 (lr=2.1685e-05) (hash(x)=152210154)
-8700 val loss 6.8737
-8700 val perplexity 966.5657
-8700 train 6.898990 (lr=2.0777e-05) (hash(x)=149631247)
-8800 val loss 6.8730
-8800 val perplexity 965.8230
-8800 train 6.727984 (lr=1.9933e-05) (hash(x)=145617784)
-8900 val loss 6.8711
-8900 val perplexity 964.0148
-8900 train 6.975079 (lr=1.9153e-05) (hash(x)=155984970)
-9000 val loss 6.8723
-9000 val perplexity 965.1191
-9000 train 6.648523 (lr=1.8439e-05) (hash(x)=143433013)
-9100 val loss 6.8737
-9100 val perplexity 966.4919
-9100 train 6.767800 (lr=1.7790e-05) (hash(x)=152507533)
-9200 val loss 6.8703
-9200 val perplexity 963.2626
-9200 train 6.991444 (lr=1.7208e-05) (hash(x)=162400650)
-9300 val loss 6.8667
-9300 val perplexity 959.7634
-9300 train 6.878949 (lr=1.6692e-05) (hash(x)=148419068)
-9400 val loss 6.8682
-9400 val perplexity 961.1745
-9400 train 6.837451 (lr=1.6245e-05) (hash(x)=145025169)
-9500 val loss 6.8684
-9500 val perplexity 961.3817
-9500 train 6.826965 (lr=1.5865e-05) (hash(x)=145104235)
-9600 val loss 6.8702
-9600 val perplexity 963.1565
-9600 train 6.798606 (lr=1.5554e-05) (hash(x)=150499797)
-9700 val loss 6.8707
-9700 val perplexity 963.6577
-9700 train 6.689303 (lr=1.5312e-05) (hash(x)=139690022)
-9800 val loss 6.8689
-9800 val perplexity 961.9287
-9800 train 6.852724 (lr=1.5139e-05) (hash(x)=145645279)
-9900 val loss 6.8711
-9900 val perplexity 964.0272
-9900 train 6.785625 (lr=1.5035e-05) (hash(x)=150103432)
-9999 val loss 6.8705
-9999 val perplexity 963.4500
+100 val loss 9.7906
+100 val perplexity 17864.8477
+100 train 9.831616 (lr=7.5750e-05) (hash(x)=149217061)
+200 val loss 7.9135
+200 val perplexity 2733.8181
+200 train 7.903269 (lr=1.5000e-04) (hash(x)=149910534)
+300 val loss 7.6195
+300 val perplexity 2037.6289
+300 train 7.489441 (lr=1.4997e-04) (hash(x)=142185643)
+400 val loss 7.5327
+400 val perplexity 1868.0796
+400 train 7.563246 (lr=1.4986e-04) (hash(x)=148123706)
+500 val loss 7.4229
+500 val perplexity 1673.8356
+500 train 7.254730 (lr=1.4969e-04) (hash(x)=149952383)
+600 val loss 7.3229
+600 val perplexity 1514.5962
+600 train 7.112319 (lr=1.4945e-04) (hash(x)=146678221)
+700 val loss 7.2195
+700 val perplexity 1365.8119
+700 train 7.256211 (lr=1.4913e-04) (hash(x)=156180736)
+800 val loss 7.1607
+800 val perplexity 1287.7596
+800 train 7.143748 (lr=1.4876e-04) (hash(x)=151700982)
+900 val loss 7.1306
+900 val perplexity 1249.6790
+900 train 6.930069 (lr=1.4831e-04) (hash(x)=147288467)
+1000 val loss 7.1058
+1000 val perplexity 1218.9619
+1000 train 7.177827 (lr=1.4779e-04) (hash(x)=156182087)
+1100 val loss 7.0983
+1100 val perplexity 1209.8513
+1100 train 6.947954 (lr=1.4721e-04) (hash(x)=147861550)
+1200 val loss 7.0880
+1200 val perplexity 1197.5005
+1200 train 6.959338 (lr=1.4656e-04) (hash(x)=149318660)
+1300 val loss 7.1193
+1300 val perplexity 1235.6134
+1300 train 6.947792 (lr=1.4585e-04) (hash(x)=151631103)
+1400 val loss 7.1081
+1400 val perplexity 1221.8605
+1400 train 7.125467 (lr=1.4507e-04) (hash(x)=150482428)
+1500 val loss 7.0467
+1500 val perplexity 1149.0096
+1500 train 6.902546 (lr=1.4422e-04) (hash(x)=144458617)
+1600 val loss 7.0057
+1600 val perplexity 1102.9209
+1600 train 6.918149 (lr=1.4332e-04) (hash(x)=143268605)
+1700 val loss 6.9909
+1700 val perplexity 1086.6871
+1700 train 6.951296 (lr=1.4235e-04) (hash(x)=150817011)
+1800 val loss 6.9886
+1800 val perplexity 1084.1615
+1800 train 6.995378 (lr=1.4131e-04) (hash(x)=152322423)
+1900 val loss 6.9769
+1900 val perplexity 1071.6195
+1900 train 6.790772 (lr=1.4022e-04) (hash(x)=143574126)
+2000 val loss 6.9831
+2000 val perplexity 1078.2130
+2000 train 6.761238 (lr=1.3907e-04) (hash(x)=147904298)
+2100 val loss 6.9866
+2100 val perplexity 1082.0898
+2100 train 6.752719 (lr=1.3786e-04) (hash(x)=144719400)
+2200 val loss 6.9856
+2200 val perplexity 1081.0084
+2200 train 7.212669 (lr=1.3660e-04) (hash(x)=154343147)
+2300 val loss 6.9550
+2300 val perplexity 1048.3564
+2300 train 7.002470 (lr=1.3527e-04) (hash(x)=154372635)
+2400 val loss 6.9724
+2400 val perplexity 1066.8245
+2400 train 6.826124 (lr=1.3390e-04) (hash(x)=141843115)
+2500 val loss 6.9567
+2500 val perplexity 1050.2083
+2500 train 6.861791 (lr=1.3247e-04) (hash(x)=149417679)
+2600 val loss 6.9538
+2600 val perplexity 1047.0795
+2600 train 6.827319 (lr=1.3099e-04) (hash(x)=145279030)
+2700 val loss 6.9492
+2700 val perplexity 1042.3495
+2700 train 6.853619 (lr=1.2946e-04) (hash(x)=146363911)
+2800 val loss 6.9600
+2800 val perplexity 1053.5844
+2800 train 6.925730 (lr=1.2788e-04) (hash(x)=152507639)
+2900 val loss 6.9717
+2900 val perplexity 1066.0504
+2900 train 6.777905 (lr=1.2626e-04) (hash(x)=139296191)
+3000 val loss 6.9777
+3000 val perplexity 1072.5018
+3000 train 6.983221 (lr=1.2459e-04) (hash(x)=148473774)
+3100 val loss 6.9501
+3100 val perplexity 1043.2361
+3100 train 7.037889 (lr=1.2287e-04) (hash(x)=154130527)
+3200 val loss 6.9663
+3200 val perplexity 1060.2772
+3200 train 6.903945 (lr=1.2112e-04) (hash(x)=151117002)
+3300 val loss 6.9904
+3300 val perplexity 1086.1484
+3300 train 7.110773 (lr=1.1932e-04) (hash(x)=151035383)
+3400 val loss 6.9912
+3400 val perplexity 1087.0308
+3400 train 7.093910 (lr=1.1749e-04) (hash(x)=150614749)
+3500 val loss 6.9915
+3500 val perplexity 1087.3533
+3500 train 7.043681 (lr=1.1562e-04) (hash(x)=151431391)
+3600 val loss 7.0193
+3600 val perplexity 1118.0074
+3600 train 6.978808 (lr=1.1372e-04) (hash(x)=151140797)
+3700 val loss 6.9775
+3700 val perplexity 1072.2477
+3700 train 7.151230 (lr=1.1179e-04) (hash(x)=154608340)
+3800 val loss 6.9801
+3800 val perplexity 1075.0132
+3800 train 6.973941 (lr=1.0982e-04) (hash(x)=169641309)
+3900 val loss 6.9675
+3900 val perplexity 1061.6168
+3900 train 6.916593 (lr=1.0783e-04) (hash(x)=150592479)
+4000 val loss 6.9695
+4000 val perplexity 1063.6959
+4000 train 6.955076 (lr=1.0581e-04) (hash(x)=155495317)
+4100 val loss 6.9653
+4100 val perplexity 1059.2281
+4100 train 7.145298 (lr=1.0377e-04) (hash(x)=170323970)
+4200 val loss 6.9714
+4200 val perplexity 1065.6687
+4200 train 6.779059 (lr=1.0171e-04) (hash(x)=140854360)
+4300 val loss 6.9555
+4300 val perplexity 1048.8595
+4300 train 6.944369 (lr=9.9622e-05) (hash(x)=148926644)
+4400 val loss 6.9527
+4400 val perplexity 1045.9672
+4400 train 6.997404 (lr=9.7520e-05) (hash(x)=149997871)
+4500 val loss 6.9458
+4500 val perplexity 1038.7681
+4500 train 6.908257 (lr=9.5403e-05) (hash(x)=149219289)
+4600 val loss 6.9330
+4600 val perplexity 1025.5349
+4600 train 6.739017 (lr=9.3273e-05) (hash(x)=139836197)
+4700 val loss 6.9361
+4700 val perplexity 1028.7106
+4700 train 6.893769 (lr=9.1132e-05) (hash(x)=150722883)
+4800 val loss 6.9405
+4800 val perplexity 1033.2831
+4800 train 7.139781 (lr=8.8982e-05) (hash(x)=166864648)
+4900 val loss 6.9288
+4900 val perplexity 1021.2498
+4900 train 7.030507 (lr=8.6825e-05) (hash(x)=157746743)
+5000 val loss 6.9255
+5000 val perplexity 1017.8948
+5000 train 6.881103 (lr=8.4663e-05) (hash(x)=153032257)
+5100 val loss 6.9227
+5100 val perplexity 1015.0491
+5100 train 6.985375 (lr=8.2500e-05) (hash(x)=151862371)
+5200 val loss 6.9293
+5200 val perplexity 1021.7968
+5200 train 7.186607 (lr=8.0337e-05) (hash(x)=168862995)
+5300 val loss 6.9301
+5300 val perplexity 1022.5923
+5300 train 6.893943 (lr=7.8175e-05) (hash(x)=148243567)
+5400 val loss 6.9281
+5400 val perplexity 1020.5435
+5400 train 6.853803 (lr=7.6018e-05) (hash(x)=145381742)
+5500 val loss 6.9387
+5500 val perplexity 1031.3790
+5500 train 7.071217 (lr=7.3868e-05) (hash(x)=155602174)
+5600 val loss 6.9308
+5600 val perplexity 1023.2815
+5600 train 7.025819 (lr=7.1727e-05) (hash(x)=160637672)
+5700 val loss 6.9346
+5700 val perplexity 1027.1803
+5700 train 6.879603 (lr=6.9597e-05) (hash(x)=152101131)
+5800 val loss 6.9140
+5800 val perplexity 1006.3104
+5800 train 6.884543 (lr=6.7480e-05) (hash(x)=152327089)
+5900 val loss 6.9178
+5900 val perplexity 1010.0659
+5900 train 6.851806 (lr=6.5378e-05) (hash(x)=151094543)
+6000 val loss 6.9214
+6000 val perplexity 1013.7050
+6000 train 7.002866 (lr=6.3294e-05) (hash(x)=160233315)
+6100 val loss 6.9204
+6100 val perplexity 1012.6880
+6100 train 7.003745 (lr=6.1230e-05) (hash(x)=158295337)
+6200 val loss 6.9196
+6200 val perplexity 1011.9175
+6200 train 6.873456 (lr=5.9188e-05) (hash(x)=151269817)
+6300 val loss 6.9151
+6300 val perplexity 1007.4050
+6300 train 6.991065 (lr=5.7169e-05) (hash(x)=156912756)
+6400 val loss 6.9218
+6400 val perplexity 1014.1149
+6400 train 6.839902 (lr=5.5177e-05) (hash(x)=125779637)
+6500 val loss 6.9173
+6500 val perplexity 1009.5565
+6500 train 6.820065 (lr=5.3213e-05) (hash(x)=148157661)
+6600 val loss 6.9251
+6600 val perplexity 1017.4473
+6600 train 6.981798 (lr=5.1279e-05) (hash(x)=152820049)
+6700 val loss 6.9197
+6700 val perplexity 1011.9740
+6700 train 6.877532 (lr=4.9377e-05) (hash(x)=155818717)
+6800 val loss 6.9144
+6800 val perplexity 1006.6727
+6800 train 6.965318 (lr=4.7509e-05) (hash(x)=158833881)
+6900 val loss 6.9193
+6900 val perplexity 1011.6271
+6900 train 6.940590 (lr=4.5676e-05) (hash(x)=164984909)
+7000 val loss 6.9143
+7000 val perplexity 1006.5340
+7000 train 6.805004 (lr=4.3882e-05) (hash(x)=151172493)
+7100 val loss 6.9159
+7100 val perplexity 1008.1864
+7100 train 6.888276 (lr=4.2128e-05) (hash(x)=154023633)
+7200 val loss 6.9209
+7200 val perplexity 1013.2715
+7200 train 6.802105 (lr=4.0414e-05) (hash(x)=139351250)
+7300 val loss 6.9159
+7300 val perplexity 1008.2215
+7300 train 6.877492 (lr=3.8745e-05) (hash(x)=158719458)
+7400 val loss 6.9136
+7400 val perplexity 1005.8325
+7400 train 7.032726 (lr=3.7120e-05) (hash(x)=162250810)
+7500 val loss 6.9135
+7500 val perplexity 1005.7472
+7500 train 6.775838 (lr=3.5541e-05) (hash(x)=142827557)
+7600 val loss 6.9058
+7600 val perplexity 998.0494
+7600 train 6.817025 (lr=3.4011e-05) (hash(x)=147830801)
+7700 val loss 6.9035
+7700 val perplexity 995.7875
+7700 train 7.017764 (lr=3.2531e-05) (hash(x)=165233314)
+7800 val loss 6.9087
+7800 val perplexity 1000.9251
+7800 train 6.919121 (lr=3.1102e-05) (hash(x)=160654717)
+7900 val loss 6.9144
+7900 val perplexity 1006.6353
+7900 train 6.797275 (lr=2.9726e-05) (hash(x)=140712586)
+8000 val loss 6.9056
+8000 val perplexity 997.8057
+8000 train 7.014383 (lr=2.8405e-05) (hash(x)=150309061)
+8100 val loss 6.9044
+8100 val perplexity 996.6664
+8100 train 6.935093 (lr=2.7138e-05) (hash(x)=158805466)
+8200 val loss 6.9134
+8200 val perplexity 1005.6608
+8200 train 6.762922 (lr=2.5929e-05) (hash(x)=142814078)
+8300 val loss 6.9123
+8300 val perplexity 1004.5633
+8300 train 6.855933 (lr=2.4778e-05) (hash(x)=148340417)
+8400 val loss 6.9173
+8400 val perplexity 1009.5709
+8400 train 6.869319 (lr=2.3686e-05) (hash(x)=157132639)
+8500 val loss 6.9167
+8500 val perplexity 1008.9866
+8500 train 6.875008 (lr=2.2655e-05) (hash(x)=152994240)
+8600 val loss 6.9130
+8600 val perplexity 1005.2601
+8600 train 6.867662 (lr=2.1685e-05) (hash(x)=152210154)
+8700 val loss 6.9102
+8700 val perplexity 1002.4277
+8700 train 6.945439 (lr=2.0777e-05) (hash(x)=149631247)
+8800 val loss 6.9110
+8800 val perplexity 1003.2904
+8800 train 6.752805 (lr=1.9933e-05) (hash(x)=145617784)
+8900 val loss 6.9149
+8900 val perplexity 1007.1778
+8900 train 7.021801 (lr=1.9153e-05) (hash(x)=155984970)
+9000 val loss 6.9155
+9000 val perplexity 1007.7278
+9000 train 6.684093 (lr=1.8439e-05) (hash(x)=143433013)
+9100 val loss 6.9143
+9100 val perplexity 1006.5181
+9100 train 6.815769 (lr=1.7790e-05) (hash(x)=152507533)
+9200 val loss 6.9159
+9200 val perplexity 1008.1912
+9200 train 7.023916 (lr=1.7208e-05) (hash(x)=162400650)
+9300 val loss 6.9134
+9300 val perplexity 1005.6873
+9300 train 6.943089 (lr=1.6692e-05) (hash(x)=148419068)
+9400 val loss 6.9130
+9400 val perplexity 1005.2514
+9400 train 6.881789 (lr=1.6245e-05) (hash(x)=145025169)
+9500 val loss 6.9133
+9500 val perplexity 1005.6000
+9500 train 6.864194 (lr=1.5865e-05) (hash(x)=145104235)
+9600 val loss 6.9163
+9600 val perplexity 1008.5455
+9600 train 6.835734 (lr=1.5554e-05) (hash(x)=150499797)
+9700 val loss 6.9179
+9700 val perplexity 1010.2282
+9700 train 6.737312 (lr=1.5312e-05) (hash(x)=139690022)
+9800 val loss 6.9151
+9800 val perplexity 1007.3925
+9800 train 6.893308 (lr=1.5139e-05) (hash(x)=145645279)
+9900 val loss 6.9154
+9900 val perplexity 1007.6418
+9900 train 6.841991 (lr=1.5035e-05) (hash(x)=150103432)
+9999 val loss 6.9172
+9999 val perplexity 1009.4544

attention_kindself_n_heads2_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3fc1b63627195057a4c942a3e917c1a8efa782e2aedfdaf127b672b7c6ea3c51
+oid sha256:495bb269f966567bc0c4a1b6dde4d76b4247ecc9dfec0e58226d786a6e26ac33
 size 38587970

attention_kindself_n_heads2_seed1339/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:43e11c87f8189366a1a631654e8af438fc6c9400b004304dc7c83f448e482398
+oid sha256:0e900cb79898a34aa43dd12ccdc5f8dfc07be2d9553d9f5d9654bc5b5ccf8c9e
 size 38587970

attention_kindself_n_heads2_seed1339/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04d5c4a169b8e36ad6535c9f1ce1eb1b912a5c2f4f09b0d7aa76882590689750
+oid sha256:ac73275343867ef02ea1c89fd2ba612301e671b0f500dfa3700943771dc8a46a
 size 38587970

attention_kindself_n_heads2_seed1339/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9e52db128e16a735f111343b41231db5a987c26b72b31b76645b4e54fc6da37
+oid sha256:19ec0d3cd9470a4f858151feaa09d711eeb7cc132ce08ba0241e132618d6c6dd
 size 38587970

attention_kindself_n_heads2_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:86f627707e6c065305b2c0d7bfdab20334ff5e0a3584495015ef8c448e8f144d
+oid sha256:9b2d2fdc0f24530cf7016c528e4cbf7ef4bc0d58a4d286fba5b245550a7a695b
 size 70895430

attention_kindself_n_heads2_seed1339/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d61fa85f6c376914274be629bc0eac5a59a534791989f372a8e8bba92dc40281
+oid sha256:baadc04fd132ee555b189b92f65de978cec8429a93489fe00dde18b7475e5dbd
 size 70895430

attention_kindself_n_heads2_seed1339/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:225354c08248313f0cc7697fc63972e363f6756c00b49b8bd7c7a75e39ec2b27
+oid sha256:14fc0781ccbae4541e4719aa83d114c7a31e7d6f49828e08ef431d61fb0ee967
 size 70895430

attention_kindself_n_heads2_seed1339/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac44cf9be14c262abd9dcaf8a209310ee9701b655acefbd539f2f83073d65709
+oid sha256:1684693033f6813206bfef9fd35016f237558a9cd9aab8773340f08ea4aea567
 size 70895430