Upload folder using huggingface_hub

attention_kindself_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1340", "n_embd": 256}

attention_kindself_n_heads4_seed1340/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.2821
 0 val perplexity 79387.0234
-0 train 11.281846 (lr=2.5000e-07) (hash(x)=164406924)
-100 val loss 9.9152
-100 val perplexity 20234.8789
-100 train 9.963033 (lr=2.5250e-05) (hash(x)=177407419)
-200 val loss 9.1560
-200 val perplexity 9470.8633
-200 train 9.125165 (lr=5.0000e-05) (hash(x)=144903932)
-300 val loss 8.3422
-300 val perplexity 4197.2910
-300 train 8.538510 (lr=4.9988e-05) (hash(x)=173839165)
-400 val loss 7.9151
-400 val perplexity 2738.4211
-400 train 7.983986 (lr=4.9954e-05) (hash(x)=167734596)
-500 val loss 7.7106
-500 val perplexity 2231.7722
-500 train 7.742835 (lr=4.9896e-05) (hash(x)=153224076)
-600 val loss 7.6163
-600 val perplexity 2031.0830
-600 train 7.545313 (lr=4.9815e-05) (hash(x)=149619098)
-700 val loss 7.5534
-700 val perplexity 1907.2063
-700 train 7.509041 (lr=4.9712e-05) (hash(x)=146539909)
-800 val loss 7.5043
-800 val perplexity 1815.8105
-800 train 7.477861 (lr=4.9585e-05) (hash(x)=153710890)
-900 val loss 7.4763
-900 val perplexity 1765.7760
-900 train 7.434323 (lr=4.9436e-05) (hash(x)=155873620)
-1000 val loss 7.4522
-1000 val perplexity 1723.6246
-1000 train 7.376773 (lr=4.9264e-05) (hash(x)=145450636)
-1100 val loss 7.4189
-1100 val perplexity 1667.2645
-1100 train 7.472945 (lr=4.9070e-05) (hash(x)=154123388)
-1200 val loss 7.3910
-1200 val perplexity 1621.3595
-1200 train 7.272326 (lr=4.8854e-05) (hash(x)=145249251)
-1300 val loss 7.3686
-1300 val perplexity 1585.4366
-1300 train 7.284389 (lr=4.8616e-05) (hash(x)=148937127)
-1400 val loss 7.3524
-1400 val perplexity 1559.8955
-1400 train 7.462795 (lr=4.8356e-05) (hash(x)=150475545)
-1500 val loss 7.3319
-1500 val perplexity 1528.2997
-1500 train 7.267529 (lr=4.8074e-05) (hash(x)=154653428)
-1600 val loss 7.3186
-1600 val perplexity 1508.0389
-1600 train 7.294810 (lr=4.7772e-05) (hash(x)=144483776)
-1700 val loss 7.2958
-1700 val perplexity 1474.1586
-1700 train 7.456839 (lr=4.7448e-05) (hash(x)=157395496)
-1800 val loss 7.2830
-1800 val perplexity 1455.2878
-1800 train 7.291935 (lr=4.7105e-05) (hash(x)=157916369)
-1900 val loss 7.2728
-1900 val perplexity 1440.6388
-1900 train 7.473686 (lr=4.6741e-05) (hash(x)=166073923)
-2000 val loss 7.2734
-2000 val perplexity 1441.5115
-2000 train 7.376596 (lr=4.6357e-05) (hash(x)=154856891)
-2100 val loss 7.2636
-2100 val perplexity 1427.3877
-2100 train 7.257746 (lr=4.5954e-05) (hash(x)=151925203)
-2200 val loss 7.2545
-2200 val perplexity 1414.4933
-2200 train 7.054421 (lr=4.5532e-05) (hash(x)=136191502)
-2300 val loss 7.2390
-2300 val perplexity 1392.6326
-2300 train 7.368625 (lr=4.5091e-05) (hash(x)=153273362)
-2400 val loss 7.2387
-2400 val perplexity 1392.3378
-2400 train 7.179043 (lr=4.4633e-05) (hash(x)=148021541)
-2500 val loss 7.2523
-2500 val perplexity 1411.2843
-2500 train 7.149913 (lr=4.4156e-05) (hash(x)=141356608)
-2600 val loss 7.2286
-2600 val perplexity 1378.2823
-2600 train 7.194962 (lr=4.3663e-05) (hash(x)=146005217)
-2700 val loss 7.2195
-2700 val perplexity 1365.8197
-2700 train 7.061798 (lr=4.3153e-05) (hash(x)=144511718)
-2800 val loss 7.2181
-2800 val perplexity 1363.8524
-2800 train 7.107300 (lr=4.2627e-05) (hash(x)=146019502)
-2900 val loss 7.2221
-2900 val perplexity 1369.3765
-2900 train 7.137704 (lr=4.2085e-05) (hash(x)=146496200)
-3000 val loss 7.2270
-3000 val perplexity 1376.1376
-3000 train 7.166032 (lr=4.1529e-05) (hash(x)=150127281)
-3100 val loss 7.1994
-3100 val perplexity 1338.6273
-3100 train 7.147545 (lr=4.0957e-05) (hash(x)=142022255)
-3200 val loss 7.1995
-3200 val perplexity 1338.7562
-3200 train 7.267565 (lr=4.0373e-05) (hash(x)=154120875)
-3300 val loss 7.1966
-3300 val perplexity 1334.8367
-3300 train 7.222846 (lr=3.9775e-05) (hash(x)=153999717)
-3400 val loss 7.1964
-3400 val perplexity 1334.6820
-3400 train 7.011579 (lr=3.9164e-05) (hash(x)=139694097)
-3500 val loss 7.1927
-3500 val perplexity 1329.6560
-3500 train 7.389040 (lr=3.8541e-05) (hash(x)=162992732)
-3600 val loss 7.1709
-3600 val perplexity 1301.0750
-3600 train 7.162068 (lr=3.7907e-05) (hash(x)=147574101)
-3700 val loss 7.1849
-3700 val perplexity 1319.3030
-3700 train 7.309031 (lr=3.7262e-05) (hash(x)=157763099)
-3800 val loss 7.1671
-3800 val perplexity 1296.1002
-3800 train 7.341320 (lr=3.6608e-05) (hash(x)=170800034)
-3900 val loss 7.1688
-3900 val perplexity 1298.2349
-3900 train 7.238739 (lr=3.5944e-05) (hash(x)=164984528)
-4000 val loss 7.1710
-4000 val perplexity 1301.0924
-4000 train 7.080540 (lr=3.5271e-05) (hash(x)=141743323)
-4100 val loss 7.1713
-4100 val perplexity 1301.4802
-4100 train 7.210657 (lr=3.4590e-05) (hash(x)=153392872)
-4200 val loss 7.1880
-4200 val perplexity 1323.4312
-4200 train 7.099106 (lr=3.3902e-05) (hash(x)=149074933)
-4300 val loss 7.1635
-4300 val perplexity 1291.4388
-4300 train 7.532630 (lr=3.3207e-05) (hash(x)=167823423)
-4400 val loss 7.1631
-4400 val perplexity 1290.9291
-4400 train 6.910741 (lr=3.2507e-05) (hash(x)=141203114)
-4500 val loss 7.1711
-4500 val perplexity 1301.2432
-4500 train 7.154581 (lr=3.1801e-05) (hash(x)=146284780)
-4600 val loss 7.1609
-4600 val perplexity 1288.0220
-4600 train 6.970417 (lr=3.1091e-05) (hash(x)=141126464)
-4700 val loss 7.1728
-4700 val perplexity 1303.4719
-4700 train 7.174458 (lr=3.0377e-05) (hash(x)=154751926)
-4800 val loss 7.1574
-4800 val perplexity 1283.5688
-4800 train 7.248188 (lr=2.9661e-05) (hash(x)=154793198)
-4900 val loss 7.1602
-4900 val perplexity 1287.1500
-4900 train 6.914657 (lr=2.8942e-05) (hash(x)=139406392)
-5000 val loss 7.1588
-5000 val perplexity 1285.3940
-5000 train 6.995086 (lr=2.8221e-05) (hash(x)=153548741)
-5100 val loss 7.1706
-5100 val perplexity 1300.6693
-5100 train 7.211829 (lr=2.7500e-05) (hash(x)=160488568)
-5200 val loss 7.1504
-5200 val perplexity 1274.6597
-5200 train 7.139657 (lr=2.6779e-05) (hash(x)=149645053)
-5300 val loss 7.1433
-5300 val perplexity 1265.6222
-5300 train 7.223660 (lr=2.6058e-05) (hash(x)=155820556)
-5400 val loss 7.1410
-5400 val perplexity 1262.6324
-5400 train 7.070198 (lr=2.5339e-05) (hash(x)=147538134)
-5500 val loss 7.1286
-5500 val perplexity 1247.1210
-5500 train 7.345302 (lr=2.4623e-05) (hash(x)=166889307)
-5600 val loss 7.1289
-5600 val perplexity 1247.5200
-5600 train 6.873375 (lr=2.3909e-05) (hash(x)=139516699)
-5700 val loss 7.1356
-5700 val perplexity 1255.9479
-5700 train 6.853154 (lr=2.3199e-05) (hash(x)=140453511)
-5800 val loss 7.1224
-5800 val perplexity 1239.3940
-5800 train 7.090032 (lr=2.2493e-05) (hash(x)=162964847)
-5900 val loss 7.1191
-5900 val perplexity 1235.3566
-5900 train 7.144404 (lr=2.1793e-05) (hash(x)=150606634)
-6000 val loss 7.1195
-6000 val perplexity 1235.8320
-6000 train 7.263775 (lr=2.1098e-05) (hash(x)=149890857)
-6100 val loss 7.1046
-6100 val perplexity 1217.5775
-6100 train 7.216160 (lr=2.0410e-05) (hash(x)=173884145)
-6200 val loss 7.1080
-6200 val perplexity 1221.7253
-6200 train 7.106386 (lr=1.9729e-05) (hash(x)=151987098)
-6300 val loss 7.1124
-6300 val perplexity 1227.0833
-6300 train 7.035869 (lr=1.9056e-05) (hash(x)=148853562)
-6400 val loss 7.1000
-6400 val perplexity 1212.0189
-6400 train 6.877523 (lr=1.8392e-05) (hash(x)=141530101)
-6500 val loss 7.0977
-6500 val perplexity 1209.2203
-6500 train 6.958074 (lr=1.7738e-05) (hash(x)=142297809)
-6600 val loss 7.0941
-6600 val perplexity 1204.7847
-6600 train 6.976329 (lr=1.7093e-05) (hash(x)=142447782)
-6700 val loss 7.0927
-6700 val perplexity 1203.1031
-6700 train 7.008992 (lr=1.6459e-05) (hash(x)=147004686)
-6800 val loss 7.0927
-6800 val perplexity 1203.2070
-6800 train 6.852241 (lr=1.5836e-05) (hash(x)=133438702)
-6900 val loss 7.0991
-6900 val perplexity 1210.9277
-6900 train 7.131525 (lr=1.5225e-05) (hash(x)=157085143)
-7000 val loss 7.0976
-7000 val perplexity 1209.0687
-7000 train 7.025262 (lr=1.4627e-05) (hash(x)=139437666)
-7100 val loss 7.0893
-7100 val perplexity 1199.0752
-7100 train 7.209750 (lr=1.4043e-05) (hash(x)=159792986)
-7200 val loss 7.0864
-7200 val perplexity 1195.6399
-7200 train 7.049824 (lr=1.3471e-05) (hash(x)=144930687)
-7300 val loss 7.0842
-7300 val perplexity 1193.0270
-7300 train 7.190523 (lr=1.2915e-05) (hash(x)=156242690)
-7400 val loss 7.0817
-7400 val perplexity 1189.9329
-7400 train 6.950914 (lr=1.2373e-05) (hash(x)=148183719)
-7500 val loss 7.0792
-7500 val perplexity 1187.0070
-7500 train 7.096027 (lr=1.1847e-05) (hash(x)=152494758)
-7600 val loss 7.0757
-7600 val perplexity 1182.8602
-7600 train 6.845621 (lr=1.1337e-05) (hash(x)=142485027)
-7700 val loss 7.0702
-7700 val perplexity 1176.4084
-7700 train 6.928816 (lr=1.0844e-05) (hash(x)=147512165)
-7800 val loss 7.0677
-7800 val perplexity 1173.4347
-7800 train 7.070253 (lr=1.0367e-05) (hash(x)=160346994)
-7900 val loss 7.0758
-7900 val perplexity 1183.0126
-7900 train 6.928421 (lr=9.9088e-06) (hash(x)=144488254)
-8000 val loss 7.0623
-8000 val perplexity 1167.1813
-8000 train 6.885472 (lr=9.4682e-06) (hash(x)=147637019)
-8100 val loss 7.0661
-8100 val perplexity 1171.6058
-8100 train 6.940768 (lr=9.0461e-06) (hash(x)=147340534)
-8200 val loss 7.0606
-8200 val perplexity 1165.0944
-8200 train 7.093674 (lr=8.6430e-06) (hash(x)=151630665)
-8300 val loss 7.0597
-8300 val perplexity 1164.1315
-8300 train 7.096731 (lr=8.2593e-06) (hash(x)=149747064)
-8400 val loss 7.0550
-8400 val perplexity 1158.6831
-8400 train 7.238415 (lr=7.8953e-06) (hash(x)=154245770)
-8500 val loss 7.0588
-8500 val perplexity 1163.1044
-8500 train 6.946722 (lr=7.5515e-06) (hash(x)=152559100)
-8600 val loss 7.0542
-8600 val perplexity 1157.6653
-8600 train 7.627717 (lr=7.2282e-06) (hash(x)=181365926)
-8700 val loss 7.0556
-8700 val perplexity 1159.3402
-8700 train 6.931863 (lr=6.9257e-06) (hash(x)=154405991)
-8800 val loss 7.0540
-8800 val perplexity 1157.4385
-8800 train 7.021280 (lr=6.6444e-06) (hash(x)=153755904)
-8900 val loss 7.0493
-8900 val perplexity 1151.9979
-8900 train 6.994501 (lr=6.3845e-06) (hash(x)=152120568)
-9000 val loss 7.0478
-9000 val perplexity 1150.2721
-9000 train 6.868895 (lr=6.1462e-06) (hash(x)=142797279)
-9100 val loss 7.0473
-9100 val perplexity 1149.7742
-9100 train 6.933502 (lr=5.9300e-06) (hash(x)=143037503)
-9200 val loss 7.0448
-9200 val perplexity 1146.8475
-9200 train 6.786018 (lr=5.7359e-06) (hash(x)=113690273)
-9300 val loss 7.0448
-9300 val perplexity 1146.8322
-9300 train 7.015091 (lr=5.5641e-06) (hash(x)=158025077)
-9400 val loss 7.0432
-9400 val perplexity 1145.0940
-9400 train 7.140065 (lr=5.4149e-06) (hash(x)=158251718)
-9500 val loss 7.0404
-9500 val perplexity 1141.8502
-9500 train 7.100505 (lr=5.2884e-06) (hash(x)=154752610)
-9600 val loss 7.0396
-9600 val perplexity 1140.9636
-9600 train 6.925082 (lr=5.1847e-06) (hash(x)=146889093)
-9700 val loss 7.0390
-9700 val perplexity 1140.2371
-9700 train 7.010257 (lr=5.1040e-06) (hash(x)=156906516)
-9800 val loss 7.0397
-9800 val perplexity 1141.0045
-9800 train 6.915371 (lr=5.0462e-06) (hash(x)=153841927)
-9900 val loss 7.0423
-9900 val perplexity 1143.9980
-9900 train 7.256087 (lr=5.0116e-06) (hash(x)=163514334)
-9999 val loss 7.0365
-9999 val perplexity 1137.4387
+0 train 11.281846 (lr=1.5000e-07) (hash(x)=164406924)
+100 val loss 10.0041
+100 val perplexity 22116.9980
+100 train 10.046823 (lr=1.5150e-05) (hash(x)=177407419)
+200 val loss 9.6264
+200 val perplexity 15160.1963
+200 train 9.603025 (lr=3.0000e-05) (hash(x)=144903932)
+300 val loss 8.9239
+300 val perplexity 7509.6455
+300 train 9.044446 (lr=2.9993e-05) (hash(x)=173839165)
+400 val loss 8.4978
+400 val perplexity 4903.9678
+400 train 8.537397 (lr=2.9972e-05) (hash(x)=167734596)
+500 val loss 8.1835
+500 val perplexity 3581.2524
+500 train 8.200040 (lr=2.9938e-05) (hash(x)=153224076)
+600 val loss 7.9486
+600 val perplexity 2831.6128
+600 train 7.921226 (lr=2.9889e-05) (hash(x)=149619098)
+700 val loss 7.7945
+700 val perplexity 2427.1655
+700 train 7.764179 (lr=2.9827e-05) (hash(x)=146539909)
+800 val loss 7.7057
+800 val perplexity 2221.0625
+800 train 7.685279 (lr=2.9751e-05) (hash(x)=153710890)
+900 val loss 7.6419
+900 val perplexity 2083.5989
+900 train 7.604532 (lr=2.9662e-05) (hash(x)=155873620)
+1000 val loss 7.5894
+1000 val perplexity 1977.1727
+1000 train 7.523177 (lr=2.9558e-05) (hash(x)=145450636)
+1100 val loss 7.5493
+1100 val perplexity 1899.3469
+1100 train 7.594795 (lr=2.9442e-05) (hash(x)=154123388)
+1200 val loss 7.5178
+1200 val perplexity 1840.5389
+1200 train 7.386674 (lr=2.9312e-05) (hash(x)=145249251)
+1300 val loss 7.4958
+1300 val perplexity 1800.5153
+1300 train 7.421271 (lr=2.9169e-05) (hash(x)=148937127)
+1400 val loss 7.4748
+1400 val perplexity 1763.0459
+1400 train 7.580665 (lr=2.9013e-05) (hash(x)=150475545)
+1500 val loss 7.4572
+1500 val perplexity 1732.2736
+1500 train 7.398860 (lr=2.8845e-05) (hash(x)=154653428)
+1600 val loss 7.4423
+1600 val perplexity 1706.6057
+1600 train 7.422663 (lr=2.8663e-05) (hash(x)=144483776)
+1700 val loss 7.4191
+1700 val perplexity 1667.5261
+1700 train 7.572146 (lr=2.8469e-05) (hash(x)=157395496)
+1800 val loss 7.4010
+1800 val perplexity 1637.5830
+1800 train 7.418879 (lr=2.8263e-05) (hash(x)=157916369)
+1900 val loss 7.3838
+1900 val perplexity 1609.6321
+1900 train 7.581061 (lr=2.8044e-05) (hash(x)=166073923)
+2000 val loss 7.3716
+2000 val perplexity 1590.1710
+2000 train 7.485497 (lr=2.7814e-05) (hash(x)=154856891)
+2100 val loss 7.3591
+2100 val perplexity 1570.4291
+2100 train 7.340837 (lr=2.7572e-05) (hash(x)=151925203)
+2200 val loss 7.3455
+2200 val perplexity 1549.2108
+2200 train 7.158240 (lr=2.7319e-05) (hash(x)=136191502)
+2300 val loss 7.3301
+2300 val perplexity 1525.4587
+2300 train 7.429245 (lr=2.7055e-05) (hash(x)=153273362)
+2400 val loss 7.3190
+2400 val perplexity 1508.7308
+2400 train 7.250983 (lr=2.6780e-05) (hash(x)=148021541)
+2500 val loss 7.3158
+2500 val perplexity 1503.9199
+2500 train 7.207904 (lr=2.6494e-05) (hash(x)=141356608)
+2600 val loss 7.2981
+2600 val perplexity 1477.4590
+2600 train 7.261382 (lr=2.6198e-05) (hash(x)=146005217)
+2700 val loss 7.2873
+2700 val perplexity 1461.6011
+2700 train 7.130173 (lr=2.5892e-05) (hash(x)=144511718)
+2800 val loss 7.2777
+2800 val perplexity 1447.6670
+2800 train 7.167113 (lr=2.5576e-05) (hash(x)=146019502)
+2900 val loss 7.2726
+2900 val perplexity 1440.2789
+2900 train 7.175721 (lr=2.5251e-05) (hash(x)=146496200)
+3000 val loss 7.2701
+3000 val perplexity 1436.6860
+3000 train 7.209612 (lr=2.4917e-05) (hash(x)=150127281)
+3100 val loss 7.2555
+3100 val perplexity 1415.9124
+3100 train 7.196756 (lr=2.4574e-05) (hash(x)=142022255)
+3200 val loss 7.2454
+3200 val perplexity 1401.6504
+3200 train 7.305396 (lr=2.4224e-05) (hash(x)=154120875)
+3300 val loss 7.2456
+3300 val perplexity 1401.8677
+3300 train 7.274103 (lr=2.3865e-05) (hash(x)=153999717)
+3400 val loss 7.2373
+3400 val perplexity 1390.3885
+3400 train 7.053738 (lr=2.3498e-05) (hash(x)=139694097)
+3500 val loss 7.2257
+3500 val perplexity 1374.2694
+3500 train 7.422082 (lr=2.3125e-05) (hash(x)=162992732)
+3600 val loss 7.2150
+3600 val perplexity 1359.6266
+3600 train 7.207018 (lr=2.2744e-05) (hash(x)=147574101)
+3700 val loss 7.2091
+3700 val perplexity 1351.6832
+3700 train 7.337052 (lr=2.2357e-05) (hash(x)=157763099)
+3800 val loss 7.2016
+3800 val perplexity 1341.6339
+3800 train 7.375456 (lr=2.1965e-05) (hash(x)=170800034)
+3900 val loss 7.1965
+3900 val perplexity 1334.7889
+3900 train 7.272295 (lr=2.1566e-05) (hash(x)=164984528)
+4000 val loss 7.1905
+4000 val perplexity 1326.7535
+4000 train 7.093181 (lr=2.1162e-05) (hash(x)=141743323)
+4100 val loss 7.1886
+4100 val perplexity 1324.2689
+4100 train 7.222679 (lr=2.0754e-05) (hash(x)=153392872)
+4200 val loss 7.1898
+4200 val perplexity 1325.8370
+4200 train 7.091339 (lr=2.0341e-05) (hash(x)=149074933)
+4300 val loss 7.1712
+4300 val perplexity 1301.3909
+4300 train 7.548721 (lr=1.9924e-05) (hash(x)=167823423)
+4400 val loss 7.1610
+4400 val perplexity 1288.2554
+4400 train 6.905305 (lr=1.9504e-05) (hash(x)=141203114)
+4500 val loss 7.1538
+4500 val perplexity 1278.9043
+4500 train 7.138118 (lr=1.9081e-05) (hash(x)=146284780)
+4600 val loss 7.1512
+4600 val perplexity 1275.6593
+4600 train 6.956368 (lr=1.8655e-05) (hash(x)=141126464)
+4700 val loss 7.1428
+4700 val perplexity 1265.0206
+4700 train 7.147374 (lr=1.8226e-05) (hash(x)=154751926)
+4800 val loss 7.1377
+4800 val perplexity 1258.5389
+4800 train 7.231942 (lr=1.7796e-05) (hash(x)=154793198)
+4900 val loss 7.1408
+4900 val perplexity 1262.4771
+4900 train 6.893873 (lr=1.7365e-05) (hash(x)=139406392)
+5000 val loss 7.1345
+5000 val perplexity 1254.4594
+5000 train 6.966713 (lr=1.6933e-05) (hash(x)=153548741)
+5100 val loss 7.1283
+5100 val perplexity 1246.7119
+5100 train 7.184055 (lr=1.6500e-05) (hash(x)=160488568)
+5200 val loss 7.1235
+5200 val perplexity 1240.7729
+5200 train 7.114891 (lr=1.6067e-05) (hash(x)=149645053)
+5300 val loss 7.1047
+5300 val perplexity 1217.6757
+5300 train 7.187130 (lr=1.5635e-05) (hash(x)=155820556)
+5400 val loss 7.1019
+5400 val perplexity 1214.2223
+5400 train 7.029991 (lr=1.5204e-05) (hash(x)=147538134)
+5500 val loss 7.1003
+5500 val perplexity 1212.3207
+5500 train 7.312305 (lr=1.4774e-05) (hash(x)=166889307)
+5600 val loss 7.0933
+5600 val perplexity 1203.8147
+5600 train 6.833846 (lr=1.4345e-05) (hash(x)=139516699)
+5700 val loss 7.0890
+5700 val perplexity 1198.7596
+5700 train 6.798667 (lr=1.3919e-05) (hash(x)=140453511)
+5800 val loss 7.0875
+5800 val perplexity 1196.8839
+5800 train 7.054469 (lr=1.3496e-05) (hash(x)=162964847)
+5900 val loss 7.0827
+5900 val perplexity 1191.2295
+5900 train 7.107747 (lr=1.3076e-05) (hash(x)=150606634)
+6000 val loss 7.0725
+6000 val perplexity 1179.0541
+6000 train 7.198762 (lr=1.2659e-05) (hash(x)=149890857)
+6100 val loss 7.0657
+6100 val perplexity 1171.0558
+6100 train 7.172532 (lr=1.2246e-05) (hash(x)=173884145)
+6200 val loss 7.0649
+6200 val perplexity 1170.1548
+6200 train 7.068169 (lr=1.1838e-05) (hash(x)=151987098)
+6300 val loss 7.0594
+6300 val perplexity 1163.7390
+6300 train 6.981870 (lr=1.1434e-05) (hash(x)=148853562)
+6400 val loss 7.0527
+6400 val perplexity 1155.9514
+6400 train 6.823884 (lr=1.1035e-05) (hash(x)=141530101)
+6500 val loss 7.0511
+6500 val perplexity 1154.1339
+6500 train 6.909740 (lr=1.0643e-05) (hash(x)=142297809)
+6600 val loss 7.0474
+6600 val perplexity 1149.8098
+6600 train 6.920826 (lr=1.0256e-05) (hash(x)=142447782)
+6700 val loss 7.0417
+6700 val perplexity 1143.3540
+6700 train 6.963796 (lr=9.8753e-06) (hash(x)=147004686)
+6800 val loss 7.0380
+6800 val perplexity 1139.0812
+6800 train 6.790812 (lr=9.5017e-06) (hash(x)=133438702)
+6900 val loss 7.0349
+6900 val perplexity 1135.5475
+6900 train 7.081897 (lr=9.1353e-06) (hash(x)=157085143)
+7000 val loss 7.0313
+7000 val perplexity 1131.4943
+7000 train 6.951317 (lr=8.7764e-06) (hash(x)=139437666)
+7100 val loss 7.0255
+7100 val perplexity 1125.0077
+7100 train 7.157989 (lr=8.4255e-06) (hash(x)=159792986)
+7200 val loss 7.0227
+7200 val perplexity 1121.8470
+7200 train 6.987632 (lr=8.0829e-06) (hash(x)=144930687)
+7300 val loss 7.0192
+7300 val perplexity 1117.8630
+7300 train 7.121083 (lr=7.7489e-06) (hash(x)=156242690)
+7400 val loss 7.0153
+7400 val perplexity 1113.5474
+7400 train 6.888875 (lr=7.4239e-06) (hash(x)=148183719)
+7500 val loss 7.0153
+7500 val perplexity 1113.5139
+7500 train 7.037551 (lr=7.1083e-06) (hash(x)=152494758)
+7600 val loss 7.0103
+7600 val perplexity 1107.9728
+7600 train 6.769261 (lr=6.8023e-06) (hash(x)=142485027)
+7700 val loss 7.0062
+7700 val perplexity 1103.4617
+7700 train 6.865348 (lr=6.5062e-06) (hash(x)=147512165)
+7800 val loss 7.0023
+7800 val perplexity 1099.1613
+7800 train 7.007141 (lr=6.2205e-06) (hash(x)=160346994)
+7900 val loss 7.0011
+7900 val perplexity 1097.7867
+7900 train 6.853342 (lr=5.9453e-06) (hash(x)=144488254)
+8000 val loss 6.9971
+8000 val perplexity 1093.4069
+8000 train 6.816508 (lr=5.6809e-06) (hash(x)=147637019)
+8100 val loss 6.9952
+8100 val perplexity 1091.3915
+8100 train 6.862648 (lr=5.4277e-06) (hash(x)=147340534)
+8200 val loss 6.9921
+8200 val perplexity 1088.0166
+8200 train 7.023407 (lr=5.1858e-06) (hash(x)=151630665)
+8300 val loss 6.9885
+8300 val perplexity 1084.0498
+8300 train 7.030170 (lr=4.9556e-06) (hash(x)=149747064)
+8400 val loss 6.9867
+8400 val perplexity 1082.1461
+8400 train 7.178265 (lr=4.7372e-06) (hash(x)=154245770)
+8500 val loss 6.9850
+8500 val perplexity 1080.3158
+8500 train 6.871959 (lr=4.5309e-06) (hash(x)=152559100)
+8600 val loss 6.9830
+8600 val perplexity 1078.1946
+8600 train 7.604201 (lr=4.3369e-06) (hash(x)=181365926)
+8700 val loss 6.9825
+8700 val perplexity 1077.6615
+8700 train 6.850633 (lr=4.1554e-06) (hash(x)=154405991)
+8800 val loss 6.9798
+8800 val perplexity 1074.7390
+8800 train 6.942220 (lr=3.9866e-06) (hash(x)=153755904)
+8900 val loss 6.9769
+8900 val perplexity 1071.5408
+8900 train 6.922254 (lr=3.8307e-06) (hash(x)=152120568)
+9000 val loss 6.9733
+9000 val perplexity 1067.7776
+9000 train 6.792783 (lr=3.6877e-06) (hash(x)=142797279)
+9100 val loss 6.9713
+9100 val perplexity 1065.6550
+9100 train 6.853178 (lr=3.5580e-06) (hash(x)=143037503)
+9200 val loss 6.9719
+9200 val perplexity 1066.2136
+9200 train 6.704218 (lr=3.4415e-06) (hash(x)=113690273)
+9300 val loss 6.9686
+9300 val perplexity 1062.7732
+9300 train 6.930370 (lr=3.3385e-06) (hash(x)=158025077)
+9400 val loss 6.9682
+9400 val perplexity 1062.3506
+9400 train 7.067931 (lr=3.2490e-06) (hash(x)=158251718)
+9500 val loss 6.9656
+9500 val perplexity 1059.5894
+9500 train 7.023955 (lr=3.1730e-06) (hash(x)=154752610)
+9600 val loss 6.9635
+9600 val perplexity 1057.3257
+9600 train 6.849610 (lr=3.1108e-06) (hash(x)=146889093)
+9700 val loss 6.9617
+9700 val perplexity 1055.4609
+9700 train 6.934623 (lr=3.0624e-06) (hash(x)=156906516)
+9800 val loss 6.9627
+9800 val perplexity 1056.5062
+9800 train 6.840199 (lr=3.0277e-06) (hash(x)=153841927)
+9900 val loss 6.9607
+9900 val perplexity 1054.3820
+9900 train 7.176472 (lr=3.0069e-06) (hash(x)=163514334)
+9999 val loss 6.9578
+9999 val perplexity 1051.3296

attention_kindself_n_heads4_seed1340/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db99a94b845a5e9e4fb8b5a5631c867ed483f60e409531d405963f2438eac0d1
+oid sha256:31c95a3e610c19cb3a8045ae6a63b4a4c09e3018612d07b44e37a34281e7ae2d
 size 92843394

attention_kindself_n_heads4_seed1340/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6724e43a86146ac7d8e4da7b7d95902f777f6815e520c3ca8fba0a1c3c12038e
+oid sha256:5b2f445b05db33d82bc81a6f5a475b1d3f8b38ab890c472d6ba4e69c8e1ca1a6
 size 92843394

attention_kindself_n_heads4_seed1340/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:424f8b21c8e1aeca8de7e6fe893bc19ad8d03ad0ae82a1a44d9eaea2d3c90802
+oid sha256:336e27d7924ad63f905e79b0e2db990c50cb12235719eca92137158ca11de1f9
 size 92843394

attention_kindself_n_heads4_seed1340/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4cecd0063c9d03220661490360e89ff0effeaad40db6b673073fef1fdce9857
+oid sha256:d20a6c8f1f42a4c116f6009adb64d54a4c5f9ff3b0cef5738a0d92df58700862
 size 92843394

attention_kindself_n_heads4_seed1340/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c3db231a87039d7e1099ed54b2babe5cea2f7fab57a86f033bcc9312c1963f34
+oid sha256:660393cd7d3c443a35399177cba4ada91e415535025c827033c684e591468dcb
 size 179406214

attention_kindself_n_heads4_seed1340/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab08a9c080f195082b8a0fb2bb7fac82cf7c8681b2148fb7974b16cfeb0ab0fe
+oid sha256:fdeec97cc7b72a1a29998577d107cd2bf57fc271394410887b71f83baf12e55e
 size 179406214

attention_kindself_n_heads4_seed1340/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88debc70f1064976faf6bebc7958e9c9cd087771c9331352e869bffd693f15d6
+oid sha256:d0ad5c4975a1968c45658cc5f905af61d2833d44001f3925de27df16808e3bb7
 size 179406214

attention_kindself_n_heads4_seed1340/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bbf1f57711eff0666b8af026156db000108d89c207451ee87477b47a4dd52965
+oid sha256:42ecd43e238626aca3f4a0e9d837b8e44c2004e0d36de088dfe1fd66032ea62f
 size 179406214