Upload folder using huggingface_hub

attention_kindself_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1341", "n_embd": 256}

attention_kindself_n_heads4_seed1341/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.3001
 0 val perplexity 80832.1328
-0 train 11.306698 (lr=2.5000e-07) (hash(x)=145079536)
-100 val loss 9.9349
-100 val perplexity 20638.1484
-100 train 9.874459 (lr=2.5250e-05) (hash(x)=154745873)
-200 val loss 9.1781
-200 val perplexity 9682.8174
-200 train 9.206763 (lr=5.0000e-05) (hash(x)=155800595)
-300 val loss 8.3602
-300 val perplexity 4273.7119
-300 train 8.323228 (lr=4.9988e-05) (hash(x)=148595389)
-400 val loss 8.0266
-400 val perplexity 3061.2158
-400 train 7.965455 (lr=4.9954e-05) (hash(x)=145606733)
-500 val loss 7.7974
-500 val perplexity 2434.1660
-500 train 7.668923 (lr=4.9896e-05) (hash(x)=138221231)
-600 val loss 7.6570
-600 val perplexity 2115.4138
-600 train 7.671975 (lr=4.9815e-05) (hash(x)=150367139)
-700 val loss 7.5883
-700 val perplexity 1974.9584
-700 train 7.727467 (lr=4.9712e-05) (hash(x)=155579314)
-800 val loss 7.5362
-800 val perplexity 1874.6230
-800 train 7.612894 (lr=4.9585e-05) (hash(x)=155747374)
-900 val loss 7.4908
-900 val perplexity 1791.4127
-900 train 7.689740 (lr=4.9436e-05) (hash(x)=159334575)
-1000 val loss 7.4582
-1000 val perplexity 1734.0844
-1000 train 7.335213 (lr=4.9264e-05) (hash(x)=140604760)
-1100 val loss 7.4387
-1100 val perplexity 1700.5392
-1100 train 7.287144 (lr=4.9070e-05) (hash(x)=146208052)
-1200 val loss 7.4228
-1200 val perplexity 1673.7517
-1200 train 7.334071 (lr=4.8854e-05) (hash(x)=148404734)
-1300 val loss 7.4098
-1300 val perplexity 1652.1472
-1300 train 7.351157 (lr=4.8616e-05) (hash(x)=155681970)
-1400 val loss 7.3949
-1400 val perplexity 1627.6387
-1400 train 7.345123 (lr=4.8356e-05) (hash(x)=148115934)
-1500 val loss 7.3776
-1500 val perplexity 1599.8215
-1500 train 7.515377 (lr=4.8074e-05) (hash(x)=157074034)
-1600 val loss 7.3625
-1600 val perplexity 1575.8352
-1600 train 7.143104 (lr=4.7772e-05) (hash(x)=137464699)
-1700 val loss 7.3544
-1700 val perplexity 1563.0792
-1700 train 7.558336 (lr=4.7448e-05) (hash(x)=166955614)
-1800 val loss 7.3451
-1800 val perplexity 1548.5469
-1800 train 7.180773 (lr=4.7105e-05) (hash(x)=143886042)
-1900 val loss 7.3411
-1900 val perplexity 1542.4170
-1900 train 8.015940 (lr=4.6741e-05) (hash(x)=193610391)
-2000 val loss 7.3353
-2000 val perplexity 1533.4343
-2000 train 7.497729 (lr=4.6357e-05) (hash(x)=163799796)
-2100 val loss 7.3237
-2100 val perplexity 1515.7485
-2100 train 7.289556 (lr=4.5954e-05) (hash(x)=154107339)
-2200 val loss 7.3198
-2200 val perplexity 1509.9255
-2200 train 7.293709 (lr=4.5532e-05) (hash(x)=144592844)
-2300 val loss 7.3144
-2300 val perplexity 1501.8445
-2300 train 7.388278 (lr=4.5091e-05) (hash(x)=154535861)
-2400 val loss 7.3082
-2400 val perplexity 1492.5593
-2400 train 8.053934 (lr=4.4633e-05) (hash(x)=204706354)
-2500 val loss 7.2990
-2500 val perplexity 1478.7517
-2500 train 7.128391 (lr=4.4156e-05) (hash(x)=143783202)
-2600 val loss 7.2942
-2600 val perplexity 1471.7782
-2600 train 7.203882 (lr=4.3663e-05) (hash(x)=150862210)
-2700 val loss 7.2886
-2700 val perplexity 1463.5914
-2700 train 7.364130 (lr=4.3153e-05) (hash(x)=150753426)
-2800 val loss 7.2890
-2800 val perplexity 1464.0759
-2800 train 7.250031 (lr=4.2627e-05) (hash(x)=147766811)
-2900 val loss 7.2737
-2900 val perplexity 1441.8539
-2900 train 7.208226 (lr=4.2085e-05) (hash(x)=151177814)
-3000 val loss 7.2692
-3000 val perplexity 1435.4247
-3000 train 7.112866 (lr=4.1529e-05) (hash(x)=135925327)
-3100 val loss 7.2730
-3100 val perplexity 1440.9088
-3100 train 6.978726 (lr=4.0957e-05) (hash(x)=124842353)
-3200 val loss 7.2753
-3200 val perplexity 1444.2098
-3200 train 7.387667 (lr=4.0373e-05) (hash(x)=160440642)
-3300 val loss 7.2647
-3300 val perplexity 1428.9902
-3300 train 7.161217 (lr=3.9775e-05) (hash(x)=143017131)
-3400 val loss 7.2511
-3400 val perplexity 1409.5935
-3400 train 7.469110 (lr=3.9164e-05) (hash(x)=148143474)
-3500 val loss 7.2516
-3500 val perplexity 1410.3889
-3500 train 7.080482 (lr=3.8541e-05) (hash(x)=148732639)
-3600 val loss 7.2410
-3600 val perplexity 1395.4398
-3600 train 7.236729 (lr=3.7907e-05) (hash(x)=146333803)
-3700 val loss 7.2483
-3700 val perplexity 1405.7367
-3700 train 7.086179 (lr=3.7262e-05) (hash(x)=143611445)
-3800 val loss 7.2378
-3800 val perplexity 1391.0444
-3800 train 7.065182 (lr=3.6608e-05) (hash(x)=141272115)
-3900 val loss 7.2511
-3900 val perplexity 1409.6836
-3900 train 7.180300 (lr=3.5944e-05) (hash(x)=149161848)
-4000 val loss 7.2304
-4000 val perplexity 1380.7662
-4000 train 7.299043 (lr=3.5271e-05) (hash(x)=152036955)
-4100 val loss 7.2417
-4100 val perplexity 1396.4901
-4100 train 7.309857 (lr=3.4590e-05) (hash(x)=153260968)
-4200 val loss 7.2438
-4200 val perplexity 1399.3445
-4200 train 7.125490 (lr=3.3902e-05) (hash(x)=147629478)
-4300 val loss 7.2442
-4300 val perplexity 1399.9458
-4300 train 7.276296 (lr=3.3207e-05) (hash(x)=161701080)
-4400 val loss 7.2339
-4400 val perplexity 1385.6603
-4400 train 7.305409 (lr=3.2507e-05) (hash(x)=155290524)
-4500 val loss 7.2393
-4500 val perplexity 1393.1593
-4500 train 7.063692 (lr=3.1801e-05) (hash(x)=144726593)
-4600 val loss 7.2351
-4600 val perplexity 1387.2800
-4600 train 7.197353 (lr=3.1091e-05) (hash(x)=149517202)
-4700 val loss 7.2225
-4700 val perplexity 1369.9283
-4700 train 7.208840 (lr=3.0377e-05) (hash(x)=139403008)
-4800 val loss 7.2164
-4800 val perplexity 1361.6372
-4800 train 7.258296 (lr=2.9661e-05) (hash(x)=153387238)
-4900 val loss 7.2094
-4900 val perplexity 1352.0862
-4900 train 6.986797 (lr=2.8942e-05) (hash(x)=139881569)
-5000 val loss 7.2091
-5000 val perplexity 1351.6555
-5000 train 7.152221 (lr=2.8221e-05) (hash(x)=152602580)
-5100 val loss 7.2024
-5100 val perplexity 1342.7072
-5100 train 7.261956 (lr=2.7500e-05) (hash(x)=150598460)
-5200 val loss 7.2082
-5200 val perplexity 1350.5076
-5200 train 7.145143 (lr=2.6779e-05) (hash(x)=147660496)
-5300 val loss 7.2034
-5300 val perplexity 1344.0082
-5300 train 6.975124 (lr=2.6058e-05) (hash(x)=147693848)
-5400 val loss 7.1950
-5400 val perplexity 1332.7544
-5400 train 6.972303 (lr=2.5339e-05) (hash(x)=136048145)
-5500 val loss 7.1836
-5500 val perplexity 1317.5847
-5500 train 7.096358 (lr=2.4623e-05) (hash(x)=140313123)
-5600 val loss 7.1728
-5600 val perplexity 1303.5310
-5600 train 7.204102 (lr=2.3909e-05) (hash(x)=156997993)
-5700 val loss 7.1696
-5700 val perplexity 1299.3303
-5700 train 7.145248 (lr=2.3199e-05) (hash(x)=143513880)
-5800 val loss 7.1668
-5800 val perplexity 1295.6720
-5800 train 7.339622 (lr=2.2493e-05) (hash(x)=153736628)
-5900 val loss 7.1585
-5900 val perplexity 1285.0411
-5900 train 7.393508 (lr=2.1793e-05) (hash(x)=166385417)
-6000 val loss 7.1613
-6000 val perplexity 1288.5822
-6000 train 7.372916 (lr=2.1098e-05) (hash(x)=153216517)
-6100 val loss 7.1614
-6100 val perplexity 1288.6780
-6100 train 7.048747 (lr=2.0410e-05) (hash(x)=145387545)
-6200 val loss 7.1572
-6200 val perplexity 1283.2543
-6200 train 7.264054 (lr=1.9729e-05) (hash(x)=156306460)
-6300 val loss 7.1461
-6300 val perplexity 1269.2065
-6300 train 7.032260 (lr=1.9056e-05) (hash(x)=148610447)
-6400 val loss 7.1415
-6400 val perplexity 1263.2659
-6400 train 7.327859 (lr=1.8392e-05) (hash(x)=158343535)
-6500 val loss 7.1378
-6500 val perplexity 1258.6140
-6500 train 7.277085 (lr=1.7738e-05) (hash(x)=157742557)
-6600 val loss 7.1429
-6600 val perplexity 1265.0973
-6600 train 7.098745 (lr=1.7093e-05) (hash(x)=150457308)
-6700 val loss 7.1355
-6700 val perplexity 1255.7797
-6700 train 7.037126 (lr=1.6459e-05) (hash(x)=142426205)
-6800 val loss 7.1432
-6800 val perplexity 1265.4321
-6800 train 6.993477 (lr=1.5836e-05) (hash(x)=151522525)
-6900 val loss 7.1385
-6900 val perplexity 1259.5896
-6900 train 7.049893 (lr=1.5225e-05) (hash(x)=152029630)
-7000 val loss 7.1301
-7000 val perplexity 1249.0457
-7000 train 7.005527 (lr=1.4627e-05) (hash(x)=142978954)
-7100 val loss 7.1271
-7100 val perplexity 1245.2415
-7100 train 7.016007 (lr=1.4043e-05) (hash(x)=148064359)
-7200 val loss 7.1236
-7200 val perplexity 1240.9304
-7200 train 7.060221 (lr=1.3471e-05) (hash(x)=155771845)
-7300 val loss 7.1249
-7300 val perplexity 1242.5178
-7300 train 6.948674 (lr=1.2915e-05) (hash(x)=141639844)
-7400 val loss 7.1217
-7400 val perplexity 1238.5835
-7400 train 7.149739 (lr=1.2373e-05) (hash(x)=153466021)
-7500 val loss 7.1157
-7500 val perplexity 1231.1477
-7500 train 7.134110 (lr=1.1847e-05) (hash(x)=148843587)
-7600 val loss 7.1180
-7600 val perplexity 1233.9883
-7600 train 7.286146 (lr=1.1337e-05) (hash(x)=152053933)
-7700 val loss 7.1133
-7700 val perplexity 1228.1533
-7700 train 6.874091 (lr=1.0844e-05) (hash(x)=141564003)
-7800 val loss 7.1118
-7800 val perplexity 1226.3030
-7800 train 7.004002 (lr=1.0367e-05) (hash(x)=148617843)
-7900 val loss 7.1108
-7900 val perplexity 1225.1328
-7900 train 7.076463 (lr=9.9088e-06) (hash(x)=146417668)
-8000 val loss 7.1130
-8000 val perplexity 1227.8202
-8000 train 6.954848 (lr=9.4682e-06) (hash(x)=150165488)
-8100 val loss 7.1062
-8100 val perplexity 1219.5206
-8100 train 7.087504 (lr=9.0461e-06) (hash(x)=144818855)
-8200 val loss 7.1025
-8200 val perplexity 1214.9938
-8200 train 7.132918 (lr=8.6430e-06) (hash(x)=143074954)
-8300 val loss 7.1034
-8300 val perplexity 1216.0747
-8300 train 7.034755 (lr=8.2593e-06) (hash(x)=145172494)
-8400 val loss 7.1012
-8400 val perplexity 1213.4415
-8400 train 7.108291 (lr=7.8953e-06) (hash(x)=152284393)
-8500 val loss 7.1001
-8500 val perplexity 1212.0831
-8500 train 6.977907 (lr=7.5515e-06) (hash(x)=147044705)
-8600 val loss 7.0980
-8600 val perplexity 1209.4879
-8600 train 7.043639 (lr=7.2282e-06) (hash(x)=147762608)
-8700 val loss 7.1013
-8700 val perplexity 1213.5769
-8700 train 7.067952 (lr=6.9257e-06) (hash(x)=141994567)
-8800 val loss 7.0941
-8800 val perplexity 1204.8772
-8800 train 7.125029 (lr=6.6444e-06) (hash(x)=152552896)
-8900 val loss 7.0935
-8900 val perplexity 1204.1541
-8900 train 6.956048 (lr=6.3845e-06) (hash(x)=139714454)
-9000 val loss 7.0907
-9000 val perplexity 1200.7579
-9000 train 7.112830 (lr=6.1462e-06) (hash(x)=152274419)
-9100 val loss 7.0909
-9100 val perplexity 1200.9858
-9100 train 7.268093 (lr=5.9300e-06) (hash(x)=160536885)
-9200 val loss 7.0907
-9200 val perplexity 1200.7115
-9200 train 6.918797 (lr=5.7359e-06) (hash(x)=138386899)
-9300 val loss 7.0895
-9300 val perplexity 1199.2856
-9300 train 7.092598 (lr=5.5641e-06) (hash(x)=149957457)
-9400 val loss 7.0853
-9400 val perplexity 1194.3065
-9400 train 7.210474 (lr=5.4149e-06) (hash(x)=161671548)
-9500 val loss 7.0858
-9500 val perplexity 1194.8624
-9500 train 7.251560 (lr=5.2884e-06) (hash(x)=171857400)
-9600 val loss 7.0832
-9600 val perplexity 1191.8232
-9600 train 6.773832 (lr=5.1847e-06) (hash(x)=135378471)
-9700 val loss 7.0826
-9700 val perplexity 1191.0563
-9700 train 6.997484 (lr=5.1040e-06) (hash(x)=149514591)
-9800 val loss 7.0818
-9800 val perplexity 1190.0991
-9800 train 7.044656 (lr=5.0462e-06) (hash(x)=152679072)
-9900 val loss 7.0810
-9900 val perplexity 1189.1620
-9900 train 7.002032 (lr=5.0116e-06) (hash(x)=150415193)
-9999 val loss 7.0800
-9999 val perplexity 1188.0104
+0 train 11.306698 (lr=1.5000e-07) (hash(x)=145079536)
+100 val loss 10.0261
+100 val perplexity 22609.2090
+100 train 9.968981 (lr=1.5150e-05) (hash(x)=154745873)
+200 val loss 9.6441
+200 val perplexity 15429.9082
+200 train 9.655743 (lr=3.0000e-05) (hash(x)=155800595)
+300 val loss 8.9667
+300 val perplexity 7837.9453
+300 train 8.934552 (lr=2.9993e-05) (hash(x)=148595389)
+400 val loss 8.4925
+400 val perplexity 4878.0337
+400 train 8.442228 (lr=2.9972e-05) (hash(x)=145606733)
+500 val loss 8.2574
+500 val perplexity 3856.0903
+500 train 8.204540 (lr=2.9938e-05) (hash(x)=138221231)
+600 val loss 8.0678
+600 val perplexity 3190.0198
+600 train 8.089054 (lr=2.9889e-05) (hash(x)=150367139)
+700 val loss 7.9177
+700 val perplexity 2745.3911
+700 train 8.045775 (lr=2.9827e-05) (hash(x)=155579314)
+800 val loss 7.7964
+800 val perplexity 2431.9211
+800 train 7.861478 (lr=2.9751e-05) (hash(x)=155747374)
+900 val loss 7.7139
+900 val perplexity 2239.3076
+900 train 7.885048 (lr=2.9662e-05) (hash(x)=159334575)
+1000 val loss 7.6516
+1000 val perplexity 2103.9319
+1000 train 7.547078 (lr=2.9558e-05) (hash(x)=140604760)
+1100 val loss 7.6129
+1100 val perplexity 2024.2212
+1100 train 7.476753 (lr=2.9442e-05) (hash(x)=146208052)
+1200 val loss 7.5828
+1200 val perplexity 1964.1543
+1200 train 7.506049 (lr=2.9312e-05) (hash(x)=148404734)
+1300 val loss 7.5548
+1300 val perplexity 1909.9128
+1300 train 7.506157 (lr=2.9169e-05) (hash(x)=155681970)
+1400 val loss 7.5313
+1400 val perplexity 1865.5293
+1400 train 7.483672 (lr=2.9013e-05) (hash(x)=148115934)
+1500 val loss 7.5048
+1500 val perplexity 1816.6732
+1500 train 7.628779 (lr=2.8845e-05) (hash(x)=157074034)
+1600 val loss 7.4882
+1600 val perplexity 1786.8827
+1600 train 7.282681 (lr=2.8663e-05) (hash(x)=137464699)
+1700 val loss 7.4733
+1700 val perplexity 1760.3569
+1700 train 7.699624 (lr=2.8469e-05) (hash(x)=166955614)
+1800 val loss 7.4555
+1800 val perplexity 1729.3726
+1800 train 7.306218 (lr=2.8263e-05) (hash(x)=143886042)
+1900 val loss 7.4465
+1900 val perplexity 1713.8367
+1900 train 8.157793 (lr=2.8044e-05) (hash(x)=193610391)
+2000 val loss 7.4311
+2000 val perplexity 1687.7231
+2000 train 7.588061 (lr=2.7814e-05) (hash(x)=163799796)
+2100 val loss 7.4155
+2100 val perplexity 1661.4758
+2100 train 7.383990 (lr=2.7572e-05) (hash(x)=154107339)
+2200 val loss 7.4025
+2200 val perplexity 1640.0312
+2200 train 7.370002 (lr=2.7319e-05) (hash(x)=144592844)
+2300 val loss 7.3893
+2300 val perplexity 1618.5370
+2300 train 7.464876 (lr=2.7055e-05) (hash(x)=154535861)
+2400 val loss 7.3804
+2400 val perplexity 1604.1582
+2400 train 8.137079 (lr=2.6780e-05) (hash(x)=204706354)
+2500 val loss 7.3751
+2500 val perplexity 1595.7661
+2500 train 7.218102 (lr=2.6494e-05) (hash(x)=143783202)
+2600 val loss 7.3654
+2600 val perplexity 1580.3450
+2600 train 7.271410 (lr=2.6198e-05) (hash(x)=150862210)
+2700 val loss 7.3560
+2700 val perplexity 1565.5422
+2700 train 7.431979 (lr=2.5892e-05) (hash(x)=150753426)
+2800 val loss 7.3492
+2800 val perplexity 1554.8976
+2800 train 7.303928 (lr=2.5576e-05) (hash(x)=147766811)
+2900 val loss 7.3390
+2900 val perplexity 1539.1718
+2900 train 7.278078 (lr=2.5251e-05) (hash(x)=151177814)
+3000 val loss 7.3307
+3000 val perplexity 1526.4302
+3000 train 7.174177 (lr=2.4917e-05) (hash(x)=135925327)
+3100 val loss 7.3244
+3100 val perplexity 1516.9069
+3100 train 7.040339 (lr=2.4574e-05) (hash(x)=124842353)
+3200 val loss 7.3266
+3200 val perplexity 1520.2181
+3200 train 7.435338 (lr=2.4224e-05) (hash(x)=160440642)
+3300 val loss 7.3115
+3300 val perplexity 1497.4510
+3300 train 7.207845 (lr=2.3865e-05) (hash(x)=143017131)
+3400 val loss 7.3044
+3400 val perplexity 1486.8070
+3400 train 7.512286 (lr=2.3498e-05) (hash(x)=148143474)
+3500 val loss 7.3002
+3500 val perplexity 1480.5734
+3500 train 7.127140 (lr=2.3125e-05) (hash(x)=148732639)
+3600 val loss 7.2905
+3600 val perplexity 1466.3738
+3600 train 7.279788 (lr=2.2744e-05) (hash(x)=146333803)
+3700 val loss 7.2896
+3700 val perplexity 1465.0208
+3700 train 7.123580 (lr=2.2357e-05) (hash(x)=143611445)
+3800 val loss 7.2885
+3800 val perplexity 1463.3877
+3800 train 7.111747 (lr=2.1965e-05) (hash(x)=141272115)
+3900 val loss 7.2697
+3900 val perplexity 1436.1346
+3900 train 7.193086 (lr=2.1566e-05) (hash(x)=149161848)
+4000 val loss 7.2576
+4000 val perplexity 1418.8943
+4000 train 7.323757 (lr=2.1162e-05) (hash(x)=152036955)
+4100 val loss 7.2537
+4100 val perplexity 1413.3195
+4100 train 7.318581 (lr=2.0754e-05) (hash(x)=153260968)
+4200 val loss 7.2432
+4200 val perplexity 1398.5120
+4200 train 7.117545 (lr=2.0341e-05) (hash(x)=147629478)
+4300 val loss 7.2383
+4300 val perplexity 1391.6953
+4300 train 7.268786 (lr=1.9924e-05) (hash(x)=161701080)
+4400 val loss 7.2286
+4400 val perplexity 1378.2469
+4400 train 7.303712 (lr=1.9504e-05) (hash(x)=155290524)
+4500 val loss 7.2303
+4500 val perplexity 1380.6550
+4500 train 7.059731 (lr=1.9081e-05) (hash(x)=144726593)
+4600 val loss 7.2247
+4600 val perplexity 1372.9601
+4600 train 7.186725 (lr=1.8655e-05) (hash(x)=149517202)
+4700 val loss 7.2116
+4700 val perplexity 1355.0332
+4700 train 7.194616 (lr=1.8226e-05) (hash(x)=139403008)
+4800 val loss 7.2014
+4800 val perplexity 1341.2859
+4800 train 7.243327 (lr=1.7796e-05) (hash(x)=153387238)
+4900 val loss 7.1937
+4900 val perplexity 1331.0828
+4900 train 6.963374 (lr=1.7365e-05) (hash(x)=139881569)
+5000 val loss 7.1992
+5000 val perplexity 1338.4148
+5000 train 7.139058 (lr=1.6933e-05) (hash(x)=152602580)
+5100 val loss 7.1824
+5100 val perplexity 1316.0294
+5100 train 7.249477 (lr=1.6500e-05) (hash(x)=150598460)
+5200 val loss 7.1743
+5200 val perplexity 1305.4244
+5200 train 7.111207 (lr=1.6067e-05) (hash(x)=147660496)
+5300 val loss 7.1752
+5300 val perplexity 1306.6388
+5300 train 6.939226 (lr=1.5635e-05) (hash(x)=147693848)
+5400 val loss 7.1752
+5400 val perplexity 1306.6394
+5400 train 6.945499 (lr=1.5204e-05) (hash(x)=136048145)
+5500 val loss 7.1585
+5500 val perplexity 1285.0258
+5500 train 7.071564 (lr=1.4774e-05) (hash(x)=140313123)
+5600 val loss 7.1523
+5600 val perplexity 1277.0269
+5600 train 7.182386 (lr=1.4345e-05) (hash(x)=156997993)
+5700 val loss 7.1460
+5700 val perplexity 1269.0305
+5700 train 7.119817 (lr=1.3919e-05) (hash(x)=143513880)
+5800 val loss 7.1424
+5800 val perplexity 1264.4019
+5800 train 7.322329 (lr=1.3496e-05) (hash(x)=153736628)
+5900 val loss 7.1384
+5900 val perplexity 1259.4106
+5900 train 7.379342 (lr=1.3076e-05) (hash(x)=166385417)
+6000 val loss 7.1305
+6000 val perplexity 1249.4418
+6000 train 7.339532 (lr=1.2659e-05) (hash(x)=153216517)
+6100 val loss 7.1324
+6100 val perplexity 1251.8959
+6100 train 7.016201 (lr=1.2246e-05) (hash(x)=145387545)
+6200 val loss 7.1294
+6200 val perplexity 1248.0878
+6200 train 7.234446 (lr=1.1838e-05) (hash(x)=156306460)
+6300 val loss 7.1173
+6300 val perplexity 1233.1659
+6300 train 6.998589 (lr=1.1434e-05) (hash(x)=148610447)
+6400 val loss 7.1144
+6400 val perplexity 1229.5426
+6400 train 7.300064 (lr=1.1035e-05) (hash(x)=158343535)
+6500 val loss 7.1075
+6500 val perplexity 1221.1411
+6500 train 7.250863 (lr=1.0643e-05) (hash(x)=157742557)
+6600 val loss 7.1070
+6600 val perplexity 1220.5369
+6600 train 7.055732 (lr=1.0256e-05) (hash(x)=150457308)
+6700 val loss 7.1003
+6700 val perplexity 1212.2936
+6700 train 7.000196 (lr=9.8753e-06) (hash(x)=142426205)
+6800 val loss 7.0989
+6800 val perplexity 1210.6316
+6800 train 6.944258 (lr=9.5017e-06) (hash(x)=151522525)
+6900 val loss 7.0960
+6900 val perplexity 1207.0739
+6900 train 7.001570 (lr=9.1353e-06) (hash(x)=152029630)
+7000 val loss 7.0854
+7000 val perplexity 1194.3942
+7000 train 6.954720 (lr=8.7764e-06) (hash(x)=142978954)
+7100 val loss 7.0819
+7100 val perplexity 1190.2830
+7100 train 6.964025 (lr=8.4255e-06) (hash(x)=148064359)
+7200 val loss 7.0801
+7200 val perplexity 1188.0596
+7200 train 7.016519 (lr=8.0829e-06) (hash(x)=155771845)
+7300 val loss 7.0784
+7300 val perplexity 1186.0615
+7300 train 6.898583 (lr=7.7489e-06) (hash(x)=141639844)
+7400 val loss 7.0750
+7400 val perplexity 1182.0111
+7400 train 7.105750 (lr=7.4239e-06) (hash(x)=153466021)
+7500 val loss 7.0698
+7500 val perplexity 1175.8695
+7500 train 7.085123 (lr=7.1083e-06) (hash(x)=148843587)
+7600 val loss 7.0659
+7600 val perplexity 1171.3831
+7600 train 7.242340 (lr=6.8023e-06) (hash(x)=152053933)
+7700 val loss 7.0645
+7700 val perplexity 1169.6594
+7700 train 6.819738 (lr=6.5062e-06) (hash(x)=141564003)
+7800 val loss 7.0610
+7800 val perplexity 1165.6084
+7800 train 6.958831 (lr=6.2205e-06) (hash(x)=148617843)
+7900 val loss 7.0582
+7900 val perplexity 1162.3824
+7900 train 7.025887 (lr=5.9453e-06) (hash(x)=146417668)
+8000 val loss 7.0567
+8000 val perplexity 1160.6235
+8000 train 6.897067 (lr=5.6809e-06) (hash(x)=150165488)
+8100 val loss 7.0525
+8100 val perplexity 1155.7068
+8100 train 7.032347 (lr=5.4277e-06) (hash(x)=144818855)
+8200 val loss 7.0502
+8200 val perplexity 1153.0596
+8200 train 7.068508 (lr=5.1858e-06) (hash(x)=143074954)
+8300 val loss 7.0468
+8300 val perplexity 1149.2288
+8300 train 6.975527 (lr=4.9556e-06) (hash(x)=145172494)
+8400 val loss 7.0459
+8400 val perplexity 1148.1432
+8400 train 7.055420 (lr=4.7372e-06) (hash(x)=152284393)
+8500 val loss 7.0425
+8500 val perplexity 1144.2218
+8500 train 6.915458 (lr=4.5309e-06) (hash(x)=147044705)
+8600 val loss 7.0424
+8600 val perplexity 1144.1121
+8600 train 6.983307 (lr=4.3369e-06) (hash(x)=147762608)
+8700 val loss 7.0410
+8700 val perplexity 1142.5272
+8700 train 7.011322 (lr=4.1554e-06) (hash(x)=141994567)
+8800 val loss 7.0352
+8800 val perplexity 1135.8756
+8800 train 7.072652 (lr=3.9866e-06) (hash(x)=152552896)
+8900 val loss 7.0333
+8900 val perplexity 1133.7501
+8900 train 6.894183 (lr=3.8307e-06) (hash(x)=139714454)
+9000 val loss 7.0328
+9000 val perplexity 1133.2528
+9000 train 7.056057 (lr=3.6877e-06) (hash(x)=152274419)
+9100 val loss 7.0316
+9100 val perplexity 1131.8865
+9100 train 7.202514 (lr=3.5580e-06) (hash(x)=160536885)
+9200 val loss 7.0298
+9200 val perplexity 1129.7684
+9200 train 6.851665 (lr=3.4415e-06) (hash(x)=138386899)
+9300 val loss 7.0302
+9300 val perplexity 1130.2065
+9300 train 7.028361 (lr=3.3385e-06) (hash(x)=149957457)
+9400 val loss 7.0258
+9400 val perplexity 1125.3226
+9400 train 7.156860 (lr=3.2490e-06) (hash(x)=161671548)
+9500 val loss 7.0248
+9500 val perplexity 1124.2178
+9500 train 7.202159 (lr=3.1730e-06) (hash(x)=171857400)
+9600 val loss 7.0241
+9600 val perplexity 1123.4290
+9600 train 6.707559 (lr=3.1108e-06) (hash(x)=135378471)
+9700 val loss 7.0229
+9700 val perplexity 1121.9984
+9700 train 6.933225 (lr=3.0624e-06) (hash(x)=149514591)
+9800 val loss 7.0205
+9800 val perplexity 1119.3560
+9800 train 6.984025 (lr=3.0277e-06) (hash(x)=152679072)
+9900 val loss 7.0192
+9900 val perplexity 1117.9419
+9900 train 6.940137 (lr=3.0069e-06) (hash(x)=150415193)
+9999 val loss 7.0191
+9999 val perplexity 1117.7302

attention_kindself_n_heads4_seed1341/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ac1d5aa92ba4713050144037ebcf63fd3e9a1a0ef11fb75b70a385d1aa5ca4b
+oid sha256:8533fe5d49e6468a64e020d2e6dca89833d23f9214fc1be9eb2ee30ef1db760e
 size 92843394

attention_kindself_n_heads4_seed1341/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:825d3da37092151cfb6fab5159d8442c013f4a85019f5356a30d2c828bb55c82
+oid sha256:d3c47e483b66f52ef3a5f50b88ea9ee3016769ed0f1aa2779e046eebeb0641ac
 size 92843394

attention_kindself_n_heads4_seed1341/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67c1d697cb0414321c4a54e6b2141ec020c319a790f62e712fa3ba3a56888c71
+oid sha256:928a4b97f0b1215e25abc5ac9938be2c276196c955e5f2e4bd4119938e3d2dee
 size 92843394

attention_kindself_n_heads4_seed1341/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2cf6289db5de1e15682ac425c55d0f01dc704a9b78e758572e7252f3a5bb6c93
+oid sha256:e126886d2370e12bdacf3942a1462616968ab2a1500b9fa89dee8ef363535dcb
 size 92843394

attention_kindself_n_heads4_seed1341/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c01df2ef9425fd5dccebc1cf0b91577579f972cdf4f0907b953e6d13ac7933e
+oid sha256:b718bdb4bfaa88b52a0da4d5bcd0142b722703bb4353691a3c0e6a89cb571c79
 size 179406214

attention_kindself_n_heads4_seed1341/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:58ee5955f79793b20bca681e1d28a5fc28cd98cf1864eaa9f06d0fe569d7829a
+oid sha256:ba858b7ea2bbf8694aaafde9ea0ddfe3c664c6803b05ee290be50a86c31a6c11
 size 179406214

attention_kindself_n_heads4_seed1341/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d8dd27a1fa4a6b7d8f3233476db26df0e11831f7d04f7eca75548086a5eb9e47
+oid sha256:c3b843f853856fbfcc3be268c818df6d800cbf52b2bf5cd06f70d819663e3fc1
 size 179406214

attention_kindself_n_heads4_seed1341/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da6de5394f11bd1e581cd639bc807a8c4d18610b171933496284661c8c7149f3
+oid sha256:c4070361dac77778acb12893408d8f260bb04f58addd60a588ca35f243df8247
 size 179406214