Upload folder using huggingface_hub

attention_kindself_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1338", "n_embd": 256}

attention_kindself_n_heads4_seed1338/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.2886
 0 val perplexity 79902.0703
-0 train 11.283466 (lr=2.5000e-07) (hash(x)=150327452)
-100 val loss 9.8544
-100 val perplexity 19041.1133
-100 train 9.863648 (lr=2.5250e-05) (hash(x)=166441190)
-200 val loss 9.0230
-200 val perplexity 8291.7314
-200 train 9.143907 (lr=5.0000e-05) (hash(x)=166780046)
-300 val loss 8.2911
-300 val perplexity 3988.2437
-300 train 8.266171 (lr=4.9988e-05) (hash(x)=159835303)
-400 val loss 7.8842
-400 val perplexity 2655.0879
-400 train 7.704815 (lr=4.9954e-05) (hash(x)=155040610)
-500 val loss 7.6806
-500 val perplexity 2166.0205
-500 train 7.444143 (lr=4.9896e-05) (hash(x)=130190460)
-600 val loss 7.5840
-600 val perplexity 1966.5450
-600 train 7.610361 (lr=4.9815e-05) (hash(x)=155504036)
-700 val loss 7.5115
-700 val perplexity 1828.8649
-700 train 7.361025 (lr=4.9712e-05) (hash(x)=137347213)
-800 val loss 7.4632
-800 val perplexity 1742.7035
-800 train 7.328137 (lr=4.9585e-05) (hash(x)=143823248)
-900 val loss 7.4254
-900 val perplexity 1678.1448
-900 train 7.541551 (lr=4.9436e-05) (hash(x)=156260416)
-1000 val loss 7.4009
-1000 val perplexity 1637.4425
-1000 train 7.370451 (lr=4.9264e-05) (hash(x)=143734685)
-1100 val loss 7.3681
-1100 val perplexity 1584.5743
-1100 train 7.314400 (lr=4.9070e-05) (hash(x)=160013925)
-1200 val loss 7.3553
-1200 val perplexity 1564.4012
-1200 train 7.117009 (lr=4.8854e-05) (hash(x)=150678249)
-1300 val loss 7.3327
-1300 val perplexity 1529.4917
-1300 train 7.139935 (lr=4.8616e-05) (hash(x)=149073315)
-1400 val loss 7.2860
-1400 val perplexity 1459.6823
-1400 train 7.564305 (lr=4.8356e-05) (hash(x)=175802021)
-1500 val loss 7.2605
-1500 val perplexity 1423.0288
-1500 train 7.520332 (lr=4.8074e-05) (hash(x)=171034639)
-1600 val loss 7.2325
-1600 val perplexity 1383.6577
-1600 train 7.434268 (lr=4.7772e-05) (hash(x)=158681215)
-1700 val loss 7.2128
-1700 val perplexity 1356.7013
-1700 train 7.162296 (lr=4.7448e-05) (hash(x)=152116061)
-1800 val loss 7.1999
-1800 val perplexity 1339.2319
-1800 train 7.145010 (lr=4.7105e-05) (hash(x)=146108145)
-1900 val loss 7.1973
-1900 val perplexity 1335.8719
-1900 train 7.039209 (lr=4.6741e-05) (hash(x)=147598108)
-2000 val loss 7.1866
-2000 val perplexity 1321.5740
-2000 train 7.076362 (lr=4.6357e-05) (hash(x)=154996086)
-2100 val loss 7.1827
-2100 val perplexity 1316.4933
-2100 train 7.007890 (lr=4.5954e-05) (hash(x)=153396183)
-2200 val loss 7.1664
-2200 val perplexity 1295.1556
-2200 train 7.154462 (lr=4.5532e-05) (hash(x)=153885445)
-2300 val loss 7.1837
-2300 val perplexity 1317.8405
-2300 train 7.172379 (lr=4.5091e-05) (hash(x)=159666385)
-2400 val loss 7.1497
-2400 val perplexity 1273.6742
-2400 train 7.117113 (lr=4.4633e-05) (hash(x)=142353087)
-2500 val loss 7.1538
-2500 val perplexity 1279.0111
-2500 train 7.165363 (lr=4.4156e-05) (hash(x)=146491718)
-2600 val loss 7.1336
-2600 val perplexity 1253.4100
-2600 train 7.067105 (lr=4.3663e-05) (hash(x)=150750353)
-2700 val loss 7.1334
-2700 val perplexity 1253.1818
-2700 train 6.869538 (lr=4.3153e-05) (hash(x)=129849193)
-2800 val loss 7.1409
-2800 val perplexity 1262.5409
-2800 train 7.030183 (lr=4.2627e-05) (hash(x)=152767913)
-2900 val loss 7.1334
-2900 val perplexity 1253.0903
-2900 train 6.959103 (lr=4.2085e-05) (hash(x)=146531140)
-3000 val loss 7.1261
-3000 val perplexity 1243.9644
-3000 train 7.178856 (lr=4.1529e-05) (hash(x)=151562048)
-3100 val loss 7.1202
-3100 val perplexity 1236.7474
-3100 train 7.116158 (lr=4.0957e-05) (hash(x)=146001424)
-3200 val loss 7.1235
-3200 val perplexity 1240.7748
-3200 train 7.121208 (lr=4.0373e-05) (hash(x)=166486165)
-3300 val loss 7.1120
-3300 val perplexity 1226.5532
-3300 train 7.035117 (lr=3.9775e-05) (hash(x)=150866680)
-3400 val loss 7.1260
-3400 val perplexity 1243.9399
-3400 train 7.041269 (lr=3.9164e-05) (hash(x)=143900419)
-3500 val loss 7.1008
-3500 val perplexity 1212.9781
-3500 train 6.948005 (lr=3.8541e-05) (hash(x)=148845794)
-3600 val loss 7.0984
-3600 val perplexity 1210.0360
-3600 train 6.968525 (lr=3.7907e-05) (hash(x)=145667796)
-3700 val loss 7.1015
-3700 val perplexity 1213.7604
-3700 train 7.115687 (lr=3.7262e-05) (hash(x)=163563851)
-3800 val loss 7.0926
-3800 val perplexity 1202.9844
-3800 train 7.065489 (lr=3.6608e-05) (hash(x)=147488689)
-3900 val loss 7.0933
-3900 val perplexity 1203.8824
-3900 train 7.070405 (lr=3.5944e-05) (hash(x)=148186608)
-4000 val loss 7.0900
-4000 val perplexity 1199.9045
-4000 train 6.936615 (lr=3.5271e-05) (hash(x)=142970187)
-4100 val loss 7.0773
-4100 val perplexity 1184.7151
-4100 train 7.097265 (lr=3.4590e-05) (hash(x)=141584883)
-4200 val loss 7.0754
-4200 val perplexity 1182.5468
-4200 train 6.886055 (lr=3.3902e-05) (hash(x)=145664585)
-4300 val loss 7.0708
-4300 val perplexity 1177.1172
-4300 train 6.912655 (lr=3.3207e-05) (hash(x)=143736499)
-4400 val loss 7.0777
-4400 val perplexity 1185.2440
-4400 train 6.899530 (lr=3.2507e-05) (hash(x)=151883322)
-4500 val loss 7.0762
-4500 val perplexity 1183.4690
-4500 train 7.021190 (lr=3.1801e-05) (hash(x)=153904871)
-4600 val loss 7.0704
-4600 val perplexity 1176.6206
-4600 train 7.104506 (lr=3.1091e-05) (hash(x)=154893521)
-4700 val loss 7.0581
-4700 val perplexity 1162.1769
-4700 train 7.123883 (lr=3.0377e-05) (hash(x)=152323949)
-4800 val loss 7.0639
-4800 val perplexity 1168.9825
-4800 train 7.007242 (lr=2.9661e-05) (hash(x)=154104619)
-4900 val loss 7.0559
-4900 val perplexity 1159.6443
-4900 train 7.077590 (lr=2.8942e-05) (hash(x)=146311426)
-5000 val loss 7.0545
-5000 val perplexity 1158.0909
-5000 train 7.086222 (lr=2.8221e-05) (hash(x)=156741847)
-5100 val loss 7.0598
-5100 val perplexity 1164.1836
-5100 train 6.874581 (lr=2.7500e-05) (hash(x)=142086346)
-5200 val loss 7.0528
-5200 val perplexity 1156.0673
-5200 train 6.862787 (lr=2.6779e-05) (hash(x)=150265428)
-5300 val loss 7.0451
-5300 val perplexity 1147.2747
-5300 train 6.953916 (lr=2.6058e-05) (hash(x)=151339108)
-5400 val loss 7.0455
-5400 val perplexity 1147.6992
-5400 train 7.044176 (lr=2.5339e-05) (hash(x)=154654372)
-5500 val loss 7.0367
-5500 val perplexity 1137.6520
-5500 train 7.023346 (lr=2.4623e-05) (hash(x)=150575051)
-5600 val loss 7.0300
-5600 val perplexity 1129.9985
-5600 train 6.926621 (lr=2.3909e-05) (hash(x)=140396423)
-5700 val loss 7.0262
-5700 val perplexity 1125.7670
-5700 train 6.930413 (lr=2.3199e-05) (hash(x)=144678758)
-5800 val loss 7.0262
-5800 val perplexity 1125.7369
-5800 train 7.138989 (lr=2.2493e-05) (hash(x)=151992743)
-5900 val loss 7.0206
-5900 val perplexity 1119.4338
-5900 train 6.823964 (lr=2.1793e-05) (hash(x)=144396927)
-6000 val loss 7.0205
-6000 val perplexity 1119.3479
-6000 train 6.976523 (lr=2.1098e-05) (hash(x)=165478625)
-6100 val loss 7.0285
-6100 val perplexity 1128.3563
-6100 train 6.817317 (lr=2.0410e-05) (hash(x)=147088621)
-6200 val loss 7.0149
-6200 val perplexity 1113.1390
-6200 train 6.848406 (lr=1.9729e-05) (hash(x)=140794994)
-6300 val loss 7.0096
-6300 val perplexity 1107.2498
-6300 train 6.922020 (lr=1.9056e-05) (hash(x)=134780906)
-6400 val loss 7.0057
-6400 val perplexity 1102.8799
-6400 train 7.010965 (lr=1.8392e-05) (hash(x)=149023655)
-6500 val loss 6.9984
-6500 val perplexity 1094.8854
-6500 train 6.877532 (lr=1.7738e-05) (hash(x)=147497796)
-6600 val loss 7.0005
-6600 val perplexity 1097.2216
-6600 train 6.884083 (lr=1.7093e-05) (hash(x)=152902689)
-6700 val loss 6.9997
-6700 val perplexity 1096.3206
-6700 train 7.056413 (lr=1.6459e-05) (hash(x)=153846046)
-6800 val loss 6.9938
-6800 val perplexity 1089.8635
-6800 train 7.118688 (lr=1.5836e-05) (hash(x)=158512738)
-6900 val loss 6.9920
-6900 val perplexity 1087.9315
-6900 train 7.292885 (lr=1.5225e-05) (hash(x)=156849968)
-7000 val loss 6.9937
-7000 val perplexity 1089.7850
-7000 train 6.811767 (lr=1.4627e-05) (hash(x)=142395855)
-7100 val loss 6.9875
-7100 val perplexity 1083.0056
-7100 train 6.884103 (lr=1.4043e-05) (hash(x)=147114884)
-7200 val loss 6.9841
-7200 val perplexity 1079.3735
-7200 train 6.994034 (lr=1.3471e-05) (hash(x)=156979839)
-7300 val loss 6.9858
-7300 val perplexity 1081.1295
-7300 train 6.760962 (lr=1.2915e-05) (hash(x)=145584373)
-7400 val loss 6.9833
-7400 val perplexity 1078.4264
-7400 train 6.740240 (lr=1.2373e-05) (hash(x)=141508204)
-7500 val loss 6.9817
-7500 val perplexity 1076.7153
-7500 train 7.028373 (lr=1.1847e-05) (hash(x)=148803965)
-7600 val loss 6.9749
-7600 val perplexity 1069.4771
-7600 train 7.015244 (lr=1.1337e-05) (hash(x)=151019676)
-7700 val loss 6.9713
-7700 val perplexity 1065.6301
-7700 train 6.945483 (lr=1.0844e-05) (hash(x)=143155750)
-7800 val loss 6.9727
-7800 val perplexity 1067.0726
-7800 train 6.974567 (lr=1.0367e-05) (hash(x)=152569653)
-7900 val loss 6.9678
-7900 val perplexity 1061.8811
-7900 train 6.818687 (lr=9.9088e-06) (hash(x)=143519455)
-8000 val loss 6.9649
-8000 val perplexity 1058.8020
-8000 train 7.139881 (lr=9.4682e-06) (hash(x)=161180944)
-8100 val loss 6.9641
-8100 val perplexity 1057.9642
-8100 train 7.000780 (lr=9.0461e-06) (hash(x)=154107345)
-8200 val loss 6.9641
-8200 val perplexity 1057.9214
-8200 train 6.970243 (lr=8.6430e-06) (hash(x)=152486517)
-8300 val loss 6.9591
-8300 val perplexity 1052.7146
-8300 train 6.985438 (lr=8.2593e-06) (hash(x)=156167749)
-8400 val loss 6.9573
-8400 val perplexity 1050.7441
-8400 train 7.019130 (lr=7.8953e-06) (hash(x)=149155006)
-8500 val loss 6.9558
-8500 val perplexity 1049.2146
-8500 train 6.930621 (lr=7.5515e-06) (hash(x)=147844390)
-8600 val loss 6.9528
-8600 val perplexity 1046.1099
-8600 train 7.217835 (lr=7.2282e-06) (hash(x)=165753320)
-8700 val loss 6.9518
-8700 val perplexity 1044.9836
-8700 train 6.754283 (lr=6.9257e-06) (hash(x)=146079979)
-8800 val loss 6.9521
-8800 val perplexity 1045.3613
-8800 train 7.165798 (lr=6.6444e-06) (hash(x)=172259509)
-8900 val loss 6.9514
-8900 val perplexity 1044.6359
-8900 train 6.651181 (lr=6.3845e-06) (hash(x)=145148314)
-9000 val loss 6.9508
-9000 val perplexity 1043.9875
-9000 train 6.883500 (lr=6.1462e-06) (hash(x)=144250633)
-9100 val loss 6.9482
-9100 val perplexity 1041.2834
-9100 train 7.060329 (lr=5.9300e-06) (hash(x)=157219797)
-9200 val loss 6.9442
-9200 val perplexity 1037.1249
-9200 train 6.898849 (lr=5.7359e-06) (hash(x)=142743778)
-9300 val loss 6.9451
-9300 val perplexity 1038.0536
-9300 train 6.823744 (lr=5.5641e-06) (hash(x)=139669771)
-9400 val loss 6.9442
-9400 val perplexity 1037.1675
-9400 train 6.878235 (lr=5.4149e-06) (hash(x)=145916843)
-9500 val loss 6.9418
-9500 val perplexity 1034.6527
-9500 train 6.851600 (lr=5.2884e-06) (hash(x)=150196125)
-9600 val loss 6.9414
-9600 val perplexity 1034.1758
-9600 train 7.224812 (lr=5.1847e-06) (hash(x)=160041419)
-9700 val loss 6.9423
-9700 val perplexity 1035.1101
-9700 train 6.753918 (lr=5.1040e-06) (hash(x)=139931627)
-9800 val loss 6.9412
-9800 val perplexity 1034.0095
-9800 train 6.870800 (lr=5.0462e-06) (hash(x)=150370792)
-9900 val loss 6.9394
-9900 val perplexity 1032.1249
-9900 train 6.819551 (lr=5.0116e-06) (hash(x)=153014886)
-9999 val loss 6.9389
-9999 val perplexity 1031.6648
+0 train 11.283466 (lr=1.5000e-07) (hash(x)=150327452)
+100 val loss 9.9426
+100 val perplexity 20797.1973
+100 train 9.951486 (lr=1.5150e-05) (hash(x)=166441190)
+200 val loss 9.5114
+200 val perplexity 13513.0186
+200 train 9.605964 (lr=3.0000e-05) (hash(x)=166780046)
+300 val loss 8.7451
+300 val perplexity 6279.5659
+300 train 8.728355 (lr=2.9993e-05) (hash(x)=159835303)
+400 val loss 8.3718
+400 val perplexity 4323.4829
+400 train 8.235453 (lr=2.9972e-05) (hash(x)=155040610)
+500 val loss 8.1024
+500 val perplexity 3302.2766
+500 train 7.927925 (lr=2.9938e-05) (hash(x)=130190460)
+600 val loss 7.8912
+600 val perplexity 2673.6594
+600 train 7.926107 (lr=2.9889e-05) (hash(x)=155504036)
+700 val loss 7.7488
+700 val perplexity 2318.7988
+700 train 7.632146 (lr=2.9827e-05) (hash(x)=137347213)
+800 val loss 7.6446
+800 val perplexity 2089.2458
+800 train 7.523963 (lr=2.9751e-05) (hash(x)=143823248)
+900 val loss 7.5762
+900 val perplexity 1951.1947
+900 train 7.693543 (lr=2.9662e-05) (hash(x)=156260416)
+1000 val loss 7.5392
+1000 val perplexity 1880.4181
+1000 train 7.516635 (lr=2.9558e-05) (hash(x)=143734685)
+1100 val loss 7.5088
+1100 val perplexity 1823.9686
+1100 train 7.466663 (lr=2.9442e-05) (hash(x)=160013925)
+1200 val loss 7.4951
+1200 val perplexity 1799.2511
+1200 train 7.284273 (lr=2.9312e-05) (hash(x)=150678249)
+1300 val loss 7.4721
+1300 val perplexity 1758.3167
+1300 train 7.295808 (lr=2.9169e-05) (hash(x)=149073315)
+1400 val loss 7.4411
+1400 val perplexity 1704.7083
+1400 train 7.834116 (lr=2.9013e-05) (hash(x)=175802021)
+1500 val loss 7.4113
+1500 val perplexity 1654.5509
+1500 train 7.657399 (lr=2.8845e-05) (hash(x)=171034639)
+1600 val loss 7.3870
+1600 val perplexity 1614.8066
+1600 train 7.606214 (lr=2.8663e-05) (hash(x)=158681215)
+1700 val loss 7.3741
+1700 val perplexity 1594.0922
+1700 train 7.319633 (lr=2.8469e-05) (hash(x)=152116061)
+1800 val loss 7.3540
+1800 val perplexity 1562.4801
+1800 train 7.298090 (lr=2.8263e-05) (hash(x)=146108145)
+1900 val loss 7.3435
+1900 val perplexity 1546.0811
+1900 train 7.185542 (lr=2.8044e-05) (hash(x)=147598108)
+2000 val loss 7.3329
+2000 val perplexity 1529.7994
+2000 train 7.227923 (lr=2.7814e-05) (hash(x)=154996086)
+2100 val loss 7.3275
+2100 val perplexity 1521.5366
+2100 train 7.156155 (lr=2.7572e-05) (hash(x)=153396183)
+2200 val loss 7.2975
+2200 val perplexity 1476.6315
+2200 train 7.291300 (lr=2.7319e-05) (hash(x)=153885445)
+2300 val loss 7.2917
+2300 val perplexity 1468.0928
+2300 train 7.280858 (lr=2.7055e-05) (hash(x)=159666385)
+2400 val loss 7.2666
+2400 val perplexity 1431.6726
+2400 train 7.232553 (lr=2.6780e-05) (hash(x)=142353087)
+2500 val loss 7.2657
+2500 val perplexity 1430.3529
+2500 train 7.271929 (lr=2.6494e-05) (hash(x)=146491718)
+2600 val loss 7.2557
+2600 val perplexity 1416.1014
+2600 train 7.198318 (lr=2.6198e-05) (hash(x)=150750353)
+2700 val loss 7.2518
+2700 val perplexity 1410.6331
+2700 train 6.991519 (lr=2.5892e-05) (hash(x)=129849193)
+2800 val loss 7.2356
+2800 val perplexity 1388.0264
+2800 train 7.123249 (lr=2.5576e-05) (hash(x)=152767913)
+2900 val loss 7.2314
+2900 val perplexity 1382.1377
+2900 train 7.050823 (lr=2.5251e-05) (hash(x)=146531140)
+3000 val loss 7.2139
+3000 val perplexity 1358.1609
+3000 train 7.264537 (lr=2.4917e-05) (hash(x)=151562048)
+3100 val loss 7.2104
+3100 val perplexity 1353.4137
+3100 train 7.202452 (lr=2.4574e-05) (hash(x)=146001424)
+3200 val loss 7.2078
+3200 val perplexity 1349.9771
+3200 train 7.196339 (lr=2.4224e-05) (hash(x)=166486165)
+3300 val loss 7.1908
+3300 val perplexity 1327.2173
+3300 train 7.114285 (lr=2.3865e-05) (hash(x)=150866680)
+3400 val loss 7.1955
+3400 val perplexity 1333.3633
+3400 train 7.111781 (lr=2.3498e-05) (hash(x)=143900419)
+3500 val loss 7.1822
+3500 val perplexity 1315.8192
+3500 train 7.032573 (lr=2.3125e-05) (hash(x)=148845794)
+3600 val loss 7.1864
+3600 val perplexity 1321.2898
+3600 train 7.056874 (lr=2.2744e-05) (hash(x)=145667796)
+3700 val loss 7.1854
+3700 val perplexity 1320.0197
+3700 train 7.198254 (lr=2.2357e-05) (hash(x)=163563851)
+3800 val loss 7.1761
+3800 val perplexity 1307.8612
+3800 train 7.136229 (lr=2.1965e-05) (hash(x)=147488689)
+3900 val loss 7.1619
+3900 val perplexity 1289.3972
+3900 train 7.139191 (lr=2.1566e-05) (hash(x)=148186608)
+4000 val loss 7.1537
+4000 val perplexity 1278.8599
+4000 train 6.994923 (lr=2.1162e-05) (hash(x)=142970187)
+4100 val loss 7.1496
+4100 val perplexity 1273.6383
+4100 train 7.151843 (lr=2.0754e-05) (hash(x)=141584883)
+4200 val loss 7.1415
+4200 val perplexity 1263.2732
+4200 train 6.962108 (lr=2.0341e-05) (hash(x)=145664585)
+4300 val loss 7.1417
+4300 val perplexity 1263.5864
+4300 train 6.984930 (lr=1.9924e-05) (hash(x)=143736499)
+4400 val loss 7.1425
+4400 val perplexity 1264.5309
+4400 train 6.965490 (lr=1.9504e-05) (hash(x)=151883322)
+4500 val loss 7.1386
+4500 val perplexity 1259.7223
+4500 train 7.083486 (lr=1.9081e-05) (hash(x)=153904871)
+4600 val loss 7.1247
+4600 val perplexity 1242.2844
+4600 train 7.154972 (lr=1.8655e-05) (hash(x)=154893521)
+4700 val loss 7.1155
+4700 val perplexity 1230.9088
+4700 train 7.179066 (lr=1.8226e-05) (hash(x)=152323949)
+4800 val loss 7.1145
+4800 val perplexity 1229.6857
+4800 train 7.053790 (lr=1.7796e-05) (hash(x)=154104619)
+4900 val loss 7.1043
+4900 val perplexity 1217.1340
+4900 train 7.153165 (lr=1.7365e-05) (hash(x)=146311426)
+5000 val loss 7.1027
+5000 val perplexity 1215.1862
+5000 train 7.146621 (lr=1.6933e-05) (hash(x)=156741847)
+5100 val loss 7.1020
+5100 val perplexity 1214.3618
+5100 train 6.915137 (lr=1.6500e-05) (hash(x)=142086346)
+5200 val loss 7.0932
+5200 val perplexity 1203.8073
+5200 train 6.918579 (lr=1.6067e-05) (hash(x)=150265428)
+5300 val loss 7.0959
+5300 val perplexity 1206.9962
+5300 train 7.013549 (lr=1.5635e-05) (hash(x)=151339108)
+5400 val loss 7.0937
+5400 val perplexity 1204.3785
+5400 train 7.089282 (lr=1.5204e-05) (hash(x)=154654372)
+5500 val loss 7.0824
+5500 val perplexity 1190.7820
+5500 train 7.066670 (lr=1.4774e-05) (hash(x)=150575051)
+5600 val loss 7.0771
+5600 val perplexity 1184.5643
+5600 train 6.975541 (lr=1.4345e-05) (hash(x)=140396423)
+5700 val loss 7.0709
+5700 val perplexity 1177.1571
+5700 train 6.969749 (lr=1.3919e-05) (hash(x)=144678758)
+5800 val loss 7.0676
+5800 val perplexity 1173.3457
+5800 train 7.194654 (lr=1.3496e-05) (hash(x)=151992743)
+5900 val loss 7.0649
+5900 val perplexity 1170.1895
+5900 train 6.870719 (lr=1.3076e-05) (hash(x)=144396927)
+6000 val loss 7.0614
+6000 val perplexity 1166.1315
+6000 train 7.017785 (lr=1.2659e-05) (hash(x)=165478625)
+6100 val loss 7.0671
+6100 val perplexity 1172.7131
+6100 train 6.864367 (lr=1.2246e-05) (hash(x)=147088621)
+6200 val loss 7.0576
+6200 val perplexity 1161.6356
+6200 train 6.893899 (lr=1.1838e-05) (hash(x)=140794994)
+6300 val loss 7.0496
+6300 val perplexity 1152.4303
+6300 train 6.960331 (lr=1.1434e-05) (hash(x)=134780906)
+6400 val loss 7.0491
+6400 val perplexity 1151.7952
+6400 train 7.059733 (lr=1.1035e-05) (hash(x)=149023655)
+6500 val loss 7.0464
+6500 val perplexity 1148.7511
+6500 train 6.926225 (lr=1.0643e-05) (hash(x)=147497796)
+6600 val loss 7.0410
+6600 val perplexity 1142.4788
+6600 train 6.931141 (lr=1.0256e-05) (hash(x)=152902689)
+6700 val loss 7.0406
+6700 val perplexity 1142.1166
+6700 train 7.096299 (lr=9.8753e-06) (hash(x)=153846046)
+6800 val loss 7.0316
+6800 val perplexity 1131.8466
+6800 train 7.157819 (lr=9.5017e-06) (hash(x)=158512738)
+6900 val loss 7.0271
+6900 val perplexity 1126.8131
+6900 train 7.321591 (lr=9.1353e-06) (hash(x)=156849968)
+7000 val loss 7.0270
+7000 val perplexity 1126.6504
+7000 train 6.840876 (lr=8.7764e-06) (hash(x)=142395855)
+7100 val loss 7.0268
+7100 val perplexity 1126.4763
+7100 train 6.930350 (lr=8.4255e-06) (hash(x)=147114884)
+7200 val loss 7.0217
+7200 val perplexity 1120.7366
+7200 train 7.040916 (lr=8.0829e-06) (hash(x)=156979839)
+7300 val loss 7.0164
+7300 val perplexity 1114.7246
+7300 train 6.797614 (lr=7.7489e-06) (hash(x)=145584373)
+7400 val loss 7.0170
+7400 val perplexity 1115.4403
+7400 train 6.778458 (lr=7.4239e-06) (hash(x)=141508204)
+7500 val loss 7.0150
+7500 val perplexity 1113.1901
+7500 train 7.063877 (lr=7.1083e-06) (hash(x)=148803965)
+7600 val loss 7.0062
+7600 val perplexity 1103.4617
+7600 train 7.052894 (lr=6.8023e-06) (hash(x)=151019676)
+7700 val loss 7.0044
+7700 val perplexity 1101.4740
+7700 train 6.963307 (lr=6.5062e-06) (hash(x)=143155750)
+7800 val loss 7.0045
+7800 val perplexity 1101.5433
+7800 train 7.010788 (lr=6.2205e-06) (hash(x)=152569653)
+7900 val loss 7.0009
+7900 val perplexity 1097.6213
+7900 train 6.854251 (lr=5.9453e-06) (hash(x)=143519455)
+8000 val loss 6.9982
+8000 val perplexity 1094.6829
+8000 train 7.195841 (lr=5.6809e-06) (hash(x)=161180944)
+8100 val loss 6.9978
+8100 val perplexity 1094.2032
+8100 train 7.047000 (lr=5.4277e-06) (hash(x)=154107345)
+8200 val loss 6.9977
+8200 val perplexity 1094.1292
+8200 train 7.006711 (lr=5.1858e-06) (hash(x)=152486517)
+8300 val loss 6.9918
+8300 val perplexity 1087.6893
+8300 train 7.017220 (lr=4.9556e-06) (hash(x)=156167749)
+8400 val loss 6.9889
+8400 val perplexity 1084.5477
+8400 train 7.048491 (lr=4.7372e-06) (hash(x)=149155006)
+8500 val loss 6.9870
+8500 val perplexity 1082.4459
+8500 train 6.974413 (lr=4.5309e-06) (hash(x)=147844390)
+8600 val loss 6.9848
+8600 val perplexity 1080.0969
+8600 train 7.249847 (lr=4.3369e-06) (hash(x)=165753320)
+8700 val loss 6.9843
+8700 val perplexity 1079.5774
+8700 train 6.793041 (lr=4.1554e-06) (hash(x)=146079979)
+8800 val loss 6.9862
+8800 val perplexity 1081.5874
+8800 train 7.210235 (lr=3.9866e-06) (hash(x)=172259509)
+8900 val loss 6.9819
+8900 val perplexity 1077.0101
+8900 train 6.689318 (lr=3.8307e-06) (hash(x)=145148314)
+9000 val loss 6.9822
+9000 val perplexity 1077.3347
+9000 train 6.923099 (lr=3.6877e-06) (hash(x)=144250633)
+9100 val loss 6.9784
+9100 val perplexity 1073.2155
+9100 train 7.097141 (lr=3.5580e-06) (hash(x)=157219797)
+9200 val loss 6.9744
+9200 val perplexity 1068.8856
+9200 train 6.931408 (lr=3.4415e-06) (hash(x)=142743778)
+9300 val loss 6.9747
+9300 val perplexity 1069.2689
+9300 train 6.859241 (lr=3.3385e-06) (hash(x)=139669771)
+9400 val loss 6.9718
+9400 val perplexity 1066.1211
+9400 train 6.897627 (lr=3.2490e-06) (hash(x)=145916843)
+9500 val loss 6.9707
+9500 val perplexity 1065.0170
+9500 train 6.886722 (lr=3.1730e-06) (hash(x)=150196125)
+9600 val loss 6.9702
+9600 val perplexity 1064.4500
+9600 train 7.268473 (lr=3.1108e-06) (hash(x)=160041419)
+9700 val loss 6.9707
+9700 val perplexity 1064.9794
+9700 train 6.782595 (lr=3.0624e-06) (hash(x)=139931627)
+9800 val loss 6.9697
+9800 val perplexity 1063.9404
+9800 train 6.904285 (lr=3.0277e-06) (hash(x)=150370792)
+9900 val loss 6.9676
+9900 val perplexity 1061.6969
+9900 train 6.856194 (lr=3.0069e-06) (hash(x)=153014886)
+9999 val loss 6.9671
+9999 val perplexity 1061.1527

attention_kindself_n_heads4_seed1338/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47ac2c378421a826d83699dc57df4d19966a9c048b77da5d9b333759351bd47f
+oid sha256:ca9200c17b09d9ce5a00f4f3921dbd191ccebd8e682587ab3b328ca5cac9d17a
 size 92843394

attention_kindself_n_heads4_seed1338/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a860aca985a0320615f1d8d8ec1acaecf432d166b3d64b59d326083f52b4c0e
+oid sha256:b98f816322613e069e2f6157ab64d6faa458aa55bc8726a20f3b558e75e3b37c
 size 92843394

attention_kindself_n_heads4_seed1338/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b2fc78bb1f9f0df027c5f5c5cffbe725e86d64853d9364af03a084edec3b6059
+oid sha256:504f025c7870d9d8bb2a09684c65f3a0362850dff43da5000a389081845dcf7b
 size 92843394

attention_kindself_n_heads4_seed1338/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b8097088a7f449b54d156c8c4b6db4b93bbd5ff37078e50a3a9779dd96a0a30
+oid sha256:013d5d288a13029430b70ee8da52fedef5bcce694e85d3b5b89b82d871036a15
 size 92843394

attention_kindself_n_heads4_seed1338/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db4d6fe753882fa50e2d46add96e2ee3d0ab48c79f1a61932f187a79112b478d
+oid sha256:de6402575793f8258f3f7f8e3813ea95e6d8419d410db0101337f90c73755b14
 size 179406214

attention_kindself_n_heads4_seed1338/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16cdb0e2b9ea2d1fb1c08e529d748f659af12105c8a875718dc07e620670adac
+oid sha256:f7ab4b4214b2cbdf83218c0b55bc5add6405ad586bacfa76d76282fbb97b44a0
 size 179406214

attention_kindself_n_heads4_seed1338/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3958e20d664099674b4732bb35171522c9108352340af6d7afb83843f10d94be
+oid sha256:2f050c209849377df876fde538e4f405d2bbfcb772a8afe1ed21dfad723c0e74
 size 179406214

attention_kindself_n_heads4_seed1338/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:30fa172455d52db805381b66dc3a2c1c812d1b5e529b0f70c0fcc0f61c17112e
+oid sha256:aa289a4a68690e4ee433b5c4431d46806fca838b4dfc1478f634439ce8fdeb70
 size 179406214