Upload folder using huggingface_hub

attention_kindself_n_heads2_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_2_1339", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_2_1339", "n_embd": 128}

attention_kindself_n_heads2_seed1339/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.7353
 0 val perplexity 124899.7500
-0 train 11.734357 (lr=2.5000e-07) (hash(x)=150724848)
-100 val loss 10.4651
-100 val perplexity 35068.8242
-100 train 10.497286 (lr=2.5250e-05) (hash(x)=149217061)
-200 val loss 9.4784
-200 val perplexity 13074.8525
-200 train 9.459652 (lr=5.0000e-05) (hash(x)=149910534)
-300 val loss 8.4916
-300 val perplexity 4873.5464
-300 train 8.435646 (lr=4.9988e-05) (hash(x)=142185643)
-400 val loss 7.9347
-400 val perplexity 2792.5461
-400 train 7.970343 (lr=4.9954e-05) (hash(x)=148123706)
-500 val loss 7.7107
-500 val perplexity 2232.1318
-500 train 7.594043 (lr=4.9896e-05) (hash(x)=149952383)
-600 val loss 7.6270
-600 val perplexity 2052.8398
-600 train 7.453488 (lr=4.9815e-05) (hash(x)=146678221)
-700 val loss 7.5568
-700 val perplexity 1913.7664
-700 train 7.582264 (lr=4.9712e-05) (hash(x)=156180736)
-800 val loss 7.5122
-800 val perplexity 1830.2686
-800 train 7.493997 (lr=4.9585e-05) (hash(x)=151700982)
-900 val loss 7.4732
-900 val perplexity 1760.2922
-900 train 7.285657 (lr=4.9436e-05) (hash(x)=147288467)
-1000 val loss 7.4353
-1000 val perplexity 1694.8337
-1000 train 7.500896 (lr=4.9264e-05) (hash(x)=156182087)
-1100 val loss 7.3997
-1100 val perplexity 1635.4713
-1100 train 7.265240 (lr=4.9070e-05) (hash(x)=147861550)
-1200 val loss 7.3593
-1200 val perplexity 1570.7631
-1200 train 7.223550 (lr=4.8854e-05) (hash(x)=149318660)
-1300 val loss 7.3372
-1300 val perplexity 1536.3575
-1300 train 7.182025 (lr=4.8616e-05) (hash(x)=151631103)
-1400 val loss 7.3177
-1400 val perplexity 1506.7767
-1400 train 7.353926 (lr=4.8356e-05) (hash(x)=150482428)
-1500 val loss 7.2556
-1500 val perplexity 1416.0056
-1500 train 7.098469 (lr=4.8074e-05) (hash(x)=144458617)
-1600 val loss 7.2306
-1600 val perplexity 1381.0382
-1600 train 7.117816 (lr=4.7772e-05) (hash(x)=143268605)
-1700 val loss 7.2024
-1700 val perplexity 1342.6124
-1700 train 7.140574 (lr=4.7448e-05) (hash(x)=150817011)
-1800 val loss 7.1735
-1800 val perplexity 1304.3784
-1800 train 7.182352 (lr=4.7105e-05) (hash(x)=152322423)
-1900 val loss 7.1587
-1900 val perplexity 1285.2250
-1900 train 6.990263 (lr=4.6741e-05) (hash(x)=143574126)
-2000 val loss 7.1590
-2000 val perplexity 1285.6001
-2000 train 6.933070 (lr=4.6357e-05) (hash(x)=147904298)
-2100 val loss 7.1400
-2100 val perplexity 1261.4673
-2100 train 6.915145 (lr=4.5954e-05) (hash(x)=144719400)
-2200 val loss 7.1296
-2200 val perplexity 1248.3776
-2200 train 7.368258 (lr=4.5532e-05) (hash(x)=154343147)
-2300 val loss 7.1069
-2300 val perplexity 1220.2990
-2300 train 7.155266 (lr=4.5091e-05) (hash(x)=154372635)
-2400 val loss 7.1004
-2400 val perplexity 1212.4403
-2400 train 6.952942 (lr=4.4633e-05) (hash(x)=141843115)
-2500 val loss 7.0758
-2500 val perplexity 1182.9872
-2500 train 6.977271 (lr=4.4156e-05) (hash(x)=149417679)
-2600 val loss 7.0675
-2600 val perplexity 1173.1707
-2600 train 6.943630 (lr=4.3663e-05) (hash(x)=145279030)
-2700 val loss 7.0585
-2700 val perplexity 1162.6952
-2700 train 6.957924 (lr=4.3153e-05) (hash(x)=146363911)
-2800 val loss 7.0487
-2800 val perplexity 1151.3960
-2800 train 7.027252 (lr=4.2627e-05) (hash(x)=152507639)
-2900 val loss 7.0534
-2900 val perplexity 1156.7444
-2900 train 6.867245 (lr=4.2085e-05) (hash(x)=139296191)
-3000 val loss 7.0461
-3000 val perplexity 1148.3628
-3000 train 7.069539 (lr=4.1529e-05) (hash(x)=148473774)
-3100 val loss 7.0199
-3100 val perplexity 1118.6302
-3100 train 7.116752 (lr=4.0957e-05) (hash(x)=154130527)
-3200 val loss 7.0328
-3200 val perplexity 1133.1583
-3200 train 6.965783 (lr=4.0373e-05) (hash(x)=151117002)
-3300 val loss 7.0206
-3300 val perplexity 1119.4948
-3300 train 7.142177 (lr=3.9775e-05) (hash(x)=151035383)
-3400 val loss 7.0035
-3400 val perplexity 1100.5216
-3400 train 7.106104 (lr=3.9164e-05) (hash(x)=150614749)
-3500 val loss 6.9936
-3500 val perplexity 1089.6199
-3500 train 7.052362 (lr=3.8541e-05) (hash(x)=151431391)
-3600 val loss 6.9841
-3600 val perplexity 1079.3010
-3600 train 6.946033 (lr=3.7907e-05) (hash(x)=151140797)
-3700 val loss 6.9774
-3700 val perplexity 1072.1720
-3700 train 7.211062 (lr=3.7262e-05) (hash(x)=154608340)
-3800 val loss 6.9552
-3800 val perplexity 1048.5380
-3800 train 6.954602 (lr=3.6608e-05) (hash(x)=169641309)
-3900 val loss 6.9448
-3900 val perplexity 1037.7443
-3900 train 6.901346 (lr=3.5944e-05) (hash(x)=150592479)
-4000 val loss 6.9374
-4000 val perplexity 1030.1282
-4000 train 6.914086 (lr=3.5271e-05) (hash(x)=155495317)
-4100 val loss 6.9406
-4100 val perplexity 1033.4181
-4100 train 7.135403 (lr=3.4590e-05) (hash(x)=170323970)
-4200 val loss 6.9227
-4200 val perplexity 1015.0854
-4200 train 6.735570 (lr=3.3902e-05) (hash(x)=140854360)
-4300 val loss 6.9131
-4300 val perplexity 1005.3391
-4300 train 6.906529 (lr=3.3207e-05) (hash(x)=148926644)
-4400 val loss 6.9095
-4400 val perplexity 1001.7760
-4400 train 6.958065 (lr=3.2507e-05) (hash(x)=149997871)
-4500 val loss 6.8977
-4500 val perplexity 989.9657
-4500 train 6.864591 (lr=3.1801e-05) (hash(x)=149219289)
-4600 val loss 6.8935
-4600 val perplexity 985.8222
-4600 train 6.694082 (lr=3.1091e-05) (hash(x)=139836197)
-4700 val loss 6.9032
-4700 val perplexity 995.4861
-4700 train 6.865824 (lr=3.0377e-05) (hash(x)=150722883)
-4800 val loss 6.8859
-4800 val perplexity 978.4028
-4800 train 7.063042 (lr=2.9661e-05) (hash(x)=166864648)
-4900 val loss 6.8857
-4900 val perplexity 978.1995
-4900 train 7.000870 (lr=2.8942e-05) (hash(x)=157746743)
-5000 val loss 6.8581
-5000 val perplexity 951.5841
-5000 train 6.818891 (lr=2.8221e-05) (hash(x)=153032257)
-5100 val loss 6.8565
-5100 val perplexity 950.0789
-5100 train 6.935815 (lr=2.7500e-05) (hash(x)=151862371)
-5200 val loss 6.8392
-5200 val perplexity 933.7040
-5200 train 7.130702 (lr=2.6779e-05) (hash(x)=168862995)
-5300 val loss 6.8410
-5300 val perplexity 935.4055
-5300 train 6.816749 (lr=2.6058e-05) (hash(x)=148243567)
-5400 val loss 6.8288
-5400 val perplexity 924.0881
-5400 train 6.752869 (lr=2.5339e-05) (hash(x)=145381742)
-5500 val loss 6.8278
-5500 val perplexity 923.1636
-5500 train 6.989860 (lr=2.4623e-05) (hash(x)=155602174)
-5600 val loss 6.8086
-5600 val perplexity 905.5921
-5600 train 6.898433 (lr=2.3909e-05) (hash(x)=160637672)
-5700 val loss 6.8047
-5700 val perplexity 902.1183
-5700 train 6.749375 (lr=2.3199e-05) (hash(x)=152101131)
-5800 val loss 6.7908
-5800 val perplexity 889.6378
-5800 train 6.761727 (lr=2.2493e-05) (hash(x)=152327089)
-5900 val loss 6.7963
-5900 val perplexity 894.5493
-5900 train 6.732069 (lr=2.1793e-05) (hash(x)=151094543)
-6000 val loss 6.7812
-6000 val perplexity 881.1279
-6000 train 6.880424 (lr=2.1098e-05) (hash(x)=160233315)
-6100 val loss 6.7784
-6100 val perplexity 878.6605
-6100 train 6.872136 (lr=2.0410e-05) (hash(x)=158295337)
-6200 val loss 6.7672
-6200 val perplexity 868.8906
-6200 train 6.724010 (lr=1.9729e-05) (hash(x)=151269817)
-6300 val loss 6.7602
-6300 val perplexity 862.8197
-6300 train 6.847940 (lr=1.9056e-05) (hash(x)=156912756)
-6400 val loss 6.7576
-6400 val perplexity 860.5812
-6400 train 6.714630 (lr=1.8392e-05) (hash(x)=125779637)
-6500 val loss 6.7565
-6500 val perplexity 859.6633
-6500 train 6.662803 (lr=1.7738e-05) (hash(x)=148157661)
-6600 val loss 6.7478
-6600 val perplexity 852.1619
-6600 train 6.795863 (lr=1.7093e-05) (hash(x)=152820049)
-6700 val loss 6.7412
-6700 val perplexity 846.5380
-6700 train 6.698226 (lr=1.6459e-05) (hash(x)=155818717)
-6800 val loss 6.7321
-6800 val perplexity 838.9412
-6800 train 6.793108 (lr=1.5836e-05) (hash(x)=158833881)
-6900 val loss 6.7339
-6900 val perplexity 840.3877
-6900 train 6.756989 (lr=1.5225e-05) (hash(x)=164984909)
-7000 val loss 6.7327
-7000 val perplexity 839.4137
-7000 train 6.607749 (lr=1.4627e-05) (hash(x)=151172493)
-7100 val loss 6.7451
-7100 val perplexity 849.8951
-7100 train 6.723945 (lr=1.4043e-05) (hash(x)=154023633)
-7200 val loss 6.7121
-7200 val perplexity 822.2639
-7200 train 6.588626 (lr=1.3471e-05) (hash(x)=139351250)
-7300 val loss 6.7147
-7300 val perplexity 824.3981
-7300 train 6.681081 (lr=1.2915e-05) (hash(x)=158719458)
-7400 val loss 6.7026
-7400 val perplexity 814.5391
-7400 train 6.810424 (lr=1.2373e-05) (hash(x)=162250810)
-7500 val loss 6.6964
-7500 val perplexity 809.4514
-7500 train 6.549837 (lr=1.1847e-05) (hash(x)=142827557)
-7600 val loss 6.6928
-7600 val perplexity 806.6149
-7600 train 6.593294 (lr=1.1337e-05) (hash(x)=147830801)
-7700 val loss 6.6912
-7700 val perplexity 805.2506
-7700 train 6.814297 (lr=1.0844e-05) (hash(x)=165233314)
-7800 val loss 6.6926
-7800 val perplexity 806.3911
-7800 train 6.703671 (lr=1.0367e-05) (hash(x)=160654717)
-7900 val loss 6.6848
-7900 val perplexity 800.1847
-7900 train 6.564507 (lr=9.9088e-06) (hash(x)=140712586)
-8000 val loss 6.6777
-8000 val perplexity 794.4768
-8000 train 6.790144 (lr=9.4682e-06) (hash(x)=150309061)
-8100 val loss 6.6745
-8100 val perplexity 791.9514
-8100 train 6.723730 (lr=9.0461e-06) (hash(x)=158805466)
-8200 val loss 6.6719
-8200 val perplexity 789.8681
-8200 train 6.532796 (lr=8.6430e-06) (hash(x)=142814078)
-8300 val loss 6.6696
-8300 val perplexity 788.1194
-8300 train 6.625739 (lr=8.2593e-06) (hash(x)=148340417)
-8400 val loss 6.6704
-8400 val perplexity 788.7062
-8400 train 6.617031 (lr=7.8953e-06) (hash(x)=157132639)
-8500 val loss 6.6660
-8500 val perplexity 785.2834
-8500 train 6.630250 (lr=7.5515e-06) (hash(x)=152994240)
-8600 val loss 6.6639
-8600 val perplexity 783.5849
-8600 train 6.615349 (lr=7.2282e-06) (hash(x)=152210154)
-8700 val loss 6.6610
-8700 val perplexity 781.3484
-8700 train 6.691710 (lr=6.9257e-06) (hash(x)=149631247)
-8800 val loss 6.6558
-8800 val perplexity 777.3157
-8800 train 6.500901 (lr=6.6444e-06) (hash(x)=145617784)
-8900 val loss 6.6552
-8900 val perplexity 776.7936
-8900 train 6.751377 (lr=6.3845e-06) (hash(x)=155984970)
-9000 val loss 6.6549
-9000 val perplexity 776.6133
-9000 train 6.419650 (lr=6.1462e-06) (hash(x)=143433013)
-9100 val loss 6.6529
-9100 val perplexity 775.0507
-9100 train 6.539464 (lr=5.9300e-06) (hash(x)=152507533)
-9200 val loss 6.6483
-9200 val perplexity 771.4741
-9200 train 6.775714 (lr=5.7359e-06) (hash(x)=162400650)
-9300 val loss 6.6472
-9300 val perplexity 770.6464
-9300 train 6.674854 (lr=5.5641e-06) (hash(x)=148419068)
-9400 val loss 6.6443
-9400 val perplexity 768.3872
-9400 train 6.611318 (lr=5.4149e-06) (hash(x)=145025169)
-9500 val loss 6.6483
-9500 val perplexity 771.4730
-9500 train 6.606385 (lr=5.2884e-06) (hash(x)=145104235)
-9600 val loss 6.6447
-9600 val perplexity 768.6980
-9600 train 6.568452 (lr=5.1847e-06) (hash(x)=150499797)
-9700 val loss 6.6428
-9700 val perplexity 767.2230
-9700 train 6.452896 (lr=5.1040e-06) (hash(x)=139690022)
-9800 val loss 6.6368
-9800 val perplexity 762.6857
-9800 train 6.622273 (lr=5.0462e-06) (hash(x)=145645279)
-9900 val loss 6.6362
-9900 val perplexity 762.1913
-9900 train 6.553835 (lr=5.0116e-06) (hash(x)=150103432)
-9999 val loss 6.6367
-9999 val perplexity 762.5792
+0 train 11.734357 (lr=1.5000e-07) (hash(x)=150724848)
+100 val loss 10.5801
+100 val perplexity 39343.8633
+100 train 10.610308 (lr=1.5150e-05) (hash(x)=149217061)
+200 val loss 10.0874
+200 val perplexity 24039.0020
+200 train 10.066890 (lr=3.0000e-05) (hash(x)=149910534)
+300 val loss 9.3288
+300 val perplexity 11257.8096
+300 train 9.309011 (lr=2.9993e-05) (hash(x)=142185643)
+400 val loss 8.8103
+400 val perplexity 6703.1270
+400 train 8.843572 (lr=2.9972e-05) (hash(x)=148123706)
+500 val loss 8.4059
+500 val perplexity 4473.6040
+500 train 8.330770 (lr=2.9938e-05) (hash(x)=149952383)
+600 val loss 8.1319
+600 val perplexity 3401.1580
+600 train 8.031683 (lr=2.9889e-05) (hash(x)=146678221)
+700 val loss 7.9376
+700 val perplexity 2800.6233
+700 train 7.953770 (lr=2.9827e-05) (hash(x)=156180736)
+800 val loss 7.8250
+800 val perplexity 2502.2854
+800 train 7.813533 (lr=2.9751e-05) (hash(x)=151700982)
+900 val loss 7.7496
+900 val perplexity 2320.6006
+900 train 7.592718 (lr=2.9662e-05) (hash(x)=147288467)
+1000 val loss 7.6949
+1000 val perplexity 2197.2209
+1000 train 7.753491 (lr=2.9558e-05) (hash(x)=156182087)
+1100 val loss 7.6572
+1100 val perplexity 2115.7932
+1100 train 7.564962 (lr=2.9442e-05) (hash(x)=147861550)
+1200 val loss 7.6225
+1200 val perplexity 2043.6482
+1200 train 7.527445 (lr=2.9312e-05) (hash(x)=149318660)
+1300 val loss 7.6060
+1300 val perplexity 2010.2833
+1300 train 7.491122 (lr=2.9169e-05) (hash(x)=151631103)
+1400 val loss 7.5915
+1400 val perplexity 1981.3442
+1400 train 7.622824 (lr=2.9013e-05) (hash(x)=150482428)
+1500 val loss 7.5671
+1500 val perplexity 1933.5757
+1500 train 7.419519 (lr=2.8845e-05) (hash(x)=144458617)
+1600 val loss 7.5522
+1600 val perplexity 1904.9077
+1600 train 7.426981 (lr=2.8663e-05) (hash(x)=143268605)
+1700 val loss 7.5397
+1700 val perplexity 1881.3500
+1700 train 7.480874 (lr=2.8469e-05) (hash(x)=150817011)
+1800 val loss 7.5290
+1800 val perplexity 1861.2963
+1800 train 7.530146 (lr=2.8263e-05) (hash(x)=152322423)
+1900 val loss 7.5200
+1900 val perplexity 1844.6526
+1900 train 7.383969 (lr=2.8044e-05) (hash(x)=143574126)
+2000 val loss 7.5181
+2000 val perplexity 1841.1428
+2000 train 7.323624 (lr=2.7814e-05) (hash(x)=147904298)
+2100 val loss 7.5116
+2100 val perplexity 1829.1091
+2100 train 7.306216 (lr=2.7572e-05) (hash(x)=144719400)
+2200 val loss 7.4975
+2200 val perplexity 1803.4436
+2200 train 7.734940 (lr=2.7319e-05) (hash(x)=154343147)
+2300 val loss 7.4830
+2300 val perplexity 1777.4812
+2300 train 7.549631 (lr=2.7055e-05) (hash(x)=154372635)
+2400 val loss 7.4680
+2400 val perplexity 1751.1493
+2400 train 7.335818 (lr=2.6780e-05) (hash(x)=141843115)
+2500 val loss 7.4653
+2500 val perplexity 1746.4386
+2500 train 7.383837 (lr=2.6494e-05) (hash(x)=149417679)
+2600 val loss 7.4516
+2600 val perplexity 1722.6050
+2600 train 7.319361 (lr=2.6198e-05) (hash(x)=145279030)
+2700 val loss 7.4430
+2700 val perplexity 1707.9359
+2700 train 7.324830 (lr=2.5892e-05) (hash(x)=146363911)
+2800 val loss 7.4419
+2800 val perplexity 1706.0630
+2800 train 7.432843 (lr=2.5576e-05) (hash(x)=152507639)
+2900 val loss 7.4337
+2900 val perplexity 1692.0431
+2900 train 7.239769 (lr=2.5251e-05) (hash(x)=139296191)
+3000 val loss 7.4352
+3000 val perplexity 1694.5986
+3000 train 7.451351 (lr=2.4917e-05) (hash(x)=148473774)
+3100 val loss 7.4152
+3100 val perplexity 1660.9879
+3100 train 7.531559 (lr=2.4574e-05) (hash(x)=154130527)
+3200 val loss 7.3971
+3200 val perplexity 1631.2493
+3200 train 7.316022 (lr=2.4224e-05) (hash(x)=151117002)
+3300 val loss 7.3957
+3300 val perplexity 1628.9562
+3300 train 7.484721 (lr=2.3865e-05) (hash(x)=151035383)
+3400 val loss 7.3850
+3400 val perplexity 1611.6335
+3400 train 7.460353 (lr=2.3498e-05) (hash(x)=150614749)
+3500 val loss 7.3748
+3500 val perplexity 1595.3119
+3500 train 7.480272 (lr=2.3125e-05) (hash(x)=151431391)
+3600 val loss 7.3699
+3600 val perplexity 1587.4148
+3600 train 7.332257 (lr=2.2744e-05) (hash(x)=151140797)
+3700 val loss 7.3623
+3700 val perplexity 1575.4994
+3700 train 7.605422 (lr=2.2357e-05) (hash(x)=154608340)
+3800 val loss 7.3553
+3800 val perplexity 1564.4565
+3800 train 7.396259 (lr=2.1965e-05) (hash(x)=169641309)
+3900 val loss 7.3462
+3900 val perplexity 1550.2402
+3900 train 7.282505 (lr=2.1566e-05) (hash(x)=150592479)
+4000 val loss 7.3375
+4000 val perplexity 1536.8381
+4000 train 7.309266 (lr=2.1162e-05) (hash(x)=155495317)
+4100 val loss 7.3349
+4100 val perplexity 1532.9006
+4100 train 7.541991 (lr=2.0754e-05) (hash(x)=170323970)
+4200 val loss 7.3297
+4200 val perplexity 1524.9293
+4200 train 7.132684 (lr=2.0341e-05) (hash(x)=140854360)
+4300 val loss 7.3218
+4300 val perplexity 1512.9619
+4300 train 7.313124 (lr=1.9924e-05) (hash(x)=148926644)
+4400 val loss 7.3151
+4400 val perplexity 1502.7915
+4400 train 7.353460 (lr=1.9504e-05) (hash(x)=149997871)
+4500 val loss 7.3015
+4500 val perplexity 1482.5260
+4500 train 7.256317 (lr=1.9081e-05) (hash(x)=149219289)
+4600 val loss 7.3045
+4600 val perplexity 1486.9587
+4600 train 7.114832 (lr=1.8655e-05) (hash(x)=139836197)
+4700 val loss 7.2981
+4700 val perplexity 1477.5386
+4700 train 7.251995 (lr=1.8226e-05) (hash(x)=150722883)
+4800 val loss 7.2928
+4800 val perplexity 1469.6428
+4800 train 7.496532 (lr=1.7796e-05) (hash(x)=166864648)
+4900 val loss 7.2853
+4900 val perplexity 1458.7443
+4900 train 7.379625 (lr=1.7365e-05) (hash(x)=157746743)
+5000 val loss 7.2742
+5000 val perplexity 1442.5511
+5000 train 7.231178 (lr=1.6933e-05) (hash(x)=153032257)
+5100 val loss 7.2707
+5100 val perplexity 1437.5378
+5100 train 7.360577 (lr=1.6500e-05) (hash(x)=151862371)
+5200 val loss 7.2671
+5200 val perplexity 1432.4524
+5200 train 7.598169 (lr=1.6067e-05) (hash(x)=168862995)
+5300 val loss 7.2619
+5300 val perplexity 1424.9681
+5300 train 7.233602 (lr=1.5635e-05) (hash(x)=148243567)
+5400 val loss 7.2560
+5400 val perplexity 1416.6168
+5400 train 7.189073 (lr=1.5204e-05) (hash(x)=145381742)
+5500 val loss 7.2543
+5500 val perplexity 1414.2053
+5500 train 7.419303 (lr=1.4774e-05) (hash(x)=155602174)
+5600 val loss 7.2437
+5600 val perplexity 1399.3024
+5600 train 7.339626 (lr=1.4345e-05) (hash(x)=160637672)
+5700 val loss 7.2383
+5700 val perplexity 1391.6992
+5700 train 7.200211 (lr=1.3919e-05) (hash(x)=152101131)
+5800 val loss 7.2325
+5800 val perplexity 1383.6643
+5800 train 7.199661 (lr=1.3496e-05) (hash(x)=152327089)
+5900 val loss 7.2286
+5900 val perplexity 1378.2699
+5900 train 7.154571 (lr=1.3076e-05) (hash(x)=151094543)
+6000 val loss 7.2269
+6000 val perplexity 1375.9749
+6000 train 7.348256 (lr=1.2659e-05) (hash(x)=160233315)
+6100 val loss 7.2183
+6100 val perplexity 1364.1685
+6100 train 7.309128 (lr=1.2246e-05) (hash(x)=158295337)
+6200 val loss 7.2133
+6200 val perplexity 1357.2998
+6200 train 7.178264 (lr=1.1838e-05) (hash(x)=151269817)
+6300 val loss 7.2138
+6300 val perplexity 1358.0261
+6300 train 7.298607 (lr=1.1434e-05) (hash(x)=156912756)
+6400 val loss 7.2091
+6400 val perplexity 1351.6993
+6400 train 7.122179 (lr=1.1035e-05) (hash(x)=125779637)
+6500 val loss 7.2083
+6500 val perplexity 1350.6543
+6500 train 7.125643 (lr=1.0643e-05) (hash(x)=148157661)
+6600 val loss 7.2025
+6600 val perplexity 1342.8083
+6600 train 7.251727 (lr=1.0256e-05) (hash(x)=152820049)
+6700 val loss 7.2053
+6700 val perplexity 1346.5375
+6700 train 7.159529 (lr=9.8753e-06) (hash(x)=155818717)
+6800 val loss 7.1916
+6800 val perplexity 1328.2531
+6800 train 7.273685 (lr=9.5017e-06) (hash(x)=158833881)
+6900 val loss 7.1892
+6900 val perplexity 1325.0187
+6900 train 7.315172 (lr=9.1353e-06) (hash(x)=164984909)
+7000 val loss 7.1880
+7000 val perplexity 1323.3959
+7000 train 7.070263 (lr=8.7764e-06) (hash(x)=151172493)
+7100 val loss 7.1887
+7100 val perplexity 1324.4097
+7100 train 7.135881 (lr=8.4255e-06) (hash(x)=154023633)
+7200 val loss 7.1816
+7200 val perplexity 1314.9869
+7200 train 7.083755 (lr=8.0829e-06) (hash(x)=139351250)
+7300 val loss 7.1815
+7300 val perplexity 1314.8822
+7300 train 7.147565 (lr=7.7489e-06) (hash(x)=158719458)
+7400 val loss 7.1737
+7400 val perplexity 1304.6241
+7400 train 7.302870 (lr=7.4239e-06) (hash(x)=162250810)
+7500 val loss 7.1700
+7500 val perplexity 1299.8119
+7500 train 7.023395 (lr=7.1083e-06) (hash(x)=142827557)
+7600 val loss 7.1681
+7600 val perplexity 1297.3591
+7600 train 7.108825 (lr=6.8023e-06) (hash(x)=147830801)
+7700 val loss 7.1686
+7700 val perplexity 1298.0386
+7700 train 7.317597 (lr=6.5062e-06) (hash(x)=165233314)
+7800 val loss 7.1652
+7800 val perplexity 1293.6176
+7800 train 7.170717 (lr=6.2205e-06) (hash(x)=160654717)
+7900 val loss 7.1636
+7900 val perplexity 1291.5756
+7900 train 7.036086 (lr=5.9453e-06) (hash(x)=140712586)
+8000 val loss 7.1592
+8000 val perplexity 1285.9176
+8000 train 7.253605 (lr=5.6809e-06) (hash(x)=150309061)
+8100 val loss 7.1588
+8100 val perplexity 1285.3352
+8100 train 7.196534 (lr=5.4277e-06) (hash(x)=158805466)
+8200 val loss 7.1534
+8200 val perplexity 1278.4166
+8200 train 6.993227 (lr=5.1858e-06) (hash(x)=142814078)
+8300 val loss 7.1530
+8300 val perplexity 1277.9552
+8300 train 7.103416 (lr=4.9556e-06) (hash(x)=148340417)
+8400 val loss 7.1520
+8400 val perplexity 1276.6713
+8400 train 7.100376 (lr=4.7372e-06) (hash(x)=157132639)
+8500 val loss 7.1520
+8500 val perplexity 1276.6281
+8500 train 7.122152 (lr=4.5309e-06) (hash(x)=152994240)
+8600 val loss 7.1468
+8600 val perplexity 1270.0638
+8600 train 7.082963 (lr=4.3369e-06) (hash(x)=152210154)
+8700 val loss 7.1454
+8700 val perplexity 1268.2241
+8700 train 7.150362 (lr=4.1554e-06) (hash(x)=149631247)
+8800 val loss 7.1453
+8800 val perplexity 1268.1292
+8800 train 7.000983 (lr=3.9866e-06) (hash(x)=145617784)
+8900 val loss 7.1438
+8900 val perplexity 1266.2615
+8900 train 7.235374 (lr=3.8307e-06) (hash(x)=155984970)
+9000 val loss 7.1425
+9000 val perplexity 1264.6105
+9000 train 6.916540 (lr=3.6877e-06) (hash(x)=143433013)
+9100 val loss 7.1428
+9100 val perplexity 1264.9344
+9100 train 7.043243 (lr=3.5580e-06) (hash(x)=152507533)
+9200 val loss 7.1385
+9200 val perplexity 1259.4893
+9200 train 7.296465 (lr=3.4415e-06) (hash(x)=162400650)
+9300 val loss 7.1372
+9300 val perplexity 1257.8801
+9300 train 7.170633 (lr=3.3385e-06) (hash(x)=148419068)
+9400 val loss 7.1358
+9400 val perplexity 1256.1599
+9400 train 7.099593 (lr=3.2490e-06) (hash(x)=145025169)
+9500 val loss 7.1338
+9500 val perplexity 1253.6897
+9500 train 7.091206 (lr=3.1730e-06) (hash(x)=145104235)
+9600 val loss 7.1343
+9600 val perplexity 1254.2380
+9600 train 7.055454 (lr=3.1108e-06) (hash(x)=150499797)
+9700 val loss 7.1347
+9700 val perplexity 1254.7023
+9700 train 6.953218 (lr=3.0624e-06) (hash(x)=139690022)
+9800 val loss 7.1308
+9800 val perplexity 1249.9108
+9800 train 7.091164 (lr=3.0277e-06) (hash(x)=145645279)
+9900 val loss 7.1307
+9900 val perplexity 1249.7272
+9900 train 7.053947 (lr=3.0069e-06) (hash(x)=150103432)
+9999 val loss 7.1295
+9999 val perplexity 1248.2246

attention_kindself_n_heads2_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:909f58a8dfeabb72fcfeeb417429e4030abaefb3b9d9a048431bae84dcc17b19
+oid sha256:c30c8701af698b957b1c4d512bee189a13af09dfe76c50286b1444f35b99f7e5
 size 38587970

attention_kindself_n_heads2_seed1339/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d12694eddb4e563c6596b3d065e6ef9d19e9fa14751a0eb96d7d909d24680519
+oid sha256:d00e31d68d7d32d620a7d4465359c33935916a0be0fd552d8c1c95b0865c6f11
 size 38587970

attention_kindself_n_heads2_seed1339/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cef669e7987742ce62ef9b50a232227f031532358a62492016abf4601b5539f3
+oid sha256:49258632dcf9ecaead6ae519abb80fbd06c9da1a03b2ba7ca9d2b240992496c2
 size 38587970

attention_kindself_n_heads2_seed1339/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e4ddc957ba2752164b50a97bdd83eea77afbc9706b30fb6d2eae3af5a68f8aa
+oid sha256:ca1c3782bb6742cf0e90d4fa5b9fc578e4275a260fb4cb834ea9ee4ddc11e025
 size 38587970

attention_kindself_n_heads2_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4e44bda42a029f004f0038fd1ffc3360799ab524869dd634d9021cd088889e57
+oid sha256:a372b2a0710b8e1d9527181586a24de9267c8bc25800c815ae5468e9d931b682
 size 70895430

attention_kindself_n_heads2_seed1339/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:803d6c0490c84281eb9ea042594da3b72013eb563c17462d39878058f7fa52d4
+oid sha256:0b916d4015a6db9d48c1500df95bdc0f4519ff4db7c7a09882952f02ea95ae55
 size 70895430

attention_kindself_n_heads2_seed1339/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae1e38720c9bc3eba40bf0e63ef1247c07917a53981537b21fff8e4b9d837a06
+oid sha256:7e1c5f537016aec15b013add9e4df15a67cfd2c65c12a7d7a0519db8770b9f55
 size 70895430

attention_kindself_n_heads2_seed1339/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b08722ea4238195b07a11c3e1371d2b9556eedf60434203bc268f4d6818d3d6
+oid sha256:014cb4f496c7dbc34902c2fb137685092a0633ed606a68d93cf7aaef49a82dbf
 size 70895430