Upload folder using huggingface_hub

attention_kindself_n_heads2_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_2_1341", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads2_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_2_1341", "n_embd": 128}

attention_kindself_n_heads2_seed1341/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.7485
 0 val perplexity 126564.2812
-0 train 11.765417 (lr=2.5000e-07) (hash(x)=145079536)
-100 val loss 10.2826
-100 val perplexity 29218.3359
-100 train 10.229090 (lr=2.5250e-05) (hash(x)=154745873)
-200 val loss 9.1425
-200 val perplexity 9344.1045
-200 train 9.139270 (lr=5.0000e-05) (hash(x)=155800595)
-300 val loss 8.2396
-300 val perplexity 3788.2061
-300 train 8.208289 (lr=4.9988e-05) (hash(x)=148595389)
-400 val loss 7.8677
-400 val perplexity 2611.6658
-400 train 7.794479 (lr=4.9954e-05) (hash(x)=145606733)
-500 val loss 7.7057
-500 val perplexity 2221.0730
-500 train 7.568192 (lr=4.9896e-05) (hash(x)=138221231)
-600 val loss 7.6270
-600 val perplexity 2052.9622
-600 train 7.637253 (lr=4.9815e-05) (hash(x)=150367139)
-700 val loss 7.5933
-700 val perplexity 1984.8761
-700 train 7.732349 (lr=4.9712e-05) (hash(x)=155579314)
-800 val loss 7.5569
-800 val perplexity 1913.9260
-800 train 7.627458 (lr=4.9585e-05) (hash(x)=155747374)
-900 val loss 7.5306
-900 val perplexity 1864.1678
-900 train 7.728004 (lr=4.9436e-05) (hash(x)=159334575)
-1000 val loss 7.5001
-1000 val perplexity 1808.2632
-1000 train 7.395680 (lr=4.9264e-05) (hash(x)=140604760)
-1100 val loss 7.4848
-1100 val perplexity 1780.7771
-1100 train 7.329884 (lr=4.9070e-05) (hash(x)=146208052)
-1200 val loss 7.4578
-1200 val perplexity 1733.3807
-1200 train 7.383160 (lr=4.8854e-05) (hash(x)=148404734)
-1300 val loss 7.4405
-1300 val perplexity 1703.6826
-1300 train 7.380110 (lr=4.8616e-05) (hash(x)=155681970)
-1400 val loss 7.4114
-1400 val perplexity 1654.7939
-1400 train 7.363786 (lr=4.8356e-05) (hash(x)=148115934)
-1500 val loss 7.3769
-1500 val perplexity 1598.6610
-1500 train 7.503820 (lr=4.8074e-05) (hash(x)=157074034)
-1600 val loss 7.3488
-1600 val perplexity 1554.3053
-1600 train 7.120208 (lr=4.7772e-05) (hash(x)=137464699)
-1700 val loss 7.3234
-1700 val perplexity 1515.3431
-1700 train 7.531859 (lr=4.7448e-05) (hash(x)=166955614)
-1800 val loss 7.3060
-1800 val perplexity 1489.2195
-1800 train 7.140155 (lr=4.7105e-05) (hash(x)=143886042)
-1900 val loss 7.2840
-1900 val perplexity 1456.8646
-1900 train 7.898643 (lr=4.6741e-05) (hash(x)=193610391)
-2000 val loss 7.2716
-2000 val perplexity 1438.8134
-2000 train 7.425419 (lr=4.6357e-05) (hash(x)=163799796)
-2100 val loss 7.2522
-2100 val perplexity 1411.1982
-2100 train 7.212413 (lr=4.5954e-05) (hash(x)=154107339)
-2200 val loss 7.2275
-2200 val perplexity 1376.7487
-2200 train 7.202817 (lr=4.5532e-05) (hash(x)=144592844)
-2300 val loss 7.2104
-2300 val perplexity 1353.4292
-2300 train 7.279191 (lr=4.5091e-05) (hash(x)=154535861)
-2400 val loss 7.1937
-2400 val perplexity 1331.0225
-2400 train 7.922030 (lr=4.4633e-05) (hash(x)=204706354)
-2500 val loss 7.1846
-2500 val perplexity 1318.9752
-2500 train 7.007863 (lr=4.4156e-05) (hash(x)=143783202)
-2600 val loss 7.1982
-2600 val perplexity 1337.0598
-2600 train 7.098747 (lr=4.3663e-05) (hash(x)=150862210)
-2700 val loss 7.1884
-2700 val perplexity 1324.0164
-2700 train 7.256421 (lr=4.3153e-05) (hash(x)=150753426)
-2800 val loss 7.1686
-2800 val perplexity 1297.9996
-2800 train 7.134176 (lr=4.2627e-05) (hash(x)=147766811)
-2900 val loss 7.1589
-2900 val perplexity 1285.5081
-2900 train 7.091248 (lr=4.2085e-05) (hash(x)=151177814)
-3000 val loss 7.1545
-3000 val perplexity 1279.8695
-3000 train 7.002457 (lr=4.1529e-05) (hash(x)=135925327)
-3100 val loss 7.1629
-3100 val perplexity 1290.6305
-3100 train 6.864954 (lr=4.0957e-05) (hash(x)=124842353)
-3200 val loss 7.1606
-3200 val perplexity 1287.6792
-3200 train 7.264678 (lr=4.0373e-05) (hash(x)=160440642)
-3300 val loss 7.1483
-3300 val perplexity 1271.9863
-3300 train 7.056784 (lr=3.9775e-05) (hash(x)=143017131)
-3400 val loss 7.1487
-3400 val perplexity 1272.4194
-3400 train 7.358580 (lr=3.9164e-05) (hash(x)=148143474)
-3500 val loss 7.1200
-3500 val perplexity 1236.3984
-3500 train 6.953858 (lr=3.8541e-05) (hash(x)=148732639)
-3600 val loss 7.1114
-3600 val perplexity 1225.9130
-3600 train 7.110910 (lr=3.7907e-05) (hash(x)=146333803)
-3700 val loss 7.1119
-3700 val perplexity 1226.4486
-3700 train 6.949524 (lr=3.7262e-05) (hash(x)=143611445)
-3800 val loss 7.1171
-3800 val perplexity 1232.8684
-3800 train 6.939259 (lr=3.6608e-05) (hash(x)=141272115)
-3900 val loss 7.0960
-3900 val perplexity 1207.1256
-3900 train 7.020997 (lr=3.5944e-05) (hash(x)=149161848)
-4000 val loss 7.0935
-4000 val perplexity 1204.0765
-4000 train 7.162819 (lr=3.5271e-05) (hash(x)=152036955)
-4100 val loss 7.0942
-4100 val perplexity 1204.9524
-4100 train 7.145968 (lr=3.4590e-05) (hash(x)=153260968)
-4200 val loss 7.0696
-4200 val perplexity 1175.6902
-4200 train 6.957745 (lr=3.3902e-05) (hash(x)=147629478)
-4300 val loss 7.0848
-4300 val perplexity 1193.7286
-4300 train 7.111011 (lr=3.3207e-05) (hash(x)=161701080)
-4400 val loss 7.0667
-4400 val perplexity 1172.2228
-4400 train 7.119734 (lr=3.2507e-05) (hash(x)=155290524)
-4500 val loss 7.0701
-4500 val perplexity 1176.2117
-4500 train 6.902558 (lr=3.1801e-05) (hash(x)=144726593)
-4600 val loss 7.0655
-4600 val perplexity 1170.8447
-4600 train 7.015216 (lr=3.1091e-05) (hash(x)=149517202)
-4700 val loss 7.0668
-4700 val perplexity 1172.4336
-4700 train 7.049967 (lr=3.0377e-05) (hash(x)=139403008)
-4800 val loss 7.0334
-4800 val perplexity 1133.9307
-4800 train 7.076432 (lr=2.9661e-05) (hash(x)=153387238)
-4900 val loss 7.0237
-4900 val perplexity 1122.9238
-4900 train 6.798825 (lr=2.8942e-05) (hash(x)=139881569)
-5000 val loss 7.0251
-5000 val perplexity 1124.5347
-5000 train 6.988297 (lr=2.8221e-05) (hash(x)=152602580)
-5100 val loss 7.0178
-5100 val perplexity 1116.3683
-5100 train 7.075103 (lr=2.7500e-05) (hash(x)=150598460)
-5200 val loss 7.0146
-5200 val perplexity 1112.7893
-5200 train 6.948948 (lr=2.6779e-05) (hash(x)=147660496)
-5300 val loss 7.0149
-5300 val perplexity 1113.0590
-5300 train 6.783306 (lr=2.6058e-05) (hash(x)=147693848)
-5400 val loss 7.0324
-5400 val perplexity 1132.6899
-5400 train 6.810908 (lr=2.5339e-05) (hash(x)=136048145)
-5500 val loss 6.9941
-5500 val perplexity 1090.2325
-5500 train 6.928900 (lr=2.4623e-05) (hash(x)=140313123)
-5600 val loss 6.9882
-5600 val perplexity 1083.7391
-5600 train 7.023926 (lr=2.3909e-05) (hash(x)=156997993)
-5700 val loss 6.9746
-5700 val perplexity 1069.1527
-5700 train 6.957600 (lr=2.3199e-05) (hash(x)=143513880)
-5800 val loss 6.9747
-5800 val perplexity 1069.2542
-5800 train 7.135317 (lr=2.2493e-05) (hash(x)=153736628)
-5900 val loss 6.9736
-5900 val perplexity 1068.0149
-5900 train 7.152683 (lr=2.1793e-05) (hash(x)=166385417)
-6000 val loss 6.9625
-6000 val perplexity 1056.2238
-6000 train 7.161615 (lr=2.1098e-05) (hash(x)=153216517)
-6100 val loss 6.9613
-6100 val perplexity 1054.9874
-6100 train 6.849194 (lr=2.0410e-05) (hash(x)=145387545)
-6200 val loss 6.9644
-6200 val perplexity 1058.3038
-6200 train 7.072074 (lr=1.9729e-05) (hash(x)=156306460)
-6300 val loss 6.9596
-6300 val perplexity 1053.2604
-6300 train 6.850439 (lr=1.9056e-05) (hash(x)=148610447)
-6400 val loss 6.9467
-6400 val perplexity 1039.7458
-6400 train 7.140009 (lr=1.8392e-05) (hash(x)=158343535)
-6500 val loss 6.9415
-6500 val perplexity 1034.2822
-6500 train 7.083022 (lr=1.7738e-05) (hash(x)=157742557)
-6600 val loss 6.9320
-6600 val perplexity 1024.5250
-6600 train 6.891453 (lr=1.7093e-05) (hash(x)=150457308)
-6700 val loss 6.9216
-6700 val perplexity 1013.9897
-6700 train 6.822219 (lr=1.6459e-05) (hash(x)=142426205)
-6800 val loss 6.9205
-6800 val perplexity 1012.8657
-6800 train 6.774536 (lr=1.5836e-05) (hash(x)=151522525)
-6900 val loss 6.9227
-6900 val perplexity 1015.0526
-6900 train 6.847309 (lr=1.5225e-05) (hash(x)=152029630)
-7000 val loss 6.9094
-7000 val perplexity 1001.6866
-7000 train 6.780774 (lr=1.4627e-05) (hash(x)=142978954)
-7100 val loss 6.9035
-7100 val perplexity 995.7990
-7100 train 6.798730 (lr=1.4043e-05) (hash(x)=148064359)
-7200 val loss 6.8999
-7200 val perplexity 992.1963
-7200 train 6.835505 (lr=1.3471e-05) (hash(x)=155771845)
-7300 val loss 6.8986
-7300 val perplexity 990.8743
-7300 train 6.715855 (lr=1.2915e-05) (hash(x)=141639844)
-7400 val loss 6.8924
-7400 val perplexity 984.7956
-7400 train 6.928619 (lr=1.2373e-05) (hash(x)=153466021)
-7500 val loss 6.8917
-7500 val perplexity 984.0934
-7500 train 6.896158 (lr=1.1847e-05) (hash(x)=148843587)
-7600 val loss 6.8860
-7600 val perplexity 978.4490
-7600 train 7.061441 (lr=1.1337e-05) (hash(x)=152053933)
-7700 val loss 6.8777
-7700 val perplexity 970.3899
-7700 train 6.651704 (lr=1.0844e-05) (hash(x)=141564003)
-7800 val loss 6.8734
-7800 val perplexity 966.2131
-7800 train 6.761744 (lr=1.0367e-05) (hash(x)=148617843)
-7900 val loss 6.8699
-7900 val perplexity 962.8681
-7900 train 6.841489 (lr=9.9088e-06) (hash(x)=146417668)
-8000 val loss 6.8715
-8000 val perplexity 964.3954
-8000 train 6.710367 (lr=9.4682e-06) (hash(x)=150165488)
-8100 val loss 6.8669
-8100 val perplexity 959.9246
-8100 train 6.835996 (lr=9.0461e-06) (hash(x)=144818855)
-8200 val loss 6.8594
-8200 val perplexity 952.8196
-8200 train 6.893314 (lr=8.6430e-06) (hash(x)=143074954)
-8300 val loss 6.8578
-8300 val perplexity 951.2951
-8300 train 6.775650 (lr=8.2593e-06) (hash(x)=145172494)
-8400 val loss 6.8549
-8400 val perplexity 948.5271
-8400 train 6.863227 (lr=7.8953e-06) (hash(x)=152284393)
-8500 val loss 6.8517
-8500 val perplexity 945.4831
-8500 train 6.727867 (lr=7.5515e-06) (hash(x)=147044705)
-8600 val loss 6.8501
-8600 val perplexity 943.9672
-8600 train 6.775654 (lr=7.2282e-06) (hash(x)=147762608)
-8700 val loss 6.8476
-8700 val perplexity 941.6030
-8700 train 6.843468 (lr=6.9257e-06) (hash(x)=141994567)
-8800 val loss 6.8425
-8800 val perplexity 936.8553
-8800 train 6.876688 (lr=6.6444e-06) (hash(x)=152552896)
-8900 val loss 6.8413
-8900 val perplexity 935.6967
-8900 train 6.693859 (lr=6.3845e-06) (hash(x)=139714454)
-9000 val loss 6.8365
-9000 val perplexity 931.2620
-9000 train 6.874582 (lr=6.1462e-06) (hash(x)=152274419)
-9100 val loss 6.8367
-9100 val perplexity 931.3975
-9100 train 6.962732 (lr=5.9300e-06) (hash(x)=160536885)
-9200 val loss 6.8362
-9200 val perplexity 930.9202
-9200 train 6.664563 (lr=5.7359e-06) (hash(x)=138386899)
-9300 val loss 6.8327
-9300 val perplexity 927.6513
-9300 train 6.842355 (lr=5.5641e-06) (hash(x)=149957457)
-9400 val loss 6.8309
-9400 val perplexity 926.0545
-9400 train 6.910855 (lr=5.4149e-06) (hash(x)=161671548)
-9500 val loss 6.8306
-9500 val perplexity 925.7362
-9500 train 6.983879 (lr=5.2884e-06) (hash(x)=171857400)
-9600 val loss 6.8278
-9600 val perplexity 923.1623
-9600 train 6.536280 (lr=5.1847e-06) (hash(x)=135378471)
-9700 val loss 6.8259
-9700 val perplexity 921.4111
-9700 train 6.745991 (lr=5.1040e-06) (hash(x)=149514591)
-9800 val loss 6.8210
-9800 val perplexity 916.8771
-9800 train 6.784397 (lr=5.0462e-06) (hash(x)=152679072)
-9900 val loss 6.8200
-9900 val perplexity 915.9756
-9900 train 6.746526 (lr=5.0116e-06) (hash(x)=150415193)
-9999 val loss 6.8178
-9999 val perplexity 913.9669
+0 train 11.765417 (lr=1.5000e-07) (hash(x)=145079536)
+100 val loss 10.3974
+100 val perplexity 32775.7656
+100 train 10.345327 (lr=1.5150e-05) (hash(x)=154745873)
+200 val loss 9.7527
+200 val perplexity 17200.3438
+200 train 9.718990 (lr=3.0000e-05) (hash(x)=155800595)
+300 val loss 8.7927
+300 val perplexity 6585.9263
+300 train 8.760824 (lr=2.9993e-05) (hash(x)=148595389)
+400 val loss 8.1923
+400 val perplexity 3613.1116
+400 train 8.128107 (lr=2.9972e-05) (hash(x)=145606733)
+500 val loss 7.8971
+500 val perplexity 2689.5447
+500 train 7.786391 (lr=2.9938e-05) (hash(x)=138221231)
+600 val loss 7.7463
+600 val perplexity 2313.0840
+600 train 7.763315 (lr=2.9889e-05) (hash(x)=150367139)
+700 val loss 7.6697
+700 val perplexity 2142.4788
+700 train 7.812234 (lr=2.9827e-05) (hash(x)=155579314)
+800 val loss 7.6175
+800 val perplexity 2033.4844
+800 train 7.687030 (lr=2.9751e-05) (hash(x)=155747374)
+900 val loss 7.5856
+900 val perplexity 1969.6241
+900 train 7.770665 (lr=2.9662e-05) (hash(x)=159334575)
+1000 val loss 7.5518
+1000 val perplexity 1904.1849
+1000 train 7.462197 (lr=2.9558e-05) (hash(x)=140604760)
+1100 val loss 7.5370
+1100 val perplexity 1876.2014
+1100 train 7.388958 (lr=2.9442e-05) (hash(x)=146208052)
+1200 val loss 7.5180
+1200 val perplexity 1840.8917
+1200 train 7.447907 (lr=2.9312e-05) (hash(x)=148404734)
+1300 val loss 7.5003
+1300 val perplexity 1808.5959
+1300 train 7.448778 (lr=2.9169e-05) (hash(x)=155681970)
+1400 val loss 7.4901
+1400 val perplexity 1790.1976
+1400 train 7.442825 (lr=2.9013e-05) (hash(x)=148115934)
+1500 val loss 7.4582
+1500 val perplexity 1734.0793
+1500 train 7.576637 (lr=2.8845e-05) (hash(x)=157074034)
+1600 val loss 7.4336
+1600 val perplexity 1691.8712
+1600 train 7.213996 (lr=2.8663e-05) (hash(x)=137464699)
+1700 val loss 7.4105
+1700 val perplexity 1653.3041
+1700 train 7.623976 (lr=2.8469e-05) (hash(x)=166955614)
+1800 val loss 7.3916
+1800 val perplexity 1622.3053
+1800 train 7.230451 (lr=2.8263e-05) (hash(x)=143886042)
+1900 val loss 7.3788
+1900 val perplexity 1601.6421
+1900 train 8.085829 (lr=2.8044e-05) (hash(x)=193610391)
+2000 val loss 7.3556
+2000 val perplexity 1564.9974
+2000 train 7.507458 (lr=2.7814e-05) (hash(x)=163799796)
+2100 val loss 7.3252
+2100 val perplexity 1518.1095
+2100 train 7.289814 (lr=2.7572e-05) (hash(x)=154107339)
+2200 val loss 7.3011
+2200 val perplexity 1481.9268
+2200 train 7.269084 (lr=2.7319e-05) (hash(x)=144592844)
+2300 val loss 7.2780
+2300 val perplexity 1448.0825
+2300 train 7.344017 (lr=2.7055e-05) (hash(x)=154535861)
+2400 val loss 7.2626
+2400 val perplexity 1425.9469
+2400 train 8.046069 (lr=2.6780e-05) (hash(x)=204706354)
+2500 val loss 7.2586
+2500 val perplexity 1420.3314
+2500 train 7.076103 (lr=2.6494e-05) (hash(x)=143783202)
+2600 val loss 7.2366
+2600 val perplexity 1389.4243
+2600 train 7.131085 (lr=2.6198e-05) (hash(x)=150862210)
+2700 val loss 7.2212
+2700 val perplexity 1368.1188
+2700 train 7.295586 (lr=2.5892e-05) (hash(x)=150753426)
+2800 val loss 7.2110
+2800 val perplexity 1354.2704
+2800 train 7.173392 (lr=2.5576e-05) (hash(x)=147766811)
+2900 val loss 7.1804
+2900 val perplexity 1313.4628
+2900 train 7.114481 (lr=2.5251e-05) (hash(x)=151177814)
+3000 val loss 7.1625
+3000 val perplexity 1290.1395
+3000 train 7.014852 (lr=2.4917e-05) (hash(x)=135925327)
+3100 val loss 7.1573
+3100 val perplexity 1283.4336
+3100 train 6.857617 (lr=2.4574e-05) (hash(x)=124842353)
+3200 val loss 7.1470
+3200 val perplexity 1270.2340
+3200 train 7.254866 (lr=2.4224e-05) (hash(x)=160440642)
+3300 val loss 7.1253
+3300 val perplexity 1242.9670
+3300 train 7.035677 (lr=2.3865e-05) (hash(x)=143017131)
+3400 val loss 7.1127
+3400 val perplexity 1227.3981
+3400 train 7.341929 (lr=2.3498e-05) (hash(x)=148143474)
+3500 val loss 7.1041
+3500 val perplexity 1216.9071
+3500 train 6.938853 (lr=2.3125e-05) (hash(x)=148732639)
+3600 val loss 7.0928
+3600 val perplexity 1203.2311
+3600 train 7.086288 (lr=2.2744e-05) (hash(x)=146333803)
+3700 val loss 7.0937
+3700 val perplexity 1204.3717
+3700 train 6.927111 (lr=2.2357e-05) (hash(x)=143611445)
+3800 val loss 7.0854
+3800 val perplexity 1194.3458
+3800 train 6.917649 (lr=2.1965e-05) (hash(x)=141272115)
+3900 val loss 7.0731
+3900 val perplexity 1179.7892
+3900 train 7.001764 (lr=2.1566e-05) (hash(x)=149161848)
+4000 val loss 7.0643
+4000 val perplexity 1169.4933
+4000 train 7.129023 (lr=2.1162e-05) (hash(x)=152036955)
+4100 val loss 7.0525
+4100 val perplexity 1155.7283
+4100 train 7.106788 (lr=2.0754e-05) (hash(x)=153260968)
+4200 val loss 7.0399
+4200 val perplexity 1141.2640
+4200 train 6.930591 (lr=2.0341e-05) (hash(x)=147629478)
+4300 val loss 7.0355
+4300 val perplexity 1136.2820
+4300 train 7.069160 (lr=1.9924e-05) (hash(x)=161701080)
+4400 val loss 7.0289
+4400 val perplexity 1128.8105
+4400 train 7.084981 (lr=1.9504e-05) (hash(x)=155290524)
+4500 val loss 7.0179
+4500 val perplexity 1116.4269
+4500 train 6.848637 (lr=1.9081e-05) (hash(x)=144726593)
+4600 val loss 7.0268
+4600 val perplexity 1126.3979
+4600 train 6.980014 (lr=1.8655e-05) (hash(x)=149517202)
+4700 val loss 7.0097
+4700 val perplexity 1107.2999
+4700 train 6.994656 (lr=1.8226e-05) (hash(x)=139403008)
+4800 val loss 6.9945
+4800 val perplexity 1090.6340
+4800 train 7.031330 (lr=1.7796e-05) (hash(x)=153387238)
+4900 val loss 6.9848
+4900 val perplexity 1080.0377
+4900 train 6.752359 (lr=1.7365e-05) (hash(x)=139881569)
+5000 val loss 6.9803
+5000 val perplexity 1075.2234
+5000 train 6.938387 (lr=1.6933e-05) (hash(x)=152602580)
+5100 val loss 6.9720
+5100 val perplexity 1066.3773
+5100 train 7.032982 (lr=1.6500e-05) (hash(x)=150598460)
+5200 val loss 6.9664
+5200 val perplexity 1060.4471
+5200 train 6.894612 (lr=1.6067e-05) (hash(x)=147660496)
+5300 val loss 6.9607
+5300 val perplexity 1054.3674
+5300 train 6.733212 (lr=1.5635e-05) (hash(x)=147693848)
+5400 val loss 6.9581
+5400 val perplexity 1051.6660
+5400 train 6.730026 (lr=1.5204e-05) (hash(x)=136048145)
+5500 val loss 6.9428
+5500 val perplexity 1035.6345
+5500 train 6.882213 (lr=1.4774e-05) (hash(x)=140313123)
+5600 val loss 6.9425
+5600 val perplexity 1035.3456
+5600 train 6.979943 (lr=1.4345e-05) (hash(x)=156997993)
+5700 val loss 6.9294
+5700 val perplexity 1021.8782
+5700 train 6.907539 (lr=1.3919e-05) (hash(x)=143513880)
+5800 val loss 6.9185
+5800 val perplexity 1010.7582
+5800 train 7.085642 (lr=1.3496e-05) (hash(x)=153736628)
+5900 val loss 6.9127
+5900 val perplexity 1004.9073
+5900 train 7.102872 (lr=1.3076e-05) (hash(x)=166385417)
+6000 val loss 6.9082
+6000 val perplexity 1000.4232
+6000 train 7.105831 (lr=1.2659e-05) (hash(x)=153216517)
+6100 val loss 6.9114
+6100 val perplexity 1003.6703
+6100 train 6.796443 (lr=1.2246e-05) (hash(x)=145387545)
+6200 val loss 6.8979
+6200 val perplexity 990.2126
+6200 train 7.003855 (lr=1.1838e-05) (hash(x)=156306460)
+6300 val loss 6.8877
+6300 val perplexity 980.1455
+6300 train 6.779792 (lr=1.1434e-05) (hash(x)=148610447)
+6400 val loss 6.8781
+6400 val perplexity 970.8212
+6400 train 7.073340 (lr=1.1035e-05) (hash(x)=158343535)
+6500 val loss 6.8763
+6500 val perplexity 969.0554
+6500 train 7.018353 (lr=1.0643e-05) (hash(x)=157742557)
+6600 val loss 6.8706
+6600 val perplexity 963.5093
+6600 train 6.827638 (lr=1.0256e-05) (hash(x)=150457308)
+6700 val loss 6.8632
+6700 val perplexity 956.4338
+6700 train 6.761833 (lr=9.8753e-06) (hash(x)=142426205)
+6800 val loss 6.8676
+6800 val perplexity 960.6696
+6800 train 6.722654 (lr=9.5017e-06) (hash(x)=151522525)
+6900 val loss 6.8535
+6900 val perplexity 947.1518
+6900 train 6.772261 (lr=9.1353e-06) (hash(x)=152029630)
+7000 val loss 6.8500
+7000 val perplexity 943.9209
+7000 train 6.727105 (lr=8.7764e-06) (hash(x)=142978954)
+7100 val loss 6.8429
+7100 val perplexity 937.1792
+7100 train 6.737306 (lr=8.4255e-06) (hash(x)=148064359)
+7200 val loss 6.8370
+7200 val perplexity 931.7009
+7200 train 6.769127 (lr=8.0829e-06) (hash(x)=155771845)
+7300 val loss 6.8348
+7300 val perplexity 929.6129
+7300 train 6.638350 (lr=7.7489e-06) (hash(x)=141639844)
+7400 val loss 6.8312
+7400 val perplexity 926.3376
+7400 train 6.870832 (lr=7.4239e-06) (hash(x)=153466021)
+7500 val loss 6.8259
+7500 val perplexity 921.4308
+7500 train 6.839567 (lr=7.1083e-06) (hash(x)=148843587)
+7600 val loss 6.8174
+7600 val perplexity 913.6078
+7600 train 6.996936 (lr=6.8023e-06) (hash(x)=152053933)
+7700 val loss 6.8139
+7700 val perplexity 910.4549
+7700 train 6.589845 (lr=6.5062e-06) (hash(x)=141564003)
+7800 val loss 6.8105
+7800 val perplexity 907.3491
+7800 train 6.699705 (lr=6.2205e-06) (hash(x)=148617843)
+7900 val loss 6.8124
+7900 val perplexity 909.0827
+7900 train 6.779389 (lr=5.9453e-06) (hash(x)=146417668)
+8000 val loss 6.8050
+8000 val perplexity 902.3446
+8000 train 6.650186 (lr=5.6809e-06) (hash(x)=150165488)
+8100 val loss 6.8024
+8100 val perplexity 899.9769
+8100 train 6.776114 (lr=5.4277e-06) (hash(x)=144818855)
+8200 val loss 6.7962
+8200 val perplexity 894.4230
+8200 train 6.834549 (lr=5.1858e-06) (hash(x)=143074954)
+8300 val loss 6.7918
+8300 val perplexity 890.5355
+8300 train 6.711532 (lr=4.9556e-06) (hash(x)=145172494)
+8400 val loss 6.7904
+8400 val perplexity 889.2421
+8400 train 6.806044 (lr=4.7372e-06) (hash(x)=152284393)
+8500 val loss 6.7883
+8500 val perplexity 887.3614
+8500 train 6.659525 (lr=4.5309e-06) (hash(x)=147044705)
+8600 val loss 6.7861
+8600 val perplexity 885.4843
+8600 train 6.711213 (lr=4.3369e-06) (hash(x)=147762608)
+8700 val loss 6.7839
+8700 val perplexity 883.4965
+8700 train 6.781090 (lr=4.1554e-06) (hash(x)=141994567)
+8800 val loss 6.7781
+8800 val perplexity 878.3743
+8800 train 6.808532 (lr=3.9866e-06) (hash(x)=152552896)
+8900 val loss 6.7773
+8900 val perplexity 877.6739
+8900 train 6.624681 (lr=3.8307e-06) (hash(x)=139714454)
+9000 val loss 6.7728
+9000 val perplexity 873.7579
+9000 train 6.808452 (lr=3.6877e-06) (hash(x)=152274419)
+9100 val loss 6.7719
+9100 val perplexity 872.9600
+9100 train 6.892161 (lr=3.5580e-06) (hash(x)=160536885)
+9200 val loss 6.7709
+9200 val perplexity 872.1387
+9200 train 6.601489 (lr=3.4415e-06) (hash(x)=138386899)
+9300 val loss 6.7689
+9300 val perplexity 870.3257
+9300 train 6.773367 (lr=3.3385e-06) (hash(x)=149957457)
+9400 val loss 6.7657
+9400 val perplexity 867.6055
+9400 train 6.860408 (lr=3.2490e-06) (hash(x)=161671548)
+9500 val loss 6.7654
+9500 val perplexity 867.3085
+9500 train 6.928828 (lr=3.1730e-06) (hash(x)=171857400)
+9600 val loss 6.7621
+9600 val perplexity 864.4633
+9600 train 6.467602 (lr=3.1108e-06) (hash(x)=135378471)
+9700 val loss 6.7593
+9700 val perplexity 862.0478
+9700 train 6.682233 (lr=3.0624e-06) (hash(x)=149514591)
+9800 val loss 6.7585
+9800 val perplexity 861.3374
+9800 train 6.721174 (lr=3.0277e-06) (hash(x)=152679072)
+9900 val loss 6.7551
+9900 val perplexity 858.4430
+9900 train 6.671714 (lr=3.0069e-06) (hash(x)=150415193)
+9999 val loss 6.7530
+9999 val perplexity 856.6614

attention_kindself_n_heads2_seed1341/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9f37868ccb8a9a3b18ff2b594b248dd93b627e135bf3db4c7f1433ab9a4b58d
+oid sha256:296dea7d8ee3e131d42a76abbb2b60a47c78c4d20b609ff1241197c9170f1f39
 size 38587970

attention_kindself_n_heads2_seed1341/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe708cdb3a1ae1687fb840f5a7e8956cb9b9dced147881782a428f6f1784574e
+oid sha256:8ce7ccd6ef7ae06a8da663b6f52e532942e899c68e28be0598ccdf1ffbba6ebb
 size 38587970

attention_kindself_n_heads2_seed1341/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:710739bd14fcc20618e82a2bbeb9ca427b9ce8836660c22d3447019556d4ddeb
+oid sha256:04c6e8f1fe0e1300ca89cfe223c99f95eb94eaacb629d8a4a35e03f031b1e504
 size 38587970

attention_kindself_n_heads2_seed1341/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09db754b0dacdd1829b4075398afe1e3433ba1a41853f8e571a6a4bdf83c5b66
+oid sha256:10242b8a6c52f5342146bdd7439961c1b49262a9d83715728d86e05a4bfcefd4
 size 38587970

attention_kindself_n_heads2_seed1341/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f3defe1d93fa733af9731e7281685a0b70e5e3008cda4cbc34e34b6f2ee15ac7
+oid sha256:1bba098d7c7ef009fe12f99ccff599bd854cf5c23fbd233789672b61607ca241
 size 70895430

attention_kindself_n_heads2_seed1341/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b53829a19019ad5c51f06b8492adc622a4d0143f13a16b832e5702910682b00
+oid sha256:f84a09402222dcdc6b4be51c39fc48b78d110580b0ab5465651191089f8822ea
 size 70895430

attention_kindself_n_heads2_seed1341/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d1f9c0f5f930db421657db03ff19d8fd8d84625f6d114179ecf168a71bf32052
+oid sha256:448ecd1e7f7292fc6ef00807247ed944bcee7a780b5b36aa0914ee986079967f
 size 70895430

attention_kindself_n_heads2_seed1341/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d0b182a10b4f8654eafc4207c594cb7a55f1fb433528ec56ba720044f8becfe
+oid sha256:af2213cdc561bbe7b5599ca0bbf2befaf5c370dcae45033220e5897669f4ca0d
 size 70895430