Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_2_1340", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_2_1340", "n_embd": 128}

attention_kindselective_n_heads2_seed1340/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.8210
 0 val perplexity 136085.8281
-0 train 11.833096 (lr=2.5000e-07) (hash(x)=164406924)
-100 val loss 10.1314
-100 val perplexity 25120.3730
-100 train 10.435864 (lr=2.5250e-05) (hash(x)=177407419)
-200 val loss 8.5877
-200 val perplexity 5365.3306
-200 train 8.530438 (lr=5.0000e-05) (hash(x)=144903932)
-300 val loss 7.9051
-300 val perplexity 2711.0681
-300 train 8.196055 (lr=4.9988e-05) (hash(x)=173839165)
-400 val loss 7.6891
-400 val perplexity 2184.3062
-400 train 7.772710 (lr=4.9954e-05) (hash(x)=167734596)
-500 val loss 7.5944
-500 val perplexity 1987.0854
-500 train 7.608729 (lr=4.9896e-05) (hash(x)=153224076)
-600 val loss 7.5572
-600 val perplexity 1914.4473
-600 train 7.472777 (lr=4.9815e-05) (hash(x)=149619098)
-700 val loss 7.5444
-700 val perplexity 1890.2152
-700 train 7.501118 (lr=4.9712e-05) (hash(x)=146539909)
-800 val loss 7.5187
-800 val perplexity 1842.2115
-800 train 7.481338 (lr=4.9585e-05) (hash(x)=153710890)
-900 val loss 7.5052
-900 val perplexity 1817.4755
-900 train 7.457563 (lr=4.9436e-05) (hash(x)=155873620)
-1000 val loss 7.4734
-1000 val perplexity 1760.6692
-1000 train 7.401876 (lr=4.9264e-05) (hash(x)=145450636)
-1100 val loss 7.4466
-1100 val perplexity 1714.0623
-1100 train 7.503413 (lr=4.9070e-05) (hash(x)=154123388)
-1200 val loss 7.4085
-1200 val perplexity 1650.0238
-1200 train 7.273043 (lr=4.8854e-05) (hash(x)=145249251)
-1300 val loss 7.3725
-1300 val perplexity 1591.5873
-1300 train 7.281743 (lr=4.8616e-05) (hash(x)=148937127)
-1400 val loss 7.3240
-1400 val perplexity 1516.2676
-1400 train 7.429045 (lr=4.8356e-05) (hash(x)=150475545)
-1500 val loss 7.2841
-1500 val perplexity 1456.8834
-1500 train 7.223948 (lr=4.8074e-05) (hash(x)=154653428)
-1600 val loss 7.2210
-1600 val perplexity 1367.9034
-1600 train 7.197338 (lr=4.7772e-05) (hash(x)=144483776)
-1700 val loss 7.1577
-1700 val perplexity 1283.9692
-1700 train 7.327583 (lr=4.7448e-05) (hash(x)=157395496)
-1800 val loss 7.0988
-1800 val perplexity 1210.5161
-1800 train 7.118510 (lr=4.7105e-05) (hash(x)=157916369)
-1900 val loss 7.0482
-1900 val perplexity 1150.7883
-1900 train 7.250220 (lr=4.6741e-05) (hash(x)=166073923)
-2000 val loss 7.0099
-2000 val perplexity 1107.5649
-2000 train 7.078530 (lr=4.6357e-05) (hash(x)=154856891)
-2100 val loss 6.9708
-2100 val perplexity 1065.0917
-2100 train 6.957214 (lr=4.5954e-05) (hash(x)=151925203)
-2200 val loss 6.9150
-2200 val perplexity 1007.2465
-2200 train 6.623221 (lr=4.5532e-05) (hash(x)=136191502)
-2300 val loss 6.8516
-2300 val perplexity 945.3943
-2300 train 6.992955 (lr=4.5091e-05) (hash(x)=153273362)
-2400 val loss 6.8002
-2400 val perplexity 898.0457
-2400 train 6.748407 (lr=4.4633e-05) (hash(x)=148021541)
-2500 val loss 6.7577
-2500 val perplexity 860.6304
-2500 train 6.699186 (lr=4.4156e-05) (hash(x)=141356608)
-2600 val loss 6.7101
-2600 val perplexity 820.6888
-2600 train 6.662726 (lr=4.3663e-05) (hash(x)=146005217)
-2700 val loss 6.6873
-2700 val perplexity 802.1870
-2700 train 6.533102 (lr=4.3153e-05) (hash(x)=144511718)
-2800 val loss 6.6459
-2800 val perplexity 769.6336
-2800 train 6.522305 (lr=4.2627e-05) (hash(x)=146019502)
-2900 val loss 6.6170
-2900 val perplexity 747.7319
-2900 train 6.526833 (lr=4.2085e-05) (hash(x)=146496200)
-3000 val loss 6.5904
-3000 val perplexity 728.0774
-3000 train 6.532321 (lr=4.1529e-05) (hash(x)=150127281)
-3100 val loss 6.5528
-3100 val perplexity 701.1899
-3100 train 6.537275 (lr=4.0957e-05) (hash(x)=142022255)
-3200 val loss 6.5277
-3200 val perplexity 683.8523
-3200 train 6.587036 (lr=4.0373e-05) (hash(x)=154120875)
-3300 val loss 6.5102
-3300 val perplexity 671.9406
-3300 train 6.563637 (lr=3.9775e-05) (hash(x)=153999717)
-3400 val loss 6.4894
-3400 val perplexity 658.1267
-3400 train 6.270551 (lr=3.9164e-05) (hash(x)=139694097)
-3500 val loss 6.4582
-3500 val perplexity 637.9100
-3500 train 6.644126 (lr=3.8541e-05) (hash(x)=162992732)
-3600 val loss 6.4246
-3600 val perplexity 616.8521
-3600 train 6.398591 (lr=3.7907e-05) (hash(x)=147574101)
-3700 val loss 6.4115
-3700 val perplexity 608.8231
-3700 train 6.528100 (lr=3.7262e-05) (hash(x)=157763099)
-3800 val loss 6.3859
-3800 val perplexity 593.4160
-3800 train 6.540219 (lr=3.6608e-05) (hash(x)=170800034)
-3900 val loss 6.3712
-3900 val perplexity 584.7700
-3900 train 6.450828 (lr=3.5944e-05) (hash(x)=164984528)
-4000 val loss 6.3565
-4000 val perplexity 576.2042
-4000 train 6.230247 (lr=3.5271e-05) (hash(x)=141743323)
-4100 val loss 6.3548
-4100 val perplexity 575.2441
-4100 train 6.368493 (lr=3.4590e-05) (hash(x)=153392872)
-4200 val loss 6.3445
-4200 val perplexity 569.3611
-4200 train 6.203356 (lr=3.3902e-05) (hash(x)=149074933)
-4300 val loss 6.3186
-4300 val perplexity 554.7869
-4300 train 6.702516 (lr=3.3207e-05) (hash(x)=167823423)
-4400 val loss 6.2979
-4400 val perplexity 543.4063
-4400 train 6.061867 (lr=3.2507e-05) (hash(x)=141203114)
-4500 val loss 6.2891
-4500 val perplexity 538.6491
-4500 train 6.289015 (lr=3.1801e-05) (hash(x)=146284780)
-4600 val loss 6.2654
-4600 val perplexity 526.0757
-4600 train 6.073493 (lr=3.1091e-05) (hash(x)=141126464)
-4700 val loss 6.2576
-4700 val perplexity 521.9659
-4700 train 6.287447 (lr=3.0377e-05) (hash(x)=154751926)
-4800 val loss 6.2461
-4800 val perplexity 516.0110
-4800 train 6.331287 (lr=2.9661e-05) (hash(x)=154793198)
-4900 val loss 6.2420
-4900 val perplexity 513.8834
-4900 train 5.956399 (lr=2.8942e-05) (hash(x)=139406392)
-5000 val loss 6.2666
-5000 val perplexity 526.6661
-5000 train 5.975211 (lr=2.8221e-05) (hash(x)=153548741)
-5100 val loss 6.2267
-5100 val perplexity 506.0816
-5100 train 6.303495 (lr=2.7500e-05) (hash(x)=160488568)
-5200 val loss 6.2044
-5200 val perplexity 494.9333
-5200 train 6.213053 (lr=2.6779e-05) (hash(x)=149645053)
-5300 val loss 6.1867
-5300 val perplexity 486.2615
-5300 train 6.276705 (lr=2.6058e-05) (hash(x)=155820556)
-5400 val loss 6.1806
-5400 val perplexity 483.2999
-5400 train 6.128071 (lr=2.5339e-05) (hash(x)=147538134)
-5500 val loss 6.1716
-5500 val perplexity 478.9390
-5500 train 6.377961 (lr=2.4623e-05) (hash(x)=166889307)
-5600 val loss 6.1622
-5600 val perplexity 474.4739
-5600 train 5.880041 (lr=2.3909e-05) (hash(x)=139516699)
-5700 val loss 6.1641
-5700 val perplexity 475.3632
-5700 train 5.843537 (lr=2.3199e-05) (hash(x)=140453511)
-5800 val loss 6.1614
-5800 val perplexity 474.1037
-5800 train 6.102824 (lr=2.2493e-05) (hash(x)=162964847)
-5900 val loss 6.1491
-5900 val perplexity 468.2912
-5900 train 6.153123 (lr=2.1793e-05) (hash(x)=150606634)
-6000 val loss 6.1313
-6000 val perplexity 460.0215
-6000 train 6.210247 (lr=2.1098e-05) (hash(x)=149890857)
-6100 val loss 6.1292
-6100 val perplexity 459.0655
-6100 train 6.234848 (lr=2.0410e-05) (hash(x)=173884145)
-6200 val loss 6.1213
-6200 val perplexity 455.4413
-6200 train 6.167700 (lr=1.9729e-05) (hash(x)=151987098)
-6300 val loss 6.1115
-6300 val perplexity 451.0340
-6300 train 6.057473 (lr=1.9056e-05) (hash(x)=148853562)
-6400 val loss 6.1064
-6400 val perplexity 448.7010
-6400 train 5.853988 (lr=1.8392e-05) (hash(x)=141530101)
-6500 val loss 6.1060
-6500 val perplexity 448.5262
-6500 train 5.964704 (lr=1.7738e-05) (hash(x)=142297809)
-6600 val loss 6.0976
-6600 val perplexity 444.7705
-6600 train 5.990374 (lr=1.7093e-05) (hash(x)=142447782)
-6700 val loss 6.0943
-6700 val perplexity 443.3159
-6700 train 6.063104 (lr=1.6459e-05) (hash(x)=147004686)
-6800 val loss 6.0880
-6800 val perplexity 440.5413
-6800 train 5.831447 (lr=1.5836e-05) (hash(x)=133438702)
-6900 val loss 6.0855
-6900 val perplexity 439.4597
-6900 train 6.103887 (lr=1.5225e-05) (hash(x)=157085143)
-7000 val loss 6.0821
-7000 val perplexity 437.9416
-7000 train 5.957407 (lr=1.4627e-05) (hash(x)=139437666)
-7100 val loss 6.0744
-7100 val perplexity 434.5800
-7100 train 6.096306 (lr=1.4043e-05) (hash(x)=159792986)
-7200 val loss 6.0653
-7200 val perplexity 430.6621
-7200 train 6.041170 (lr=1.3471e-05) (hash(x)=144930687)
-7300 val loss 6.0612
-7300 val perplexity 428.8893
-7300 train 6.113462 (lr=1.2915e-05) (hash(x)=156242690)
-7400 val loss 6.0626
-7400 val perplexity 429.4924
-7400 train 5.896077 (lr=1.2373e-05) (hash(x)=148183719)
-7500 val loss 6.0577
-7500 val perplexity 427.3765
-7500 train 6.091302 (lr=1.1847e-05) (hash(x)=152494758)
-7600 val loss 6.0547
-7600 val perplexity 426.1260
-7600 train 5.754051 (lr=1.1337e-05) (hash(x)=142485027)
-7700 val loss 6.0560
-7700 val perplexity 426.6740
-7700 train 5.928751 (lr=1.0844e-05) (hash(x)=147512165)
-7800 val loss 6.0404
-7800 val perplexity 420.0570
-7800 train 5.927812 (lr=1.0367e-05) (hash(x)=160346994)
-7900 val loss 6.0410
-7900 val perplexity 420.3325
-7900 train 5.907988 (lr=9.9088e-06) (hash(x)=144488254)
-8000 val loss 6.0358
-8000 val perplexity 418.1136
-8000 train 5.854425 (lr=9.4682e-06) (hash(x)=147637019)
-8100 val loss 6.0361
-8100 val perplexity 418.2736
-8100 train 5.901888 (lr=9.0461e-06) (hash(x)=147340534)
-8200 val loss 6.0326
-8200 val perplexity 416.7949
-8200 train 6.087666 (lr=8.6430e-06) (hash(x)=151630665)
-8300 val loss 6.0267
-8300 val perplexity 414.3367
-8300 train 6.126692 (lr=8.2593e-06) (hash(x)=149747064)
-8400 val loss 6.0232
-8400 val perplexity 412.8936
-8400 train 6.212832 (lr=7.8953e-06) (hash(x)=154245770)
-8500 val loss 6.0230
-8500 val perplexity 412.8336
-8500 train 5.904833 (lr=7.5515e-06) (hash(x)=152559100)
-8600 val loss 6.0187
-8600 val perplexity 411.0398
-8600 train 6.535484 (lr=7.2282e-06) (hash(x)=181365926)
-8700 val loss 6.0188
-8700 val perplexity 411.0829
-8700 train 5.863358 (lr=6.9257e-06) (hash(x)=154405991)
-8800 val loss 6.0153
-8800 val perplexity 409.6678
-8800 train 5.970211 (lr=6.6444e-06) (hash(x)=153755904)
-8900 val loss 6.0107
-8900 val perplexity 407.7833
-8900 train 5.926088 (lr=6.3845e-06) (hash(x)=152120568)
-9000 val loss 6.0077
-9000 val perplexity 406.5505
-9000 train 5.799891 (lr=6.1462e-06) (hash(x)=142797279)
-9100 val loss 6.0051
-9100 val perplexity 405.5031
-9100 train 5.871176 (lr=5.9300e-06) (hash(x)=143037503)
-9200 val loss 6.0051
-9200 val perplexity 405.4734
-9200 train 5.892231 (lr=5.7359e-06) (hash(x)=113690273)
-9300 val loss 6.0039
-9300 val perplexity 405.0020
-9300 train 5.939885 (lr=5.5641e-06) (hash(x)=158025077)
-9400 val loss 6.0019
-9400 val perplexity 404.1820
-9400 train 6.068281 (lr=5.4149e-06) (hash(x)=158251718)
-9500 val loss 5.9988
-9500 val perplexity 402.9359
-9500 train 6.039095 (lr=5.2884e-06) (hash(x)=154752610)
-9600 val loss 5.9971
-9600 val perplexity 402.2651
-9600 train 5.874845 (lr=5.1847e-06) (hash(x)=146889093)
-9700 val loss 5.9940
-9700 val perplexity 401.0011
-9700 train 6.001687 (lr=5.1040e-06) (hash(x)=156906516)
-9800 val loss 5.9932
-9800 val perplexity 400.7096
-9800 train 5.831883 (lr=5.0462e-06) (hash(x)=153841927)
-9900 val loss 5.9949
-9900 val perplexity 401.3906
-9900 train 6.192818 (lr=5.0116e-06) (hash(x)=163514334)
-9999 val loss 5.9891
-9999 val perplexity 399.0517
+0 train 11.833096 (lr=1.5000e-07) (hash(x)=164406924)
+100 val loss 10.2860
+100 val perplexity 29318.0957
+100 train 10.581880 (lr=1.5150e-05) (hash(x)=177407419)
+200 val loss 9.2449
+200 val perplexity 10351.6846
+200 train 9.195401 (lr=3.0000e-05) (hash(x)=144903932)
+300 val loss 8.1777
+300 val perplexity 3560.5066
+300 train 8.432213 (lr=2.9993e-05) (hash(x)=173839165)
+400 val loss 7.8301
+400 val perplexity 2515.1199
+400 train 7.897438 (lr=2.9972e-05) (hash(x)=167734596)
+500 val loss 7.6721
+500 val perplexity 2147.6790
+500 train 7.686492 (lr=2.9938e-05) (hash(x)=153224076)
+600 val loss 7.6069
+600 val perplexity 2012.0172
+600 train 7.531812 (lr=2.9889e-05) (hash(x)=149619098)
+700 val loss 7.5730
+700 val perplexity 1944.9812
+700 train 7.526173 (lr=2.9827e-05) (hash(x)=146539909)
+800 val loss 7.5481
+800 val perplexity 1897.0706
+800 train 7.518998 (lr=2.9751e-05) (hash(x)=153710890)
+900 val loss 7.5388
+900 val perplexity 1879.5011
+900 train 7.499132 (lr=2.9662e-05) (hash(x)=155873620)
+1000 val loss 7.5291
+1000 val perplexity 1861.4382
+1000 train 7.460474 (lr=2.9558e-05) (hash(x)=145450636)
+1100 val loss 7.5105
+1100 val perplexity 1827.0963
+1100 train 7.569565 (lr=2.9442e-05) (hash(x)=154123388)
+1200 val loss 7.4993
+1200 val perplexity 1806.8583
+1200 train 7.362889 (lr=2.9312e-05) (hash(x)=145249251)
+1300 val loss 7.4861
+1300 val perplexity 1783.1503
+1300 train 7.407315 (lr=2.9169e-05) (hash(x)=148937127)
+1400 val loss 7.4802
+1400 val perplexity 1772.6321
+1400 train 7.583092 (lr=2.9013e-05) (hash(x)=150475545)
+1500 val loss 7.4730
+1500 val perplexity 1759.8701
+1500 train 7.411776 (lr=2.8845e-05) (hash(x)=154653428)
+1600 val loss 7.4660
+1600 val perplexity 1747.5458
+1600 train 7.437787 (lr=2.8663e-05) (hash(x)=144483776)
+1700 val loss 7.4535
+1700 val perplexity 1725.8730
+1700 train 7.627961 (lr=2.8469e-05) (hash(x)=157395496)
+1800 val loss 7.4387
+1800 val perplexity 1700.5133
+1800 train 7.462626 (lr=2.8263e-05) (hash(x)=157916369)
+1900 val loss 7.4201
+1900 val perplexity 1669.2788
+1900 train 7.636841 (lr=2.8044e-05) (hash(x)=166073923)
+2000 val loss 7.4135
+2000 val perplexity 1658.2711
+2000 train 7.522213 (lr=2.7814e-05) (hash(x)=154856891)
+2100 val loss 7.4018
+2100 val perplexity 1638.9025
+2100 train 7.377049 (lr=2.7572e-05) (hash(x)=151925203)
+2200 val loss 7.3915
+2200 val perplexity 1622.1089
+2200 train 7.193004 (lr=2.7319e-05) (hash(x)=136191502)
+2300 val loss 7.3719
+2300 val perplexity 1590.5979
+2300 train 7.487554 (lr=2.7055e-05) (hash(x)=153273362)
+2400 val loss 7.3512
+2400 val perplexity 1557.9932
+2400 train 7.283143 (lr=2.6780e-05) (hash(x)=148021541)
+2500 val loss 7.3460
+2500 val perplexity 1550.0237
+2500 train 7.250226 (lr=2.6494e-05) (hash(x)=141356608)
+2600 val loss 7.3185
+2600 val perplexity 1507.8693
+2600 train 7.283971 (lr=2.6198e-05) (hash(x)=146005217)
+2700 val loss 7.3159
+2700 val perplexity 1504.0547
+2700 train 7.160429 (lr=2.5892e-05) (hash(x)=144511718)
+2800 val loss 7.2898
+2800 val perplexity 1465.3499
+2800 train 7.166070 (lr=2.5576e-05) (hash(x)=146019502)
+2900 val loss 7.2652
+2900 val perplexity 1429.6077
+2900 train 7.165171 (lr=2.5251e-05) (hash(x)=146496200)
+3000 val loss 7.2538
+3000 val perplexity 1413.4381
+3000 train 7.196369 (lr=2.4917e-05) (hash(x)=150127281)
+3100 val loss 7.2288
+3100 val perplexity 1378.5229
+3100 train 7.172277 (lr=2.4574e-05) (hash(x)=142022255)
+3200 val loss 7.2119
+3200 val perplexity 1355.4843
+3200 train 7.291126 (lr=2.4224e-05) (hash(x)=154120875)
+3300 val loss 7.1995
+3300 val perplexity 1338.7019
+3300 train 7.225865 (lr=2.3865e-05) (hash(x)=153999717)
+3400 val loss 7.1853
+3400 val perplexity 1319.9038
+3400 train 6.986575 (lr=2.3498e-05) (hash(x)=139694097)
+3500 val loss 7.1647
+3500 val perplexity 1292.9854
+3500 train 7.357235 (lr=2.3125e-05) (hash(x)=162992732)
+3600 val loss 7.1354
+3600 val perplexity 1255.6628
+3600 train 7.120846 (lr=2.2744e-05) (hash(x)=147574101)
+3700 val loss 7.1180
+3700 val perplexity 1233.9659
+3700 train 7.231690 (lr=2.2357e-05) (hash(x)=157763099)
+3800 val loss 7.0878
+3800 val perplexity 1197.3303
+3800 train 7.244596 (lr=2.1965e-05) (hash(x)=170800034)
+3900 val loss 7.0743
+3900 val perplexity 1181.2009
+3900 train 7.151352 (lr=2.1566e-05) (hash(x)=164984528)
+4000 val loss 7.0556
+4000 val perplexity 1159.3071
+4000 train 6.946763 (lr=2.1162e-05) (hash(x)=141743323)
+4100 val loss 7.0423
+4100 val perplexity 1143.9823
+4100 train 7.050414 (lr=2.0754e-05) (hash(x)=153392872)
+4200 val loss 7.0286
+4200 val perplexity 1128.3999
+4200 train 6.904306 (lr=2.0341e-05) (hash(x)=149074933)
+4300 val loss 7.0063
+4300 val perplexity 1103.5922
+4300 train 7.419280 (lr=1.9924e-05) (hash(x)=167823423)
+4400 val loss 6.9891
+4400 val perplexity 1084.7200
+4400 train 6.721108 (lr=1.9504e-05) (hash(x)=141203114)
+4500 val loss 6.9697
+4500 val perplexity 1063.9399
+4500 train 6.962101 (lr=1.9081e-05) (hash(x)=146284780)
+4600 val loss 6.9558
+4600 val perplexity 1049.2501
+4600 train 6.758272 (lr=1.8655e-05) (hash(x)=141126464)
+4700 val loss 6.9420
+4700 val perplexity 1034.8446
+4700 train 6.943596 (lr=1.8226e-05) (hash(x)=154751926)
+4800 val loss 6.9285
+4800 val perplexity 1020.9679
+4800 train 7.011858 (lr=1.7796e-05) (hash(x)=154793198)
+4900 val loss 6.9213
+4900 val perplexity 1013.5986
+4900 train 6.667045 (lr=1.7365e-05) (hash(x)=139406392)
+5000 val loss 6.9092
+5000 val perplexity 1001.4541
+5000 train 6.710131 (lr=1.6933e-05) (hash(x)=153548741)
+5100 val loss 6.8940
+5100 val perplexity 986.2919
+5100 train 6.959525 (lr=1.6500e-05) (hash(x)=160488568)
+5200 val loss 6.8850
+5200 val perplexity 977.5457
+5200 train 6.886682 (lr=1.6067e-05) (hash(x)=149645053)
+5300 val loss 6.8517
+5300 val perplexity 945.4849
+5300 train 6.939456 (lr=1.5635e-05) (hash(x)=155820556)
+5400 val loss 6.8374
+5400 val perplexity 932.0350
+5400 train 6.763284 (lr=1.5204e-05) (hash(x)=147538134)
+5500 val loss 6.8217
+5500 val perplexity 917.5742
+5500 train 7.024829 (lr=1.4774e-05) (hash(x)=166889307)
+5600 val loss 6.8077
+5600 val perplexity 904.7902
+5600 train 6.529876 (lr=1.4345e-05) (hash(x)=139516699)
+5700 val loss 6.7965
+5700 val perplexity 894.6815
+5700 train 6.495327 (lr=1.3919e-05) (hash(x)=140453511)
+5800 val loss 6.7922
+5800 val perplexity 890.8710
+5800 train 6.750346 (lr=1.3496e-05) (hash(x)=162964847)
+5900 val loss 6.7734
+5900 val perplexity 874.2501
+5900 train 6.787580 (lr=1.3076e-05) (hash(x)=150606634)
+6000 val loss 6.7546
+6000 val perplexity 857.9982
+6000 train 6.869712 (lr=1.2659e-05) (hash(x)=149890857)
+6100 val loss 6.7394
+6100 val perplexity 845.0160
+6100 train 6.851167 (lr=1.2246e-05) (hash(x)=173884145)
+6200 val loss 6.7288
+6200 val perplexity 836.1551
+6200 train 6.760945 (lr=1.1838e-05) (hash(x)=151987098)
+6300 val loss 6.7160
+6300 val perplexity 825.5145
+6300 train 6.630625 (lr=1.1434e-05) (hash(x)=148853562)
+6400 val loss 6.7048
+6400 val perplexity 816.2783
+6400 train 6.468220 (lr=1.1035e-05) (hash(x)=141530101)
+6500 val loss 6.6959
+6500 val perplexity 809.1172
+6500 train 6.547406 (lr=1.0643e-05) (hash(x)=142297809)
+6600 val loss 6.6835
+6600 val perplexity 799.1030
+6600 train 6.547137 (lr=1.0256e-05) (hash(x)=142447782)
+6700 val loss 6.6707
+6700 val perplexity 788.9225
+6700 train 6.593633 (lr=9.8753e-06) (hash(x)=147004686)
+6800 val loss 6.6594
+6800 val perplexity 780.0958
+6800 train 6.405754 (lr=9.5017e-06) (hash(x)=133438702)
+6900 val loss 6.6530
+6900 val perplexity 775.0946
+6900 train 6.670481 (lr=9.1353e-06) (hash(x)=157085143)
+7000 val loss 6.6447
+7000 val perplexity 768.6899
+7000 train 6.533543 (lr=8.7764e-06) (hash(x)=139437666)
+7100 val loss 6.6337
+7100 val perplexity 760.2664
+7100 train 6.748162 (lr=8.4255e-06) (hash(x)=159792986)
+7200 val loss 6.6230
+7200 val perplexity 752.1608
+7200 train 6.588307 (lr=8.0829e-06) (hash(x)=144930687)
+7300 val loss 6.6153
+7300 val perplexity 746.4230
+7300 train 6.699518 (lr=7.7489e-06) (hash(x)=156242690)
+7400 val loss 6.6093
+7400 val perplexity 741.9413
+7400 train 6.480586 (lr=7.4239e-06) (hash(x)=148183719)
+7500 val loss 6.6042
+7500 val perplexity 738.1967
+7500 train 6.624214 (lr=7.1083e-06) (hash(x)=152494758)
+7600 val loss 6.5976
+7600 val perplexity 733.3229
+7600 train 6.321466 (lr=6.8023e-06) (hash(x)=142485027)
+7700 val loss 6.5898
+7700 val perplexity 727.6033
+7700 train 6.445391 (lr=6.5062e-06) (hash(x)=147512165)
+7800 val loss 6.5780
+7800 val perplexity 719.1275
+7800 train 6.539083 (lr=6.2205e-06) (hash(x)=160346994)
+7900 val loss 6.5764
+7900 val perplexity 717.9341
+7900 train 6.418441 (lr=5.9453e-06) (hash(x)=144488254)
+8000 val loss 6.5700
+8000 val perplexity 713.3625
+8000 train 6.375028 (lr=5.6809e-06) (hash(x)=147637019)
+8100 val loss 6.5666
+8100 val perplexity 710.9796
+8100 train 6.427360 (lr=5.4277e-06) (hash(x)=147340534)
+8200 val loss 6.5623
+8200 val perplexity 707.9001
+8200 train 6.574669 (lr=5.1858e-06) (hash(x)=151630665)
+8300 val loss 6.5526
+8300 val perplexity 701.0833
+8300 train 6.621968 (lr=4.9556e-06) (hash(x)=149747064)
+8400 val loss 6.5468
+8400 val perplexity 696.9837
+8400 train 6.719865 (lr=4.7372e-06) (hash(x)=154245770)
+8500 val loss 6.5429
+8500 val perplexity 694.2892
+8500 train 6.417419 (lr=4.5309e-06) (hash(x)=152559100)
+8600 val loss 6.5389
+8600 val perplexity 691.4964
+8600 train 7.097158 (lr=4.3369e-06) (hash(x)=181365926)
+8700 val loss 6.5363
+8700 val perplexity 689.7556
+8700 train 6.383374 (lr=4.1554e-06) (hash(x)=154405991)
+8800 val loss 6.5326
+8800 val perplexity 687.1642
+8800 train 6.495862 (lr=3.9866e-06) (hash(x)=153755904)
+8900 val loss 6.5266
+8900 val perplexity 683.0411
+8900 train 6.463516 (lr=3.8307e-06) (hash(x)=152120568)
+9000 val loss 6.5208
+9000 val perplexity 679.1530
+9000 train 6.330629 (lr=3.6877e-06) (hash(x)=142797279)
+9100 val loss 6.5182
+9100 val perplexity 677.3309
+9100 train 6.386018 (lr=3.5580e-06) (hash(x)=143037503)
+9200 val loss 6.5147
+9200 val perplexity 674.9849
+9200 train 6.315207 (lr=3.4415e-06) (hash(x)=113690273)
+9300 val loss 6.5132
+9300 val perplexity 673.9931
+9300 train 6.458709 (lr=3.3385e-06) (hash(x)=158025077)
+9400 val loss 6.5109
+9400 val perplexity 672.4445
+9400 train 6.597129 (lr=3.2490e-06) (hash(x)=158251718)
+9500 val loss 6.5048
+9500 val perplexity 668.3381
+9500 train 6.567007 (lr=3.1730e-06) (hash(x)=154752610)
+9600 val loss 6.5019
+9600 val perplexity 666.3765
+9600 train 6.378345 (lr=3.1108e-06) (hash(x)=146889093)
+9700 val loss 6.4992
+9700 val perplexity 664.6182
+9700 train 6.483963 (lr=3.0624e-06) (hash(x)=156906516)
+9800 val loss 6.4974
+9800 val perplexity 663.4178
+9800 train 6.365656 (lr=3.0277e-06) (hash(x)=153841927)
+9900 val loss 6.4961
+9900 val perplexity 662.5817
+9900 train 6.691042 (lr=3.0069e-06) (hash(x)=163514334)
+9999 val loss 6.4912
+9999 val perplexity 659.2943

attention_kindselective_n_heads2_seed1340/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e5aeb7b7022a7ef095259330e13212b1ff8ec10d9fe91f9cfe5321726439400e
+oid sha256:2c6723edebcb92ca2e72054a32701bc25d1fc927f93c9e3ffc3e865f8b7415e5
 size 38587970

attention_kindselective_n_heads2_seed1340/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad1dcff0aef7e71a44d4bafe3d6fa2064b4970bceaa1db35e56cacedd4bce39b
+oid sha256:03f9fd53f979fe8cf9bf929f3f9ccb04af6199807a56cf30e6401e200e0c1bf4
 size 38587970

attention_kindselective_n_heads2_seed1340/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df381c5d857ee4e2185137efa7971ff6ecca01a7071f0e3523bebefff488b94d
+oid sha256:dc16680a7b7cd0eb8bab693bc1270640af820af29ff05c8484576c829dca657b
 size 38587970

attention_kindselective_n_heads2_seed1340/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3260dd2ac1f7c16a62ed6d3c8b157a3f0a33efe14c610d63fec80ece19e6295c
+oid sha256:bee304b6d21be05c3175629877a7130ec354483d6f47805582d72d4e56b8b5f5
 size 38587970

attention_kindselective_n_heads2_seed1340/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f494e8b5205e169278e98dc46247c57b4eecddb1d447131b03a3e8d70073d9e
+oid sha256:eaf36019f143c8ca2d052bd8495bde7f6853829697fcff4a1af930a2ed55ee8a
 size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c7f3fc56d8050fdf69a5ced1176d19cd3d7141742bed03ead99504850e90b4be
+oid sha256:7a683404880ddecba30f3c5ded4e51f96de7555ad774fd34351ff9d5bc661c4b
 size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4c05973832a45530c4593bc265a28c7ae61596a93a88b25f70012a7d34e37954
+oid sha256:51bfb9ae26523dc1ea5f3a0fad4a6e93f8c6ce10ef08bbcd9d25e43431a4005b
 size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2084254e2cd8f9f5a63850ea73b259adc84ee7fc54797ca79e71efee5351e261
+oid sha256:8966865e8fa15c38fb89509c163897db98961bbf4dcf7cb885b3b3ff5e1a6ddd
 size 70895430