Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_4_1339", "n_embd": 256}

attention_kindselective_n_heads4_seed1339/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.2810
 0 val perplexity 79298.4922
-0 train 11.295413 (lr=7.5000e-07) (hash(x)=150724848)
-100 val loss 8.5817
-100 val perplexity 5332.9116
-100 train 8.631360 (lr=7.5750e-05) (hash(x)=149217061)
-200 val loss 7.5932
-200 val perplexity 1984.5913
-200 train 7.581491 (lr=1.5000e-04) (hash(x)=149910534)
-300 val loss 7.4699
-300 val perplexity 1754.4465
-300 train 7.326682 (lr=1.4997e-04) (hash(x)=142185643)
-400 val loss 7.3927
-400 val perplexity 1624.1583
-400 train 7.405558 (lr=1.4986e-04) (hash(x)=148123706)
-500 val loss 7.3282
-500 val perplexity 1522.6230
-500 train 7.174585 (lr=1.4969e-04) (hash(x)=149952383)
-600 val loss 7.2701
-600 val perplexity 1436.7457
-600 train 7.065297 (lr=1.4945e-04) (hash(x)=146678221)
-700 val loss 7.1607
-700 val perplexity 1287.7505
-700 train 7.205720 (lr=1.4913e-04) (hash(x)=156180736)
-800 val loss 7.0975
-800 val perplexity 1209.0013
-800 train 7.072469 (lr=1.4876e-04) (hash(x)=151700982)
-900 val loss 7.0589
-900 val perplexity 1163.1676
-900 train 6.836958 (lr=1.4831e-04) (hash(x)=147288467)
-1000 val loss 7.0223
-1000 val perplexity 1121.3513
-1000 train 7.066115 (lr=1.4779e-04) (hash(x)=156182087)
-1100 val loss 7.0102
-1100 val perplexity 1107.9083
-1100 train 6.860521 (lr=1.4721e-04) (hash(x)=147861550)
-1200 val loss 6.9932
-1200 val perplexity 1089.1798
-1200 train 6.866424 (lr=1.4656e-04) (hash(x)=149318660)
-1300 val loss 7.0027
-1300 val perplexity 1099.6331
-1300 train 6.823798 (lr=1.4585e-04) (hash(x)=151631103)
-1400 val loss 7.0197
-1400 val perplexity 1118.4031
-1400 train 7.047281 (lr=1.4507e-04) (hash(x)=150482428)
-1500 val loss 6.9541
-1500 val perplexity 1047.3906
-1500 train 6.805685 (lr=1.4422e-04) (hash(x)=144458617)
-1600 val loss 6.8616
-1600 val perplexity 954.9236
-1600 train 6.756378 (lr=1.4332e-04) (hash(x)=143268605)
-1700 val loss 6.8326
-1700 val perplexity 927.6204
-1700 train 6.767980 (lr=1.4235e-04) (hash(x)=150817011)
-1800 val loss 6.8296
-1800 val perplexity 924.8635
-1800 train 6.823513 (lr=1.4131e-04) (hash(x)=152322423)
-1900 val loss 6.8104
-1900 val perplexity 907.2120
-1900 train 6.606685 (lr=1.4022e-04) (hash(x)=143574126)
-2000 val loss 6.8044
-2000 val perplexity 901.8112
-2000 train 6.589591 (lr=1.3907e-04) (hash(x)=147904298)
-2100 val loss 6.7997
-2100 val perplexity 897.5615
-2100 train 6.542164 (lr=1.3786e-04) (hash(x)=144719400)
-2200 val loss 6.7954
-2200 val perplexity 893.6961
-2200 train 7.016779 (lr=1.3660e-04) (hash(x)=154343147)
-2300 val loss 6.7536
-2300 val perplexity 857.1774
-2300 train 6.790805 (lr=1.3527e-04) (hash(x)=154372635)
-2400 val loss 6.7246
-2400 val perplexity 832.6526
-2400 train 6.568871 (lr=1.3390e-04) (hash(x)=141843115)
-2500 val loss 6.6865
-2500 val perplexity 801.4880
-2500 train 6.583528 (lr=1.3247e-04) (hash(x)=149417679)
-2600 val loss 6.6823
-2600 val perplexity 798.1768
-2600 train 6.545222 (lr=1.3099e-04) (hash(x)=145279030)
-2700 val loss 6.6431
-2700 val perplexity 767.4857
-2700 train 6.573290 (lr=1.2946e-04) (hash(x)=146363911)
-2800 val loss 6.6954
-2800 val perplexity 808.7022
-2800 train 6.648605 (lr=1.2788e-04) (hash(x)=152507639)
-2900 val loss 6.6802
-2900 val perplexity 796.4598
-2900 train 6.488822 (lr=1.2626e-04) (hash(x)=139296191)
-3000 val loss 6.6775
-3000 val perplexity 794.3075
-3000 train 6.686038 (lr=1.2459e-04) (hash(x)=148473774)
-3100 val loss 6.6638
-3100 val perplexity 783.4847
-3100 train 6.742999 (lr=1.2287e-04) (hash(x)=154130527)
-3200 val loss 6.6479
-3200 val perplexity 771.1449
-3200 train 6.599888 (lr=1.2112e-04) (hash(x)=151117002)
-3300 val loss 6.6349
-3300 val perplexity 761.2346
-3300 train 6.778743 (lr=1.1932e-04) (hash(x)=151035383)
-3400 val loss 6.6253
-3400 val perplexity 753.8972
-3400 train 6.759344 (lr=1.1749e-04) (hash(x)=150614749)
-3500 val loss 6.6271
-3500 val perplexity 755.2767
-3500 train 6.640333 (lr=1.1562e-04) (hash(x)=151431391)
-3600 val loss 6.5977
-3600 val perplexity 733.3723
-3600 train 6.536980 (lr=1.1372e-04) (hash(x)=151140797)
-3700 val loss 6.6127
-3700 val perplexity 744.4752
-3700 train 6.792097 (lr=1.1179e-04) (hash(x)=154608340)
-3800 val loss 6.6042
-3800 val perplexity 738.1901
-3800 train 6.620290 (lr=1.0982e-04) (hash(x)=169641309)
-3900 val loss 6.5557
-3900 val perplexity 703.2455
-3900 train 6.523308 (lr=1.0783e-04) (hash(x)=150592479)
-4000 val loss 6.5413
-4000 val perplexity 693.1867
-4000 train 6.511438 (lr=1.0581e-04) (hash(x)=155495317)
-4100 val loss 6.5415
-4100 val perplexity 693.3563
-4100 train 6.727324 (lr=1.0377e-04) (hash(x)=170323970)
-4200 val loss 6.5481
-4200 val perplexity 697.9368
-4200 train 6.369248 (lr=1.0171e-04) (hash(x)=140854360)
-4300 val loss 6.5413
-4300 val perplexity 693.1804
-4300 train 6.532832 (lr=9.9622e-05) (hash(x)=148926644)
-4400 val loss 6.5165
-4400 val perplexity 676.1769
-4400 train 6.555954 (lr=9.7520e-05) (hash(x)=149997871)
-4500 val loss 6.4623
-4500 val perplexity 640.5529
-4500 train 6.415502 (lr=9.5403e-05) (hash(x)=149219289)
-4600 val loss 6.4509
-4600 val perplexity 633.2473
-4600 train 6.264147 (lr=9.3273e-05) (hash(x)=139836197)
-4700 val loss 6.4489
-4700 val perplexity 631.9807
-4700 train 6.410443 (lr=9.1132e-05) (hash(x)=150722883)
-4800 val loss 6.4401
-4800 val perplexity 626.4460
-4800 train 6.639719 (lr=8.8982e-05) (hash(x)=166864648)
-4900 val loss 6.4142
-4900 val perplexity 610.4495
-4900 train 6.536889 (lr=8.6825e-05) (hash(x)=157746743)
-5000 val loss 6.3877
-5000 val perplexity 594.4929
-5000 train 6.344072 (lr=8.4663e-05) (hash(x)=153032257)
-5100 val loss 6.3855
-5100 val perplexity 593.1589
-5100 train 6.445783 (lr=8.2500e-05) (hash(x)=151862371)
-5200 val loss 6.3667
-5200 val perplexity 582.1065
-5200 train 6.667581 (lr=8.0337e-05) (hash(x)=168862995)
-5300 val loss 6.3696
-5300 val perplexity 583.8088
-5300 train 6.328732 (lr=7.8175e-05) (hash(x)=148243567)
-5400 val loss 6.3508
-5400 val perplexity 572.9389
-5400 train 6.276412 (lr=7.6018e-05) (hash(x)=145381742)
-5500 val loss 6.3349
-5500 val perplexity 563.9183
-5500 train 6.471667 (lr=7.3868e-05) (hash(x)=155602174)
-5600 val loss 6.3206
-5600 val perplexity 555.9157
-5600 train 6.392186 (lr=7.1727e-05) (hash(x)=160637672)
-5700 val loss 6.3222
-5700 val perplexity 556.7697
-5700 train 6.250708 (lr=6.9597e-05) (hash(x)=152101131)
-5800 val loss 6.3087
-5800 val perplexity 549.3273
-5800 train 6.284104 (lr=6.7480e-05) (hash(x)=152327089)
-5900 val loss 6.3093
-5900 val perplexity 549.6396
-5900 train 6.256981 (lr=6.5378e-05) (hash(x)=151094543)
-6000 val loss 6.3130
-6000 val perplexity 551.7166
-6000 train 6.390558 (lr=6.3294e-05) (hash(x)=160233315)
-6100 val loss 6.2926
-6100 val perplexity 540.5599
-6100 train 6.394215 (lr=6.1230e-05) (hash(x)=158295337)
-6200 val loss 6.2789
-6200 val perplexity 533.2273
-6200 train 6.239454 (lr=5.9188e-05) (hash(x)=151269817)
-6300 val loss 6.2694
-6300 val perplexity 528.1562
-6300 train 6.401673 (lr=5.7169e-05) (hash(x)=156912756)
-6400 val loss 6.2619
-6400 val perplexity 524.2102
-6400 train 6.249959 (lr=5.5177e-05) (hash(x)=125779637)
-6500 val loss 6.2738
-6500 val perplexity 530.4722
-6500 train 6.169670 (lr=5.3213e-05) (hash(x)=148157661)
-6600 val loss 6.2593
-6600 val perplexity 522.8664
-6600 train 6.271819 (lr=5.1279e-05) (hash(x)=152820049)
-6700 val loss 6.2570
-6700 val perplexity 521.6345
-6700 train 6.216044 (lr=4.9377e-05) (hash(x)=155818717)
-6800 val loss 6.2368
-6800 val perplexity 511.1964
-6800 train 6.283014 (lr=4.7509e-05) (hash(x)=158833881)
-6900 val loss 6.2285
-6900 val perplexity 506.9743
-6900 train 6.179168 (lr=4.5676e-05) (hash(x)=164984909)
-7000 val loss 6.2311
-7000 val perplexity 508.3391
-7000 train 6.126690 (lr=4.3882e-05) (hash(x)=151172493)
-7100 val loss 6.2327
-7100 val perplexity 509.1287
-7100 train 6.266987 (lr=4.2128e-05) (hash(x)=154023633)
-7200 val loss 6.2152
-7200 val perplexity 500.3194
-7200 train 6.093438 (lr=4.0414e-05) (hash(x)=139351250)
-7300 val loss 6.2126
-7300 val perplexity 498.9968
-7300 train 6.167672 (lr=3.8745e-05) (hash(x)=158719458)
-7400 val loss 6.2025
-7400 val perplexity 494.0034
-7400 train 6.270429 (lr=3.7120e-05) (hash(x)=162250810)
-7500 val loss 6.1907
-7500 val perplexity 488.2101
-7500 train 6.070210 (lr=3.5541e-05) (hash(x)=142827557)
-7600 val loss 6.1884
-7600 val perplexity 487.0707
-7600 train 6.085032 (lr=3.4011e-05) (hash(x)=147830801)
-7700 val loss 6.1889
-7700 val perplexity 487.3230
-7700 train 6.269209 (lr=3.2531e-05) (hash(x)=165233314)
-7800 val loss 6.1830
-7800 val perplexity 484.4642
-7800 train 6.176233 (lr=3.1102e-05) (hash(x)=160654717)
-7900 val loss 6.1814
-7900 val perplexity 483.6577
-7900 train 6.050529 (lr=2.9726e-05) (hash(x)=140712586)
-8000 val loss 6.1706
-8000 val perplexity 478.4890
-8000 train 6.319698 (lr=2.8405e-05) (hash(x)=150309061)
-8100 val loss 6.1642
-8100 val perplexity 475.4011
-8100 train 6.150677 (lr=2.7138e-05) (hash(x)=158805466)
-8200 val loss 6.1579
-8200 val perplexity 472.4475
-8200 train 6.029180 (lr=2.5929e-05) (hash(x)=142814078)
-8300 val loss 6.1567
-8300 val perplexity 471.8569
-8300 train 6.110010 (lr=2.4778e-05) (hash(x)=148340417)
-8400 val loss 6.1595
-8400 val perplexity 473.1881
-8400 train 6.101810 (lr=2.3686e-05) (hash(x)=157132639)
-8500 val loss 6.1568
-8500 val perplexity 471.9323
-8500 train 6.116642 (lr=2.2655e-05) (hash(x)=152994240)
-8600 val loss 6.1439
-8600 val perplexity 465.8618
-8600 train 6.103668 (lr=2.1685e-05) (hash(x)=152210154)
-8700 val loss 6.1421
-8700 val perplexity 465.0104
-8700 train 6.190649 (lr=2.0777e-05) (hash(x)=149631247)
-8800 val loss 6.1399
-8800 val perplexity 464.0101
-8800 train 5.996617 (lr=1.9933e-05) (hash(x)=145617784)
-8900 val loss 6.1353
-8900 val perplexity 461.8795
-8900 train 6.206507 (lr=1.9153e-05) (hash(x)=155984970)
-9000 val loss 6.1374
-9000 val perplexity 462.8537
-9000 train 5.890743 (lr=1.8439e-05) (hash(x)=143433013)
-9100 val loss 6.1376
-9100 val perplexity 462.9261
-9100 train 6.009195 (lr=1.7790e-05) (hash(x)=152507533)
-9200 val loss 6.1288
-9200 val perplexity 458.8862
-9200 train 6.234875 (lr=1.7208e-05) (hash(x)=162400650)
-9300 val loss 6.1247
-9300 val perplexity 457.0203
-9300 train 6.179266 (lr=1.6692e-05) (hash(x)=148419068)
-9400 val loss 6.1257
-9400 val perplexity 457.4461
-9400 train 6.100061 (lr=1.6245e-05) (hash(x)=145025169)
-9500 val loss 6.1228
-9500 val perplexity 456.1235
-9500 train 6.096772 (lr=1.5865e-05) (hash(x)=145104235)
-9600 val loss 6.1236
-9600 val perplexity 456.4965
-9600 train 6.053626 (lr=1.5554e-05) (hash(x)=150499797)
-9700 val loss 6.1195
-9700 val perplexity 454.6506
-9700 train 5.945662 (lr=1.5312e-05) (hash(x)=139690022)
-9800 val loss 6.1149
-9800 val perplexity 452.5396
-9800 train 6.116775 (lr=1.5139e-05) (hash(x)=145645279)
-9900 val loss 6.1106
-9900 val perplexity 450.5890
-9900 train 6.013401 (lr=1.5035e-05) (hash(x)=150103432)
-9999 val loss 6.1084
-9999 val perplexity 449.6328
+0 train 11.295413 (lr=2.5000e-07) (hash(x)=150724848)
+100 val loss 9.7855
+100 val perplexity 17773.2871
+100 train 9.836302 (lr=2.5250e-05) (hash(x)=149217061)
+200 val loss 8.0909
+200 val perplexity 3264.4639
+200 train 8.104568 (lr=5.0000e-05) (hash(x)=149910534)
+300 val loss 7.6193
+300 val perplexity 2037.1655
+300 train 7.494644 (lr=4.9988e-05) (hash(x)=142185643)
+400 val loss 7.5177
+400 val perplexity 1840.3098
+400 train 7.550296 (lr=4.9954e-05) (hash(x)=148123706)
+500 val loss 7.4518
+500 val perplexity 1722.9491
+500 train 7.302660 (lr=4.9896e-05) (hash(x)=149952383)
+600 val loss 7.4002
+600 val perplexity 1636.2810
+600 train 7.193367 (lr=4.9815e-05) (hash(x)=146678221)
+700 val loss 7.2899
+700 val perplexity 1465.4797
+700 train 7.324069 (lr=4.9712e-05) (hash(x)=156180736)
+800 val loss 7.1962
+800 val perplexity 1334.2875
+800 train 7.182938 (lr=4.9585e-05) (hash(x)=151700982)
+900 val loss 7.1326
+900 val perplexity 1252.1842
+900 train 6.912815 (lr=4.9436e-05) (hash(x)=147288467)
+1000 val loss 7.0566
+1000 val perplexity 1160.5161
+1000 train 7.103452 (lr=4.9264e-05) (hash(x)=156182087)
+1100 val loss 7.0104
+1100 val perplexity 1108.0488
+1100 train 6.875510 (lr=4.9070e-05) (hash(x)=147861550)
+1200 val loss 6.9584
+1200 val perplexity 1051.9824
+1200 train 6.816287 (lr=4.8854e-05) (hash(x)=149318660)
+1300 val loss 6.9151
+1300 val perplexity 1007.3896
+1300 train 6.738190 (lr=4.8616e-05) (hash(x)=151631103)
+1400 val loss 6.8571
+1400 val perplexity 950.6181
+1400 train 6.885700 (lr=4.8356e-05) (hash(x)=150482428)
+1500 val loss 6.7695
+1500 val perplexity 870.8608
+1500 train 6.605503 (lr=4.8074e-05) (hash(x)=144458617)
+1600 val loss 6.6986
+1600 val perplexity 811.2830
+1600 train 6.586299 (lr=4.7772e-05) (hash(x)=143268605)
+1700 val loss 6.6478
+1700 val perplexity 771.0643
+1700 train 6.589181 (lr=4.7448e-05) (hash(x)=150817011)
+1800 val loss 6.6028
+1800 val perplexity 737.1257
+1800 train 6.595220 (lr=4.7105e-05) (hash(x)=152322423)
+1900 val loss 6.5588
+1900 val perplexity 705.4356
+1900 train 6.328050 (lr=4.6741e-05) (hash(x)=143574126)
+2000 val loss 6.5234
+2000 val perplexity 680.9199
+2000 train 6.262884 (lr=4.6357e-05) (hash(x)=147904298)
+2100 val loss 6.4996
+2100 val perplexity 664.8712
+2100 train 6.229161 (lr=4.5954e-05) (hash(x)=144719400)
+2200 val loss 6.4638
+2200 val perplexity 641.5072
+2200 train 6.717620 (lr=4.5532e-05) (hash(x)=154343147)
+2300 val loss 6.4179
+2300 val perplexity 612.6993
+2300 train 6.457769 (lr=4.5091e-05) (hash(x)=154372635)
+2400 val loss 6.3894
+2400 val perplexity 595.4929
+2400 train 6.222154 (lr=4.4633e-05) (hash(x)=141843115)
+2500 val loss 6.3550
+2500 val perplexity 575.3881
+2500 train 6.258588 (lr=4.4156e-05) (hash(x)=149417679)
+2600 val loss 6.3298
+2600 val perplexity 561.0241
+2600 train 6.210278 (lr=4.3663e-05) (hash(x)=145279030)
+2700 val loss 6.3072
+2700 val perplexity 548.5043
+2700 train 6.232386 (lr=4.3153e-05) (hash(x)=146363911)
+2800 val loss 6.3028
+2800 val perplexity 546.0763
+2800 train 6.250120 (lr=4.2627e-05) (hash(x)=152507639)
+2900 val loss 6.2756
+2900 val perplexity 531.4626
+2900 train 6.065167 (lr=4.2085e-05) (hash(x)=139296191)
+3000 val loss 6.2584
+3000 val perplexity 522.3799
+3000 train 6.259926 (lr=4.1529e-05) (hash(x)=148473774)
+3100 val loss 6.2200
+3100 val perplexity 502.7139
+3100 train 6.297199 (lr=4.0957e-05) (hash(x)=154130527)
+3200 val loss 6.2027
+3200 val perplexity 494.1042
+3200 train 6.156054 (lr=4.0373e-05) (hash(x)=151117002)
+3300 val loss 6.2222
+3300 val perplexity 503.8253
+3300 train 6.403162 (lr=3.9775e-05) (hash(x)=151035383)
+3400 val loss 6.1724
+3400 val perplexity 479.3145
+3400 train 6.324723 (lr=3.9164e-05) (hash(x)=150614749)
+3500 val loss 6.1556
+3500 val perplexity 471.3615
+3500 train 6.130349 (lr=3.8541e-05) (hash(x)=151431391)
+3600 val loss 6.1415
+3600 val perplexity 464.7550
+3600 train 6.086444 (lr=3.7907e-05) (hash(x)=151140797)
+3700 val loss 6.1257
+3700 val perplexity 457.4801
+3700 train 6.282626 (lr=3.7262e-05) (hash(x)=154608340)
+3800 val loss 6.1110
+3800 val perplexity 450.7861
+3800 train 6.130027 (lr=3.6608e-05) (hash(x)=169641309)
+3900 val loss 6.0976
+3900 val perplexity 444.7820
+3900 train 6.078045 (lr=3.5944e-05) (hash(x)=150592479)
+4000 val loss 6.0864
+4000 val perplexity 439.8175
+4000 train 6.084772 (lr=3.5271e-05) (hash(x)=155495317)
+4100 val loss 6.0752
+4100 val perplexity 434.9162
+4100 train 6.258116 (lr=3.4590e-05) (hash(x)=170323970)
+4200 val loss 6.0694
+4200 val perplexity 432.4155
+4200 train 5.917346 (lr=3.3902e-05) (hash(x)=140854360)
+4300 val loss 6.0450
+4300 val perplexity 421.9954
+4300 train 6.074870 (lr=3.3207e-05) (hash(x)=148926644)
+4400 val loss 6.0332
+4400 val perplexity 417.0492
+4400 train 6.078179 (lr=3.2507e-05) (hash(x)=149997871)
+4500 val loss 6.0181
+4500 val perplexity 410.7877
+4500 train 5.947857 (lr=3.1801e-05) (hash(x)=149219289)
+4600 val loss 6.0075
+4600 val perplexity 406.4592
+4600 train 5.819511 (lr=3.1091e-05) (hash(x)=139836197)
+4700 val loss 6.0009
+4700 val perplexity 403.7760
+4700 train 5.956812 (lr=3.0377e-05) (hash(x)=150722883)
+4800 val loss 5.9937
+4800 val perplexity 400.8761
+4800 train 6.216718 (lr=2.9661e-05) (hash(x)=166864648)
+4900 val loss 5.9782
+4900 val perplexity 394.7388
+4900 train 6.082317 (lr=2.8942e-05) (hash(x)=157746743)
+5000 val loss 5.9667
+5000 val perplexity 390.2183
+5000 train 5.909554 (lr=2.8221e-05) (hash(x)=153032257)
+5100 val loss 5.9598
+5100 val perplexity 387.5401
+5100 train 6.027834 (lr=2.7500e-05) (hash(x)=151862371)
+5200 val loss 5.9436
+5200 val perplexity 381.3076
+5200 train 6.246490 (lr=2.6779e-05) (hash(x)=168862995)
+5300 val loss 5.9406
+5300 val perplexity 380.1586
+5300 train 5.897988 (lr=2.6058e-05) (hash(x)=148243567)
+5400 val loss 5.9296
+5400 val perplexity 376.0054
+5400 train 5.878036 (lr=2.5339e-05) (hash(x)=145381742)
+5500 val loss 5.9223
+5500 val perplexity 373.2592
+5500 train 6.065330 (lr=2.4623e-05) (hash(x)=155602174)
+5600 val loss 5.9051
+5600 val perplexity 366.8907
+5600 train 5.973073 (lr=2.3909e-05) (hash(x)=160637672)
+5700 val loss 5.8967
+5700 val perplexity 363.8202
+5700 train 5.849224 (lr=2.3199e-05) (hash(x)=152101131)
+5800 val loss 5.8853
+5800 val perplexity 359.7047
+5800 train 5.849683 (lr=2.2493e-05) (hash(x)=152327089)
+5900 val loss 5.8840
+5900 val perplexity 359.2561
+5900 train 5.854714 (lr=2.1793e-05) (hash(x)=151094543)
+6000 val loss 5.8786
+6000 val perplexity 357.3181
+6000 train 5.959620 (lr=2.1098e-05) (hash(x)=160233315)
+6100 val loss 5.8675
+6100 val perplexity 353.3617
+6100 train 5.975004 (lr=2.0410e-05) (hash(x)=158295337)
+6200 val loss 5.8583
+6200 val perplexity 350.1454
+6200 train 5.810185 (lr=1.9729e-05) (hash(x)=151269817)
+6300 val loss 5.8503
+6300 val perplexity 347.3304
+6300 train 6.021831 (lr=1.9056e-05) (hash(x)=156912756)
+6400 val loss 5.8446
+6400 val perplexity 345.3590
+6400 train 5.762765 (lr=1.8392e-05) (hash(x)=125779637)
+6500 val loss 5.8434
+6500 val perplexity 344.9615
+6500 train 5.746417 (lr=1.7738e-05) (hash(x)=148157661)
+6600 val loss 5.8375
+6600 val perplexity 342.9185
+6600 train 5.843159 (lr=1.7093e-05) (hash(x)=152820049)
+6700 val loss 5.8293
+6700 val perplexity 340.1296
+6700 train 5.762090 (lr=1.6459e-05) (hash(x)=155818717)
+6800 val loss 5.8213
+6800 val perplexity 337.4054
+6800 train 5.823565 (lr=1.5836e-05) (hash(x)=158833881)
+6900 val loss 5.8178
+6900 val perplexity 336.2453
+6900 train 5.699900 (lr=1.5225e-05) (hash(x)=164984909)
+7000 val loss 5.8136
+7000 val perplexity 334.8174
+7000 train 5.715562 (lr=1.4627e-05) (hash(x)=151172493)
+7100 val loss 5.8125
+7100 val perplexity 334.4544
+7100 train 5.878971 (lr=1.4043e-05) (hash(x)=154023633)
+7200 val loss 5.8041
+7200 val perplexity 331.6638
+7200 train 5.654086 (lr=1.3471e-05) (hash(x)=139351250)
+7300 val loss 5.7993
+7300 val perplexity 330.0705
+7300 train 5.760409 (lr=1.2915e-05) (hash(x)=158719458)
+7400 val loss 5.7935
+7400 val perplexity 328.1453
+7400 train 5.858016 (lr=1.2373e-05) (hash(x)=162250810)
+7500 val loss 5.7854
+7500 val perplexity 325.4996
+7500 train 5.671573 (lr=1.1847e-05) (hash(x)=142827557)
+7600 val loss 5.7819
+7600 val perplexity 324.3608
+7600 train 5.689194 (lr=1.1337e-05) (hash(x)=147830801)
+7700 val loss 5.7770
+7700 val perplexity 322.7880
+7700 train 5.886009 (lr=1.0844e-05) (hash(x)=165233314)
+7800 val loss 5.7758
+7800 val perplexity 322.3985
+7800 train 5.762441 (lr=1.0367e-05) (hash(x)=160654717)
+7900 val loss 5.7727
+7900 val perplexity 321.4012
+7900 train 5.651993 (lr=9.9088e-06) (hash(x)=140712586)
+8000 val loss 5.7648
+8000 val perplexity 318.8694
+8000 train 5.921185 (lr=9.4682e-06) (hash(x)=150309061)
+8100 val loss 5.7624
+8100 val perplexity 318.1208
+8100 train 5.754895 (lr=9.0461e-06) (hash(x)=158805466)
+8200 val loss 5.7590
+8200 val perplexity 317.0285
+8200 train 5.649700 (lr=8.6430e-06) (hash(x)=142814078)
+8300 val loss 5.7564
+8300 val perplexity 316.1991
+8300 train 5.721898 (lr=8.2593e-06) (hash(x)=148340417)
+8400 val loss 5.7526
+8400 val perplexity 315.0060
+8400 train 5.684734 (lr=7.8953e-06) (hash(x)=157132639)
+8500 val loss 5.7524
+8500 val perplexity 314.9457
+8500 train 5.686018 (lr=7.5515e-06) (hash(x)=152994240)
+8600 val loss 5.7467
+8600 val perplexity 313.1505
+8600 train 5.707251 (lr=7.2282e-06) (hash(x)=152210154)
+8700 val loss 5.7439
+8700 val perplexity 312.2925
+8700 train 5.840783 (lr=6.9257e-06) (hash(x)=149631247)
+8800 val loss 5.7405
+8800 val perplexity 311.2222
+8800 train 5.616950 (lr=6.6444e-06) (hash(x)=145617784)
+8900 val loss 5.7388
+8900 val perplexity 310.6878
+8900 train 5.849643 (lr=6.3845e-06) (hash(x)=155984970)
+9000 val loss 5.7392
+9000 val perplexity 310.8018
+9000 train 5.479745 (lr=6.1462e-06) (hash(x)=143433013)
+9100 val loss 5.7369
+9100 val perplexity 310.1082
+9100 train 5.641852 (lr=5.9300e-06) (hash(x)=152507533)
+9200 val loss 5.7322
+9200 val perplexity 308.6447
+9200 train 5.855296 (lr=5.7359e-06) (hash(x)=162400650)
+9300 val loss 5.7293
+9300 val perplexity 307.7554
+9300 train 5.773619 (lr=5.5641e-06) (hash(x)=148419068)
+9400 val loss 5.7281
+9400 val perplexity 307.3799
+9400 train 5.693515 (lr=5.4149e-06) (hash(x)=145025169)
+9500 val loss 5.7252
+9500 val perplexity 306.4956
+9500 train 5.701646 (lr=5.2884e-06) (hash(x)=145104235)
+9600 val loss 5.7255
+9600 val perplexity 306.5889
+9600 train 5.670694 (lr=5.1847e-06) (hash(x)=150499797)
+9700 val loss 5.7246
+9700 val perplexity 306.3200
+9700 train 5.538178 (lr=5.1040e-06) (hash(x)=139690022)
+9800 val loss 5.7195
+9800 val perplexity 304.7672
+9800 train 5.719058 (lr=5.0462e-06) (hash(x)=145645279)
+9900 val loss 5.7182
+9900 val perplexity 304.3503
+9900 train 5.652002 (lr=5.0116e-06) (hash(x)=150103432)
+9999 val loss 5.7170
+9999 val perplexity 303.9936

attention_kindselective_n_heads4_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9294532d9d2d4c7db014f7c1f5323cf5a4a2a16513b50cbc1b1039191702c59
+oid sha256:ced26739fb82bc69f7ee9c640198949689a798ab740c86dbf2eadbd39203dd98
 size 92843394

attention_kindselective_n_heads4_seed1339/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:71bdb80b8d183c8b995a678d3b770c11ac172bc535692db54ed7801762c64c1b
+oid sha256:1ebac561767c2492d8e1fb6761bc84e4ec2fbb1367b033d71d0098b4bc2ecaca
 size 92843394

attention_kindselective_n_heads4_seed1339/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce765557cd271eef6368a61da11f4937b2fa47747c4d797eb63cb9c6318ee3dc
+oid sha256:c5d2fda2d90c5ec03013f47a1c3364954d533cc0251601d3a41f6a83cdb40612
 size 92843394

attention_kindselective_n_heads4_seed1339/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9bf858df261be317cfb63794b256e4a8d2da111417ac052e97654337dcdab941
+oid sha256:94b6f80f738e19cf0304a1d5d46415fcf9e9d41b48032c77a9af321381704eaa
 size 92843394

attention_kindselective_n_heads4_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:202df9b4ec19476f10069e10e4da133d7a1786cb35ccd20c1a5c3bf5c23dc384
+oid sha256:858364f02fec2cbc5fc2be92e99cf9942bf42be3bcff923e0b45ccff657789e6
 size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d94d2d71ffe79e132aabc8895a5207065010391979dca509b8e3d88bad546633
+oid sha256:b6f8978b05d5a7af2625d71ea3a5d8eb6b6cea10bd5679bc91fe43f405c87e70
 size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bba53bf30935669f2bae08d31556a325db6d8c9f1cb754cb0ff4ef1cbe276934
+oid sha256:4ef6fdae2c457a60886109c85a62b83253e66cfe65e65678fe7b4625a3f391a4
 size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6fa363931f59651b3dfe70400beca8db629ec599879e5ac22dc68db271c6bb19
+oid sha256:0bf90d410f6b64353348c768bb6fd59b5318b1e1dc467572b0b69c5931e62c7e
 size 179406214