Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1344/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4.5e-5_61440_4_1344", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1344", "n_embd": 256}

attention_kindselective_n_heads4_seed1344/log2.txt

@@ -1,267 +1,522 @@
 max_steps: 8750
+200 val loss 9.5917
+200 val perplexity 14642.5957
+200 train 9.630045 (lr=1.6080e-05) (hash(x)=146764602)
 0 val loss 11.2831
-0 val perplexity 79465.6484
-0 train 11.285273 (lr=9.0000e-08) (hash(x)=145830960)
-100 val loss 10.0638
-100 val perplexity 23478.2871
-100 train 10.097435 (lr=9.0900e-06) (hash(x)=144157453)
-200 val loss 9.5202
-200 val perplexity 13631.6973
-200 train 9.558424 (lr=1.8090e-05) (hash(x)=146764602)
-300 val loss 8.5081
-300 val perplexity 4954.6196
-300 train 8.403627 (lr=2.7090e-05) (hash(x)=140559124)
-400 val loss 7.7386
-400 val perplexity 2295.2202
-400 train 8.096967 (lr=3.6090e-05) (hash(x)=166024176)
-500 val loss 7.4794
-500 val perplexity 1771.2329
-500 train 7.450180 (lr=4.5000e-05) (hash(x)=161040668)
-600 val loss 7.3790
-600 val perplexity 1601.9789
-600 train 7.393307 (lr=4.4985e-05) (hash(x)=148878990)
-700 val loss 7.2895
-700 val perplexity 1464.8370
-700 train 7.270169 (lr=4.4941e-05) (hash(x)=150420695)
-800 val loss 7.1978
-800 val perplexity 1336.4874
-800 train 7.170339 (lr=4.4868e-05) (hash(x)=151995229)
-900 val loss 7.1269
-900 val perplexity 1244.9921
-900 train 6.876092 (lr=4.4766e-05) (hash(x)=139853932)
-1000 val loss 7.0380
-1000 val perplexity 1139.1506
-1000 train 7.244468 (lr=4.4634e-05) (hash(x)=174104207)
-1100 val loss 6.9615
-1100 val perplexity 1055.2466
-1100 train 6.792953 (lr=4.4474e-05) (hash(x)=146275038)
-1200 val loss 6.8973
-1200 val perplexity 989.6367
-1200 train 6.665742 (lr=4.4285e-05) (hash(x)=141403655)
-1300 val loss 6.8148
-1300 val perplexity 911.2523
-1300 train 6.898851 (lr=4.4068e-05) (hash(x)=173675040)
-1400 val loss 6.7374
-1400 val perplexity 843.3451
-1400 train 6.708657 (lr=4.3822e-05) (hash(x)=162475906)
-1500 val loss 6.6799
-1500 val perplexity 796.2518
-1500 train 6.506414 (lr=4.3549e-05) (hash(x)=152785612)
-1600 val loss 6.6006
-1600 val perplexity 735.5471
-1600 train 6.552176 (lr=4.3249e-05) (hash(x)=151987383)
-1700 val loss 6.5444
-1700 val perplexity 695.3262
-1700 train 6.532230 (lr=4.2922e-05) (hash(x)=143781605)
-1800 val loss 6.4969
-1800 val perplexity 663.1117
-1800 train 6.485873 (lr=4.2569e-05) (hash(x)=151040203)
-1900 val loss 6.4521
-1900 val perplexity 634.0513
-1900 train 6.349230 (lr=4.2190e-05) (hash(x)=149414572)
-2000 val loss 6.4051
-2000 val perplexity 604.9101
-2000 train 6.363278 (lr=4.1785e-05) (hash(x)=156479674)
-2100 val loss 6.3630
-2100 val perplexity 579.9941
-2100 train 6.231485 (lr=4.1356e-05) (hash(x)=137861481)
-2200 val loss 6.3339
-2200 val perplexity 563.3536
-2200 train 6.395198 (lr=4.0903e-05) (hash(x)=155937443)
-2300 val loss 6.3157
-2300 val perplexity 553.1810
-2300 train 5.989519 (lr=4.0426e-05) (hash(x)=131035715)
-2400 val loss 6.2738
-2400 val perplexity 530.4802
-2400 train 6.270514 (lr=3.9927e-05) (hash(x)=146568981)
-2500 val loss 6.2432
-2500 val perplexity 514.5201
-2500 train 6.221696 (lr=3.9406e-05) (hash(x)=163317586)
-2600 val loss 6.2184
-2600 val perplexity 501.8752
-2600 train 6.037910 (lr=3.8863e-05) (hash(x)=144201060)
-2700 val loss 6.1925
-2700 val perplexity 489.0673
-2700 train 6.092707 (lr=3.8300e-05) (hash(x)=141825701)
-2800 val loss 6.1621
-2800 val perplexity 474.4416
-2800 train 6.274689 (lr=3.7717e-05) (hash(x)=160561627)
-2900 val loss 6.1349
-2900 val perplexity 461.6791
-2900 train 6.078423 (lr=3.7116e-05) (hash(x)=151758176)
-3000 val loss 6.1149
-3000 val perplexity 452.5696
-3000 train 6.131371 (lr=3.6496e-05) (hash(x)=155815751)
-3100 val loss 6.1001
-3100 val perplexity 445.9056
-3100 train 5.759576 (lr=3.5860e-05) (hash(x)=142307043)
-3200 val loss 6.0838
-3200 val perplexity 438.6731
-3200 train 6.193019 (lr=3.5207e-05) (hash(x)=156310690)
-3300 val loss 6.0547
-3300 val perplexity 426.1009
-3300 train 6.055473 (lr=3.4539e-05) (hash(x)=151344506)
-3400 val loss 6.0379
-3400 val perplexity 419.0226
-3400 train 6.073301 (lr=3.3857e-05) (hash(x)=162911881)
-3500 val loss 6.0226
-3500 val perplexity 412.6454
-3500 train 5.966004 (lr=3.3162e-05) (hash(x)=142889971)
-3600 val loss 6.0021
-3600 val perplexity 404.2900
-3600 train 5.993401 (lr=3.2455e-05) (hash(x)=149470354)
-3700 val loss 5.9825
-3700 val perplexity 396.4442
-3700 train 5.922817 (lr=3.1736e-05) (hash(x)=154194821)
-3800 val loss 5.9734
-3800 val perplexity 392.8425
-3800 train 5.903098 (lr=3.1008e-05) (hash(x)=148885848)
-3900 val loss 5.9517
-3900 val perplexity 384.4179
-3900 train 5.866324 (lr=3.0270e-05) (hash(x)=148308484)
-4000 val loss 5.9353
-4000 val perplexity 378.1353
-4000 train 5.775120 (lr=2.9524e-05) (hash(x)=139828564)
-4100 val loss 5.9260
-4100 val perplexity 374.6640
-4100 train 5.837074 (lr=2.8771e-05) (hash(x)=139981997)
-4200 val loss 5.9099
-4200 val perplexity 368.6822
-4200 train 6.059356 (lr=2.8013e-05) (hash(x)=150738447)
-4300 val loss 5.8959
-4300 val perplexity 363.5316
-4300 train 5.676573 (lr=2.7250e-05) (hash(x)=142198107)
-4400 val loss 5.8850
-4400 val perplexity 359.6176
-4400 train 5.762465 (lr=2.6483e-05) (hash(x)=142731201)
-4500 val loss 5.8668
-4500 val perplexity 353.1304
-4500 train 5.892882 (lr=2.5714e-05) (hash(x)=154814426)
-4600 val loss 5.8548
-4600 val perplexity 348.9009
-4600 train 5.830213 (lr=2.4943e-05) (hash(x)=155922230)
-4700 val loss 5.8471
-4700 val perplexity 346.2455
-4700 train 5.597148 (lr=2.4172e-05) (hash(x)=139398510)
-4800 val loss 5.8393
-4800 val perplexity 343.5542
-4800 train 5.886129 (lr=2.3402e-05) (hash(x)=140893236)
-4900 val loss 5.8202
-4900 val perplexity 337.0312
-4900 train 5.712579 (lr=2.2633e-05) (hash(x)=153747830)
-5000 val loss 5.8103
-5000 val perplexity 333.7246
-5000 train 6.062975 (lr=2.1868e-05) (hash(x)=148919005)
-5100 val loss 5.8004
-5100 val perplexity 330.4405
-5100 train 5.591538 (lr=2.1107e-05) (hash(x)=142281936)
-5200 val loss 5.8013
-5200 val perplexity 330.7337
-5200 train 5.609389 (lr=2.0351e-05) (hash(x)=143162650)
-5300 val loss 5.7807
-5300 val perplexity 323.9874
-5300 train 5.845477 (lr=1.9602e-05) (hash(x)=148546849)
-5400 val loss 5.7698
-5400 val perplexity 320.4624
-5400 train 5.628488 (lr=1.8860e-05) (hash(x)=143492259)
-5500 val loss 5.7640
-5500 val perplexity 318.6333
-5500 train 5.525702 (lr=1.8127e-05) (hash(x)=141023941)
-5600 val loss 5.7623
-5600 val perplexity 318.0780
-5600 train 5.642804 (lr=1.7403e-05) (hash(x)=142065021)
-5700 val loss 5.7467
-5700 val perplexity 313.1555
-5700 train 5.629653 (lr=1.6690e-05) (hash(x)=145749913)
-5800 val loss 5.7393
-5800 val perplexity 310.8448
-5800 train 5.421354 (lr=1.5989e-05) (hash(x)=140035522)
-5900 val loss 5.7349
-5900 val perplexity 309.4912
-5900 train 5.485237 (lr=1.5300e-05) (hash(x)=154752726)
-6000 val loss 5.7325
-6000 val perplexity 308.7275
-6000 train 5.766155 (lr=1.4625e-05) (hash(x)=158715824)
-6100 val loss 5.7190
-6100 val perplexity 304.5998
-6100 train 5.739907 (lr=1.3965e-05) (hash(x)=137413820)
-6200 val loss 5.7107
-6200 val perplexity 302.0692
-6200 train 5.631349 (lr=1.3320e-05) (hash(x)=151507523)
-6300 val loss 5.7095
-6300 val perplexity 301.7145
-6300 train 5.488567 (lr=1.2692e-05) (hash(x)=147514617)
-6400 val loss 5.7060
-6400 val perplexity 300.6701
-6400 train 5.711531 (lr=1.2081e-05) (hash(x)=151604465)
-6500 val loss 5.6965
-6500 val perplexity 297.8287
-6500 train 5.751291 (lr=1.1489e-05) (hash(x)=144515881)
-6600 val loss 5.6902
-6600 val perplexity 295.9584
-6600 train 5.482191 (lr=1.0916e-05) (hash(x)=136948374)
-6700 val loss 5.6892
-6700 val perplexity 295.6615
-6700 train 5.490501 (lr=1.0363e-05) (hash(x)=146268592)
-6800 val loss 5.6796
-6800 val perplexity 292.8444
-6800 train 5.632364 (lr=9.8310e-06) (hash(x)=152676836)
-6900 val loss 5.6722
-6900 val perplexity 290.6861
-6900 train 5.596709 (lr=9.3205e-06) (hash(x)=134657776)
-7000 val loss 5.6706
-7000 val perplexity 290.2227
-7000 train 5.630913 (lr=8.8324e-06) (hash(x)=166721861)
-7100 val loss 5.6703
-7100 val perplexity 290.1279
-7100 train 5.472641 (lr=8.3674e-06) (hash(x)=135496702)
-7200 val loss 5.6653
-7200 val perplexity 288.6681
-7200 train 5.777382 (lr=7.9261e-06) (hash(x)=155567461)
-7300 val loss 5.6578
-7300 val perplexity 286.5287
-7300 train 5.460151 (lr=7.5093e-06) (hash(x)=142803829)
-7400 val loss 5.6564
-7400 val perplexity 286.1036
-7400 train 5.478210 (lr=7.1174e-06) (hash(x)=145294178)
-7500 val loss 5.6543
-7500 val perplexity 285.5166
-7500 train 5.422940 (lr=6.7511e-06) (hash(x)=150573713)
-7600 val loss 5.6515
-7600 val perplexity 284.7068
-7600 train 5.663387 (lr=6.4109e-06) (hash(x)=142771511)
-7700 val loss 5.6450
-7700 val perplexity 282.8737
-7700 train 5.550337 (lr=6.0972e-06) (hash(x)=143602175)
-7800 val loss 5.6422
-7800 val perplexity 282.0916
-7800 train 5.701168 (lr=5.8107e-06) (hash(x)=152379862)
-7900 val loss 5.6413
-7900 val perplexity 281.8354
-7900 train 5.475989 (lr=5.5515e-06) (hash(x)=146655921)
-8000 val loss 5.6420
-8000 val perplexity 282.0280
-8000 train 5.775498 (lr=5.3203e-06) (hash(x)=148262482)
-8100 val loss 5.6343
-8100 val perplexity 279.8623
-8100 train 5.575891 (lr=5.1172e-06) (hash(x)=147683655)
-8200 val loss 5.6332
-8200 val perplexity 279.5538
-8200 train 5.693173 (lr=4.9425e-06) (hash(x)=164975934)
-8300 val loss 5.6301
-8300 val perplexity 278.6893
-8300 train 5.533537 (lr=4.7966e-06) (hash(x)=145300550)
-8400 val loss 5.6282
-8400 val perplexity 278.1678
-8400 train 5.545886 (lr=4.6796e-06) (hash(x)=150679400)
-8500 val loss 5.6290
-8500 val perplexity 278.3904
-8500 train 5.625071 (lr=4.5917e-06) (hash(x)=164109401)
-8600 val loss 5.6257
-8600 val perplexity 277.4749
-8600 train 5.575358 (lr=4.5330e-06) (hash(x)=161036376)
-8700 val loss 5.6213
-8700 val perplexity 276.2414
-8700 train 5.549177 (lr=4.5037e-06) (hash(x)=153828820)
-8749 val loss 5.6201
-8749 val perplexity 275.9221
+0 val perplexity 79466.0234
+300 val loss 8.5257
+300 val perplexity 5042.8242
+300 train 8.421991 (lr=2.4080e-05) (hash(x)=140559124)
+0 train 11.285278 (lr=1.0000e-07) (hash(x)=145830960)
+400 val loss 7.7028
+400 val perplexity 2214.5776
+400 train 8.064939 (lr=3.2080e-05) (hash(x)=166024176)
+100 val loss 10.0381
+100 val perplexity 22882.0020
+100 train 10.072891 (lr=1.0100e-05) (hash(x)=144157453)
+500 val loss 7.4824
+500 val perplexity 1776.5195
+500 train 7.453581 (lr=4.0000e-05) (hash(x)=161040668)
+200 val loss 9.3307
+200 val perplexity 11278.5498
+200 train 9.366409 (lr=2.0100e-05) (hash(x)=146764602)
+600 val loss 7.3866
+600 val perplexity 1614.1600
+600 train 7.406942 (lr=3.9987e-05) (hash(x)=148878990)
+300 val loss 8.1109
+300 val perplexity 3330.7200
+300 train 7.960665 (lr=3.0100e-05) (hash(x)=140559124)
+700 val loss 7.2836
+700 val perplexity 1456.2139
+700 train 7.265226 (lr=3.9948e-05) (hash(x)=150420695)
+400 val loss 7.5387
+400 val perplexity 1879.4033
+400 train 7.929288 (lr=4.0100e-05) (hash(x)=166024176)
+800 val loss 7.1846
+800 val perplexity 1319.0261
+800 train 7.157286 (lr=3.9883e-05) (hash(x)=151995229)
+500 val loss 7.4256
+500 val perplexity 1678.3735
+500 train 7.395220 (lr=5.0000e-05) (hash(x)=161040668)
+900 val loss 7.0973
+900 val perplexity 1208.6497
+900 train 6.848862 (lr=3.9792e-05) (hash(x)=139853932)
+600 val loss 7.3370
+600 val perplexity 1536.0286
+600 train 7.340587 (lr=4.9984e-05) (hash(x)=148878990)
+1000 val loss 6.9982
+1000 val perplexity 1094.6145
+1000 train 7.201624 (lr=3.9675e-05) (hash(x)=174104207)
+700 val loss 7.2297
+700 val perplexity 1379.8744
+700 train 7.214864 (lr=4.9935e-05) (hash(x)=150420695)
+1100 val loss 6.9161
+1100 val perplexity 1008.4128
+1100 train 6.749304 (lr=3.9532e-05) (hash(x)=146275038)
+800 val loss 7.1343
+800 val perplexity 1254.2781
+800 train 7.107428 (lr=4.9853e-05) (hash(x)=151995229)
+1200 val loss 6.8534
+1200 val perplexity 947.0867
+1200 train 6.625364 (lr=3.9364e-05) (hash(x)=141403655)
+900 val loss 7.0693
+900 val perplexity 1175.3370
+900 train 6.814147 (lr=4.9739e-05) (hash(x)=139853932)
+1300 val loss 6.7709
+1300 val perplexity 872.0862
+1300 train 6.850709 (lr=3.9171e-05) (hash(x)=173675040)
+1000 val loss 7.0031
+1000 val perplexity 1100.0358
+1000 train 7.203287 (lr=4.9593e-05) (hash(x)=174104207)
+1400 val loss 6.6960
+1400 val perplexity 809.1636
+1400 train 6.668308 (lr=3.8953e-05) (hash(x)=162475906)
+1100 val loss 6.9378
+1100 val perplexity 1030.4529
+1100 train 6.766367 (lr=4.9415e-05) (hash(x)=146275038)
+1500 val loss 6.6402
+1500 val perplexity 765.2175
+1500 train 6.466898 (lr=3.8711e-05) (hash(x)=152785612)
+1200 val loss 6.8749
+1200 val perplexity 967.6586
+1200 train 6.643046 (lr=4.9205e-05) (hash(x)=141403655)
+1600 val loss 6.5700
+1600 val perplexity 713.3383
+1600 train 6.522436 (lr=3.8444e-05) (hash(x)=151987383)
+1300 val loss 6.7905
+1300 val perplexity 889.3807
+1300 train 6.872322 (lr=4.8964e-05) (hash(x)=173675040)
+1700 val loss 6.5158
+1700 val perplexity 675.7247
+1700 train 6.502824 (lr=3.8153e-05) (hash(x)=143781605)
+1400 val loss 6.7308
+1400 val perplexity 837.7838
+1400 train 6.700823 (lr=4.8691e-05) (hash(x)=162475906)
+1800 val loss 6.4719
+1800 val perplexity 646.7280
+1800 train 6.459326 (lr=3.7839e-05) (hash(x)=151040203)
+1500 val loss 6.6808
+1500 val perplexity 796.9203
+1500 train 6.506968 (lr=4.8388e-05) (hash(x)=152785612)
+1900 val loss 6.4342
+1900 val perplexity 622.7968
+1900 train 6.334003 (lr=3.7502e-05) (hash(x)=149414572)
+1600 val loss 6.6151
+1600 val perplexity 746.2864
+1600 train 6.567549 (lr=4.8055e-05) (hash(x)=151987383)
+2000 val loss 6.3950
+2000 val perplexity 598.8578
+2000 train 6.353477 (lr=3.7143e-05) (hash(x)=156479674)
+1700 val loss 6.5682
+1700 val perplexity 712.0578
+1700 train 6.554473 (lr=4.7691e-05) (hash(x)=143781605)
+2100 val loss 6.3528
+2100 val perplexity 574.1195
+2100 train 6.224854 (lr=3.6761e-05) (hash(x)=137861481)
+1800 val loss 6.5217
+1800 val perplexity 679.7147
+1800 train 6.508534 (lr=4.7299e-05) (hash(x)=151040203)
+2200 val loss 6.3209
+2200 val perplexity 556.0743
+2200 train 6.377265 (lr=3.6358e-05) (hash(x)=155937443)
+1900 val loss 6.4849
+1900 val perplexity 655.1425
+1900 train 6.383931 (lr=4.6878e-05) (hash(x)=149414572)
+2300 val loss 6.3188
+2300 val perplexity 554.9308
+2300 train 5.995306 (lr=3.5935e-05) (hash(x)=131035715)
+2000 val loss 6.4401
+2000 val perplexity 626.4890
+2000 train 6.394539 (lr=4.6428e-05) (hash(x)=156479674)
+2400 val loss 6.2713
+2400 val perplexity 529.1736
+2400 train 6.266420 (lr=3.5491e-05) (hash(x)=146568981)
+2100 val loss 6.4051
+2100 val perplexity 604.8986
+2100 train 6.270535 (lr=4.5951e-05) (hash(x)=137861481)
+2500 val loss 6.2430
+2500 val perplexity 514.4131
+2500 train 6.220254 (lr=3.5027e-05) (hash(x)=163317586)
+2200 val loss 6.3759
+2200 val perplexity 587.5141
+2200 train 6.435637 (lr=4.5448e-05) (hash(x)=155937443)
+2600 val loss 6.2233
+2600 val perplexity 504.3846
+2600 train 6.038826 (lr=3.4545e-05) (hash(x)=144201060)
+2300 val loss 6.3613
+2300 val perplexity 579.0132
+2300 train 6.040595 (lr=4.4918e-05) (hash(x)=131035715)
+2700 val loss 6.2014
+2700 val perplexity 493.4525
+2700 train 6.095143 (lr=3.4044e-05) (hash(x)=141825701)
+2400 val loss 6.3159
+2400 val perplexity 553.3100
+2400 train 6.307761 (lr=4.4363e-05) (hash(x)=146568981)
+2800 val loss 6.1755
+2800 val perplexity 480.8045
+2800 train 6.286294 (lr=3.3526e-05) (hash(x)=160561627)
+2500 val loss 6.2862
+2500 val perplexity 537.1021
+2500 train 6.260865 (lr=4.3784e-05) (hash(x)=163317586)
+2900 val loss 6.1511
+2900 val perplexity 469.2112
+2900 train 6.096576 (lr=3.2992e-05) (hash(x)=151758176)
+2600 val loss 6.2591
+2600 val perplexity 522.7268
+2600 train 6.077036 (lr=4.3181e-05) (hash(x)=144201060)
+3000 val loss 6.1326
+3000 val perplexity 460.6170
+3000 train 6.152384 (lr=3.2441e-05) (hash(x)=155815751)
+2700 val loss 6.2398
+2700 val perplexity 512.7802
+2700 train 6.131793 (lr=4.2555e-05) (hash(x)=141825701)
+3100 val loss 6.1248
+3100 val perplexity 457.0416
+3100 train 5.794218 (lr=3.1875e-05) (hash(x)=142307043)
+2800 val loss 6.2099
+2800 val perplexity 497.6453
+2800 train 6.320862 (lr=4.1908e-05) (hash(x)=160561627)
+3200 val loss 6.1004
+3200 val perplexity 446.0268
+3200 train 6.209918 (lr=3.1295e-05) (hash(x)=156310690)
+2900 val loss 6.1826
+2900 val perplexity 484.2572
+2900 train 6.125389 (lr=4.1240e-05) (hash(x)=151758176)
+3300 val loss 6.0788
+3300 val perplexity 436.5081
+3300 train 6.077555 (lr=3.0702e-05) (hash(x)=151344506)
+3000 val loss 6.1635
+3000 val perplexity 475.1087
+3000 train 6.178281 (lr=4.0551e-05) (hash(x)=155815751)
+3400 val loss 6.0629
+3400 val perplexity 429.6288
+3400 train 6.093707 (lr=3.0095e-05) (hash(x)=162911881)
+3100 val loss 6.1495
+3100 val perplexity 468.4661
+3100 train 5.808314 (lr=3.9844e-05) (hash(x)=142307043)
+3500 val loss 6.0512
+3500 val perplexity 424.6271
+3500 train 5.990658 (lr=2.9477e-05) (hash(x)=142889971)
+3200 val loss 6.1270
+3200 val perplexity 458.0616
+3200 train 6.234677 (lr=3.9119e-05) (hash(x)=156310690)
+3600 val loss 6.0319
+3600 val perplexity 416.4943
+3600 train 6.020875 (lr=2.8849e-05) (hash(x)=149470354)
+3300 val loss 6.0978
+3300 val perplexity 444.8636
+3300 train 6.097316 (lr=3.8377e-05) (hash(x)=151344506)
+3700 val loss 6.0184
+3700 val perplexity 410.9171
+3700 train 5.961654 (lr=2.8210e-05) (hash(x)=154194821)
+3400 val loss 6.0849
+3400 val perplexity 439.1918
+3400 train 6.120546 (lr=3.7619e-05) (hash(x)=162911881)
+3800 val loss 6.0059
+3800 val perplexity 405.8116
+3800 train 5.937997 (lr=2.7562e-05) (hash(x)=148885848)
+3500 val loss 6.0766
+3500 val perplexity 435.5587
+3500 train 6.014277 (lr=3.6847e-05) (hash(x)=142889971)
+3900 val loss 5.9894
+3900 val perplexity 399.1929
+3900 train 5.899461 (lr=2.6907e-05) (hash(x)=148308484)
+3600 val loss 6.0508
+3600 val perplexity 424.4692
+3600 train 6.039314 (lr=3.6061e-05) (hash(x)=149470354)
+4000 val loss 5.9717
+4000 val perplexity 392.1893
+4000 train 5.813733 (lr=2.6244e-05) (hash(x)=139828564)
+3700 val loss 6.0343
+3700 val perplexity 417.5173
+3700 train 5.979168 (lr=3.5263e-05) (hash(x)=154194821)
+4100 val loss 5.9616
+4100 val perplexity 388.2494
+4100 train 5.875154 (lr=2.5575e-05) (hash(x)=139981997)
+3800 val loss 6.0246
+3800 val perplexity 413.4932
+3800 train 5.954288 (lr=3.4453e-05) (hash(x)=148885848)
+4200 val loss 5.9504
+4200 val perplexity 383.9165
+4200 train 6.103284 (lr=2.4900e-05) (hash(x)=150738447)
+3900 val loss 6.0049
+3900 val perplexity 405.4062
+3900 train 5.915174 (lr=3.3633e-05) (hash(x)=148308484)
+4300 val loss 5.9363
+4300 val perplexity 378.5393
+4300 train 5.712550 (lr=2.4222e-05) (hash(x)=142198107)
+4000 val loss 5.9892
+4000 val perplexity 399.0954
+4000 train 5.824629 (lr=3.2805e-05) (hash(x)=139828564)
+4400 val loss 5.9295
+4400 val perplexity 375.9687
+4400 train 5.798333 (lr=2.3540e-05) (hash(x)=142731201)
+4100 val loss 5.9779
+4100 val perplexity 394.6185
+4100 train 5.888789 (lr=3.1968e-05) (hash(x)=139981997)
+4500 val loss 5.9109
+4500 val perplexity 369.0209
+4500 train 5.931695 (lr=2.2856e-05) (hash(x)=154814426)
+4200 val loss 5.9643
+4200 val perplexity 389.2749
+4200 train 6.117281 (lr=3.1126e-05) (hash(x)=150738447)
+4600 val loss 5.8980
+4600 val perplexity 364.3122
+4600 train 5.870726 (lr=2.2171e-05) (hash(x)=155922230)
+4300 val loss 5.9482
+4300 val perplexity 383.0560
+4300 train 5.727036 (lr=3.0277e-05) (hash(x)=142198107)
+4700 val loss 5.8929
+4700 val perplexity 362.4561
+4700 train 5.643782 (lr=2.1486e-05) (hash(x)=139398510)
+4400 val loss 5.9390
+4400 val perplexity 379.5455
+4400 train 5.811529 (lr=2.9425e-05) (hash(x)=142731201)
+4800 val loss 5.8803
+4800 val perplexity 357.9199
+4800 train 5.943650 (lr=2.0801e-05) (hash(x)=140893236)
+4500 val loss 5.9237
+4500 val perplexity 373.7953
+4500 train 5.939521 (lr=2.8571e-05) (hash(x)=154814426)
+4900 val loss 5.8631
+4900 val perplexity 351.8223
+4900 train 5.758955 (lr=2.0118e-05) (hash(x)=153747830)
+4600 val loss 5.9083
+4600 val perplexity 368.0674
+4600 train 5.877394 (lr=2.7714e-05) (hash(x)=155922230)
+5000 val loss 5.8568
+5000 val perplexity 349.6060
+5000 train 6.100023 (lr=1.9438e-05) (hash(x)=148919005)
+4700 val loss 5.9007
+4700 val perplexity 365.2940
+4700 train 5.648995 (lr=2.6857e-05) (hash(x)=139398510)
+5100 val loss 5.8488
+5100 val perplexity 346.8314
+5100 train 5.637376 (lr=1.8762e-05) (hash(x)=142281936)
+4800 val loss 5.8867
+4800 val perplexity 360.2307
+4800 train 5.943976 (lr=2.6002e-05) (hash(x)=140893236)
+5200 val loss 5.8502
+5200 val perplexity 347.3190
+5200 train 5.655197 (lr=1.8090e-05) (hash(x)=143162650)
+4900 val loss 5.8698
+4900 val perplexity 354.1944
+4900 train 5.770061 (lr=2.5148e-05) (hash(x)=153747830)
+5300 val loss 5.8282
+5300 val perplexity 339.7482
+5300 train 5.886518 (lr=1.7424e-05) (hash(x)=148546849)
+5000 val loss 5.8572
+5000 val perplexity 349.7294
+5000 train 6.099062 (lr=2.4298e-05) (hash(x)=148919005)
+5400 val loss 5.8173
+5400 val perplexity 336.0493
+5400 train 5.671466 (lr=1.6765e-05) (hash(x)=143492259)
+5100 val loss 5.8483
+5100 val perplexity 346.6322
+5100 train 5.639922 (lr=2.3452e-05) (hash(x)=142281936)
+5500 val loss 5.8143
+5500 val perplexity 335.0696
+5500 train 5.568772 (lr=1.6113e-05) (hash(x)=141023941)
+5200 val loss 5.8518
+5200 val perplexity 347.8689
+5200 train 5.653669 (lr=2.2613e-05) (hash(x)=143162650)
+5600 val loss 5.8109
+5600 val perplexity 333.9080
+5600 train 5.691159 (lr=1.5469e-05) (hash(x)=142065021)
+5300 val loss 5.8263
+5300 val perplexity 339.1058
+5300 train 5.889111 (lr=2.1780e-05) (hash(x)=148546849)
+5700 val loss 5.7951
+5700 val perplexity 328.6996
+5700 train 5.668637 (lr=1.4836e-05) (hash(x)=145749913)
+5400 val loss 5.8158
+5400 val perplexity 335.5734
+5400 train 5.676393 (lr=2.0956e-05) (hash(x)=143492259)
+5800 val loss 5.7882
+5800 val perplexity 326.4294
+5800 train 5.470370 (lr=1.4212e-05) (hash(x)=140035522)
+5500 val loss 5.8084
+5500 val perplexity 333.0947
+5500 train 5.565637 (lr=2.0141e-05) (hash(x)=141023941)
+5900 val loss 5.7848
+5900 val perplexity 325.3286
+5900 train 5.537515 (lr=1.3600e-05) (hash(x)=154752726)
+5600 val loss 5.8085
+5600 val perplexity 333.1090
+5600 train 5.685212 (lr=1.9337e-05) (hash(x)=142065021)
+6000 val loss 5.7838
+6000 val perplexity 324.9914
+6000 train 5.811926 (lr=1.3000e-05) (hash(x)=158715824)
+5700 val loss 5.7899
+5700 val perplexity 326.9874
+5700 train 5.669137 (lr=1.8545e-05) (hash(x)=145749913)
+6100 val loss 5.7704
+6100 val perplexity 320.6667
+6100 train 5.785007 (lr=1.2413e-05) (hash(x)=137413820)
+5800 val loss 5.7829
+5800 val perplexity 324.7016
+5800 train 5.465841 (lr=1.7765e-05) (hash(x)=140035522)
+6200 val loss 5.7627
+6200 val perplexity 318.1954
+6200 train 5.680292 (lr=1.1840e-05) (hash(x)=151507523)
+5900 val loss 5.7770
+5900 val perplexity 322.7901
+5900 train 5.525473 (lr=1.7000e-05) (hash(x)=154752726)
+6300 val loss 5.7615
+6300 val perplexity 317.8233
+6300 train 5.529540 (lr=1.1282e-05) (hash(x)=147514617)
+6000 val loss 5.7735
+6000 val perplexity 321.6488
+6000 train 5.798147 (lr=1.6250e-05) (hash(x)=158715824)
+6400 val loss 5.7574
+6400 val perplexity 316.5249
+6400 train 5.762690 (lr=1.0739e-05) (hash(x)=151604465)
+6100 val loss 5.7618
+6100 val perplexity 317.9291
+6100 train 5.776758 (lr=1.5516e-05) (hash(x)=137413820)
+6500 val loss 5.7473
+6500 val perplexity 313.3486
+6500 train 5.805442 (lr=1.0213e-05) (hash(x)=144515881)
+6200 val loss 5.7522
+6200 val perplexity 314.8702
+6200 train 5.669681 (lr=1.4800e-05) (hash(x)=151507523)
+6600 val loss 5.7416
+6600 val perplexity 311.5598
+6300 val loss 5.7512
+6300 val perplexity 314.5645
+6600 train 5.532074 (lr=9.7032e-06) (hash(x)=136948374)
+6300 train 5.526007 (lr=1.4102e-05) (hash(x)=147514617)
+6700 val loss 5.7411
+6700 val perplexity 311.3926
+6400 val loss 5.7485
+6400 val perplexity 313.7257
+6700 train 5.550056 (lr=9.2116e-06) (hash(x)=146268592)
+6400 train 5.760475 (lr=1.3424e-05) (hash(x)=151604465)
+6800 val loss 5.7322
+6800 val perplexity 308.6383
+6500 val loss 5.7388
+6500 val perplexity 310.6908
+6800 train 5.683815 (lr=8.7387e-06) (hash(x)=152676836)
+6500 train 5.794615 (lr=1.2766e-05) (hash(x)=144515881)
+6900 val loss 5.7256
+6900 val perplexity 306.6284
+6900 train 5.647960 (lr=8.2849e-06) (hash(x)=134657776)
+6600 val loss 5.7305
+6600 val perplexity 308.1154
+6600 train 5.523883 (lr=1.2129e-05) (hash(x)=136948374)
+7000 val loss 5.7219
+7000 val perplexity 305.4925
+7000 train 5.691176 (lr=7.8510e-06) (hash(x)=166721861)
+6700 val loss 5.7292
+6700 val perplexity 307.7284
+6700 train 5.533874 (lr=1.1515e-05) (hash(x)=146268592)
+7100 val loss 5.7214
+7100 val perplexity 305.3211
+7100 train 5.526393 (lr=7.4377e-06) (hash(x)=135496702)
+6800 val loss 5.7191
+6800 val perplexity 304.6219
+6800 train 5.669891 (lr=1.0923e-05) (hash(x)=152676836)
+7200 val loss 5.7178
+7200 val perplexity 304.2230
+7200 train 5.824883 (lr=7.0455e-06) (hash(x)=155567461)
+6900 val loss 5.7130
+6900 val perplexity 302.7735
+6900 train 5.640803 (lr=1.0356e-05) (hash(x)=134657776)
+7300 val loss 5.7101
+7300 val perplexity 301.8895
+7300 train 5.511367 (lr=6.6749e-06) (hash(x)=142803829)
+7000 val loss 5.7093
+7000 val perplexity 301.6670
+7000 train 5.675343 (lr=9.8138e-06) (hash(x)=166721861)
+7400 val loss 5.7082
+7400 val perplexity 301.3404
+7400 train 5.528187 (lr=6.3266e-06) (hash(x)=145294178)
+7100 val loss 5.7074
+7100 val perplexity 301.0902
+7100 train 5.510521 (lr=9.2971e-06) (hash(x)=135496702)
+7500 val loss 5.7073
+7500 val perplexity 301.0652
+7500 train 5.473443 (lr=6.0010e-06) (hash(x)=150573713)
+7200 val loss 5.7045
+7200 val perplexity 300.2151
+7200 train 5.806618 (lr=8.8068e-06) (hash(x)=155567461)
+7600 val loss 5.7047
+7600 val perplexity 300.2834
+7600 train 5.723767 (lr=5.6986e-06) (hash(x)=142771511)
+7300 val loss 5.6948
+7300 val perplexity 297.3138
+7300 train 5.498550 (lr=8.3436e-06) (hash(x)=142803829)
+7700 val loss 5.6985
+7700 val perplexity 298.4202
+7700 train 5.598147 (lr=5.4198e-06) (hash(x)=143602175)
+7400 val loss 5.6940
+7400 val perplexity 297.0836
+7400 train 5.514047 (lr=7.9082e-06) (hash(x)=145294178)
+7800 val loss 5.6950
+7800 val perplexity 297.3798
+7800 train 5.744207 (lr=5.1650e-06) (hash(x)=152379862)
+7500 val loss 5.6917
+7500 val perplexity 296.3914
+7500 train 5.456120 (lr=7.5012e-06) (hash(x)=150573713)
+7900 val loss 5.6939
+7900 val perplexity 297.0622
+7900 train 5.528719 (lr=4.9347e-06) (hash(x)=146655921)
+7600 val loss 5.6894
+7600 val perplexity 295.7183
+7600 train 5.704326 (lr=7.1232e-06) (hash(x)=142771511)
+8000 val loss 5.6958
+8000 val perplexity 297.6096
+8000 train 5.821932 (lr=4.7291e-06) (hash(x)=148262482)
+7700 val loss 5.6819
+7700 val perplexity 293.5073
+7700 train 5.588364 (lr=6.7747e-06) (hash(x)=143602175)
+8100 val loss 5.6871
+8100 val perplexity 295.0270
+7800 val loss 5.6794
+7800 val perplexity 292.7674
+8100 train 5.627559 (lr=4.5486e-06) (hash(x)=147683655)
+7800 train 5.731502 (lr=6.4563e-06) (hash(x)=152379862)
+7900 val loss 5.6787
+7900 val perplexity 292.5810
+7900 train 5.505094 (lr=6.1684e-06) (hash(x)=146655921)
+8200 val loss 5.6863
+8200 val perplexity 294.8073
+8200 train 5.751357 (lr=4.3933e-06) (hash(x)=164975934)
+8000 val loss 5.6797
+8000 val perplexity 292.8628
+8000 train 5.804450 (lr=5.9114e-06) (hash(x)=148262482)
+8300 val loss 5.6828
+8300 val perplexity 293.7621
+8300 train 5.594470 (lr=4.2636e-06) (hash(x)=145300550)
+8100 val loss 5.6704
+8100 val perplexity 290.1621
+8100 train 5.615933 (lr=5.6857e-06) (hash(x)=147683655)
+8400 val loss 5.6817
+8400 val perplexity 293.4604
+8400 train 5.595119 (lr=4.1596e-06) (hash(x)=150679400)
+8500 val loss 5.6823
+8500 val perplexity 293.6205
+8200 val loss 5.6696
+8200 val perplexity 289.9252
+8500 train 5.679973 (lr=4.0815e-06) (hash(x)=164109401)
+8200 train 5.733337 (lr=5.4917e-06) (hash(x)=164975934)
+8300 val loss 5.6657
+8300 val perplexity 288.7932
+8600 val loss 5.6785
+8600 val perplexity 292.5168
+8300 train 5.576839 (lr=5.3295e-06) (hash(x)=145300550)
+8600 train 5.644332 (lr=4.0294e-06) (hash(x)=161036376)
+8400 val loss 5.6644
+8400 val perplexity 288.4171
+8700 val loss 5.6741
+8700 val perplexity 291.2169
+8400 train 5.575017 (lr=5.1995e-06) (hash(x)=150679400)
+8700 train 5.603286 (lr=4.0033e-06) (hash(x)=153828820)
+8749 val loss 5.6732
+8749 val perplexity 290.9644
+8500 val loss 5.6648
+8500 val perplexity 288.5241
+8500 train 5.657299 (lr=5.1019e-06) (hash(x)=164109401)
+8600 val loss 5.6618
+8600 val perplexity 287.6606
+8600 train 5.614815 (lr=5.0367e-06) (hash(x)=161036376)

attention_kindselective_n_heads4_seed1344/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:559937116dc87cad8d5b0c4793cd92eae070e7ad9289ee0b597d496211e882f3
+oid sha256:6fa79b55f69ee7ccd3222fa1718354c3d95a36aeed030536bfd5e741e3808e92
 size 92843394

attention_kindselective_n_heads4_seed1344/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:846e83e3fb29ba9606086848e1d93051cf0e43d2ce03044226aad14ec7f885ea
+oid sha256:fd0902096483a36ffa43b976db8b5fec301777b4e34a8cbccb0bcd76e124dc3e
 size 179406214