Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1344/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1344", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_4_1344", "n_embd": 256}

attention_kindselective_n_heads4_seed1344/log2.txt

@@ -1,527 +1,267 @@
 max_steps: 8750
-200 val loss 9.5917
-200 val perplexity 14642.5957
-200 train 9.630045 (lr=1.6080e-05) (hash(x)=146764602)
 0 val loss 11.2831
-0 val perplexity 79466.0234
-300 val loss 8.5257
-300 val perplexity 5042.8242
-300 train 8.421991 (lr=2.4080e-05) (hash(x)=140559124)
-0 train 11.285278 (lr=1.0000e-07) (hash(x)=145830960)
-400 val loss 7.7028
-400 val perplexity 2214.5776
-400 train 8.064939 (lr=3.2080e-05) (hash(x)=166024176)
-100 val loss 10.0381
-100 val perplexity 22882.0020
-100 train 10.072891 (lr=1.0100e-05) (hash(x)=144157453)
-500 val loss 7.4824
-500 val perplexity 1776.5195
-500 train 7.453581 (lr=4.0000e-05) (hash(x)=161040668)
-200 val loss 9.3307
-200 val perplexity 11278.5498
-200 train 9.366409 (lr=2.0100e-05) (hash(x)=146764602)
-600 val loss 7.3866
-600 val perplexity 1614.1600
-600 train 7.406942 (lr=3.9987e-05) (hash(x)=148878990)
-300 val loss 8.1109
-300 val perplexity 3330.7200
-300 train 7.960665 (lr=3.0100e-05) (hash(x)=140559124)
-700 val loss 7.2836
-700 val perplexity 1456.2139
-700 train 7.265226 (lr=3.9948e-05) (hash(x)=150420695)
-400 val loss 7.5387
-400 val perplexity 1879.4033
-400 train 7.929288 (lr=4.0100e-05) (hash(x)=166024176)
-800 val loss 7.1846
-800 val perplexity 1319.0261
-800 train 7.157286 (lr=3.9883e-05) (hash(x)=151995229)
-500 val loss 7.4256
-500 val perplexity 1678.3735
-500 train 7.395220 (lr=5.0000e-05) (hash(x)=161040668)
-900 val loss 7.0973
-900 val perplexity 1208.6497
-900 train 6.848862 (lr=3.9792e-05) (hash(x)=139853932)
-600 val loss 7.3370
-600 val perplexity 1536.0286
-600 train 7.340587 (lr=4.9984e-05) (hash(x)=148878990)
-1000 val loss 6.9982
-1000 val perplexity 1094.6145
-1000 train 7.201624 (lr=3.9675e-05) (hash(x)=174104207)
-700 val loss 7.2297
-700 val perplexity 1379.8744
-700 train 7.214864 (lr=4.9935e-05) (hash(x)=150420695)
-1100 val loss 6.9161
-1100 val perplexity 1008.4128
-1100 train 6.749304 (lr=3.9532e-05) (hash(x)=146275038)
-800 val loss 7.1343
-800 val perplexity 1254.2781
-800 train 7.107428 (lr=4.9853e-05) (hash(x)=151995229)
-1200 val loss 6.8534
-1200 val perplexity 947.0867
-1200 train 6.625364 (lr=3.9364e-05) (hash(x)=141403655)
-900 val loss 7.0693
-900 val perplexity 1175.3370
-900 train 6.814147 (lr=4.9739e-05) (hash(x)=139853932)
-1300 val loss 6.7709
-1300 val perplexity 872.0862
-1300 train 6.850709 (lr=3.9171e-05) (hash(x)=173675040)
-1000 val loss 7.0031
-1000 val perplexity 1100.0358
-1000 train 7.203287 (lr=4.9593e-05) (hash(x)=174104207)
-1400 val loss 6.6960
-1400 val perplexity 809.1636
-1400 train 6.668308 (lr=3.8953e-05) (hash(x)=162475906)
-1100 val loss 6.9378
-1100 val perplexity 1030.4529
-1100 train 6.766367 (lr=4.9415e-05) (hash(x)=146275038)
-1500 val loss 6.6402
-1500 val perplexity 765.2175
-1500 train 6.466898 (lr=3.8711e-05) (hash(x)=152785612)
-1200 val loss 6.8749
-1200 val perplexity 967.6586
-1200 train 6.643046 (lr=4.9205e-05) (hash(x)=141403655)
-1600 val loss 6.5700
-1600 val perplexity 713.3383
-1600 train 6.522436 (lr=3.8444e-05) (hash(x)=151987383)
-1300 val loss 6.7905
-1300 val perplexity 889.3807
-1300 train 6.872322 (lr=4.8964e-05) (hash(x)=173675040)
-1700 val loss 6.5158
-1700 val perplexity 675.7247
-1700 train 6.502824 (lr=3.8153e-05) (hash(x)=143781605)
-1400 val loss 6.7308
-1400 val perplexity 837.7838
-1400 train 6.700823 (lr=4.8691e-05) (hash(x)=162475906)
-1800 val loss 6.4719
-1800 val perplexity 646.7280
-1800 train 6.459326 (lr=3.7839e-05) (hash(x)=151040203)
-1500 val loss 6.6808
-1500 val perplexity 796.9203
-1500 train 6.506968 (lr=4.8388e-05) (hash(x)=152785612)
-1900 val loss 6.4342
-1900 val perplexity 622.7968
-1900 train 6.334003 (lr=3.7502e-05) (hash(x)=149414572)
-1600 val loss 6.6151
-1600 val perplexity 746.2864
-1600 train 6.567549 (lr=4.8055e-05) (hash(x)=151987383)
-2000 val loss 6.3950
-2000 val perplexity 598.8578
-2000 train 6.353477 (lr=3.7143e-05) (hash(x)=156479674)
-1700 val loss 6.5682
-1700 val perplexity 712.0578
-1700 train 6.554473 (lr=4.7691e-05) (hash(x)=143781605)
-2100 val loss 6.3528
-2100 val perplexity 574.1195
-2100 train 6.224854 (lr=3.6761e-05) (hash(x)=137861481)
-1800 val loss 6.5217
-1800 val perplexity 679.7147
-1800 train 6.508534 (lr=4.7299e-05) (hash(x)=151040203)
-2200 val loss 6.3209
-2200 val perplexity 556.0743
-2200 train 6.377265 (lr=3.6358e-05) (hash(x)=155937443)
-1900 val loss 6.4849
-1900 val perplexity 655.1425
-1900 train 6.383931 (lr=4.6878e-05) (hash(x)=149414572)
-2300 val loss 6.3188
-2300 val perplexity 554.9308
-2300 train 5.995306 (lr=3.5935e-05) (hash(x)=131035715)
-2000 val loss 6.4401
-2000 val perplexity 626.4890
-2000 train 6.394539 (lr=4.6428e-05) (hash(x)=156479674)
-2400 val loss 6.2713
-2400 val perplexity 529.1736
-2400 train 6.266420 (lr=3.5491e-05) (hash(x)=146568981)
-2100 val loss 6.4051
-2100 val perplexity 604.8986
-2100 train 6.270535 (lr=4.5951e-05) (hash(x)=137861481)
-2500 val loss 6.2430
-2500 val perplexity 514.4131
-2500 train 6.220254 (lr=3.5027e-05) (hash(x)=163317586)
-2200 val loss 6.3759
-2200 val perplexity 587.5141
-2200 train 6.435637 (lr=4.5448e-05) (hash(x)=155937443)
-2600 val loss 6.2233
-2600 val perplexity 504.3846
-2600 train 6.038826 (lr=3.4545e-05) (hash(x)=144201060)
-2300 val loss 6.3613
-2300 val perplexity 579.0132
-2300 train 6.040595 (lr=4.4918e-05) (hash(x)=131035715)
-2700 val loss 6.2014
-2700 val perplexity 493.4525
-2700 train 6.095143 (lr=3.4044e-05) (hash(x)=141825701)
-2400 val loss 6.3159
-2400 val perplexity 553.3100
-2400 train 6.307761 (lr=4.4363e-05) (hash(x)=146568981)
-2800 val loss 6.1755
-2800 val perplexity 480.8045
-2800 train 6.286294 (lr=3.3526e-05) (hash(x)=160561627)
-2500 val loss 6.2862
-2500 val perplexity 537.1021
-2500 train 6.260865 (lr=4.3784e-05) (hash(x)=163317586)
-2900 val loss 6.1511
-2900 val perplexity 469.2112
-2900 train 6.096576 (lr=3.2992e-05) (hash(x)=151758176)
-2600 val loss 6.2591
-2600 val perplexity 522.7268
-2600 train 6.077036 (lr=4.3181e-05) (hash(x)=144201060)
-3000 val loss 6.1326
-3000 val perplexity 460.6170
-3000 train 6.152384 (lr=3.2441e-05) (hash(x)=155815751)
-2700 val loss 6.2398
-2700 val perplexity 512.7802
-2700 train 6.131793 (lr=4.2555e-05) (hash(x)=141825701)
-3100 val loss 6.1248
-3100 val perplexity 457.0416
-3100 train 5.794218 (lr=3.1875e-05) (hash(x)=142307043)
-2800 val loss 6.2099
-2800 val perplexity 497.6453
-2800 train 6.320862 (lr=4.1908e-05) (hash(x)=160561627)
-3200 val loss 6.1004
-3200 val perplexity 446.0268
-3200 train 6.209918 (lr=3.1295e-05) (hash(x)=156310690)
-2900 val loss 6.1826
-2900 val perplexity 484.2572
-2900 train 6.125389 (lr=4.1240e-05) (hash(x)=151758176)
-3300 val loss 6.0788
-3300 val perplexity 436.5081
-3300 train 6.077555 (lr=3.0702e-05) (hash(x)=151344506)
-3000 val loss 6.1635
-3000 val perplexity 475.1087
-3000 train 6.178281 (lr=4.0551e-05) (hash(x)=155815751)
-3400 val loss 6.0629
-3400 val perplexity 429.6288
-3400 train 6.093707 (lr=3.0095e-05) (hash(x)=162911881)
-3100 val loss 6.1495
-3100 val perplexity 468.4661
-3100 train 5.808314 (lr=3.9844e-05) (hash(x)=142307043)
-3500 val loss 6.0512
-3500 val perplexity 424.6271
-3500 train 5.990658 (lr=2.9477e-05) (hash(x)=142889971)
-3200 val loss 6.1270
-3200 val perplexity 458.0616
-3200 train 6.234677 (lr=3.9119e-05) (hash(x)=156310690)
-3600 val loss 6.0319
-3600 val perplexity 416.4943
-3600 train 6.020875 (lr=2.8849e-05) (hash(x)=149470354)
-3300 val loss 6.0978
-3300 val perplexity 444.8636
-3300 train 6.097316 (lr=3.8377e-05) (hash(x)=151344506)
-3700 val loss 6.0184
-3700 val perplexity 410.9171
-3700 train 5.961654 (lr=2.8210e-05) (hash(x)=154194821)
-3400 val loss 6.0849
-3400 val perplexity 439.1918
-3400 train 6.120546 (lr=3.7619e-05) (hash(x)=162911881)
-3800 val loss 6.0059
-3800 val perplexity 405.8116
-3800 train 5.937997 (lr=2.7562e-05) (hash(x)=148885848)
-3500 val loss 6.0766
-3500 val perplexity 435.5587
-3500 train 6.014277 (lr=3.6847e-05) (hash(x)=142889971)
-3900 val loss 5.9894
-3900 val perplexity 399.1929
-3900 train 5.899461 (lr=2.6907e-05) (hash(x)=148308484)
-3600 val loss 6.0508
-3600 val perplexity 424.4692
-3600 train 6.039314 (lr=3.6061e-05) (hash(x)=149470354)
-4000 val loss 5.9717
-4000 val perplexity 392.1893
-4000 train 5.813733 (lr=2.6244e-05) (hash(x)=139828564)
-3700 val loss 6.0343
-3700 val perplexity 417.5173
-3700 train 5.979168 (lr=3.5263e-05) (hash(x)=154194821)
-4100 val loss 5.9616
-4100 val perplexity 388.2494
-4100 train 5.875154 (lr=2.5575e-05) (hash(x)=139981997)
-3800 val loss 6.0246
-3800 val perplexity 413.4932
-3800 train 5.954288 (lr=3.4453e-05) (hash(x)=148885848)
-4200 val loss 5.9504
-4200 val perplexity 383.9165
-4200 train 6.103284 (lr=2.4900e-05) (hash(x)=150738447)
-3900 val loss 6.0049
-3900 val perplexity 405.4062
-3900 train 5.915174 (lr=3.3633e-05) (hash(x)=148308484)
-4300 val loss 5.9363
-4300 val perplexity 378.5393
-4300 train 5.712550 (lr=2.4222e-05) (hash(x)=142198107)
-4000 val loss 5.9892
-4000 val perplexity 399.0954
-4000 train 5.824629 (lr=3.2805e-05) (hash(x)=139828564)
-4400 val loss 5.9295
-4400 val perplexity 375.9687
-4400 train 5.798333 (lr=2.3540e-05) (hash(x)=142731201)
-4100 val loss 5.9779
-4100 val perplexity 394.6185
-4100 train 5.888789 (lr=3.1968e-05) (hash(x)=139981997)
-4500 val loss 5.9109
-4500 val perplexity 369.0209
-4500 train 5.931695 (lr=2.2856e-05) (hash(x)=154814426)
-4200 val loss 5.9643
-4200 val perplexity 389.2749
-4200 train 6.117281 (lr=3.1126e-05) (hash(x)=150738447)
-4600 val loss 5.8980
-4600 val perplexity 364.3122
-4600 train 5.870726 (lr=2.2171e-05) (hash(x)=155922230)
-4300 val loss 5.9482
-4300 val perplexity 383.0560
-4300 train 5.727036 (lr=3.0277e-05) (hash(x)=142198107)
-4700 val loss 5.8929
-4700 val perplexity 362.4561
-4700 train 5.643782 (lr=2.1486e-05) (hash(x)=139398510)
-4400 val loss 5.9390
-4400 val perplexity 379.5455
-4400 train 5.811529 (lr=2.9425e-05) (hash(x)=142731201)
-4800 val loss 5.8803
-4800 val perplexity 357.9199
-4800 train 5.943650 (lr=2.0801e-05) (hash(x)=140893236)
-4500 val loss 5.9237
-4500 val perplexity 373.7953
-4500 train 5.939521 (lr=2.8571e-05) (hash(x)=154814426)
-4900 val loss 5.8631
-4900 val perplexity 351.8223
-4900 train 5.758955 (lr=2.0118e-05) (hash(x)=153747830)
-4600 val loss 5.9083
-4600 val perplexity 368.0674
-4600 train 5.877394 (lr=2.7714e-05) (hash(x)=155922230)
-5000 val loss 5.8568
-5000 val perplexity 349.6060
-5000 train 6.100023 (lr=1.9438e-05) (hash(x)=148919005)
-4700 val loss 5.9007
-4700 val perplexity 365.2940
-4700 train 5.648995 (lr=2.6857e-05) (hash(x)=139398510)
-5100 val loss 5.8488
-5100 val perplexity 346.8314
-5100 train 5.637376 (lr=1.8762e-05) (hash(x)=142281936)
-4800 val loss 5.8867
-4800 val perplexity 360.2307
-4800 train 5.943976 (lr=2.6002e-05) (hash(x)=140893236)
-5200 val loss 5.8502
-5200 val perplexity 347.3190
-5200 train 5.655197 (lr=1.8090e-05) (hash(x)=143162650)
-4900 val loss 5.8698
-4900 val perplexity 354.1944
-4900 train 5.770061 (lr=2.5148e-05) (hash(x)=153747830)
-5300 val loss 5.8282
-5300 val perplexity 339.7482
-5300 train 5.886518 (lr=1.7424e-05) (hash(x)=148546849)
-5000 val loss 5.8572
-5000 val perplexity 349.7294
-5000 train 6.099062 (lr=2.4298e-05) (hash(x)=148919005)
-5400 val loss 5.8173
-5400 val perplexity 336.0493
-5400 train 5.671466 (lr=1.6765e-05) (hash(x)=143492259)
-5100 val loss 5.8483
-5100 val perplexity 346.6322
-5100 train 5.639922 (lr=2.3452e-05) (hash(x)=142281936)
-5500 val loss 5.8143
-5500 val perplexity 335.0696
-5500 train 5.568772 (lr=1.6113e-05) (hash(x)=141023941)
-5200 val loss 5.8518
-5200 val perplexity 347.8689
-5200 train 5.653669 (lr=2.2613e-05) (hash(x)=143162650)
-5600 val loss 5.8109
-5600 val perplexity 333.9080
-5600 train 5.691159 (lr=1.5469e-05) (hash(x)=142065021)
-5300 val loss 5.8263
-5300 val perplexity 339.1058
-5300 train 5.889111 (lr=2.1780e-05) (hash(x)=148546849)
-5700 val loss 5.7951
-5700 val perplexity 328.6996
-5700 train 5.668637 (lr=1.4836e-05) (hash(x)=145749913)
-5400 val loss 5.8158
-5400 val perplexity 335.5734
-5400 train 5.676393 (lr=2.0956e-05) (hash(x)=143492259)
-5800 val loss 5.7882
-5800 val perplexity 326.4294
-5800 train 5.470370 (lr=1.4212e-05) (hash(x)=140035522)
-5500 val loss 5.8084
-5500 val perplexity 333.0947
-5500 train 5.565637 (lr=2.0141e-05) (hash(x)=141023941)
-5900 val loss 5.7848
-5900 val perplexity 325.3286
-5900 train 5.537515 (lr=1.3600e-05) (hash(x)=154752726)
-5600 val loss 5.8085
-5600 val perplexity 333.1090
-5600 train 5.685212 (lr=1.9337e-05) (hash(x)=142065021)
-6000 val loss 5.7838
-6000 val perplexity 324.9914
-6000 train 5.811926 (lr=1.3000e-05) (hash(x)=158715824)
-5700 val loss 5.7899
-5700 val perplexity 326.9874
-5700 train 5.669137 (lr=1.8545e-05) (hash(x)=145749913)
-6100 val loss 5.7704
-6100 val perplexity 320.6667
-6100 train 5.785007 (lr=1.2413e-05) (hash(x)=137413820)
-5800 val loss 5.7829
-5800 val perplexity 324.7016
-5800 train 5.465841 (lr=1.7765e-05) (hash(x)=140035522)
-6200 val loss 5.7627
-6200 val perplexity 318.1954
-6200 train 5.680292 (lr=1.1840e-05) (hash(x)=151507523)
-5900 val loss 5.7770
-5900 val perplexity 322.7901
-5900 train 5.525473 (lr=1.7000e-05) (hash(x)=154752726)
-6300 val loss 5.7615
-6300 val perplexity 317.8233
-6300 train 5.529540 (lr=1.1282e-05) (hash(x)=147514617)
-6000 val loss 5.7735
-6000 val perplexity 321.6488
-6000 train 5.798147 (lr=1.6250e-05) (hash(x)=158715824)
-6400 val loss 5.7574
-6400 val perplexity 316.5249
-6400 train 5.762690 (lr=1.0739e-05) (hash(x)=151604465)
-6100 val loss 5.7618
-6100 val perplexity 317.9291
-6100 train 5.776758 (lr=1.5516e-05) (hash(x)=137413820)
-6500 val loss 5.7473
-6500 val perplexity 313.3486
-6500 train 5.805442 (lr=1.0213e-05) (hash(x)=144515881)
-6200 val loss 5.7522
-6200 val perplexity 314.8702
-6200 train 5.669681 (lr=1.4800e-05) (hash(x)=151507523)
-6600 val loss 5.7416
-6600 val perplexity 311.5598
-6300 val loss 5.7512
-6300 val perplexity 314.5645
-6600 train 5.532074 (lr=9.7032e-06) (hash(x)=136948374)
-6300 train 5.526007 (lr=1.4102e-05) (hash(x)=147514617)
-6700 val loss 5.7411
-6700 val perplexity 311.3926
-6400 val loss 5.7485
-6400 val perplexity 313.7257
-6700 train 5.550056 (lr=9.2116e-06) (hash(x)=146268592)
-6400 train 5.760475 (lr=1.3424e-05) (hash(x)=151604465)
-6800 val loss 5.7322
-6800 val perplexity 308.6383
-6500 val loss 5.7388
-6500 val perplexity 310.6908
-6800 train 5.683815 (lr=8.7387e-06) (hash(x)=152676836)
-6500 train 5.794615 (lr=1.2766e-05) (hash(x)=144515881)
-6900 val loss 5.7256
-6900 val perplexity 306.6284
-6900 train 5.647960 (lr=8.2849e-06) (hash(x)=134657776)
-6600 val loss 5.7305
-6600 val perplexity 308.1154
-6600 train 5.523883 (lr=1.2129e-05) (hash(x)=136948374)
-7000 val loss 5.7219
-7000 val perplexity 305.4925
-7000 train 5.691176 (lr=7.8510e-06) (hash(x)=166721861)
-6700 val loss 5.7292
-6700 val perplexity 307.7284
-6700 train 5.533874 (lr=1.1515e-05) (hash(x)=146268592)
-7100 val loss 5.7214
-7100 val perplexity 305.3211
-7100 train 5.526393 (lr=7.4377e-06) (hash(x)=135496702)
-6800 val loss 5.7191
-6800 val perplexity 304.6219
-6800 train 5.669891 (lr=1.0923e-05) (hash(x)=152676836)
-7200 val loss 5.7178
-7200 val perplexity 304.2230
-7200 train 5.824883 (lr=7.0455e-06) (hash(x)=155567461)
-6900 val loss 5.7130
-6900 val perplexity 302.7735
-6900 train 5.640803 (lr=1.0356e-05) (hash(x)=134657776)
-7300 val loss 5.7101
-7300 val perplexity 301.8895
-7300 train 5.511367 (lr=6.6749e-06) (hash(x)=142803829)
-7000 val loss 5.7093
-7000 val perplexity 301.6670
-7000 train 5.675343 (lr=9.8138e-06) (hash(x)=166721861)
-7400 val loss 5.7082
-7400 val perplexity 301.3404
-7400 train 5.528187 (lr=6.3266e-06) (hash(x)=145294178)
-7100 val loss 5.7074
-7100 val perplexity 301.0902
-7100 train 5.510521 (lr=9.2971e-06) (hash(x)=135496702)
-7500 val loss 5.7073
-7500 val perplexity 301.0652
-7500 train 5.473443 (lr=6.0010e-06) (hash(x)=150573713)
-7200 val loss 5.7045
-7200 val perplexity 300.2151
-7200 train 5.806618 (lr=8.8068e-06) (hash(x)=155567461)
-7600 val loss 5.7047
-7600 val perplexity 300.2834
-7600 train 5.723767 (lr=5.6986e-06) (hash(x)=142771511)
-7300 val loss 5.6948
-7300 val perplexity 297.3138
-7300 train 5.498550 (lr=8.3436e-06) (hash(x)=142803829)
-7700 val loss 5.6985
-7700 val perplexity 298.4202
-7700 train 5.598147 (lr=5.4198e-06) (hash(x)=143602175)
-7400 val loss 5.6940
-7400 val perplexity 297.0836
-7400 train 5.514047 (lr=7.9082e-06) (hash(x)=145294178)
-7800 val loss 5.6950
-7800 val perplexity 297.3798
-7800 train 5.744207 (lr=5.1650e-06) (hash(x)=152379862)
-7500 val loss 5.6917
-7500 val perplexity 296.3914
-7500 train 5.456120 (lr=7.5012e-06) (hash(x)=150573713)
-7900 val loss 5.6939
-7900 val perplexity 297.0622
-7900 train 5.528719 (lr=4.9347e-06) (hash(x)=146655921)
-7600 val loss 5.6894
-7600 val perplexity 295.7183
-7600 train 5.704326 (lr=7.1232e-06) (hash(x)=142771511)
-8000 val loss 5.6958
-8000 val perplexity 297.6096
-8000 train 5.821932 (lr=4.7291e-06) (hash(x)=148262482)
-7700 val loss 5.6819
-7700 val perplexity 293.5073
-7700 train 5.588364 (lr=6.7747e-06) (hash(x)=143602175)
-8100 val loss 5.6871
-8100 val perplexity 295.0270
-7800 val loss 5.6794
-7800 val perplexity 292.7674
-8100 train 5.627559 (lr=4.5486e-06) (hash(x)=147683655)
-7800 train 5.731502 (lr=6.4563e-06) (hash(x)=152379862)
-7900 val loss 5.6787
-7900 val perplexity 292.5810
-7900 train 5.505094 (lr=6.1684e-06) (hash(x)=146655921)
-8200 val loss 5.6863
-8200 val perplexity 294.8073
-8200 train 5.751357 (lr=4.3933e-06) (hash(x)=164975934)
-8000 val loss 5.6797
-8000 val perplexity 292.8628
-8000 train 5.804450 (lr=5.9114e-06) (hash(x)=148262482)
-8300 val loss 5.6828
-8300 val perplexity 293.7621
-8300 train 5.594470 (lr=4.2636e-06) (hash(x)=145300550)
-8100 val loss 5.6704
-8100 val perplexity 290.1621
-8100 train 5.615933 (lr=5.6857e-06) (hash(x)=147683655)
-8400 val loss 5.6817
-8400 val perplexity 293.4604
-8400 train 5.595119 (lr=4.1596e-06) (hash(x)=150679400)
-8500 val loss 5.6823
-8500 val perplexity 293.6205
-8200 val loss 5.6696
-8200 val perplexity 289.9252
-8500 train 5.679973 (lr=4.0815e-06) (hash(x)=164109401)
-8200 train 5.733337 (lr=5.4917e-06) (hash(x)=164975934)
-8300 val loss 5.6657
-8300 val perplexity 288.7932
-8600 val loss 5.6785
-8600 val perplexity 292.5168
-8300 train 5.576839 (lr=5.3295e-06) (hash(x)=145300550)
-8600 train 5.644332 (lr=4.0294e-06) (hash(x)=161036376)
-8400 val loss 5.6644
-8400 val perplexity 288.4171
-8700 val loss 5.6741
-8700 val perplexity 291.2169
-8400 train 5.575017 (lr=5.1995e-06) (hash(x)=150679400)
-8700 train 5.603286 (lr=4.0033e-06) (hash(x)=153828820)
-8749 val loss 5.6732
-8749 val perplexity 290.9644
-8500 val loss 5.6648
-8500 val perplexity 288.5241
-8500 train 5.657299 (lr=5.1019e-06) (hash(x)=164109401)
-8600 val loss 5.6618
-8600 val perplexity 287.6606
-8600 train 5.614815 (lr=5.0367e-06) (hash(x)=161036376)
-8700 val loss 5.6567
-8700 val perplexity 286.2138
-8700 train 5.585802 (lr=5.0041e-06) (hash(x)=153828820)
-8749 val loss 5.6550
-8749 val perplexity 285.7292
+0 val perplexity 79465.5000
+0 train 11.285261 (lr=1.2000e-07) (hash(x)=145830960)
+100 val loss 9.9936
+100 val perplexity 21885.0879
+100 train 10.028351 (lr=1.2120e-05) (hash(x)=144157453)
+200 val loss 9.2065
+200 val perplexity 9961.6328
+200 train 9.241407 (lr=2.4120e-05) (hash(x)=146764602)
+300 val loss 7.9260
+300 val perplexity 2768.3066
+300 train 7.760632 (lr=3.6120e-05) (hash(x)=140559124)
+400 val loss 7.4926
+400 val perplexity 1794.7747
+400 train 7.893311 (lr=4.8120e-05) (hash(x)=166024176)
+500 val loss 7.3529
+500 val perplexity 1560.6521
+500 train 7.319139 (lr=6.0000e-05) (hash(x)=161040668)
+600 val loss 7.2451
+600 val perplexity 1401.2542
+600 train 7.242630 (lr=5.9980e-05) (hash(x)=148878990)
+700 val loss 7.1206
+700 val perplexity 1237.2394
+700 train 7.109270 (lr=5.9922e-05) (hash(x)=150420695)
+800 val loss 7.0173
+800 val perplexity 1115.8207
+800 train 6.990163 (lr=5.9824e-05) (hash(x)=151995229)
+900 val loss 6.9273
+900 val perplexity 1019.6912
+900 train 6.674723 (lr=5.9687e-05) (hash(x)=139853932)
+1000 val loss 6.8337
+1000 val perplexity 928.5917
+1000 train 7.040174 (lr=5.9512e-05) (hash(x)=174104207)
+1100 val loss 6.7474
+1100 val perplexity 851.8145
+1100 train 6.573821 (lr=5.9298e-05) (hash(x)=146275038)
+1200 val loss 6.6759
+1200 val perplexity 793.0219
+1200 train 6.453066 (lr=5.9046e-05) (hash(x)=141403655)
+1300 val loss 6.6006
+1300 val perplexity 735.5050
+1300 train 6.682136 (lr=5.8757e-05) (hash(x)=173675040)
+1400 val loss 6.5371
+1400 val perplexity 690.2772
+1400 train 6.508105 (lr=5.8430e-05) (hash(x)=162475906)
+1500 val loss 6.4942
+1500 val perplexity 661.3112
+1500 train 6.322554 (lr=5.8066e-05) (hash(x)=152785612)
+1600 val loss 6.4288
+1600 val perplexity 619.4235
+1600 train 6.379230 (lr=5.7666e-05) (hash(x)=151987383)
+1700 val loss 6.3818
+1700 val perplexity 591.0200
+1700 train 6.376427 (lr=5.7230e-05) (hash(x)=143781605)
+1800 val loss 6.3489
+1800 val perplexity 571.8567
+1800 train 6.338639 (lr=5.6759e-05) (hash(x)=151040203)
+1900 val loss 6.3207
+1900 val perplexity 555.9756
+1900 train 6.220351 (lr=5.6253e-05) (hash(x)=149414572)
+2000 val loss 6.2809
+2000 val perplexity 534.2764
+2000 train 6.233247 (lr=5.5714e-05) (hash(x)=156479674)
+2100 val loss 6.2493
+2100 val perplexity 517.6708
+2100 train 6.118375 (lr=5.5142e-05) (hash(x)=137861481)
+2200 val loss 6.2233
+2200 val perplexity 504.3856
+2200 train 6.286711 (lr=5.4537e-05) (hash(x)=155937443)
+2300 val loss 6.2125
+2300 val perplexity 498.9350
+2300 train 5.889745 (lr=5.3902e-05) (hash(x)=131035715)
+2400 val loss 6.1760
+2400 val perplexity 481.0644
+2400 train 6.170713 (lr=5.3236e-05) (hash(x)=146568981)
+2500 val loss 6.1466
+2500 val perplexity 467.1402
+2500 train 6.122752 (lr=5.2541e-05) (hash(x)=163317586)
+2600 val loss 6.1279
+2600 val perplexity 458.4825
+2600 train 5.950536 (lr=5.1817e-05) (hash(x)=144201060)
+2700 val loss 6.1110
+2700 val perplexity 450.7988
+2700 train 6.010400 (lr=5.1067e-05) (hash(x)=141825701)
+2800 val loss 6.0889
+2800 val perplexity 440.9305
+2800 train 6.206187 (lr=5.0290e-05) (hash(x)=160561627)
+2900 val loss 6.0610
+2900 val perplexity 428.8093
+2900 train 6.007767 (lr=4.9487e-05) (hash(x)=151758176)
+3000 val loss 6.0450
+3000 val perplexity 421.9950
+3000 train 6.072838 (lr=4.8662e-05) (hash(x)=155815751)
+3100 val loss 6.0350
+3100 val perplexity 417.8105
+3100 train 5.686655 (lr=4.7813e-05) (hash(x)=142307043)
+3200 val loss 6.0137
+3200 val perplexity 409.0127
+3200 train 6.125471 (lr=4.6943e-05) (hash(x)=156310690)
+3300 val loss 5.9909
+3300 val perplexity 399.7701
+3300 train 5.994750 (lr=4.6052e-05) (hash(x)=151344506)
+3400 val loss 5.9736
+3400 val perplexity 392.9013
+3400 train 6.005388 (lr=4.5143e-05) (hash(x)=162911881)
+3500 val loss 5.9642
+3500 val perplexity 389.2571
+3500 train 5.905776 (lr=4.4216e-05) (hash(x)=142889971)
+3600 val loss 5.9455
+3600 val perplexity 382.0471
+3600 train 5.939862 (lr=4.3273e-05) (hash(x)=149470354)
+3700 val loss 5.9241
+3700 val perplexity 373.9290
+3700 train 5.866091 (lr=4.2315e-05) (hash(x)=154194821)
+3800 val loss 5.9158
+3800 val perplexity 370.8608
+3800 train 5.844939 (lr=4.1343e-05) (hash(x)=148885848)
+3900 val loss 5.8975
+3900 val perplexity 364.1401
+3900 train 5.812170 (lr=4.0360e-05) (hash(x)=148308484)
+4000 val loss 5.8831
+4000 val perplexity 358.9251
+4000 train 5.724350 (lr=3.9365e-05) (hash(x)=139828564)
+4100 val loss 5.8734
+4100 val perplexity 355.4681
+4100 train 5.774945 (lr=3.8362e-05) (hash(x)=139981997)
+4200 val loss 5.8649
+4200 val perplexity 352.4458
+4200 train 6.013971 (lr=3.7351e-05) (hash(x)=150738447)
+4300 val loss 5.8415
+4300 val perplexity 344.2853
+4300 train 5.615357 (lr=3.6333e-05) (hash(x)=142198107)
+4400 val loss 5.8334
+4400 val perplexity 341.5034
+4400 train 5.707536 (lr=3.5311e-05) (hash(x)=142731201)
+4500 val loss 5.8160
+4500 val perplexity 335.6200
+4500 train 5.847620 (lr=3.4285e-05) (hash(x)=154814426)
+4600 val loss 5.8043
+4600 val perplexity 331.7113
+4600 train 5.779626 (lr=3.3257e-05) (hash(x)=155922230)
+4700 val loss 5.7968
+4700 val perplexity 329.2366
+4700 train 5.551564 (lr=3.2229e-05) (hash(x)=139398510)
+4800 val loss 5.7882
+4800 val perplexity 326.4132
+4800 train 5.837641 (lr=3.1202e-05) (hash(x)=140893236)
+4900 val loss 5.7662
+4900 val perplexity 319.3077
+4900 train 5.658298 (lr=3.0178e-05) (hash(x)=153747830)
+5000 val loss 5.7569
+5000 val perplexity 316.3622
+5000 train 6.012340 (lr=2.9157e-05) (hash(x)=148919005)
+5100 val loss 5.7471
+5100 val perplexity 313.2700
+5100 train 5.541666 (lr=2.8143e-05) (hash(x)=142281936)
+5200 val loss 5.7479
+5200 val perplexity 313.5405
+5200 train 5.553550 (lr=2.7135e-05) (hash(x)=143162650)
+5300 val loss 5.7266
+5300 val perplexity 306.9259
+5300 train 5.788898 (lr=2.6136e-05) (hash(x)=148546849)
+5400 val loss 5.7142
+5400 val perplexity 303.1394
+5400 train 5.574027 (lr=2.5147e-05) (hash(x)=143492259)
+5500 val loss 5.7102
+5500 val perplexity 301.9269
+5500 train 5.464751 (lr=2.4169e-05) (hash(x)=141023941)
+5600 val loss 5.7076
+5600 val perplexity 301.1327
+5600 train 5.591500 (lr=2.3204e-05) (hash(x)=142065021)
+5700 val loss 5.6908
+5700 val perplexity 296.1168
+5700 train 5.580994 (lr=2.2253e-05) (hash(x)=145749913)
+5800 val loss 5.6849
+5800 val perplexity 294.3895
+5800 train 5.364305 (lr=2.1318e-05) (hash(x)=140035522)
+5900 val loss 5.6792
+5900 val perplexity 292.7043
+5900 train 5.426435 (lr=2.0400e-05) (hash(x)=154752726)
+6000 val loss 5.6766
+6000 val perplexity 291.9586
+6000 train 5.706806 (lr=1.9500e-05) (hash(x)=158715824)
+6100 val loss 5.6645
+6100 val perplexity 288.4511
+6100 train 5.684230 (lr=1.8620e-05) (hash(x)=137413820)
+6200 val loss 5.6567
+6200 val perplexity 286.2024
+6200 train 5.568646 (lr=1.7760e-05) (hash(x)=151507523)
+6300 val loss 5.6565
+6300 val perplexity 286.1344
+6300 train 5.428356 (lr=1.6923e-05) (hash(x)=147514617)
+6400 val loss 5.6518
+6400 val perplexity 284.8132
+6400 train 5.656651 (lr=1.6108e-05) (hash(x)=151604465)
+6500 val loss 5.6394
+6500 val perplexity 281.2830
+6500 train 5.699336 (lr=1.5319e-05) (hash(x)=144515881)
+6600 val loss 5.6334
+6600 val perplexity 279.6182
+6600 train 5.427759 (lr=1.4555e-05) (hash(x)=136948374)
+6700 val loss 5.6313
+6700 val perplexity 279.0251
+6700 train 5.432090 (lr=1.3817e-05) (hash(x)=146268592)
+6800 val loss 5.6226
+6800 val perplexity 276.6176
+6800 train 5.572458 (lr=1.3108e-05) (hash(x)=152676836)
+6900 val loss 5.6163
+6900 val perplexity 274.8722
+6900 train 5.546123 (lr=1.2427e-05) (hash(x)=134657776)
+7000 val loss 5.6137
+7000 val perplexity 274.1497
+7000 train 5.578875 (lr=1.1777e-05) (hash(x)=166721861)
+7100 val loss 5.6102
+7100 val perplexity 273.2121
+7100 train 5.411637 (lr=1.1157e-05) (hash(x)=135496702)
+7200 val loss 5.6081
+7200 val perplexity 272.6226
+7200 train 5.722853 (lr=1.0568e-05) (hash(x)=155567461)
+7300 val loss 5.6003
+7300 val perplexity 270.5051
+7300 train 5.405878 (lr=1.0012e-05) (hash(x)=142803829)
+7400 val loss 5.5992
+7400 val perplexity 270.2187
+7400 train 5.418221 (lr=9.4899e-06) (hash(x)=145294178)
+7500 val loss 5.5960
+7500 val perplexity 269.3583
+7500 train 5.358080 (lr=9.0014e-06) (hash(x)=150573713)
+7600 val loss 5.5934
+7600 val perplexity 268.6475
+7600 train 5.610190 (lr=8.5478e-06) (hash(x)=142771511)
+7700 val loss 5.5872
+7700 val perplexity 266.9865
+7700 train 5.489188 (lr=8.1297e-06) (hash(x)=143602175)
+7800 val loss 5.5840
+7800 val perplexity 266.1398
+7800 train 5.650937 (lr=7.7476e-06) (hash(x)=152379862)
+7900 val loss 5.5834
+7900 val perplexity 265.9798
+7900 train 5.406669 (lr=7.4021e-06) (hash(x)=146655921)
+8000 val loss 5.5843
+8000 val perplexity 266.2111
+8000 train 5.718188 (lr=7.0937e-06) (hash(x)=148262482)
+8100 val loss 5.5750
+8100 val perplexity 263.7508
+8100 train 5.523913 (lr=6.8229e-06) (hash(x)=147683655)
+8200 val loss 5.5740
+8200 val perplexity 263.4843
+8200 train 5.637914 (lr=6.5900e-06) (hash(x)=164975934)
+8300 val loss 5.5708
+8300 val perplexity 262.6368
+8300 train 5.479691 (lr=6.3954e-06) (hash(x)=145300550)
+8400 val loss 5.5692
+8400 val perplexity 262.2281
+8400 train 5.482785 (lr=6.2395e-06) (hash(x)=150679400)
+8500 val loss 5.5699
+8500 val perplexity 262.4032
+8500 train 5.551933 (lr=6.1223e-06) (hash(x)=164109401)
+8600 val loss 5.5665
+8600 val perplexity 261.5091
+8600 train 5.501308 (lr=6.0440e-06) (hash(x)=161036376)
+8700 val loss 5.5612
+8700 val perplexity 260.1292
+8700 train 5.481901 (lr=6.0049e-06) (hash(x)=153828820)
+8749 val loss 5.5600
+8749 val perplexity 259.8274

attention_kindselective_n_heads4_seed1344/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7f203f360e5c1b74018e436c01930f0970f110dc443580ee0515a9ac032a65c
+oid sha256:9ddff7320ee65678a2b50a72c78cd50cfd2bf2009a28ef223a000cf8ae65ce84
 size 92843394

attention_kindselective_n_heads4_seed1344/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a7b318059b66e32fb7098d752dc4c7be604ac4babeef3f868d2e9feed5a7021
+oid sha256:519f510f379da84d77fadc0c34bfc0112d75ba6e799520fce58803c05c6c911d
 size 179406214