Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1342/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_4_1342", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5.5e-5_61440_4_1342", "n_embd": 256}

attention_kindselective_n_heads4_seed1342/log2.txt

@@ -1,267 +1,267 @@
 max_steps: 8750
 0 val loss 11.2898
 0 val perplexity 80004.5469
-0 train 11.289051 (lr=1.2000e-07) (hash(x)=150332693)
-100 val loss 9.9615
-100 val perplexity 21195.4160
-100 train 9.912132 (lr=1.2120e-05) (hash(x)=149277319)
-200 val loss 9.0154
-200 val perplexity 8229.0254
-200 train 9.076880 (lr=2.4120e-05) (hash(x)=155934075)
-300 val loss 7.8700
-300 val perplexity 2617.4729
-300 train 7.893194 (lr=3.6120e-05) (hash(x)=165402628)
-400 val loss 7.5153
-400 val perplexity 1835.8740
-400 train 7.302748 (lr=4.8120e-05) (hash(x)=139168939)
-500 val loss 7.3772
-500 val perplexity 1599.0566
-500 train 7.295662 (lr=6.0000e-05) (hash(x)=147134338)
-600 val loss 7.2687
-600 val perplexity 1434.7521
-600 train 7.172646 (lr=5.9980e-05) (hash(x)=159086459)
-700 val loss 7.1673
-700 val perplexity 1296.3660
-700 train 7.202695 (lr=5.9922e-05) (hash(x)=152399099)
-800 val loss 7.0257
-800 val perplexity 1125.1729
-800 train 7.027829 (lr=5.9824e-05) (hash(x)=156548843)
-900 val loss 6.9291
-900 val perplexity 1021.5352
-900 train 6.900503 (lr=5.9687e-05) (hash(x)=157889911)
-1000 val loss 6.8540
-1000 val perplexity 947.6541
-1000 train 6.645953 (lr=5.9512e-05) (hash(x)=142943707)
-1100 val loss 6.7872
-1100 val perplexity 886.4179
-1100 train 6.784300 (lr=5.9298e-05) (hash(x)=148938689)
-1200 val loss 6.6919
-1200 val perplexity 805.8444
-1200 train 6.832088 (lr=5.9046e-05) (hash(x)=161962903)
-1300 val loss 6.6179
-1300 val perplexity 748.3896
-1300 train 6.603671 (lr=5.8757e-05) (hash(x)=145979235)
-1400 val loss 6.5546
-1400 val perplexity 702.4713
-1400 train 6.501323 (lr=5.8430e-05) (hash(x)=159157507)
-1500 val loss 6.5041
-1500 val perplexity 667.8551
-1500 train 6.537813 (lr=5.8066e-05) (hash(x)=145943130)
-1600 val loss 6.4396
-1600 val perplexity 626.1458
-1600 train 6.243019 (lr=5.7666e-05) (hash(x)=143249173)
-1700 val loss 6.3910
-1700 val perplexity 596.4265
-1700 train 6.449312 (lr=5.7230e-05) (hash(x)=173907143)
-1800 val loss 6.3457
-1800 val perplexity 570.0468
-1800 train 6.273961 (lr=5.6759e-05) (hash(x)=138363755)
-1900 val loss 6.3148
-1900 val perplexity 552.6835
-1900 train 6.127532 (lr=5.6253e-05) (hash(x)=146341390)
-2000 val loss 6.2714
-2000 val perplexity 529.2251
-2000 train 6.300304 (lr=5.5714e-05) (hash(x)=154559671)
-2100 val loss 6.2476
-2100 val perplexity 516.7497
-2100 train 6.296726 (lr=5.5142e-05) (hash(x)=155607137)
-2200 val loss 6.2124
-2200 val perplexity 498.9062
-2200 train 6.073236 (lr=5.4537e-05) (hash(x)=153010221)
-2300 val loss 6.1781
-2300 val perplexity 482.0536
-2300 train 6.189386 (lr=5.3902e-05) (hash(x)=160097777)
-2400 val loss 6.1516
-2400 val perplexity 469.4609
-2400 train 6.133095 (lr=5.3236e-05) (hash(x)=135590094)
-2500 val loss 6.1327
-2500 val perplexity 460.6616
-2500 train 6.251341 (lr=5.2541e-05) (hash(x)=147042929)
-2600 val loss 6.1032
-2600 val perplexity 447.2693
-2600 train 6.064587 (lr=5.1817e-05) (hash(x)=163793881)
-2700 val loss 6.0817
-2700 val perplexity 437.7921
-2700 train 6.115384 (lr=5.1067e-05) (hash(x)=153598698)
-2800 val loss 6.0655
-2800 val perplexity 430.7336
-2800 train 5.855768 (lr=5.0290e-05) (hash(x)=133439803)
-2900 val loss 6.0305
-2900 val perplexity 415.9322
-2900 train 5.968013 (lr=4.9487e-05) (hash(x)=148239158)
-3000 val loss 6.0177
-3000 val perplexity 410.6143
-3000 train 6.027388 (lr=4.8662e-05) (hash(x)=150991971)
-3100 val loss 6.0013
-3100 val perplexity 403.9464
-3100 train 6.011987 (lr=4.7813e-05) (hash(x)=157791832)
-3200 val loss 5.9744
-3200 val perplexity 393.2289
-3200 train 6.042794 (lr=4.6943e-05) (hash(x)=163746305)
-3300 val loss 5.9654
-3300 val perplexity 389.7220
-3300 train 5.886434 (lr=4.6052e-05) (hash(x)=155120269)
-3400 val loss 5.9279
-3400 val perplexity 375.3517
-3400 train 5.871515 (lr=4.5143e-05) (hash(x)=152748451)
-3500 val loss 5.9162
-3500 val perplexity 371.0053
-3500 train 5.804614 (lr=4.4216e-05) (hash(x)=147395004)
-3600 val loss 5.9043
-3600 val perplexity 366.6275
-3600 train 5.918603 (lr=4.3273e-05) (hash(x)=157554250)
-3700 val loss 5.8756
-3700 val perplexity 356.2478
-3700 train 5.730301 (lr=4.2315e-05) (hash(x)=147953321)
-3800 val loss 5.8586
-3800 val perplexity 350.2188
-3800 train 5.712625 (lr=4.1343e-05) (hash(x)=146898609)
-3900 val loss 5.8522
-3900 val perplexity 347.9937
-3900 train 5.886589 (lr=4.0360e-05) (hash(x)=150115282)
-4000 val loss 5.8241
-4000 val perplexity 338.3399
-4000 train 5.707231 (lr=3.9365e-05) (hash(x)=142448619)
-4100 val loss 5.8301
-4100 val perplexity 340.3960
-4100 train 5.576627 (lr=3.8362e-05) (hash(x)=141244123)
-4200 val loss 5.8160
-4200 val perplexity 335.6147
-4200 train 5.818020 (lr=3.7351e-05) (hash(x)=165913661)
-4300 val loss 5.7864
-4300 val perplexity 325.8335
-4300 train 5.816000 (lr=3.6333e-05) (hash(x)=155744823)
-4400 val loss 5.7760
-4400 val perplexity 322.4549
-4400 train 5.771632 (lr=3.5311e-05) (hash(x)=160461243)
-4500 val loss 5.7639
-4500 val perplexity 318.5829
-4500 train 5.661434 (lr=3.4285e-05) (hash(x)=156073129)
-4600 val loss 5.7462
-4600 val perplexity 312.9993
-4600 train 5.798613 (lr=3.3257e-05) (hash(x)=152294132)
-4700 val loss 5.7369
-4700 val perplexity 310.0911
-4700 train 5.671713 (lr=3.2229e-05) (hash(x)=140904569)
-4800 val loss 5.7324
-4800 val perplexity 308.7122
-4800 train 5.635421 (lr=3.1202e-05) (hash(x)=156191635)
-4900 val loss 5.7176
-4900 val perplexity 304.1834
-4900 train 5.841964 (lr=3.0178e-05) (hash(x)=145063976)
-5000 val loss 5.7085
-5000 val perplexity 301.4318
-5000 train 5.654516 (lr=2.9157e-05) (hash(x)=160110619)
-5100 val loss 5.6953
-5100 val perplexity 297.4629
-5100 train 5.728802 (lr=2.8143e-05) (hash(x)=156270070)
-5200 val loss 5.6893
-5200 val perplexity 295.6986
-5200 train 5.585282 (lr=2.7135e-05) (hash(x)=138954242)
-5300 val loss 5.6777
-5300 val perplexity 292.2853
-5300 train 5.522610 (lr=2.6136e-05) (hash(x)=146472367)
-5400 val loss 5.6648
-5400 val perplexity 288.5367
-5400 train 5.598588 (lr=2.5147e-05) (hash(x)=146753405)
-5500 val loss 5.6572
-5500 val perplexity 286.3393
-5500 train 5.606112 (lr=2.4169e-05) (hash(x)=147757398)
-5600 val loss 5.6523
-5600 val perplexity 284.9454
-5600 train 5.521980 (lr=2.3204e-05) (hash(x)=151291757)
-5700 val loss 5.6385
-5700 val perplexity 281.0490
-5700 train 5.692122 (lr=2.2253e-05) (hash(x)=156274392)
-5800 val loss 5.6329
-5800 val perplexity 279.4854
-5800 train 5.609037 (lr=2.1318e-05) (hash(x)=152612266)
-5900 val loss 5.6271
-5900 val perplexity 277.8664
-5900 train 5.795071 (lr=2.0400e-05) (hash(x)=148973201)
-6000 val loss 5.6178
-6000 val perplexity 275.2767
-6000 train 5.579207 (lr=1.9500e-05) (hash(x)=158311490)
-6100 val loss 5.6121
-6100 val perplexity 273.7303
-6100 train 5.554811 (lr=1.8620e-05) (hash(x)=147541319)
-6200 val loss 5.6111
-6200 val perplexity 273.4489
-6200 train 5.679410 (lr=1.7760e-05) (hash(x)=154753980)
-6300 val loss 5.6028
-6300 val perplexity 271.1972
-6300 train 5.494817 (lr=1.6923e-05) (hash(x)=140798998)
-6400 val loss 5.5951
-6400 val perplexity 269.1085
-6400 train 5.487599 (lr=1.6108e-05) (hash(x)=149299516)
-6500 val loss 5.5900
-6500 val perplexity 267.7465
-6500 train 5.562661 (lr=1.5319e-05) (hash(x)=148495806)
-6600 val loss 5.5818
-6600 val perplexity 265.5546
-6600 train 5.563025 (lr=1.4555e-05) (hash(x)=153664387)
-6700 val loss 5.5788
-6700 val perplexity 264.7507
-6700 train 5.554018 (lr=1.3817e-05) (hash(x)=153333491)
-6800 val loss 5.5798
-6800 val perplexity 265.0316
-6800 train 5.324367 (lr=1.3108e-05) (hash(x)=150149382)
-6900 val loss 5.5716
-6900 val perplexity 262.8433
-6900 train 5.661313 (lr=1.2427e-05) (hash(x)=152379767)
-7000 val loss 5.5638
-7000 val perplexity 260.8132
-7000 train 5.611398 (lr=1.1777e-05) (hash(x)=150432537)
-7100 val loss 5.5639
-7100 val perplexity 260.8436
-7100 train 5.440166 (lr=1.1157e-05) (hash(x)=148277836)
-7200 val loss 5.5617
-7200 val perplexity 260.2775
-7200 train 5.550549 (lr=1.0568e-05) (hash(x)=168591659)
-7300 val loss 5.5562
-7300 val perplexity 258.8360
-7300 train 5.538674 (lr=1.0012e-05) (hash(x)=152764851)
-7400 val loss 5.5513
-7400 val perplexity 257.5660
-7400 train 5.519680 (lr=9.4899e-06) (hash(x)=142920131)
-7500 val loss 5.5492
-7500 val perplexity 257.0265
-7500 train 5.550600 (lr=9.0014e-06) (hash(x)=146954649)
-7600 val loss 5.5489
-7600 val perplexity 256.9645
-7600 train 5.316984 (lr=8.5478e-06) (hash(x)=144529566)
-7700 val loss 5.5435
-7700 val perplexity 255.5719
-7700 train 5.502102 (lr=8.1297e-06) (hash(x)=142669945)
-7800 val loss 5.5391
-7800 val perplexity 254.4444
-7800 train 5.590072 (lr=7.7476e-06) (hash(x)=161563764)
-7900 val loss 5.5373
-7900 val perplexity 253.9967
-7900 train 5.333921 (lr=7.4021e-06) (hash(x)=141516532)
-8000 val loss 5.5385
-8000 val perplexity 254.3059
-8000 train 5.403804 (lr=7.0937e-06) (hash(x)=152418457)
-8100 val loss 5.5318
-8100 val perplexity 252.5901
-8100 train 5.520045 (lr=6.8229e-06) (hash(x)=161109470)
-8200 val loss 5.5277
-8200 val perplexity 251.5600
-8200 train 5.655939 (lr=6.5900e-06) (hash(x)=154408822)
-8300 val loss 5.5258
-8300 val perplexity 251.0971
-8300 train 5.419258 (lr=6.3954e-06) (hash(x)=146431691)
-8400 val loss 5.5267
-8400 val perplexity 251.3187
-8400 train 5.305446 (lr=6.2395e-06) (hash(x)=150182527)
-8500 val loss 5.5204
-8500 val perplexity 249.7403
-8500 train 5.583275 (lr=6.1223e-06) (hash(x)=154828709)
-8600 val loss 5.5201
-8600 val perplexity 249.6640
-8600 train 5.437249 (lr=6.0440e-06) (hash(x)=154995428)
-8700 val loss 5.5193
-8700 val perplexity 249.4662
-8700 train 5.418331 (lr=6.0049e-06) (hash(x)=142111043)
-8749 val loss 5.5172
-8749 val perplexity 248.9265
+0 train 11.289022 (lr=1.1000e-07) (hash(x)=150332693)
+100 val loss 9.9785
+100 val perplexity 21557.6309
+100 train 9.928879 (lr=1.1110e-05) (hash(x)=149277319)
+200 val loss 9.0998
+200 val perplexity 8953.7422
+200 train 9.162449 (lr=2.2110e-05) (hash(x)=155934075)
+300 val loss 7.9244
+300 val perplexity 2764.0276
+300 train 7.947826 (lr=3.3110e-05) (hash(x)=165402628)
+400 val loss 7.5156
+400 val perplexity 1836.4186
+400 train 7.303900 (lr=4.4110e-05) (hash(x)=139168939)
+500 val loss 7.3663
+500 val perplexity 1581.7880
+500 train 7.281931 (lr=5.5000e-05) (hash(x)=147134338)
+600 val loss 7.2562
+600 val perplexity 1416.9106
+600 train 7.163249 (lr=5.4982e-05) (hash(x)=159086459)
+700 val loss 7.1584
+700 val perplexity 1284.9026
+700 train 7.195020 (lr=5.4928e-05) (hash(x)=152399099)
+800 val loss 7.0560
+800 val perplexity 1159.8362
+800 train 7.059469 (lr=5.4839e-05) (hash(x)=156548843)
+900 val loss 6.9738
+900 val perplexity 1068.2446
+900 train 6.945956 (lr=5.4713e-05) (hash(x)=157889911)
+1000 val loss 6.8933
+1000 val perplexity 985.6544
+1000 train 6.688016 (lr=5.4553e-05) (hash(x)=142943707)
+1100 val loss 6.8225
+1100 val perplexity 918.3061
+1100 train 6.816679 (lr=5.4357e-05) (hash(x)=148938689)
+1200 val loss 6.7309
+1200 val perplexity 837.9177
+1200 train 6.869128 (lr=5.4126e-05) (hash(x)=161962903)
+1300 val loss 6.6476
+1300 val perplexity 770.9232
+1300 train 6.630394 (lr=5.3860e-05) (hash(x)=145979235)
+1400 val loss 6.5794
+1400 val perplexity 720.1292
+1400 train 6.523628 (lr=5.3561e-05) (hash(x)=159157507)
+1500 val loss 6.5286
+1500 val perplexity 684.4431
+1500 train 6.563924 (lr=5.3227e-05) (hash(x)=145943130)
+1600 val loss 6.4595
+1600 val perplexity 638.7619
+1600 train 6.259445 (lr=5.2860e-05) (hash(x)=143249173)
+1700 val loss 6.4041
+1700 val perplexity 604.3295
+1700 train 6.467296 (lr=5.2461e-05) (hash(x)=173907143)
+1800 val loss 6.3679
+1800 val perplexity 582.8306
+1800 train 6.292572 (lr=5.2029e-05) (hash(x)=138363755)
+1900 val loss 6.3335
+1900 val perplexity 563.1382
+1900 train 6.143126 (lr=5.1565e-05) (hash(x)=146341390)
+2000 val loss 6.2870
+2000 val perplexity 537.5458
+2000 train 6.311212 (lr=5.1071e-05) (hash(x)=154559671)
+2100 val loss 6.2628
+2100 val perplexity 524.6801
+2100 train 6.310268 (lr=5.0547e-05) (hash(x)=155607137)
+2200 val loss 6.2281
+2200 val perplexity 506.7824
+2200 train 6.084721 (lr=4.9993e-05) (hash(x)=153010221)
+2300 val loss 6.1992
+2300 val perplexity 492.3499
+2300 train 6.212214 (lr=4.9410e-05) (hash(x)=160097777)
+2400 val loss 6.1760
+2400 val perplexity 481.0809
+2400 train 6.155311 (lr=4.8800e-05) (hash(x)=135590094)
+2500 val loss 6.1477
+2500 val perplexity 467.6450
+2500 train 6.267031 (lr=4.8162e-05) (hash(x)=147042929)
+2600 val loss 6.1151
+2600 val perplexity 452.6300
+2600 train 6.078469 (lr=4.7499e-05) (hash(x)=163793881)
+2700 val loss 6.0944
+2700 val perplexity 443.3677
+2700 train 6.127896 (lr=4.6811e-05) (hash(x)=153598698)
+2800 val loss 6.0819
+2800 val perplexity 437.8585
+2800 train 5.872927 (lr=4.6099e-05) (hash(x)=133439803)
+2900 val loss 6.0548
+2900 val perplexity 426.1370
+2900 train 5.988275 (lr=4.5364e-05) (hash(x)=148239158)
+3000 val loss 6.0374
+3000 val perplexity 418.8012
+3000 train 6.044248 (lr=4.4606e-05) (hash(x)=150991971)
+3100 val loss 6.0165
+3100 val perplexity 410.1316
+3100 train 6.021707 (lr=4.3828e-05) (hash(x)=157791832)
+3200 val loss 5.9870
+3200 val perplexity 398.2261
+3200 train 6.056021 (lr=4.3031e-05) (hash(x)=163746305)
+3300 val loss 5.9798
+3300 val perplexity 395.3527
+3300 train 5.901730 (lr=4.2215e-05) (hash(x)=155120269)
+3400 val loss 5.9487
+3400 val perplexity 383.2661
+3400 train 5.896263 (lr=4.1381e-05) (hash(x)=152748451)
+3500 val loss 5.9354
+3500 val perplexity 378.2086
+3500 train 5.818613 (lr=4.0532e-05) (hash(x)=147395004)
+3600 val loss 5.9183
+3600 val perplexity 371.7606
+3600 train 5.936450 (lr=3.9667e-05) (hash(x)=157554250)
+3700 val loss 5.8993
+3700 val perplexity 364.7847
+3700 train 5.752002 (lr=3.8789e-05) (hash(x)=147953321)
+3800 val loss 5.8818
+3800 val perplexity 358.4565
+3800 train 5.737122 (lr=3.7898e-05) (hash(x)=146898609)
+3900 val loss 5.8784
+3900 val perplexity 357.2249
+3900 train 5.921919 (lr=3.6996e-05) (hash(x)=150115282)
+4000 val loss 5.8495
+4000 val perplexity 347.0485
+4000 train 5.720140 (lr=3.6085e-05) (hash(x)=142448619)
+4100 val loss 5.8489
+4100 val perplexity 346.8653
+4100 train 5.590994 (lr=3.5165e-05) (hash(x)=141244123)
+4200 val loss 5.8404
+4200 val perplexity 343.9145
+4200 train 5.850804 (lr=3.4238e-05) (hash(x)=165913661)
+4300 val loss 5.8065
+4300 val perplexity 332.4451
+4300 train 5.839415 (lr=3.3305e-05) (hash(x)=155744823)
+4400 val loss 5.7989
+4400 val perplexity 329.9486
+4400 train 5.796937 (lr=3.2368e-05) (hash(x)=160461243)
+4500 val loss 5.7862
+4500 val perplexity 325.7645
+4500 train 5.693707 (lr=3.1428e-05) (hash(x)=156073129)
+4600 val loss 5.7709
+4600 val perplexity 320.8156
+4600 train 5.820288 (lr=3.0486e-05) (hash(x)=152294132)
+4700 val loss 5.7620
+4700 val perplexity 317.9967
+4700 train 5.701264 (lr=2.9543e-05) (hash(x)=140904569)
+4800 val loss 5.7532
+4800 val perplexity 315.2115
+4800 train 5.662505 (lr=2.8602e-05) (hash(x)=156191635)
+4900 val loss 5.7442
+4900 val perplexity 312.3885
+4900 train 5.856668 (lr=2.7663e-05) (hash(x)=145063976)
+5000 val loss 5.7323
+5000 val perplexity 308.6840
+5000 train 5.685281 (lr=2.6728e-05) (hash(x)=160110619)
+5100 val loss 5.7166
+5100 val perplexity 303.8726
+5100 train 5.745248 (lr=2.5798e-05) (hash(x)=156270070)
+5200 val loss 5.7070
+5200 val perplexity 300.9782
+5200 train 5.602131 (lr=2.4874e-05) (hash(x)=138954242)
+5300 val loss 5.6998
+5300 val perplexity 298.8035
+5300 train 5.549146 (lr=2.3958e-05) (hash(x)=146472367)
+5400 val loss 5.6868
+5400 val perplexity 294.9346
+5400 train 5.617336 (lr=2.3051e-05) (hash(x)=146753405)
+5500 val loss 5.6801
+5500 val perplexity 292.9851
+5500 train 5.628195 (lr=2.2155e-05) (hash(x)=147757398)
+5600 val loss 5.6737
+5600 val perplexity 291.1140
+5600 train 5.540354 (lr=2.1271e-05) (hash(x)=151291757)
+5700 val loss 5.6622
+5700 val perplexity 287.7889
+5700 train 5.716890 (lr=2.0399e-05) (hash(x)=156274392)
+5800 val loss 5.6544
+5800 val perplexity 285.5433
+5800 train 5.625447 (lr=1.9542e-05) (hash(x)=152612266)
+5900 val loss 5.6490
+5900 val perplexity 284.0074
+5900 train 5.817714 (lr=1.8700e-05) (hash(x)=148973201)
+6000 val loss 5.6398
+6000 val perplexity 281.4183
+6000 train 5.600852 (lr=1.7875e-05) (hash(x)=158311490)
+6100 val loss 5.6321
+6100 val perplexity 279.2485
+6100 train 5.568927 (lr=1.7068e-05) (hash(x)=147541319)
+6200 val loss 5.6330
+6200 val perplexity 279.5024
+6200 train 5.703368 (lr=1.6280e-05) (hash(x)=154753980)
+6300 val loss 5.6232
+6300 val perplexity 276.7728
+6300 train 5.516932 (lr=1.5512e-05) (hash(x)=140798998)
+6400 val loss 5.6174
+6400 val perplexity 275.1835
+6400 train 5.505464 (lr=1.4766e-05) (hash(x)=149299516)
+6500 val loss 5.6123
+6500 val perplexity 273.7734
+6500 train 5.594793 (lr=1.4042e-05) (hash(x)=148495806)
+6600 val loss 5.6055
+6600 val perplexity 271.9237
+6600 train 5.585682 (lr=1.3342e-05) (hash(x)=153664387)
+6700 val loss 5.6028
+6700 val perplexity 271.1919
+6700 train 5.585989 (lr=1.2666e-05) (hash(x)=153333491)
+6800 val loss 5.6044
+6800 val perplexity 271.6152
+6800 train 5.347058 (lr=1.2016e-05) (hash(x)=150149382)
+6900 val loss 5.5961
+6900 val perplexity 269.3717
+6900 train 5.685107 (lr=1.1392e-05) (hash(x)=152379767)
+7000 val loss 5.5877
+7000 val perplexity 267.1243
+7000 train 5.639709 (lr=1.0795e-05) (hash(x)=150432537)
+7100 val loss 5.5885
+7100 val perplexity 267.3259
+7100 train 5.467541 (lr=1.0227e-05) (hash(x)=148277836)
+7200 val loss 5.5859
+7200 val perplexity 266.6448
+7200 train 5.576658 (lr=9.6875e-06) (hash(x)=168591659)
+7300 val loss 5.5788
+7300 val perplexity 264.7546
+7300 train 5.557188 (lr=9.1780e-06) (hash(x)=152764851)
+7400 val loss 5.5746
+7400 val perplexity 263.6494
+7400 train 5.541438 (lr=8.6990e-06) (hash(x)=142920131)
+7500 val loss 5.5725
+7500 val perplexity 263.0795
+7500 train 5.570638 (lr=8.2513e-06) (hash(x)=146954649)
+7600 val loss 5.5725
+7600 val perplexity 263.0981
+7600 train 5.347485 (lr=7.8355e-06) (hash(x)=144529566)
+7700 val loss 5.5669
+7700 val perplexity 261.6309
+7700 train 5.527046 (lr=7.4522e-06) (hash(x)=142669945)
+7800 val loss 5.5628
+7800 val perplexity 260.5567
+7800 train 5.617598 (lr=7.1019e-06) (hash(x)=161563764)
+7900 val loss 5.5607
+7900 val perplexity 260.0027
+7900 train 5.355335 (lr=6.7852e-06) (hash(x)=141516532)
+8000 val loss 5.5610
+8000 val perplexity 260.0852
+8000 train 5.424737 (lr=6.5025e-06) (hash(x)=152418457)
+8100 val loss 5.5556
+8100 val perplexity 258.6718
+8100 train 5.545348 (lr=6.2543e-06) (hash(x)=161109470)
+8200 val loss 5.5520
+8200 val perplexity 257.7588
+8200 train 5.681771 (lr=6.0408e-06) (hash(x)=154408822)
+8300 val loss 5.5499
+8300 val perplexity 257.2069
+8300 train 5.438511 (lr=5.8625e-06) (hash(x)=146431691)
+8400 val loss 5.5507
+8400 val perplexity 257.4096
+8400 train 5.328045 (lr=5.7195e-06) (hash(x)=150182527)
+8500 val loss 5.5443
+8500 val perplexity 255.7633
+8500 train 5.612017 (lr=5.6121e-06) (hash(x)=154828709)
+8600 val loss 5.5440
+8600 val perplexity 255.7007
+8600 train 5.459042 (lr=5.5404e-06) (hash(x)=154995428)
+8700 val loss 5.5439
+8700 val perplexity 255.6694
+8700 train 5.444529 (lr=5.5045e-06) (hash(x)=142111043)
+8749 val loss 5.5424
+8749 val perplexity 255.2784

attention_kindselective_n_heads4_seed1342/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed254524176e23c51719c63044e63e56c50e88d2fa6b66c49078d9e232a82ccd
+oid sha256:957e725a82dcd9ed1bd037691b360b3021a01371484023e99935b6d8a8215591
 size 92843394

attention_kindselective_n_heads4_seed1342/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ec491cf0d3e027e7f68059046708aaa88b97538de7373c8531f904bf4345641
+oid sha256:72bda2c05a567a2bc11a95bfe22019cdc84340ef8797165189edcf66c1a6746e
 size 179406214