Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1345/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_4_1345", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5.5e-5_61440_4_1345", "n_embd": 256}

attention_kindselective_n_heads4_seed1345/log2.txt

@@ -1,267 +1,267 @@
 max_steps: 8750
 0 val loss 11.2669
-0 val perplexity 78187.7734
-0 train 11.258430 (lr=1.2000e-07) (hash(x)=134751525)
-100 val loss 9.9839
-100 val perplexity 21675.4258
-100 train 10.008630 (lr=1.2120e-05) (hash(x)=150358957)
-200 val loss 9.1733
-200 val perplexity 9635.9756
-200 train 9.176708 (lr=2.4120e-05) (hash(x)=126319983)
-300 val loss 7.9611
-300 val perplexity 2867.2952
-300 train 7.902992 (lr=3.6120e-05) (hash(x)=159305874)
-400 val loss 7.5449
-400 val perplexity 1891.1628
-400 train 7.521968 (lr=4.8120e-05) (hash(x)=149855631)
-500 val loss 7.3894
-500 val perplexity 1618.7214
-500 train 7.420051 (lr=6.0000e-05) (hash(x)=150706760)
-600 val loss 7.2891
-600 val perplexity 1464.2700
-600 train 7.219808 (lr=5.9980e-05) (hash(x)=146858163)
-700 val loss 7.2209
-700 val perplexity 1367.7762
-700 train 7.094349 (lr=5.9922e-05) (hash(x)=144262072)
-800 val loss 7.1403
-800 val perplexity 1261.8079
-800 train 7.109925 (lr=5.9824e-05) (hash(x)=135443880)
-900 val loss 7.0661
-900 val perplexity 1171.6042
-900 train 7.008464 (lr=5.9687e-05) (hash(x)=153147388)
-1000 val loss 7.0039
-1000 val perplexity 1100.8654
-1000 train 6.929332 (lr=5.9512e-05) (hash(x)=151448445)
-1100 val loss 6.9303
-1100 val perplexity 1022.8132
-1100 train 6.935490 (lr=5.9298e-05) (hash(x)=153161010)
-1200 val loss 6.8744
-1200 val perplexity 967.1544
-1200 train 6.700087 (lr=5.9046e-05) (hash(x)=143227423)
-1300 val loss 6.8295
-1300 val perplexity 924.6844
-1300 train 7.299239 (lr=5.8757e-05) (hash(x)=176373796)
-1400 val loss 6.7762
-1400 val perplexity 876.6993
-1400 train 6.702026 (lr=5.8430e-05) (hash(x)=155989503)
-1500 val loss 6.7162
-1500 val perplexity 825.6920
-1500 train 6.698337 (lr=5.8066e-05) (hash(x)=156507542)
-1600 val loss 6.6620
-1600 val perplexity 782.1313
-1600 train 6.663013 (lr=5.7666e-05) (hash(x)=156078901)
-1700 val loss 6.6072
-1700 val perplexity 740.3943
-1700 train 6.513906 (lr=5.7230e-05) (hash(x)=156889457)
-1800 val loss 6.5688
-1800 val perplexity 712.5136
-1800 train 6.808304 (lr=5.6759e-05) (hash(x)=162477906)
-1900 val loss 6.5196
-1900 val perplexity 678.3251
-1900 train 6.565875 (lr=5.6253e-05) (hash(x)=158648033)
-2000 val loss 6.4788
-2000 val perplexity 651.1573
-2000 train 6.401828 (lr=5.5714e-05) (hash(x)=156732586)
-2100 val loss 6.4433
-2100 val perplexity 628.4625
-2100 train 6.465071 (lr=5.5142e-05) (hash(x)=159721084)
-2200 val loss 6.3954
-2200 val perplexity 599.0789
-2200 train 6.372261 (lr=5.4537e-05) (hash(x)=144907001)
-2300 val loss 6.3756
-2300 val perplexity 587.3514
-2300 train 6.315606 (lr=5.3902e-05) (hash(x)=154541837)
-2400 val loss 6.3586
-2400 val perplexity 577.4647
-2400 train 6.445983 (lr=5.3236e-05) (hash(x)=161706790)
-2500 val loss 6.3219
-2500 val perplexity 556.6255
-2500 train 6.244786 (lr=5.2541e-05) (hash(x)=143556162)
-2600 val loss 6.2845
-2600 val perplexity 536.1711
-2600 train 6.136559 (lr=5.1817e-05) (hash(x)=142425078)
-2700 val loss 6.2612
-2700 val perplexity 523.8307
-2700 train 6.278818 (lr=5.1067e-05) (hash(x)=163195606)
-2800 val loss 6.2290
-2800 val perplexity 507.2432
-2800 train 6.131935 (lr=5.0290e-05) (hash(x)=154078337)
-2900 val loss 6.2198
-2900 val perplexity 502.5972
-2900 train 6.084310 (lr=4.9487e-05) (hash(x)=145186687)
-3000 val loss 6.1924
-3000 val perplexity 489.0109
-3000 train 6.149713 (lr=4.8662e-05) (hash(x)=149726716)
-3100 val loss 6.1746
-3100 val perplexity 480.3961
-3100 train 6.102753 (lr=4.7813e-05) (hash(x)=151122509)
-3200 val loss 6.1655
-3200 val perplexity 476.0258
-3200 train 6.135470 (lr=4.6943e-05) (hash(x)=152172187)
-3300 val loss 6.1460
-3300 val perplexity 466.8445
-3300 train 6.080579 (lr=4.6052e-05) (hash(x)=150581974)
-3400 val loss 6.1241
-3400 val perplexity 456.7542
-3400 train 6.051082 (lr=4.5143e-05) (hash(x)=160061666)
-3500 val loss 6.1072
-3500 val perplexity 449.0678
-3500 train 6.054682 (lr=4.4216e-05) (hash(x)=150556913)
-3600 val loss 6.0908
-3600 val perplexity 441.7588
-3600 train 5.995741 (lr=4.3273e-05) (hash(x)=148311961)
-3700 val loss 6.0760
-3700 val perplexity 435.2701
-3700 train 5.968667 (lr=4.2315e-05) (hash(x)=150305284)
-3800 val loss 6.0523
-3800 val perplexity 425.0813
-3800 train 6.064778 (lr=4.1343e-05) (hash(x)=148428531)
-3900 val loss 6.0355
-3900 val perplexity 418.0211
-3900 train 5.900431 (lr=4.0360e-05) (hash(x)=142448374)
-4000 val loss 6.0270
-4000 val perplexity 414.4525
-4000 train 5.948363 (lr=3.9365e-05) (hash(x)=158002288)
-4100 val loss 6.0058
-4100 val perplexity 405.7936
-4100 train 5.948852 (lr=3.8362e-05) (hash(x)=151388626)
-4200 val loss 5.9921
-4200 val perplexity 400.2397
-4200 train 5.912879 (lr=3.7351e-05) (hash(x)=145123294)
-4300 val loss 5.9760
-4300 val perplexity 393.8557
-4300 train 5.848454 (lr=3.6333e-05) (hash(x)=145869150)
-4400 val loss 5.9646
-4400 val perplexity 389.3833
-4400 train 5.987700 (lr=3.5311e-05) (hash(x)=146923320)
-4500 val loss 5.9459
-4500 val perplexity 382.1999
-4500 train 5.768764 (lr=3.4285e-05) (hash(x)=146720819)
-4600 val loss 5.9323
-4600 val perplexity 377.0094
-4600 train 5.676044 (lr=3.3257e-05) (hash(x)=140342909)
-4700 val loss 5.9234
-4700 val perplexity 373.6711
-4700 train 6.046507 (lr=3.2229e-05) (hash(x)=163866463)
-4800 val loss 5.9108
-4800 val perplexity 369.0188
-4800 train 5.595890 (lr=3.1202e-05) (hash(x)=133459145)
-4900 val loss 5.8975
-4900 val perplexity 364.1420
-4900 train 5.693576 (lr=3.0178e-05) (hash(x)=143144356)
-5000 val loss 5.9056
-5000 val perplexity 367.0866
-5000 train 5.802285 (lr=2.9157e-05) (hash(x)=132636494)
-5100 val loss 5.8668
-5100 val perplexity 353.1207
-5100 train 5.786910 (lr=2.8143e-05) (hash(x)=157278728)
-5200 val loss 5.8582
-5200 val perplexity 350.0958
-5200 train 5.717509 (lr=2.7135e-05) (hash(x)=148198434)
-5300 val loss 5.8443
-5300 val perplexity 345.2503
-5300 train 5.772146 (lr=2.6136e-05) (hash(x)=138380906)
-5400 val loss 5.8313
-5400 val perplexity 340.8168
-5400 train 5.920127 (lr=2.5147e-05) (hash(x)=163441464)
-5500 val loss 5.8272
-5500 val perplexity 339.4170
-5500 train 5.771481 (lr=2.4169e-05) (hash(x)=154347714)
-5600 val loss 5.8127
-5600 val perplexity 334.5254
-5600 train 5.842416 (lr=2.3204e-05) (hash(x)=148449981)
-5700 val loss 5.8027
-5700 val perplexity 331.1774
-5700 train 5.693919 (lr=2.2253e-05) (hash(x)=139300274)
-5800 val loss 5.7951
-5800 val perplexity 328.6917
-5800 train 5.783244 (lr=2.1318e-05) (hash(x)=154857144)
-5900 val loss 5.7837
-5900 val perplexity 324.9723
-5900 train 5.845992 (lr=2.0400e-05) (hash(x)=151756013)
-6000 val loss 5.7775
-6000 val perplexity 322.9481
-6000 train 5.678282 (lr=1.9500e-05) (hash(x)=145414657)
-6100 val loss 5.7698
-6100 val perplexity 320.4596
-6100 train 5.592708 (lr=1.8620e-05) (hash(x)=139501217)
-6200 val loss 5.7579
-6200 val perplexity 316.6911
-6200 train 5.798489 (lr=1.7760e-05) (hash(x)=155783358)
-6300 val loss 5.7509
-6300 val perplexity 314.4768
-6300 train 5.809797 (lr=1.6923e-05) (hash(x)=157033091)
-6400 val loss 5.7465
-6400 val perplexity 313.1020
-6400 train 5.552113 (lr=1.6108e-05) (hash(x)=144475330)
-6500 val loss 5.7366
-6500 val perplexity 310.0059
-6500 train 5.773449 (lr=1.5319e-05) (hash(x)=124041822)
-6600 val loss 5.7329
-6600 val perplexity 308.8680
-6600 train 5.667017 (lr=1.4555e-05) (hash(x)=142131981)
-6700 val loss 5.7213
-6700 val perplexity 305.3057
-6700 train 5.823481 (lr=1.3817e-05) (hash(x)=154916248)
-6800 val loss 5.7211
-6800 val perplexity 305.2263
-6800 train 5.719387 (lr=1.3108e-05) (hash(x)=147996387)
-6900 val loss 5.7133
-6900 val perplexity 302.8786
-6900 train 5.747306 (lr=1.2427e-05) (hash(x)=153422428)
-7000 val loss 5.7054
-7000 val perplexity 300.4824
-7000 train 5.728880 (lr=1.1777e-05) (hash(x)=174135078)
-7100 val loss 5.7028
-7100 val perplexity 299.7173
-7100 train 5.778639 (lr=1.1157e-05) (hash(x)=160382475)
-7200 val loss 5.6978
-7200 val perplexity 298.2030
-7200 train 5.788217 (lr=1.0568e-05) (hash(x)=155310085)
-7300 val loss 5.6913
-7300 val perplexity 296.2772
-7300 train 5.824478 (lr=1.0012e-05) (hash(x)=159329031)
-7400 val loss 5.6893
-7400 val perplexity 295.6850
-7400 train 5.633309 (lr=9.4899e-06) (hash(x)=151577269)
-7500 val loss 5.6847
-7500 val perplexity 294.3328
-7500 train 5.734163 (lr=9.0014e-06) (hash(x)=160620287)
-7600 val loss 5.6801
-7600 val perplexity 292.9817
-7600 train 5.569936 (lr=8.5478e-06) (hash(x)=143711597)
-7700 val loss 5.6764
-7700 val perplexity 291.8844
-7700 train 5.676998 (lr=8.1297e-06) (hash(x)=149243577)
-7800 val loss 5.6739
-7800 val perplexity 291.1786
-7800 train 5.664146 (lr=7.7476e-06) (hash(x)=147677247)
-7900 val loss 5.6663
-7900 val perplexity 288.9625
-7900 train 5.614513 (lr=7.4021e-06) (hash(x)=145801236)
-8000 val loss 5.6648
-8000 val perplexity 288.5190
-8000 train 5.780615 (lr=7.0937e-06) (hash(x)=158007655)
-8100 val loss 5.6641
-8100 val perplexity 288.3269
-8100 train 5.724568 (lr=6.8229e-06) (hash(x)=158030048)
-8200 val loss 5.6592
-8200 val perplexity 286.9110
-8200 train 5.510582 (lr=6.5900e-06) (hash(x)=150674725)
-8300 val loss 5.6571
-8300 val perplexity 286.3263
-8300 train 5.562442 (lr=6.3954e-06) (hash(x)=150805711)
-8400 val loss 5.6521
-8400 val perplexity 284.8779
-8400 train 5.526485 (lr=6.2395e-06) (hash(x)=138396210)
-8500 val loss 5.6513
-8500 val perplexity 284.6625
-8500 train 5.528487 (lr=6.1223e-06) (hash(x)=156172740)
-8600 val loss 5.6482
-8600 val perplexity 283.7753
-8600 train 5.684916 (lr=6.0440e-06) (hash(x)=154911352)
-8700 val loss 5.6456
-8700 val perplexity 283.0405
-8700 train 5.704347 (lr=6.0049e-06) (hash(x)=153446789)
-8749 val loss 5.6443
-8749 val perplexity 282.6697
+0 val perplexity 78187.8516
+0 train 11.258394 (lr=1.1000e-07) (hash(x)=134751525)
+100 val loss 9.9976
+100 val perplexity 21974.2227
+100 train 10.021976 (lr=1.1110e-05) (hash(x)=150358957)
+200 val loss 9.2695
+200 val perplexity 10609.2920
+200 train 9.278807 (lr=2.2110e-05) (hash(x)=126319983)
+300 val loss 8.0105
+300 val perplexity 3012.2820
+300 train 7.949356 (lr=3.3110e-05) (hash(x)=159305874)
+400 val loss 7.5227
+400 val perplexity 1849.6167
+400 train 7.498507 (lr=4.4110e-05) (hash(x)=149855631)
+500 val loss 7.3664
+500 val perplexity 1581.9810
+500 train 7.399071 (lr=5.5000e-05) (hash(x)=150706760)
+600 val loss 7.2326
+600 val perplexity 1383.7897
+600 train 7.160625 (lr=5.4982e-05) (hash(x)=146858163)
+700 val loss 7.1190
+700 val perplexity 1235.2375
+700 train 6.990807 (lr=5.4928e-05) (hash(x)=144262072)
+800 val loss 6.9961
+800 val perplexity 1092.3348
+800 train 6.965110 (lr=5.4839e-05) (hash(x)=135443880)
+900 val loss 6.8929
+900 val perplexity 985.2578
+900 train 6.839639 (lr=5.4713e-05) (hash(x)=153147388)
+1000 val loss 6.8104
+1000 val perplexity 907.2111
+1000 train 6.732587 (lr=5.4553e-05) (hash(x)=151448445)
+1100 val loss 6.7108
+1100 val perplexity 821.2675
+1100 train 6.725699 (lr=5.4357e-05) (hash(x)=153161010)
+1200 val loss 6.6303
+1200 val perplexity 757.6766
+1200 train 6.457338 (lr=5.4126e-05) (hash(x)=143227423)
+1300 val loss 6.5620
+1300 val perplexity 707.6649
+1300 train 7.021236 (lr=5.3860e-05) (hash(x)=176373796)
+1400 val loss 6.5022
+1400 val perplexity 666.6238
+1400 train 6.430148 (lr=5.3561e-05) (hash(x)=155989503)
+1500 val loss 6.4467
+1500 val perplexity 630.6347
+1500 train 6.422615 (lr=5.3227e-05) (hash(x)=156507542)
+1600 val loss 6.4011
+1600 val perplexity 602.5052
+1600 train 6.387425 (lr=5.2860e-05) (hash(x)=156078901)
+1700 val loss 6.3629
+1700 val perplexity 579.9485
+1700 train 6.269570 (lr=5.2461e-05) (hash(x)=156889457)
+1800 val loss 6.3314
+1800 val perplexity 561.9170
+1800 train 6.587677 (lr=5.2029e-05) (hash(x)=162477906)
+1900 val loss 6.2904
+1900 val perplexity 539.3497
+1900 train 6.326413 (lr=5.1565e-05) (hash(x)=158648033)
+2000 val loss 6.2677
+2000 val perplexity 527.2448
+2000 train 6.195924 (lr=5.1071e-05) (hash(x)=156732586)
+2100 val loss 6.2429
+2100 val perplexity 514.3626
+2100 train 6.257056 (lr=5.0547e-05) (hash(x)=159721084)
+2200 val loss 6.2077
+2200 val perplexity 496.5814
+2200 train 6.200092 (lr=4.9993e-05) (hash(x)=144907001)
+2300 val loss 6.2001
+2300 val perplexity 492.7748
+2300 train 6.144407 (lr=4.9410e-05) (hash(x)=154541837)
+2400 val loss 6.1663
+2400 val perplexity 476.4173
+2400 train 6.260852 (lr=4.8800e-05) (hash(x)=161706790)
+2500 val loss 6.1369
+2500 val perplexity 462.6117
+2500 train 6.061900 (lr=4.8162e-05) (hash(x)=143556162)
+2600 val loss 6.1219
+2600 val perplexity 455.7222
+2600 train 5.966206 (lr=4.7499e-05) (hash(x)=142425078)
+2700 val loss 6.0938
+2700 val perplexity 443.0906
+2700 train 6.117737 (lr=4.6811e-05) (hash(x)=163195606)
+2800 val loss 6.0680
+2800 val perplexity 431.8334
+2800 train 5.982263 (lr=4.6099e-05) (hash(x)=154078337)
+2900 val loss 6.0549
+2900 val perplexity 426.1941
+2900 train 5.913872 (lr=4.5364e-05) (hash(x)=145186687)
+3000 val loss 6.0264
+3000 val perplexity 414.2376
+3000 train 5.994296 (lr=4.4606e-05) (hash(x)=149726716)
+3100 val loss 6.0056
+3100 val perplexity 405.6932
+3100 train 5.927385 (lr=4.3828e-05) (hash(x)=151122509)
+3200 val loss 5.9966
+3200 val perplexity 402.0695
+3200 train 5.966908 (lr=4.3031e-05) (hash(x)=152172187)
+3300 val loss 5.9800
+3300 val perplexity 395.4515
+3300 train 5.916352 (lr=4.2215e-05) (hash(x)=150581974)
+3400 val loss 5.9623
+3400 val perplexity 388.5039
+3400 train 5.884938 (lr=4.1381e-05) (hash(x)=160061666)
+3500 val loss 5.9419
+3500 val perplexity 380.6590
+3500 train 5.894887 (lr=4.0532e-05) (hash(x)=150556913)
+3600 val loss 5.9305
+3600 val perplexity 376.3459
+3600 train 5.835590 (lr=3.9667e-05) (hash(x)=148311961)
+3700 val loss 5.9138
+3700 val perplexity 370.1077
+3700 train 5.802270 (lr=3.8789e-05) (hash(x)=150305284)
+3800 val loss 5.8978
+3800 val perplexity 364.2344
+3800 train 5.900220 (lr=3.7898e-05) (hash(x)=148428531)
+3900 val loss 5.8802
+3900 val perplexity 357.8729
+3900 train 5.757717 (lr=3.6996e-05) (hash(x)=142448374)
+4000 val loss 5.8676
+4000 val perplexity 353.4018
+4000 train 5.796079 (lr=3.6085e-05) (hash(x)=158002288)
+4100 val loss 5.8497
+4100 val perplexity 347.1451
+4100 train 5.784432 (lr=3.5165e-05) (hash(x)=151388626)
+4200 val loss 5.8321
+4200 val perplexity 341.0719
+4200 train 5.756764 (lr=3.4238e-05) (hash(x)=145123294)
+4300 val loss 5.8195
+4300 val perplexity 336.8081
+4300 train 5.703814 (lr=3.3305e-05) (hash(x)=145869150)
+4400 val loss 5.8130
+4400 val perplexity 334.6342
+4400 train 5.843296 (lr=3.2368e-05) (hash(x)=146923320)
+4500 val loss 5.7975
+4500 val perplexity 329.4873
+4500 train 5.628051 (lr=3.1428e-05) (hash(x)=146720819)
+4600 val loss 5.7863
+4600 val perplexity 325.7993
+4600 train 5.543585 (lr=3.0486e-05) (hash(x)=140342909)
+4700 val loss 5.7761
+4700 val perplexity 322.4923
+4700 train 5.914660 (lr=2.9543e-05) (hash(x)=163866463)
+4800 val loss 5.7574
+4800 val perplexity 316.5242
+4800 train 5.449148 (lr=2.8602e-05) (hash(x)=133459145)
+4900 val loss 5.7496
+4900 val perplexity 314.0524
+4900 train 5.549150 (lr=2.7663e-05) (hash(x)=143144356)
+5000 val loss 5.7556
+5000 val perplexity 315.9481
+5000 train 5.664907 (lr=2.6728e-05) (hash(x)=132636494)
+5100 val loss 5.7271
+5100 val perplexity 307.0887
+5100 train 5.646737 (lr=2.5798e-05) (hash(x)=157278728)
+5200 val loss 5.7195
+5200 val perplexity 304.7547
+5200 train 5.568845 (lr=2.4874e-05) (hash(x)=148198434)
+5300 val loss 5.7074
+5300 val perplexity 301.0816
+5300 train 5.628862 (lr=2.3958e-05) (hash(x)=138380906)
+5400 val loss 5.6969
+5400 val perplexity 297.9376
+5400 train 5.790673 (lr=2.3051e-05) (hash(x)=163441464)
+5500 val loss 5.6898
+5500 val perplexity 295.8444
+5500 train 5.624358 (lr=2.2155e-05) (hash(x)=154347714)
+5600 val loss 5.6831
+5600 val perplexity 293.8553
+5600 train 5.732700 (lr=2.1271e-05) (hash(x)=148449981)
+5700 val loss 5.6700
+5700 val perplexity 290.0321
+5700 train 5.560979 (lr=2.0399e-05) (hash(x)=139300274)
+5800 val loss 5.6666
+5800 val perplexity 289.0583
+5800 train 5.662941 (lr=1.9542e-05) (hash(x)=154857144)
+5900 val loss 5.6567
+5900 val perplexity 286.1906
+5900 train 5.713377 (lr=1.8700e-05) (hash(x)=151756013)
+6000 val loss 5.6536
+6000 val perplexity 285.3265
+6000 train 5.558116 (lr=1.7875e-05) (hash(x)=145414657)
+6100 val loss 5.6455
+6100 val perplexity 283.0208
+6100 train 5.462373 (lr=1.7068e-05) (hash(x)=139501217)
+6200 val loss 5.6348
+6200 val perplexity 279.9936
+6200 train 5.671405 (lr=1.6280e-05) (hash(x)=155783358)
+6300 val loss 5.6287
+6300 val perplexity 278.2988
+6300 train 5.696047 (lr=1.5512e-05) (hash(x)=157033091)
+6400 val loss 5.6260
+6400 val perplexity 277.5588
+6400 train 5.424657 (lr=1.4766e-05) (hash(x)=144475330)
+6500 val loss 5.6181
+6500 val perplexity 275.3672
+6500 train 5.671980 (lr=1.4042e-05) (hash(x)=124041822)
+6600 val loss 5.6151
+6600 val perplexity 274.5500
+6600 train 5.548001 (lr=1.3342e-05) (hash(x)=142131981)
+6700 val loss 5.6043
+6700 val perplexity 271.5820
+6700 train 5.703641 (lr=1.2666e-05) (hash(x)=154916248)
+6800 val loss 5.6080
+6800 val perplexity 272.5901
+6800 train 5.602819 (lr=1.2016e-05) (hash(x)=147996387)
+6900 val loss 5.5998
+6900 val perplexity 270.3646
+6900 train 5.625703 (lr=1.1392e-05) (hash(x)=153422428)
+7000 val loss 5.5910
+7000 val perplexity 268.0145
+7000 train 5.609641 (lr=1.0795e-05) (hash(x)=174135078)
+7100 val loss 5.5899
+7100 val perplexity 267.7013
+7100 train 5.663852 (lr=1.0227e-05) (hash(x)=160382475)
+7200 val loss 5.5857
+7200 val perplexity 266.5980
+7200 train 5.676028 (lr=9.6875e-06) (hash(x)=155310085)
+7300 val loss 5.5798
+7300 val perplexity 265.0145
+7300 train 5.710726 (lr=9.1780e-06) (hash(x)=159329031)
+7400 val loss 5.5787
+7400 val perplexity 264.7280
+7400 train 5.517731 (lr=8.6990e-06) (hash(x)=151577269)
+7500 val loss 5.5763
+7500 val perplexity 264.0891
+7500 train 5.629300 (lr=8.2513e-06) (hash(x)=160620287)
+7600 val loss 5.5709
+7600 val perplexity 262.6826
+7600 train 5.452636 (lr=7.8355e-06) (hash(x)=143711597)
+7700 val loss 5.5694
+7700 val perplexity 262.2819
+7700 train 5.560287 (lr=7.4522e-06) (hash(x)=149243577)
+7800 val loss 5.5679
+7800 val perplexity 261.8705
+7800 train 5.552493 (lr=7.1019e-06) (hash(x)=147677247)
+7900 val loss 5.5609
+7900 val perplexity 260.0510
+7900 train 5.508999 (lr=6.7852e-06) (hash(x)=145801236)
+8000 val loss 5.5601
+8000 val perplexity 259.8535
+8000 train 5.676331 (lr=6.5025e-06) (hash(x)=158007655)
+8100 val loss 5.5592
+8100 val perplexity 259.6133
+8100 train 5.619223 (lr=6.2543e-06) (hash(x)=158030048)
+8200 val loss 5.5549
+8200 val perplexity 258.5128
+8200 train 5.401045 (lr=6.0408e-06) (hash(x)=150674725)
+8300 val loss 5.5543
+8300 val perplexity 258.3459
+8300 train 5.462399 (lr=5.8625e-06) (hash(x)=150805711)
+8400 val loss 5.5496
+8400 val perplexity 257.1231
+8400 train 5.412580 (lr=5.7195e-06) (hash(x)=138396210)
+8500 val loss 5.5492
+8500 val perplexity 257.0238
+8500 train 5.425281 (lr=5.6121e-06) (hash(x)=156172740)
+8600 val loss 5.5468
+8600 val perplexity 256.4105
+8600 train 5.577955 (lr=5.5404e-06) (hash(x)=154911352)
+8700 val loss 5.5442
+8700 val perplexity 255.7428
+8700 train 5.603814 (lr=5.5045e-06) (hash(x)=153446789)
+8749 val loss 5.5424
+8749 val perplexity 255.2849

attention_kindselective_n_heads4_seed1345/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cbe65aa68ce8a1b132bdcc93dd306f10bd3d23a968a6861cbaed251ab68445e7
+oid sha256:a1de0ab893d2a408a9264a4952538b0c26197f2b53386e7e3c4c157047ba65d4
 size 92843394

attention_kindselective_n_heads4_seed1345/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c491b43071fa7701cbaa61eb611374932299ce3707e387499ccfc8d42d33b12
+oid sha256:85575ed77760a9c620403c462e5192c0729b3c617206a46c275ae26232b8f68b
 size 179406214