Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1345/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4.5e-5_61440_4_1345", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1345", "n_embd": 256}

attention_kindselective_n_heads4_seed1345/log2.txt

@@ -1,267 +1,531 @@
 max_steps: 8750
 0 val loss 11.2669
 0 val perplexity 78187.7734
-0 train 11.258421 (lr=9.0000e-08) (hash(x)=134751525)
-100 val loss 10.0591
-100 val perplexity 23368.2949
-100 train 10.084015 (lr=9.0900e-06) (hash(x)=150358957)
-200 val loss 9.4623
-200 val perplexity 12865.2119
-200 train 9.482208 (lr=1.8090e-05) (hash(x)=126319983)
-300 val loss 8.2846
-300 val perplexity 3962.3010
-300 train 8.218966 (lr=2.7090e-05) (hash(x)=159305874)
-400 val loss 7.6156
-400 val perplexity 2029.5901
-400 train 7.596677 (lr=3.6090e-05) (hash(x)=149855631)
-500 val loss 7.4200
-500 val perplexity 1669.0735
-500 train 7.448377 (lr=4.5000e-05) (hash(x)=150706760)
-600 val loss 7.3022
-600 val perplexity 1483.5238
-600 train 7.233404 (lr=4.4985e-05) (hash(x)=146858163)
-700 val loss 7.1946
-700 val perplexity 1332.2479
-700 train 7.066607 (lr=4.4941e-05) (hash(x)=144262072)
-800 val loss 7.0810
-800 val perplexity 1189.1654
-800 train 7.053042 (lr=4.4868e-05) (hash(x)=135443880)
-900 val loss 6.9834
-900 val perplexity 1078.5498
-900 train 6.929296 (lr=4.4766e-05) (hash(x)=153147388)
-1000 val loss 6.9159
-1000 val perplexity 1008.2147
-1000 train 6.843293 (lr=4.4634e-05) (hash(x)=151448445)
-1100 val loss 6.8259
-1100 val perplexity 921.3812
-1100 train 6.837877 (lr=4.4474e-05) (hash(x)=153161010)
-1200 val loss 6.7506
-1200 val perplexity 854.6096
-1200 train 6.578292 (lr=4.4285e-05) (hash(x)=143227423)
-1300 val loss 6.6868
-1300 val perplexity 801.7579
-1300 train 7.172554 (lr=4.4068e-05) (hash(x)=176373796)
-1400 val loss 6.6160
-1400 val perplexity 746.9671
-1400 train 6.540329 (lr=4.3822e-05) (hash(x)=155989503)
-1500 val loss 6.5556
-1500 val perplexity 703.1462
-1500 train 6.532267 (lr=4.3549e-05) (hash(x)=156507542)
-1600 val loss 6.5053
-1600 val perplexity 668.6906
-1600 train 6.507467 (lr=4.3249e-05) (hash(x)=156078901)
-1700 val loss 6.4618
-1700 val perplexity 640.2064
-1700 train 6.365796 (lr=4.2922e-05) (hash(x)=156889457)
-1800 val loss 6.4338
-1800 val perplexity 622.5391
-1800 train 6.691582 (lr=4.2569e-05) (hash(x)=162477906)
-1900 val loss 6.3811
-1900 val perplexity 590.5602
-1900 train 6.422180 (lr=4.2190e-05) (hash(x)=158648033)
-2000 val loss 6.3547
-2000 val perplexity 575.2073
-2000 train 6.281474 (lr=4.1785e-05) (hash(x)=156732586)
-2100 val loss 6.3244
-2100 val perplexity 558.0005
-2100 train 6.340858 (lr=4.1356e-05) (hash(x)=159721084)
-2200 val loss 6.2931
-2200 val perplexity 540.8015
-2200 train 6.273417 (lr=4.0903e-05) (hash(x)=144907001)
-2300 val loss 6.2683
-2300 val perplexity 527.5813
-2300 train 6.208333 (lr=4.0426e-05) (hash(x)=154541837)
-2400 val loss 6.2421
-2400 val perplexity 513.9549
-2400 train 6.331748 (lr=3.9927e-05) (hash(x)=161706790)
-2500 val loss 6.2153
-2500 val perplexity 500.3525
-2500 train 6.141446 (lr=3.9406e-05) (hash(x)=143556162)
-2600 val loss 6.1963
-2600 val perplexity 490.9108
-2600 train 6.050023 (lr=3.8863e-05) (hash(x)=142425078)
-2700 val loss 6.1650
-2700 val perplexity 475.8134
-2700 train 6.189079 (lr=3.8300e-05) (hash(x)=163195606)
-2800 val loss 6.1348
-2800 val perplexity 461.6595
-2800 train 6.041891 (lr=3.7717e-05) (hash(x)=154078337)
-2900 val loss 6.1197
-2900 val perplexity 454.7504
-2900 train 5.981575 (lr=3.7116e-05) (hash(x)=145186687)
-3000 val loss 6.0966
-3000 val perplexity 444.3332
-3000 train 6.058323 (lr=3.6496e-05) (hash(x)=149726716)
-3100 val loss 6.0758
-3100 val perplexity 435.2021
-3100 train 6.000450 (lr=3.5860e-05) (hash(x)=151122509)
-3200 val loss 6.0591
-3200 val perplexity 427.9922
-3200 train 6.028750 (lr=3.5207e-05) (hash(x)=152172187)
-3300 val loss 6.0385
-3300 val perplexity 419.2482
-3300 train 5.972092 (lr=3.4539e-05) (hash(x)=150581974)
-3400 val loss 6.0271
-3400 val perplexity 414.4937
-3400 train 5.958289 (lr=3.3857e-05) (hash(x)=160061666)
-3500 val loss 6.0029
-3500 val perplexity 404.6082
-3500 train 5.951034 (lr=3.3162e-05) (hash(x)=150556913)
-3600 val loss 5.9885
-3600 val perplexity 398.8278
-3600 train 5.896544 (lr=3.2455e-05) (hash(x)=148311961)
-3700 val loss 5.9747
-3700 val perplexity 393.3694
-3700 train 5.866710 (lr=3.1736e-05) (hash(x)=150305284)
-3800 val loss 5.9561
-3800 val perplexity 386.0872
-3800 train 5.966131 (lr=3.1008e-05) (hash(x)=148428531)
-3900 val loss 5.9435
-3900 val perplexity 381.2489
-3900 train 5.814651 (lr=3.0270e-05) (hash(x)=142448374)
-4000 val loss 5.9283
-4000 val perplexity 375.5012
-4000 train 5.862759 (lr=2.9524e-05) (hash(x)=158002288)
-4100 val loss 5.9091
-4100 val perplexity 368.3666
-4100 train 5.850963 (lr=2.8771e-05) (hash(x)=151388626)
-4200 val loss 5.8978
-4200 val perplexity 364.2400
-4200 train 5.821980 (lr=2.8013e-05) (hash(x)=145123294)
-4300 val loss 5.8837
-4300 val perplexity 359.1506
-4300 train 5.765946 (lr=2.7250e-05) (hash(x)=145869150)
-4400 val loss 5.8761
-4400 val perplexity 356.4175
-4400 train 5.911475 (lr=2.6483e-05) (hash(x)=146923320)
-4500 val loss 5.8595
-4500 val perplexity 350.5503
-4500 train 5.688518 (lr=2.5714e-05) (hash(x)=146720819)
-4600 val loss 5.8489
-4600 val perplexity 346.8638
-4600 train 5.593976 (lr=2.4943e-05) (hash(x)=140342909)
-4700 val loss 5.8383
-4700 val perplexity 343.2022
-4700 train 5.974589 (lr=2.4172e-05) (hash(x)=163866463)
-4800 val loss 5.8217
-4800 val perplexity 337.5406
-4800 train 5.515379 (lr=2.3402e-05) (hash(x)=133459145)
-4900 val loss 5.8140
-4900 val perplexity 334.9486
-4900 train 5.611025 (lr=2.2633e-05) (hash(x)=143144356)
-5000 val loss 5.8290
-5000 val perplexity 340.0303
-5000 train 5.728374 (lr=2.1868e-05) (hash(x)=132636494)
-5100 val loss 5.7898
-5100 val perplexity 326.9575
-5100 train 5.715200 (lr=2.1107e-05) (hash(x)=157278728)
-5200 val loss 5.7817
-5200 val perplexity 324.3232
-5200 train 5.637352 (lr=2.0351e-05) (hash(x)=148198434)
-5300 val loss 5.7699
-5300 val perplexity 320.5101
-5300 train 5.696755 (lr=1.9602e-05) (hash(x)=138380906)
-5400 val loss 5.7576
-5400 val perplexity 316.5839
-5400 train 5.838938 (lr=1.8860e-05) (hash(x)=163441464)
-5500 val loss 5.7551
-5500 val perplexity 315.7874
-5500 train 5.701233 (lr=1.8127e-05) (hash(x)=154347714)
-5600 val loss 5.7429
-5600 val perplexity 311.9672
-5600 train 5.776368 (lr=1.7403e-05) (hash(x)=148449981)
-5700 val loss 5.7320
-5700 val perplexity 308.5851
-5700 train 5.623940 (lr=1.6690e-05) (hash(x)=139300274)
-5800 val loss 5.7271
-5800 val perplexity 307.0853
-5800 train 5.716210 (lr=1.5989e-05) (hash(x)=154857144)
-5900 val loss 5.7195
-5900 val perplexity 304.7662
-5900 train 5.779558 (lr=1.5300e-05) (hash(x)=151756013)
-6000 val loss 5.7133
-6000 val perplexity 302.8814
-6000 train 5.619116 (lr=1.4625e-05) (hash(x)=145414657)
-6100 val loss 5.7073
-6100 val perplexity 301.0667
-6100 train 5.534505 (lr=1.3965e-05) (hash(x)=139501217)
-6200 val loss 5.6980
-6200 val perplexity 298.2719
-6200 train 5.732094 (lr=1.3320e-05) (hash(x)=155783358)
-6300 val loss 5.6917
-6300 val perplexity 296.4115
-6300 train 5.751144 (lr=1.2692e-05) (hash(x)=157033091)
-6400 val loss 5.6877
-6400 val perplexity 295.2075
-6400 train 5.495005 (lr=1.2081e-05) (hash(x)=144475330)
-6500 val loss 5.6813
-6500 val perplexity 293.3268
-6500 train 5.727438 (lr=1.1489e-05) (hash(x)=124041822)
-6600 val loss 5.6776
-6600 val perplexity 292.2538
-6600 train 5.614210 (lr=1.0916e-05) (hash(x)=142131981)
-6700 val loss 5.6687
-6700 val perplexity 289.6444
-6700 train 5.769709 (lr=1.0363e-05) (hash(x)=154916248)
-6800 val loss 5.6677
-6800 val perplexity 289.3799
-6800 train 5.666794 (lr=9.8310e-06) (hash(x)=147996387)
-6900 val loss 5.6623
-6900 val perplexity 287.8063
-6900 train 5.690933 (lr=9.3205e-06) (hash(x)=153422428)
-7000 val loss 5.6558
-7000 val perplexity 285.9500
-7000 train 5.677885 (lr=8.8324e-06) (hash(x)=174135078)
-7100 val loss 5.6527
-7100 val perplexity 285.0608
-7100 train 5.732208 (lr=8.3674e-06) (hash(x)=160382475)
-7200 val loss 5.6492
-7200 val perplexity 284.0699
-7200 train 5.743230 (lr=7.9261e-06) (hash(x)=155310085)
-7300 val loss 5.6438
-7300 val perplexity 282.5283
-7300 train 5.782816 (lr=7.5093e-06) (hash(x)=159329031)
-7400 val loss 5.6415
-7400 val perplexity 281.8864
-7400 train 5.581890 (lr=7.1174e-06) (hash(x)=151577269)
-7500 val loss 5.6393
-7500 val perplexity 281.2724
-7500 train 5.690196 (lr=6.7511e-06) (hash(x)=160620287)
-7600 val loss 5.6345
-7600 val perplexity 279.9241
-7600 train 5.519984 (lr=6.4109e-06) (hash(x)=143711597)
-7700 val loss 5.6324
-7700 val perplexity 279.3272
-7700 train 5.632235 (lr=6.0972e-06) (hash(x)=149243577)
-7800 val loss 5.6312
-7800 val perplexity 278.9935
-7800 train 5.617033 (lr=5.8107e-06) (hash(x)=147677247)
-7900 val loss 5.6240
-7900 val perplexity 276.9847
-7900 train 5.574734 (lr=5.5515e-06) (hash(x)=145801236)
-8000 val loss 5.6234
-8000 val perplexity 276.8374
-8000 train 5.743947 (lr=5.3203e-06) (hash(x)=158007655)
-8100 val loss 5.6221
-8100 val perplexity 276.4651
-8100 train 5.681450 (lr=5.1172e-06) (hash(x)=158030048)
-8200 val loss 5.6184
-8200 val perplexity 275.4516
-8200 train 5.468052 (lr=4.9425e-06) (hash(x)=150674725)
-8300 val loss 5.6173
-8300 val perplexity 275.1565
-8300 train 5.523213 (lr=4.7966e-06) (hash(x)=150805711)
-8400 val loss 5.6133
-8400 val perplexity 274.0571
-8400 train 5.488098 (lr=4.6796e-06) (hash(x)=138396210)
-8500 val loss 5.6127
-8500 val perplexity 273.8857
-8500 train 5.489274 (lr=4.5917e-06) (hash(x)=156172740)
-8600 val loss 5.6100
-8600 val perplexity 273.1501
-8600 train 5.649384 (lr=4.5330e-06) (hash(x)=154911352)
-8700 val loss 5.6074
-8700 val perplexity 272.4272
-8700 train 5.664300 (lr=4.5037e-06) (hash(x)=153446789)
-8749 val loss 5.6057
-8749 val perplexity 271.9800
+0 val loss 11.2669
+0 val perplexity 78187.7734
+0 train 11.258430 (lr=8.0000e-08) (hash(x)=134751525)
+0 train 11.258430 (lr=1.0000e-07) (hash(x)=134751525)
+100 val loss 10.1048
+100 val perplexity 24460.9590
+100 train 10.127542 (lr=8.0800e-06) (hash(x)=150358957)
+100 val loss 10.0231
+100 val perplexity 22540.2520
+100 train 10.047743 (lr=1.0100e-05) (hash(x)=150358957)
+200 val loss 9.5683
+200 val perplexity 14303.5908
+200 train 9.594391 (lr=1.6080e-05) (hash(x)=126319983)
+200 val loss 9.3888
+200 val perplexity 11953.4688
+200 train 9.405663 (lr=2.0100e-05) (hash(x)=126319983)
+300 val loss 8.5065
+300 val perplexity 4946.5840
+300 train 8.438013 (lr=2.4080e-05) (hash(x)=159305874)
+300 val loss 8.1438
+300 val perplexity 3441.8569
+300 train 8.080306 (lr=3.0100e-05) (hash(x)=159305874)
+400 val loss 7.7059
+400 val perplexity 2221.3674
+400 train 7.694429 (lr=3.2080e-05) (hash(x)=149855631)
+400 val loss 7.5543
+400 val perplexity 1909.0006
+400 train 7.531188 (lr=4.0100e-05) (hash(x)=149855631)
+500 val loss 7.4746
+500 val perplexity 1762.6592
+500 train 7.505083 (lr=4.0000e-05) (hash(x)=150706760)
+500 val loss 7.3794
+500 val perplexity 1602.6191
+500 train 7.408833 (lr=5.0000e-05) (hash(x)=150706760)
+600 val loss 7.3726
+600 val perplexity 1591.7163
+600 train 7.307174 (lr=3.9987e-05) (hash(x)=146858163)
+600 val loss 7.2431
+600 val perplexity 1398.3760
+600 train 7.172870 (lr=4.9984e-05) (hash(x)=146858163)
+700 val loss 7.2818
+700 val perplexity 1453.6379
+700 train 7.155026 (lr=3.9948e-05) (hash(x)=144262072)
+700 val loss 7.1254
+700 val perplexity 1243.1661
+700 train 6.993992 (lr=4.9935e-05) (hash(x)=144262072)
+800 val loss 7.1949
+800 val perplexity 1332.6610
+800 train 7.168702 (lr=3.9883e-05) (hash(x)=135443880)
+800 val loss 7.0023
+800 val perplexity 1099.1837
+800 train 6.974597 (lr=4.9853e-05) (hash(x)=135443880)
+900 val loss 7.1091
+900 val perplexity 1222.9949
+900 train 7.050885 (lr=3.9792e-05) (hash(x)=153147388)
+900 val loss 6.8980
+900 val perplexity 990.2542
+900 train 6.843626 (lr=4.9739e-05) (hash(x)=153147388)
+1000 val loss 7.0470
+1000 val perplexity 1149.3538
+1000 train 6.974381 (lr=3.9675e-05) (hash(x)=151448445)
+1000 val loss 6.8099
+1000 val perplexity 906.8176
+1000 train 6.733419 (lr=4.9593e-05) (hash(x)=151448445)
+1100 val loss 6.9672
+1100 val perplexity 1061.2783
+1100 train 6.973591 (lr=3.9532e-05) (hash(x)=153161010)
+1100 val loss 6.7060
+1100 val perplexity 817.2753
+1100 train 6.728254 (lr=4.9415e-05) (hash(x)=153161010)
+1200 val loss 6.8929
+1200 val perplexity 985.2737
+1200 train 6.717103 (lr=3.9364e-05) (hash(x)=143227423)
+1200 val loss 6.6174
+1200 val perplexity 748.0264
+1200 train 6.445128 (lr=4.9205e-05) (hash(x)=143227423)
+1300 val loss 6.8303
+1300 val perplexity 925.4383
+1300 train 7.318541 (lr=3.9171e-05) (hash(x)=176373796)
+1300 val loss 6.5548
+1300 val perplexity 702.6147
+1300 train 7.003553 (lr=4.8964e-05) (hash(x)=176373796)
+1400 val loss 6.7688
+1400 val perplexity 870.2381
+1400 train 6.693253 (lr=3.8953e-05) (hash(x)=155989503)
+1400 val loss 6.4960
+1400 val perplexity 662.5030
+1400 train 6.429008 (lr=4.8691e-05) (hash(x)=155989503)
+1500 val loss 6.6972
+1500 val perplexity 810.0951
+1500 train 6.671696 (lr=3.8711e-05) (hash(x)=156507542)
+1500 val loss 6.4365
+1500 val perplexity 624.1983
+1500 train 6.412195 (lr=4.8388e-05) (hash(x)=156507542)
+1600 val loss 6.6402
+1600 val perplexity 765.2156
+1600 train 6.649072 (lr=3.8444e-05) (hash(x)=156078901)
+1600 val loss 6.3925
+1600 val perplexity 597.3754
+1600 train 6.381972 (lr=4.8055e-05) (hash(x)=156078901)
+1700 val loss 6.5829
+1700 val perplexity 722.6072
+1700 train 6.486052 (lr=3.8153e-05) (hash(x)=156889457)
+1700 val loss 6.3520
+1700 val perplexity 573.6584
+1700 train 6.256339 (lr=4.7691e-05) (hash(x)=156889457)
+1800 val loss 6.5346
+1800 val perplexity 688.5756
+1800 train 6.774926 (lr=3.7839e-05) (hash(x)=162477906)
+1800 val loss 6.3242
+1800 val perplexity 557.9255
+1800 train 6.584904 (lr=4.7299e-05) (hash(x)=162477906)
+1900 val loss 6.4822
+1900 val perplexity 653.4048
+1900 train 6.524128 (lr=3.7502e-05) (hash(x)=158648033)
+1900 val loss 6.2800
+1900 val perplexity 533.7811
+1900 train 6.317149 (lr=4.6878e-05) (hash(x)=158648033)
+2000 val loss 6.4540
+2000 val perplexity 635.2576
+2000 train 6.377429 (lr=3.7143e-05) (hash(x)=156732586)
+2000 val loss 6.2506
+2000 val perplexity 518.3123
+2000 train 6.174955 (lr=4.6428e-05) (hash(x)=156732586)
+2100 val loss 6.4112
+2100 val perplexity 608.6367
+2100 train 6.429165 (lr=3.6761e-05) (hash(x)=159721084)
+2100 val loss 6.2153
+2100 val perplexity 500.3592
+2100 train 6.225609 (lr=4.5951e-05) (hash(x)=159721084)
+2200 val loss 6.3710
+2200 val perplexity 584.6622
+2200 train 6.346819 (lr=3.6358e-05) (hash(x)=144907001)
+2200 val loss 6.1789
+2200 val perplexity 482.4542
+2200 train 6.165885 (lr=4.5448e-05) (hash(x)=144907001)
+2300 val loss 6.3493
+2300 val perplexity 572.0708
+2300 train 6.289486 (lr=3.5935e-05) (hash(x)=154541837)
+2300 val loss 6.1619
+2300 val perplexity 474.3160
+2300 train 6.103507 (lr=4.4918e-05) (hash(x)=154541837)
+2400 val loss 6.3174
+2400 val perplexity 554.1542
+2400 train 6.411627 (lr=3.5491e-05) (hash(x)=161706790)
+2400 val loss 6.1369
+2400 val perplexity 462.5943
+2400 train 6.238992 (lr=4.4363e-05) (hash(x)=161706790)
+2500 val loss 6.2840
+2500 val perplexity 535.9188
+2500 train 6.210105 (lr=3.5027e-05) (hash(x)=143556162)
+2500 val loss 6.1030
+2500 val perplexity 447.1863
+2500 train 6.026715 (lr=4.3784e-05) (hash(x)=143556162)
+2600 val loss 6.2586
+2600 val perplexity 522.4983
+2600 train 6.108171 (lr=3.4545e-05) (hash(x)=142425078)
+2600 val loss 6.0869
+2600 val perplexity 440.0374
+2600 train 5.936298 (lr=4.3181e-05) (hash(x)=142425078)
+2700 val loss 6.2373
+2700 val perplexity 511.4839
+2700 train 6.257928 (lr=3.4044e-05) (hash(x)=163195606)
+2700 val loss 6.0633
+2700 val perplexity 429.8103
+2700 train 6.086275 (lr=4.2555e-05) (hash(x)=163195606)
+2800 val loss 6.2056
+2800 val perplexity 495.5253
+2800 train 6.111760 (lr=3.3526e-05) (hash(x)=154078337)
+2800 val loss 6.0311
+2800 val perplexity 416.1743
+2800 train 5.942109 (lr=4.1908e-05) (hash(x)=154078337)
+2900 val loss 6.1913
+2900 val perplexity 488.5019
+2900 train 6.053054 (lr=3.2992e-05) (hash(x)=145186687)
+2900 val loss 6.0149
+2900 val perplexity 409.4986
+2900 train 5.887829 (lr=4.1240e-05) (hash(x)=145186687)
+3000 val loss 6.1616
+3000 val perplexity 474.1754
+3000 train 6.120342 (lr=3.2441e-05) (hash(x)=149726716)
+3000 val loss 5.9870
+3000 val perplexity 398.2309
+3000 train 5.952100 (lr=4.0551e-05) (hash(x)=149726716)
+3100 val loss 6.1420
+3100 val perplexity 465.0009
+3100 train 6.068071 (lr=3.1875e-05) (hash(x)=151122509)
+3100 val loss 5.9691
+3100 val perplexity 391.1497
+3100 train 5.889432 (lr=3.9844e-05) (hash(x)=151122509)
+3200 val loss 6.1261
+3200 val perplexity 457.6258
+3200 train 6.096629 (lr=3.1295e-05) (hash(x)=152172187)
+3200 val loss 5.9580
+3200 val perplexity 386.8179
+3200 train 5.925971 (lr=3.9119e-05) (hash(x)=152172187)
+3300 val loss 6.1072
+3300 val perplexity 449.0725
+3300 train 6.041915 (lr=3.0702e-05) (hash(x)=150581974)
+3300 val loss 5.9352
+3300 val perplexity 378.1105
+3300 train 5.872858 (lr=3.8377e-05) (hash(x)=150581974)
+3400 val loss 6.0943
+3400 val perplexity 443.3019
+3400 train 6.023142 (lr=3.0095e-05) (hash(x)=160061666)
+3400 val loss 5.9239
+3400 val perplexity 373.8839
+3400 train 5.847596 (lr=3.7619e-05) (hash(x)=160061666)
+3500 val loss 6.0703
+3500 val perplexity 432.8308
+3500 train 6.021031 (lr=2.9477e-05) (hash(x)=150556913)
+3500 val loss 5.8993
+3500 val perplexity 364.7906
+3500 train 5.848571 (lr=3.6847e-05) (hash(x)=150556913)
+3600 val loss 6.0600
+3600 val perplexity 428.3789
+3600 train 5.965510 (lr=2.8849e-05) (hash(x)=148311961)
+3600 val loss 5.8786
+3600 val perplexity 357.2937
+3600 train 5.785236 (lr=3.6061e-05) (hash(x)=148311961)
+3700 val loss 6.0418
+3700 val perplexity 420.6604
+3700 train 5.936149 (lr=2.8210e-05) (hash(x)=150305284)
+3700 val loss 5.8685
+3700 val perplexity 353.7125
+3700 train 5.754406 (lr=3.5263e-05) (hash(x)=150305284)
+3800 val loss 6.0221
+3800 val perplexity 412.4275
+3800 train 6.032154 (lr=2.7562e-05) (hash(x)=148428531)
+3800 val loss 5.8474
+3800 val perplexity 346.3270
+3800 train 5.850537 (lr=3.4453e-05) (hash(x)=148428531)
+3900 val loss 6.0072
+3900 val perplexity 406.3344
+3900 train 5.885365 (lr=2.6907e-05) (hash(x)=142448374)
+3900 val loss 5.8296
+3900 val perplexity 340.2376
+3900 train 5.704201 (lr=3.3633e-05) (hash(x)=142448374)
+4000 val loss 5.9996
+4000 val perplexity 403.2542
+4000 train 5.928854 (lr=2.6244e-05) (hash(x)=158002288)
+4000 val loss 5.8199
+4000 val perplexity 336.9466
+4000 train 5.747924 (lr=3.2805e-05) (hash(x)=158002288)
+4100 val loss 5.9774
+4100 val perplexity 394.4323
+4100 train 5.919015 (lr=2.5575e-05) (hash(x)=151388626)
+4100 val loss 5.7989
+4100 val perplexity 329.9272
+4100 train 5.736006 (lr=3.1968e-05) (hash(x)=151388626)
+4200 val loss 5.9657
+4200 val perplexity 389.8166
+4200 train 5.885614 (lr=2.4900e-05) (hash(x)=145123294)
+4200 val loss 5.7855
+4200 val perplexity 325.5608
+4200 train 5.711619 (lr=3.1126e-05) (hash(x)=145123294)
+4300 val loss 5.9532
+4300 val perplexity 384.9809
+4300 train 5.826245 (lr=2.4222e-05) (hash(x)=145869150)
+4300 val loss 5.7755
+4300 val perplexity 322.3113
+4300 train 5.656160 (lr=3.0277e-05) (hash(x)=145869150)
+4400 val loss 5.9471
+4400 val perplexity 382.6427
+4400 train 5.974758 (lr=2.3540e-05) (hash(x)=146923320)
+4400 val loss 5.7648
+4400 val perplexity 318.8671
+4400 train 5.801609 (lr=2.9425e-05) (hash(x)=146923320)
+4500 val loss 5.9267
+4500 val perplexity 374.9101
+4500 train 5.747364 (lr=2.2856e-05) (hash(x)=146720819)
+4500 val loss 5.7480
+4500 val perplexity 313.5577
+4500 train 5.575530 (lr=2.8571e-05) (hash(x)=146720819)
+4600 val loss 5.9185
+4600 val perplexity 371.8680
+4600 train 5.661109 (lr=2.2171e-05) (hash(x)=140342909)
+4600 val loss 5.7397
+4600 val perplexity 310.9650
+4600 train 5.495009 (lr=2.7714e-05) (hash(x)=140342909)
+4700 val loss 5.9109
+4700 val perplexity 369.0468
+4700 train 6.058608 (lr=2.1486e-05) (hash(x)=163866463)
+4700 val loss 5.7257
+4700 val perplexity 306.6404
+4700 train 5.864868 (lr=2.6857e-05) (hash(x)=163866463)
+4800 val loss 5.8930
+4800 val perplexity 362.4748
+4800 train 5.589387 (lr=2.0801e-05) (hash(x)=133459145)
+4800 val loss 5.7127
+4800 val perplexity 302.6829
+4800 train 5.398318 (lr=2.6002e-05) (hash(x)=133459145)
+4900 val loss 5.8848
+4900 val perplexity 359.5341
+4900 train 5.686441 (lr=2.0118e-05) (hash(x)=143144356)
+4900 val loss 5.7034
+4900 val perplexity 299.8943
+4900 train 5.506179 (lr=2.5148e-05) (hash(x)=143144356)
+5000 val loss 5.9014
+5000 val perplexity 365.5357
+5000 train 5.793544 (lr=1.9438e-05) (hash(x)=132636494)
+5000 val loss 5.7167
+5000 val perplexity 303.8913
+5000 train 5.627277 (lr=2.4298e-05) (hash(x)=132636494)
+5100 val loss 5.8627
+5100 val perplexity 351.6631
+5100 train 5.787260 (lr=1.8762e-05) (hash(x)=157278728)
+5100 val loss 5.6822
+5100 val perplexity 293.6040
+5100 train 5.602773 (lr=2.3452e-05) (hash(x)=157278728)
+5200 val loss 5.8544
+5200 val perplexity 348.7502
+5200 train 5.707648 (lr=1.8090e-05) (hash(x)=148198434)
+5200 val loss 5.6738
+5200 val perplexity 291.1431
+5200 train 5.524666 (lr=2.2613e-05) (hash(x)=148198434)
+5300 val loss 5.8439
+5300 val perplexity 345.1328
+5300 train 5.773888 (lr=1.7424e-05) (hash(x)=138380906)
+5300 val loss 5.6625
+5300 val perplexity 287.8815
+5300 train 5.588348 (lr=2.1780e-05) (hash(x)=138380906)
+5400 val loss 5.8330
+5400 val perplexity 341.3846
+5400 train 5.928821 (lr=1.6765e-05) (hash(x)=163441464)
+5400 val loss 5.6512
+5400 val perplexity 284.6234
+5400 train 5.733276 (lr=2.0956e-05) (hash(x)=163441464)
+5500 val loss 5.8272
+5500 val perplexity 339.3984
+5500 train 5.776112 (lr=1.6113e-05) (hash(x)=154347714)
+5500 val loss 5.6481
+5500 val perplexity 283.7407
+5500 train 5.582100 (lr=2.0141e-05) (hash(x)=154347714)
+5600 val loss 5.8178
+5600 val perplexity 336.2433
+5600 train 5.856583 (lr=1.5469e-05) (hash(x)=148449981)
+5600 val loss 5.6360
+5600 val perplexity 280.3305
+5600 train 5.680454 (lr=1.9337e-05) (hash(x)=148449981)
+5700 val loss 5.8072
+5700 val perplexity 332.6860
+5700 train 5.697080 (lr=1.4836e-05) (hash(x)=139300274)
+5700 val loss 5.6259
+5700 val perplexity 277.5210
+5700 train 5.518164 (lr=1.8545e-05) (hash(x)=139300274)
+5800 val loss 5.8016
+5800 val perplexity 330.8387
+5800 train 5.794398 (lr=1.4212e-05) (hash(x)=154857144)
+5800 val loss 5.6236
+5800 val perplexity 276.8877
+5800 train 5.618228 (lr=1.7765e-05) (hash(x)=154857144)
+5900 val loss 5.7934
+5900 val perplexity 328.1333
+5900 train 5.859982 (lr=1.3600e-05) (hash(x)=151756013)
+5900 val loss 5.6159
+5900 val perplexity 274.7690
+5900 train 5.677189 (lr=1.7000e-05) (hash(x)=151756013)
+6000 val loss 5.7904
+6000 val perplexity 327.1291
+6000 train 5.691472 (lr=1.3000e-05) (hash(x)=145414657)
+6000 val loss 5.6072
+6000 val perplexity 272.3825
+6000 train 5.513993 (lr=1.6250e-05) (hash(x)=145414657)
+6100 val loss 5.7825
+6100 val perplexity 324.5720
+6100 train 5.605465 (lr=1.2413e-05) (hash(x)=139501217)
+6100 val loss 5.6022
+6100 val perplexity 271.0273
+6100 train 5.420915 (lr=1.5516e-05) (hash(x)=139501217)
+6200 val loss 5.7733
+6200 val perplexity 321.6091
+6200 train 5.814078 (lr=1.1840e-05) (hash(x)=155783358)
+6200 val loss 5.5946
+6200 val perplexity 268.9805
+6200 train 5.631000 (lr=1.4800e-05) (hash(x)=155783358)
+6300 val loss 5.7674
+6300 val perplexity 319.6986
+6300 train 5.833146 (lr=1.1282e-05) (hash(x)=157033091)
+6300 val loss 5.5866
+6300 val perplexity 266.8292
+6300 train 5.654327 (lr=1.4102e-05) (hash(x)=157033091)
+6400 val loss 5.7638
+6400 val perplexity 318.5564
+6400 train 5.564855 (lr=1.0739e-05) (hash(x)=144475330)
+6400 val loss 5.5839
+6400 val perplexity 266.1018
+6400 train 5.383296 (lr=1.3424e-05) (hash(x)=144475330)
+6500 val loss 5.7558
+6500 val perplexity 316.0177
+6500 train 5.806682 (lr=1.0213e-05) (hash(x)=124041822)
+6500 val loss 5.5759
+6500 val perplexity 263.9772
+6500 train 5.622413 (lr=1.2766e-05) (hash(x)=124041822)
+6600 val loss 5.7531
+6600 val perplexity 315.1798
+6600 train 5.685970 (lr=9.7032e-06) (hash(x)=142131981)
+6600 val loss 5.5739
+6600 val perplexity 263.4679
+6600 train 5.510099 (lr=1.2129e-05) (hash(x)=142131981)
+6700 val loss 5.7443
+6700 val perplexity 312.4091
+6700 train 5.842805 (lr=9.2116e-06) (hash(x)=154916248)
+6700 val loss 5.5630
+6700 val perplexity 260.6121
+6700 train 5.672098 (lr=1.1515e-05) (hash(x)=154916248)
+6800 val loss 5.7442
+6800 val perplexity 312.3599
+6800 train 5.740576 (lr=8.7387e-06) (hash(x)=147996387)
+6800 val loss 5.5651
+6800 val perplexity 261.1384
+6800 train 5.563013 (lr=1.0923e-05) (hash(x)=147996387)
+6900 val loss 5.7386
+6900 val perplexity 310.6446
+6900 train 5.776214 (lr=8.2849e-06) (hash(x)=153422428)
+6900 val loss 5.5579
+6900 val perplexity 259.2890
+6900 train 5.582366 (lr=1.0356e-05) (hash(x)=153422428)
+7000 val loss 5.7312
+7000 val perplexity 308.3365
+7000 train 5.751740 (lr=7.8510e-06) (hash(x)=174135078)
+7000 val loss 5.5507
+7000 val perplexity 257.4088
+7000 train 5.575917 (lr=9.8138e-06) (hash(x)=174135078)
+7100 val loss 5.7284
+7100 val perplexity 307.4814
+7100 train 5.801340 (lr=7.4377e-06) (hash(x)=160382475)
+7100 val loss 5.5490
+7100 val perplexity 256.9679
+7100 train 5.626064 (lr=9.2971e-06) (hash(x)=160382475)
+7200 val loss 5.7261
+7200 val perplexity 306.7845
+7200 train 5.817417 (lr=7.0455e-06) (hash(x)=155310085)
+7200 val loss 5.5455
+7200 val perplexity 256.0913
+7200 train 5.644667 (lr=8.8068e-06) (hash(x)=155310085)
+7300 val loss 5.7205
+7300 val perplexity 305.0573
+7300 train 5.856584 (lr=6.6749e-06) (hash(x)=159329031)
+7300 val loss 5.5398
+7300 val perplexity 254.6357
+7300 train 5.675587 (lr=8.3436e-06) (hash(x)=159329031)
+7400 val loss 5.7178
+7400 val perplexity 304.2255
+7400 train 5.658116 (lr=6.3266e-06) (hash(x)=151577269)
+7400 val loss 5.5378
+7400 val perplexity 254.1247
+7400 train 5.481563 (lr=7.9082e-06) (hash(x)=151577269)
+7500 val loss 5.7159
+7500 val perplexity 303.6710
+7500 train 5.764712 (lr=6.0010e-06) (hash(x)=160620287)
+7500 val loss 5.5361
+7500 val perplexity 253.6919
+7500 train 5.596354 (lr=7.5012e-06) (hash(x)=160620287)
+7600 val loss 5.7108
+7600 val perplexity 302.1122
+7600 train 5.597404 (lr=5.6986e-06) (hash(x)=143711597)
+7600 val loss 5.5301
+7600 val perplexity 252.1762
+7600 train 5.417143 (lr=7.1232e-06) (hash(x)=143711597)
+7700 val loss 5.7093
+7700 val perplexity 301.6568
+7700 train 5.714295 (lr=5.4198e-06) (hash(x)=149243577)
+7700 val loss 5.5294
+7700 val perplexity 251.9845
+7700 train 5.518233 (lr=6.7747e-06) (hash(x)=149243577)
+7800 val loss 5.7086
+7800 val perplexity 301.4565
+7800 train 5.695864 (lr=5.1650e-06) (hash(x)=147677247)
+7800 val loss 5.5276
+7800 val perplexity 251.5379
+7800 train 5.511157 (lr=6.4563e-06) (hash(x)=147677247)
+7900 val loss 5.7003
+7900 val perplexity 298.9606
+7900 train 5.654427 (lr=4.9347e-06) (hash(x)=145801236)
+7900 val loss 5.5207
+7900 val perplexity 249.8004
+7900 train 5.467436 (lr=6.1684e-06) (hash(x)=145801236)
+8000 val loss 5.7006
+8000 val perplexity 299.0564
+8000 train 5.812949 (lr=4.7291e-06) (hash(x)=158007655)
+8000 val loss 5.5199
+8000 val perplexity 249.6133
+8000 train 5.639585 (lr=5.9114e-06) (hash(x)=158007655)
+8100 val loss 5.6989
+8100 val perplexity 298.5511
+8100 train 5.762550 (lr=4.5486e-06) (hash(x)=158030048)
+8100 val loss 5.5189
+8100 val perplexity 249.3497
+8100 train 5.581952 (lr=5.6857e-06) (hash(x)=158030048)
+8200 val loss 5.6953
+8200 val perplexity 297.4649
+8200 train 5.538460 (lr=4.3933e-06) (hash(x)=150674725)
+8200 val loss 5.5148
+8200 val perplexity 248.3339
+8200 train 5.359712 (lr=5.4917e-06) (hash(x)=150674725)
+8300 val loss 5.6944
+8300 val perplexity 297.2130
+8300 train 5.608293 (lr=4.2636e-06) (hash(x)=150805711)
+8300 val loss 5.5138
+8300 val perplexity 248.0881
+8300 train 5.418227 (lr=5.3295e-06) (hash(x)=150805711)
+8400 val loss 5.6895
+8400 val perplexity 295.7561
+8400 train 5.560624 (lr=4.1596e-06) (hash(x)=138396210)
+8400 val loss 5.5093
+8400 val perplexity 246.9719
+8400 train 5.380278 (lr=5.1995e-06) (hash(x)=138396210)
+8500 val loss 5.6898
+8500 val perplexity 295.8447
+8500 train 5.565089 (lr=4.0815e-06) (hash(x)=156172740)
+8500 val loss 5.5093
+8500 val perplexity 246.9775
+8500 train 5.386551 (lr=5.1019e-06) (hash(x)=156172740)
+8600 val loss 5.6870
+8600 val perplexity 295.0010
+8600 train 5.733992 (lr=4.0294e-06) (hash(x)=154911352)
+8600 val loss 5.5065
+8600 val perplexity 246.2987
+8600 train 5.535190 (lr=5.0367e-06) (hash(x)=154911352)
+8700 val loss 5.6839
+8700 val perplexity 294.1030
+8700 train 5.744620 (lr=4.0033e-06) (hash(x)=153446789)
+8749 val loss 5.6830
+8749 val perplexity 293.8351
+8700 val loss 5.5038
+8700 val perplexity 245.6131
+8700 train 5.561337 (lr=5.0041e-06) (hash(x)=153446789)

attention_kindselective_n_heads4_seed1345/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63a5d75f3d50c1e38bd49d0b3ffde6667deba50fb9d33f7f876d2ce34040e7cc
+oid sha256:4116fa61c7f1cdf137ea00df9bdd7b917474dd43ad1ed65e218ac759467c44e8
 size 92843394

attention_kindselective_n_heads4_seed1345/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48e14c02fa70197c42cfb86a6c654015643809abd3d0f2330ae92703b10052ff
+oid sha256:5f79873e3c061c0420e145d23845da2a9f7a57bbb91c03038039c018ea71fc52
 size 179406214