Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4.5e-5_61440_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_61440_4_1340", "n_embd": 256}

attention_kindselective_n_heads4_seed1340/log2.txt

@@ -1,267 +1,533 @@
 max_steps: 8750
 0 val loss 11.2709
-0 val perplexity 78502.7734
-0 train 11.277069 (lr=9.0000e-08) (hash(x)=164406924)
-100 val loss 10.0557
-100 val perplexity 23287.3145
-100 train 10.019081 (lr=9.0900e-06) (hash(x)=144903932)
-200 val loss 9.4995
-200 val perplexity 13352.6445
-200 train 9.559769 (lr=1.8090e-05) (hash(x)=167734596)
-300 val loss 8.3418
-300 val perplexity 4195.8223
-300 train 8.368614 (lr=2.7090e-05) (hash(x)=149619098)
-400 val loss 7.7028
-400 val perplexity 2214.4700
-400 train 7.609691 (lr=3.6090e-05) (hash(x)=153710890)
-500 val loss 7.5150
-500 val perplexity 1835.3279
-500 train 7.606066 (lr=4.5000e-05) (hash(x)=145450636)
-600 val loss 7.3772
-600 val perplexity 1599.0978
-600 train 7.234566 (lr=4.4985e-05) (hash(x)=145249251)
-700 val loss 7.2820
-700 val perplexity 1453.9449
-700 train 7.401809 (lr=4.4941e-05) (hash(x)=150475545)
-800 val loss 7.1954
-800 val perplexity 1333.3405
-800 train 7.163744 (lr=4.4868e-05) (hash(x)=144483776)
-900 val loss 7.1004
-900 val perplexity 1212.4415
-900 train 7.208779 (lr=4.4766e-05) (hash(x)=157916369)
-1000 val loss 7.0209
-1000 val perplexity 1119.8268
-1000 train 7.043412 (lr=4.4634e-05) (hash(x)=154856891)
-1100 val loss 6.9413
-1100 val perplexity 1034.0712
-1100 train 6.648606 (lr=4.4474e-05) (hash(x)=136191502)
-1200 val loss 6.8589
-1200 val perplexity 952.3258
-1200 train 6.815243 (lr=4.4285e-05) (hash(x)=148021541)
-1300 val loss 6.7793
-1300 val perplexity 879.4661
-1300 train 6.710860 (lr=4.4068e-05) (hash(x)=146005217)
-1400 val loss 6.7046
-1400 val perplexity 816.1409
-1400 train 6.612201 (lr=4.3822e-05) (hash(x)=146019502)
-1500 val loss 6.6382
-1500 val perplexity 763.7422
-1500 train 6.578260 (lr=4.3549e-05) (hash(x)=150127281)
-1600 val loss 6.5751
-1600 val perplexity 716.9893
-1600 train 6.544516 (lr=4.3249e-05) (hash(x)=154120875)
-1700 val loss 6.5345
-1700 val perplexity 688.4567
-1700 train 6.748356 (lr=4.2922e-05) (hash(x)=155797680)
-1800 val loss 6.4693
-1800 val perplexity 645.0002
-1800 train 6.460943 (lr=4.2569e-05) (hash(x)=156809396)
-1900 val loss 6.4339
-1900 val perplexity 622.6234
-1900 train 6.314532 (lr=4.2190e-05) (hash(x)=144640294)
-2000 val loss 6.3959
-2000 val perplexity 599.3755
-2000 train 6.258166 (lr=4.1785e-05) (hash(x)=162831106)
-2100 val loss 6.3713
-2100 val perplexity 584.7957
-2100 train 6.357996 (lr=4.1356e-05) (hash(x)=158239484)
-2200 val loss 6.3303
-2200 val perplexity 561.3037
-2200 train 6.209711 (lr=4.0903e-05) (hash(x)=140504180)
-2300 val loss 6.2935
-2300 val perplexity 541.0540
-2300 train 6.283567 (lr=4.0426e-05) (hash(x)=142234024)
-2400 val loss 6.2662
-2400 val perplexity 526.4846
-2400 train 6.264317 (lr=3.9927e-05) (hash(x)=143091562)
-2500 val loss 6.2603
-2500 val perplexity 523.3643
-2500 train 6.165597 (lr=3.9406e-05) (hash(x)=149857456)
-2600 val loss 6.2165
-2600 val perplexity 500.9319
-2600 train 6.254084 (lr=3.8863e-05) (hash(x)=146191551)
-2700 val loss 6.1897
-2700 val perplexity 487.6854
-2700 train 6.109442 (lr=3.8300e-05) (hash(x)=145375752)
-2800 val loss 6.1660
-2800 val perplexity 476.2601
-2800 train 6.046856 (lr=3.7717e-05) (hash(x)=151568014)
-2900 val loss 6.1587
-2900 val perplexity 472.7968
-2900 train 5.982612 (lr=3.7116e-05) (hash(x)=149366597)
-3000 val loss 6.1218
-3000 val perplexity 455.6800
-3000 train 6.048753 (lr=3.6496e-05) (hash(x)=150464442)
-3100 val loss 6.1017
-3100 val perplexity 446.6161
-3100 train 6.231345 (lr=3.5860e-05) (hash(x)=182449036)
-3200 val loss 6.0852
-3200 val perplexity 439.3019
-3200 train 6.040099 (lr=3.5207e-05) (hash(x)=140141286)
-3300 val loss 6.0630
-3300 val perplexity 429.6579
-3300 train 6.028925 (lr=3.4539e-05) (hash(x)=148099414)
-3400 val loss 6.0475
-3400 val perplexity 423.0616
-3400 train 5.908727 (lr=3.3857e-05) (hash(x)=142633951)
-3500 val loss 6.0416
-3500 val perplexity 420.5567
-3500 train 5.899892 (lr=3.3162e-05) (hash(x)=148368965)
-3600 val loss 6.0151
-3600 val perplexity 409.5803
-3600 train 6.040414 (lr=3.2455e-05) (hash(x)=152372067)
-3700 val loss 6.0002
-3700 val perplexity 403.4992
-3700 train 6.086949 (lr=3.1736e-05) (hash(x)=168885609)
-3800 val loss 5.9895
-3800 val perplexity 399.2296
-3800 train 5.802959 (lr=3.1008e-05) (hash(x)=141633734)
-3900 val loss 5.9654
-3900 val perplexity 389.6895
-3900 train 5.856780 (lr=3.0270e-05) (hash(x)=153141007)
-4000 val loss 5.9524
-4000 val perplexity 384.6906
-4000 train 5.884473 (lr=2.9524e-05) (hash(x)=160577202)
-4100 val loss 5.9442
-4100 val perplexity 381.5477
-4100 train 6.007852 (lr=2.8771e-05) (hash(x)=153858169)
-4200 val loss 5.9247
-4200 val perplexity 374.1573
-4200 train 5.942840 (lr=2.8013e-05) (hash(x)=155889149)
-4300 val loss 5.9165
-4300 val perplexity 371.1189
-4300 train 5.729906 (lr=2.7250e-05) (hash(x)=152294662)
-4400 val loss 5.8993
-4400 val perplexity 364.7923
-4400 train 5.796800 (lr=2.6483e-05) (hash(x)=141804386)
-4500 val loss 5.8835
-4500 val perplexity 359.0594
-4500 train 5.779256 (lr=2.5714e-05) (hash(x)=151095242)
-4600 val loss 5.8884
-4600 val perplexity 360.8189
-4600 train 5.833579 (lr=2.4943e-05) (hash(x)=156414699)
-4700 val loss 5.8623
-4700 val perplexity 351.5250
-4700 train 5.885815 (lr=2.4172e-05) (hash(x)=161556686)
-4800 val loss 5.8483
-4800 val perplexity 346.6370
-4800 train 5.760074 (lr=2.3402e-05) (hash(x)=149000293)
-4900 val loss 5.8378
-4900 val perplexity 343.0105
-4900 train 5.630357 (lr=2.2633e-05) (hash(x)=154349989)
-5000 val loss 5.8264
-5000 val perplexity 339.1286
-5000 train 5.662440 (lr=2.1868e-05) (hash(x)=131475967)
-5100 val loss 5.8129
-5100 val perplexity 334.5789
-5100 train 5.719586 (lr=2.1107e-05) (hash(x)=149717902)
-5200 val loss 5.8060
-5200 val perplexity 332.2750
-5200 train 5.670409 (lr=2.0351e-05) (hash(x)=151407999)
-5300 val loss 5.7981
-5300 val perplexity 329.6660
-5300 train 5.900123 (lr=1.9602e-05) (hash(x)=168602728)
-5400 val loss 5.7835
-5400 val perplexity 324.8809
-5400 train 5.742907 (lr=1.8860e-05) (hash(x)=158344511)
-5500 val loss 5.7781
-5500 val perplexity 323.1493
-5500 train 5.736913 (lr=1.8127e-05) (hash(x)=148350057)
-5600 val loss 5.7689
-5600 val perplexity 320.1697
-5600 train 5.838607 (lr=1.7403e-05) (hash(x)=153847323)
-5700 val loss 5.7593
-5700 val perplexity 317.1218
-5700 train 5.706427 (lr=1.6690e-05) (hash(x)=156607405)
-5800 val loss 5.7527
-5800 val perplexity 315.0317
-5800 train 5.520199 (lr=1.5989e-05) (hash(x)=145115031)
-5900 val loss 5.7456
-5900 val perplexity 312.8043
-5900 train 5.628171 (lr=1.5300e-05) (hash(x)=141584622)
-6000 val loss 5.7354
-6000 val perplexity 309.6448
-6000 train 5.666881 (lr=1.4625e-05) (hash(x)=146613857)
-6100 val loss 5.7322
-6100 val perplexity 308.6477
-6100 train 5.669149 (lr=1.3965e-05) (hash(x)=144621768)
-6200 val loss 5.7294
-6200 val perplexity 307.7820
-6200 train 5.664270 (lr=1.3320e-05) (hash(x)=146521760)
-6300 val loss 5.7176
-6300 val perplexity 304.1604
-6300 train 5.547387 (lr=1.2692e-05) (hash(x)=161378136)
-6400 val loss 5.7128
-6400 val perplexity 302.7064
-6400 train 5.478628 (lr=1.2081e-05) (hash(x)=141624235)
-6500 val loss 5.7064
-6500 val perplexity 300.7927
-6500 train 5.628902 (lr=1.1489e-05) (hash(x)=151197095)
-6600 val loss 5.7043
-6600 val perplexity 300.1533
-6600 train 5.721037 (lr=1.0916e-05) (hash(x)=153269571)
-6700 val loss 5.6970
-6700 val perplexity 297.9862
-6700 train 5.661511 (lr=1.0363e-05) (hash(x)=146111181)
-6800 val loss 5.6897
-6800 val perplexity 295.8090
-6800 train 5.722671 (lr=9.8310e-06) (hash(x)=147269760)
-6900 val loss 5.6870
-6900 val perplexity 295.0048
-6900 train 5.759125 (lr=9.3205e-06) (hash(x)=152912762)
-7000 val loss 5.6839
-7000 val perplexity 294.0924
-7000 train 5.766317 (lr=8.8324e-06) (hash(x)=165412343)
-7100 val loss 5.6741
-7100 val perplexity 291.2299
-7100 train 5.744351 (lr=8.3674e-06) (hash(x)=162866028)
-7200 val loss 5.6704
-7200 val perplexity 290.1535
-7200 train 5.638232 (lr=7.9261e-06) (hash(x)=142998115)
-7300 val loss 5.6689
-7300 val perplexity 289.7291
-7300 train 5.438991 (lr=7.5093e-06) (hash(x)=145486999)
-7400 val loss 5.6625
-7400 val perplexity 287.8556
-7400 train 5.632413 (lr=7.1174e-06) (hash(x)=155325873)
-7500 val loss 5.6589
-7500 val perplexity 286.8464
-7500 train 5.621664 (lr=6.7511e-06) (hash(x)=145131256)
-7600 val loss 5.6627
-7600 val perplexity 287.9161
-7600 train 5.585990 (lr=6.4109e-06) (hash(x)=144008365)
-7700 val loss 5.6527
-7700 val perplexity 285.0522
-7700 train 5.748820 (lr=6.0972e-06) (hash(x)=148848532)
-7800 val loss 5.6501
-7800 val perplexity 284.3308
-7800 train 5.638553 (lr=5.8107e-06) (hash(x)=150391642)
-7900 val loss 5.6506
-7900 val perplexity 284.4709
-7900 train 5.408098 (lr=5.5515e-06) (hash(x)=152191414)
-8000 val loss 5.6456
-8000 val perplexity 283.0550
-8000 train 5.726780 (lr=5.3203e-06) (hash(x)=159755587)
-8100 val loss 5.6430
-8100 val perplexity 282.2966
-8100 train 5.665362 (lr=5.1172e-06) (hash(x)=156664468)
-8200 val loss 5.6412
-8200 val perplexity 281.7932
-8200 train 5.488082 (lr=4.9425e-06) (hash(x)=139457379)
-8300 val loss 5.6457
-8300 val perplexity 283.0753
-8300 train 5.412615 (lr=4.7966e-06) (hash(x)=145478564)
-8400 val loss 5.6364
-8400 val perplexity 280.4597
-8400 train 5.574693 (lr=4.6796e-06) (hash(x)=154982769)
-8500 val loss 5.6342
-8500 val perplexity 279.8371
-8500 train 5.599327 (lr=4.5917e-06) (hash(x)=145798118)
-8600 val loss 5.6341
-8600 val perplexity 279.8044
-8600 train 5.540295 (lr=4.5330e-06) (hash(x)=143231551)
-8700 val loss 5.6379
-8700 val perplexity 280.8729
-8700 train 5.537607 (lr=4.5037e-06) (hash(x)=154780112)
-8749 val loss 5.6298
-8749 val perplexity 278.5986
+0 val perplexity 78503.8281
+0 val loss 11.2709
+0 val perplexity 78503.8281
+0 train 11.277061 (lr=1.0000e-07) (hash(x)=164406924)
+0 train 11.277061 (lr=8.0000e-08) (hash(x)=164406924)
+100 val loss 10.0260
+100 val perplexity 22607.1387
+100 train 9.988957 (lr=1.0100e-05) (hash(x)=144903932)
+100 val loss 10.0925
+100 val perplexity 24160.8594
+100 train 10.055962 (lr=8.0800e-06) (hash(x)=144903932)
+200 val loss 9.5168
+200 val perplexity 13586.6348
+200 train 9.577106 (lr=2.0100e-05) (hash(x)=167734596)
+200 val loss 9.6049
+200 val perplexity 14837.5244
+200 train 9.662758 (lr=1.6080e-05) (hash(x)=167734596)
+300 val loss 8.3659
+300 val perplexity 4298.1421
+300 train 8.395082 (lr=3.0100e-05) (hash(x)=149619098)
+300 val loss 8.5073
+300 val perplexity 4950.7749
+300 train 8.541262 (lr=2.4080e-05) (hash(x)=149619098)
+400 val loss 7.6858
+400 val perplexity 2177.2568
+400 train 7.588470 (lr=4.0100e-05) (hash(x)=153710890)
+400 val loss 7.7435
+400 val perplexity 2306.5437
+400 train 7.660269 (lr=3.2080e-05) (hash(x)=153710890)
+500 val loss 7.5081
+500 val perplexity 1822.7219
+500 val loss 7.4949
+500 val perplexity 1798.8890
+500 train 7.599987 (lr=5.0000e-05) (hash(x)=145450636)
+500 train 7.588236 (lr=4.0000e-05) (hash(x)=145450636)
+600 val loss 7.3629
+600 val perplexity 1576.3440
+600 train 7.224650 (lr=3.9987e-05) (hash(x)=145249251)
+600 val loss 7.3735
+600 val perplexity 1593.1895
+600 train 7.235652 (lr=4.9984e-05) (hash(x)=145249251)
+700 val loss 7.2708
+700 val perplexity 1437.7072
+700 train 7.395315 (lr=3.9948e-05) (hash(x)=150475545)
+700 val loss 7.2930
+700 val perplexity 1469.9099
+700 train 7.411776 (lr=4.9935e-05) (hash(x)=150475545)
+800 val loss 7.1754
+800 val perplexity 1306.9037
+800 train 7.144771 (lr=3.9883e-05) (hash(x)=144483776)
+800 val loss 7.2191
+800 val perplexity 1365.2955
+800 train 7.187092 (lr=4.9853e-05) (hash(x)=144483776)
+900 val loss 7.0760
+900 val perplexity 1183.2659
+900 train 7.180530 (lr=3.9792e-05) (hash(x)=157916369)
+900 val loss 7.1320
+900 val perplexity 1251.3336
+900 train 7.237772 (lr=4.9739e-05) (hash(x)=157916369)
+1000 val loss 6.9951
+1000 val perplexity 1091.2765
+1000 train 7.012112 (lr=3.9675e-05) (hash(x)=154856891)
+1000 val loss 7.0521
+1000 val perplexity 1155.2704
+1000 train 7.068480 (lr=4.9593e-05) (hash(x)=154856891)
+1100 val loss 6.9135
+1100 val perplexity 1005.7716
+1100 train 6.621021 (lr=3.9532e-05) (hash(x)=136191502)
+1100 val loss 6.9700
+1100 val perplexity 1064.1713
+1100 train 6.674323 (lr=4.9415e-05) (hash(x)=136191502)
+1200 val loss 6.8271
+1200 val perplexity 922.5089
+1200 train 6.779061 (lr=3.9364e-05) (hash(x)=148021541)
+1200 val loss 6.8753
+1200 val perplexity 968.0394
+1200 train 6.832675 (lr=4.9205e-05) (hash(x)=148021541)
+1300 val loss 6.7479
+1300 val perplexity 852.2403
+1300 train 6.678041 (lr=3.9171e-05) (hash(x)=146005217)
+1300 val loss 6.8000
+1300 val perplexity 897.8826
+1300 train 6.726591 (lr=4.8964e-05) (hash(x)=146005217)
+1400 val loss 6.6751
+1400 val perplexity 792.3956
+1400 train 6.583837 (lr=3.8953e-05) (hash(x)=146019502)
+1400 val loss 6.7348
+1400 val perplexity 841.1787
+1400 train 6.644874 (lr=4.8691e-05) (hash(x)=146019502)
+1500 val loss 6.6206
+1500 val perplexity 750.3671
+1500 train 6.558818 (lr=3.8711e-05) (hash(x)=150127281)
+1500 val loss 6.6657
+1500 val perplexity 785.0457
+1500 train 6.603220 (lr=4.8388e-05) (hash(x)=150127281)
+1600 val loss 6.5518
+1600 val perplexity 700.5061
+1600 train 6.517587 (lr=3.8444e-05) (hash(x)=154120875)
+1600 val loss 6.5995
+1600 val perplexity 734.7468
+1600 train 6.566146 (lr=4.8055e-05) (hash(x)=154120875)
+1700 val loss 6.5082
+1700 val perplexity 670.6404
+1700 train 6.723195 (lr=3.8153e-05) (hash(x)=155797680)
+1700 val loss 6.5612
+1700 val perplexity 707.1252
+1700 train 6.770807 (lr=4.7691e-05) (hash(x)=155797680)
+1800 val loss 6.4488
+1800 val perplexity 631.9665
+1800 train 6.439053 (lr=3.7839e-05) (hash(x)=156809396)
+1800 val loss 6.4928
+1800 val perplexity 660.3954
+1800 train 6.483476 (lr=4.7299e-05) (hash(x)=156809396)
+1900 val loss 6.4066
+1900 val perplexity 605.8390
+1900 train 6.291262 (lr=3.7502e-05) (hash(x)=144640294)
+1900 val loss 6.4444
+1900 val perplexity 629.1414
+1900 train 6.327821 (lr=4.6878e-05) (hash(x)=144640294)
+2000 val loss 6.3715
+2000 val perplexity 584.9480
+2000 train 6.235487 (lr=3.7143e-05) (hash(x)=162831106)
+2000 val loss 6.4098
+2000 val perplexity 607.7933
+2000 train 6.276045 (lr=4.6428e-05) (hash(x)=162831106)
+2100 val loss 6.3520
+2100 val perplexity 573.6329
+2100 train 6.336426 (lr=3.6761e-05) (hash(x)=158239484)
+2100 val loss 6.3949
+2100 val perplexity 598.7888
+2100 train 6.386225 (lr=4.5951e-05) (hash(x)=158239484)
+2200 val loss 6.3100
+2200 val perplexity 550.0421
+2200 train 6.191505 (lr=3.6358e-05) (hash(x)=140504180)
+2200 val loss 6.3454
+2200 val perplexity 569.8500
+2200 train 6.221703 (lr=4.5448e-05) (hash(x)=140504180)
+2300 val loss 6.2763
+2300 val perplexity 531.8421
+2300 train 6.261601 (lr=3.5935e-05) (hash(x)=142234024)
+2300 val loss 6.3166
+2300 val perplexity 553.7072
+2300 train 6.301832 (lr=4.4918e-05) (hash(x)=142234024)
+2400 val loss 6.2521
+2400 val perplexity 519.1253
+2400 train 6.250567 (lr=3.5491e-05) (hash(x)=143091562)
+2400 val loss 6.2905
+2400 val perplexity 539.4141
+2400 train 6.288221 (lr=4.4363e-05) (hash(x)=143091562)
+2500 val loss 6.2441
+2500 val perplexity 514.9432
+2500 train 6.147487 (lr=3.5027e-05) (hash(x)=149857456)
+2500 val loss 6.2896
+2500 val perplexity 538.9595
+2500 train 6.186120 (lr=4.3784e-05) (hash(x)=149857456)
+2600 val loss 6.2019
+2600 val perplexity 493.6617
+2600 train 6.236071 (lr=3.4545e-05) (hash(x)=146191551)
+2600 val loss 6.2425
+2600 val perplexity 514.1449
+2600 train 6.275596 (lr=4.3181e-05) (hash(x)=146191551)
+2700 val loss 6.1777
+2700 val perplexity 481.8874
+2700 train 6.097025 (lr=3.4044e-05) (hash(x)=145375752)
+2700 val loss 6.2147
+2700 val perplexity 500.0365
+2700 train 6.136672 (lr=4.2555e-05) (hash(x)=145375752)
+2800 val loss 6.1567
+2800 val perplexity 471.8585
+2800 train 6.042354 (lr=3.3526e-05) (hash(x)=151568014)
+2800 val loss 6.1907
+2800 val perplexity 488.1791
+2800 train 6.074316 (lr=4.1908e-05) (hash(x)=151568014)
+2900 val loss 6.1475
+2900 val perplexity 467.5582
+2900 train 5.972951 (lr=3.2992e-05) (hash(x)=149366597)
+2900 val loss 6.1772
+2900 val perplexity 481.6508
+2900 train 6.004710 (lr=4.1240e-05) (hash(x)=149366597)
+3000 val loss 6.1128
+3000 val perplexity 451.5853
+3000 train 6.041257 (lr=3.2441e-05) (hash(x)=150464442)
+3000 val loss 6.1488
+3000 val perplexity 468.1472
+3000 train 6.076155 (lr=4.0551e-05) (hash(x)=150464442)
+3100 val loss 6.0997
+3100 val perplexity 445.7368
+3100 train 6.224766 (lr=3.1875e-05) (hash(x)=182449036)
+3100 val loss 6.1301
+3100 val perplexity 459.4888
+3100 train 6.255580 (lr=3.9844e-05) (hash(x)=182449036)
+3200 val loss 6.0814
+3200 val perplexity 437.6281
+3200 train 6.035765 (lr=3.1295e-05) (hash(x)=140141286)
+3200 val loss 6.1066
+3200 val perplexity 448.8311
+3200 train 6.063962 (lr=3.9119e-05) (hash(x)=140141286)
+3300 val loss 6.0583
+3300 val perplexity 427.6649
+3300 train 6.022595 (lr=3.0702e-05) (hash(x)=148099414)
+3300 val loss 6.0904
+3300 val perplexity 441.6101
+3300 train 6.056362 (lr=3.8377e-05) (hash(x)=148099414)
+3400 val loss 6.0437
+3400 val perplexity 421.4331
+3400 train 5.909395 (lr=3.0095e-05) (hash(x)=142633951)
+3400 val loss 6.0702
+3400 val perplexity 432.7883
+3400 train 5.931591 (lr=3.7619e-05) (hash(x)=142633951)
+3500 val loss 6.0337
+3500 val perplexity 417.2721
+3500 train 5.888732 (lr=2.9477e-05) (hash(x)=148368965)
+3500 val loss 6.0616
+3500 val perplexity 429.0826
+3500 train 5.915489 (lr=3.6847e-05) (hash(x)=148368965)
+3600 val loss 6.0122
+3600 val perplexity 408.4012
+3600 train 6.034571 (lr=2.8849e-05) (hash(x)=152372067)
+3600 val loss 6.0408
+3600 val perplexity 420.2325
+3600 train 6.061450 (lr=3.6061e-05) (hash(x)=152372067)
+3700 val loss 5.9993
+3700 val perplexity 403.1573
+3700 train 6.084296 (lr=2.8210e-05) (hash(x)=168885609)
+3700 val loss 6.0215
+3700 val perplexity 412.1892
+3700 train 6.104209 (lr=3.5263e-05) (hash(x)=168885609)
+3800 val loss 5.9861
+3800 val perplexity 397.8702
+3800 train 5.802641 (lr=2.7562e-05) (hash(x)=141633734)
+3800 val loss 6.0121
+3800 val perplexity 408.3385
+3800 train 5.825030 (lr=3.4453e-05) (hash(x)=141633734)
+3900 val loss 5.9647
+3900 val perplexity 389.4314
+3900 train 5.857396 (lr=2.6907e-05) (hash(x)=153141007)
+3900 val loss 5.9845
+3900 val perplexity 397.2269
+3900 train 5.879739 (lr=3.3633e-05) (hash(x)=153141007)
+4000 val loss 5.9538
+4000 val perplexity 385.2009
+4000 train 5.884251 (lr=2.6244e-05) (hash(x)=160577202)
+4000 val loss 5.9714
+4000 val perplexity 392.0666
+4000 train 5.898576 (lr=3.2805e-05) (hash(x)=160577202)
+4100 val loss 5.9447
+4100 val perplexity 381.7100
+4100 train 6.008455 (lr=2.5575e-05) (hash(x)=153858169)
+4100 val loss 5.9632
+4100 val perplexity 388.8482
+4100 train 6.025754 (lr=3.1968e-05) (hash(x)=153858169)
+4200 val loss 5.9262
+4200 val perplexity 374.7319
+4200 train 5.932951 (lr=2.4900e-05) (hash(x)=155889149)
+4200 val loss 5.9439
+4200 val perplexity 381.4120
+4200 train 5.955914 (lr=3.1126e-05) (hash(x)=155889149)
+4300 val loss 5.9161
+4300 val perplexity 370.9474
+4300 train 5.727002 (lr=2.4222e-05) (hash(x)=152294662)
+4300 val loss 5.9319
+4300 val perplexity 376.8553
+4300 train 5.746829 (lr=3.0277e-05) (hash(x)=152294662)
+4400 val loss 5.9034
+4400 val perplexity 366.2634
+4400 train 5.796988 (lr=2.3540e-05) (hash(x)=141804386)
+4400 val loss 5.9182
+4400 val perplexity 371.7475
+4400 train 5.814868 (lr=2.9425e-05) (hash(x)=141804386)
+4500 val loss 5.8863
+4500 val perplexity 360.0661
+4500 train 5.783825 (lr=2.2856e-05) (hash(x)=151095242)
+4500 val loss 5.8995
+4500 val perplexity 364.8424
+4500 train 5.797462 (lr=2.8571e-05) (hash(x)=151095242)
+4600 val loss 5.8843
+4600 val perplexity 359.3385
+4600 train 5.827990 (lr=2.2171e-05) (hash(x)=156414699)
+4600 val loss 5.9050
+4600 val perplexity 366.8577
+4600 train 5.844274 (lr=2.7714e-05) (hash(x)=156414699)
+4700 val loss 5.8653
+4700 val perplexity 352.5865
+4700 train 5.883523 (lr=2.1486e-05) (hash(x)=161556686)
+4700 val loss 5.8770
+4700 val perplexity 356.7452
+4700 train 5.898721 (lr=2.6857e-05) (hash(x)=161556686)
+4800 val loss 5.8517
+4800 val perplexity 347.8271
+4800 train 5.748372 (lr=2.0801e-05) (hash(x)=149000293)
+4800 val loss 5.8659
+4800 val perplexity 352.8071
+4800 train 5.770058 (lr=2.6002e-05) (hash(x)=149000293)
+4900 val loss 5.8437
+4900 val perplexity 345.0453
+4900 train 5.636767 (lr=2.0118e-05) (hash(x)=154349989)
+4900 val loss 5.8568
+4900 val perplexity 349.6093
+4900 train 5.650864 (lr=2.5148e-05) (hash(x)=154349989)
+5000 val loss 5.8322
+5000 val perplexity 341.1129
+5000 train 5.672789 (lr=1.9438e-05) (hash(x)=131475967)
+5000 val loss 5.8438
+5000 val perplexity 345.0920
+5000 train 5.681104 (lr=2.4298e-05) (hash(x)=131475967)
+5100 val loss 5.8227
+5100 val perplexity 337.8868
+5100 train 5.723958 (lr=1.8762e-05) (hash(x)=149717902)
+5100 val loss 5.8335
+5100 val perplexity 341.5510
+5100 train 5.733714 (lr=2.3452e-05) (hash(x)=149717902)
+5200 val loss 5.8152
+5200 val perplexity 335.3440
+5200 train 5.678337 (lr=1.8090e-05) (hash(x)=151407999)
+5200 val loss 5.8262
+5200 val perplexity 339.0665
+5200 train 5.690392 (lr=2.2613e-05) (hash(x)=151407999)
+5300 val loss 5.8050
+5300 val perplexity 331.9452
+5300 train 5.897745 (lr=1.7424e-05) (hash(x)=168602728)
+5300 val loss 5.8148
+5300 val perplexity 335.2257
+5300 train 5.912063 (lr=2.1780e-05) (hash(x)=168602728)
+5400 val loss 5.7930
+5400 val perplexity 328.0000
+5400 train 5.753254 (lr=1.6765e-05) (hash(x)=158344511)
+5400 val loss 5.8038
+5400 val perplexity 331.5522
+5400 train 5.758777 (lr=2.0956e-05) (hash(x)=158344511)
+5500 val loss 5.7865
+5500 val perplexity 325.8689
+5500 train 5.752546 (lr=1.6113e-05) (hash(x)=148350057)
+5500 val loss 5.7960
+5500 val perplexity 328.9738
+5500 train 5.756084 (lr=2.0141e-05) (hash(x)=148350057)
+5600 val loss 5.7780
+5600 val perplexity 323.0984
+5600 train 5.847509 (lr=1.5469e-05) (hash(x)=153847323)
+5600 val loss 5.7851
+5600 val perplexity 325.4178
+5600 train 5.857274 (lr=1.9337e-05) (hash(x)=153847323)
+5700 val loss 5.7694
+5700 val perplexity 320.3582
+5700 train 5.716361 (lr=1.4836e-05) (hash(x)=156607405)
+5700 val loss 5.7761
+5700 val perplexity 322.4830
+5700 train 5.717851 (lr=1.8545e-05) (hash(x)=156607405)
+5800 val loss 5.7644
+5800 val perplexity 318.7543
+5800 train 5.527573 (lr=1.4212e-05) (hash(x)=145115031)
+5800 val loss 5.7714
+5800 val perplexity 320.9943
+5800 train 5.540751 (lr=1.7765e-05) (hash(x)=145115031)
+5900 val loss 5.7564
+5900 val perplexity 316.2131
+5900 train 5.637622 (lr=1.3600e-05) (hash(x)=141584622)
+5900 val loss 5.7621
+5900 val perplexity 318.0196
+5900 train 5.637635 (lr=1.7000e-05) (hash(x)=141584622)
+6000 val loss 5.7476
+6000 val perplexity 313.4452
+6000 train 5.677502 (lr=1.3000e-05) (hash(x)=146613857)
+6000 val loss 5.7531
+6000 val perplexity 315.1696
+6000 train 5.681298 (lr=1.6250e-05) (hash(x)=146613857)
+6100 val loss 5.7435
+6100 val perplexity 312.1638
+6100 train 5.675444 (lr=1.2413e-05) (hash(x)=144621768)
+6100 val loss 5.7493
+6100 val perplexity 313.9653
+6100 train 5.690016 (lr=1.5516e-05) (hash(x)=144621768)
+6200 val loss 5.7394
+6200 val perplexity 310.8896
+6200 train 5.668490 (lr=1.1840e-05) (hash(x)=146521760)
+6200 val loss 5.7437
+6200 val perplexity 312.2157
+6200 train 5.672497 (lr=1.4800e-05) (hash(x)=146521760)
+6300 val loss 5.7298
+6300 val perplexity 307.8989
+6300 train 5.561487 (lr=1.1282e-05) (hash(x)=161378136)
+6300 val loss 5.7348
+6300 val perplexity 309.4624
+6300 train 5.566812 (lr=1.4102e-05) (hash(x)=161378136)
+6400 val loss 5.7250
+6400 val perplexity 306.4334
+6400 train 5.492900 (lr=1.0739e-05) (hash(x)=141624235)
+6400 val loss 5.7307
+6400 val perplexity 308.1824
+6400 train 5.490386 (lr=1.3424e-05) (hash(x)=141624235)
+6500 val loss 5.7193
+6500 val perplexity 304.6893
+6500 train 5.635794 (lr=1.0213e-05) (hash(x)=151197095)
+6500 val loss 5.7225
+6500 val perplexity 305.6655
+6500 train 5.639839 (lr=1.2766e-05) (hash(x)=151197095)
+6600 val loss 5.7163
+6600 val perplexity 303.7909
+6600 train 5.730577 (lr=9.7032e-06) (hash(x)=153269571)
+6600 val loss 5.7207
+6600 val perplexity 305.1150
+6600 train 5.732325 (lr=1.2129e-05) (hash(x)=153269571)
+6700 val loss 5.7106
+6700 val perplexity 302.0526
+6700 train 5.674960 (lr=9.2116e-06) (hash(x)=146111181)
+6700 val loss 5.7165
+6700 val perplexity 303.8293
+6700 train 5.676014 (lr=1.1515e-05) (hash(x)=146111181)
+6800 val loss 5.7021
+6800 val perplexity 299.4932
+6800 train 5.730585 (lr=8.7387e-06) (hash(x)=147269760)
+6800 val loss 5.7072
+6800 val perplexity 301.0196
+6800 train 5.740689 (lr=1.0923e-05) (hash(x)=147269760)
+6900 val loss 5.6998
+6900 val perplexity 298.8089
+6900 train 5.770253 (lr=8.2849e-06) (hash(x)=152912762)
+6900 val loss 5.7036
+6900 val perplexity 299.9584
+6900 train 5.775711 (lr=1.0356e-05) (hash(x)=152912762)
+7000 val loss 5.6976
+7000 val perplexity 298.1481
+7000 train 5.774178 (lr=7.8510e-06) (hash(x)=165412343)
+7000 val loss 5.7015
+7000 val perplexity 299.3077
+7000 train 5.780069 (lr=9.8138e-06) (hash(x)=165412343)
+7100 val loss 5.6879
+7100 val perplexity 295.2649
+7100 train 5.759478 (lr=7.4377e-06) (hash(x)=162866028)
+7100 val loss 5.6917
+7100 val perplexity 296.3834
+7100 train 5.758303 (lr=9.2971e-06) (hash(x)=162866028)
+7200 val loss 5.6843
+7200 val perplexity 294.2133
+7200 train 5.651568 (lr=7.0455e-06) (hash(x)=142998115)
+7200 val loss 5.6882
+7200 val perplexity 295.3568
+7200 train 5.657072 (lr=8.8068e-06) (hash(x)=142998115)
+7300 val loss 5.6823
+7300 val perplexity 293.6291
+7300 train 5.454581 (lr=6.6749e-06) (hash(x)=145486999)
+7300 val loss 5.6855
+7300 val perplexity 294.5671
+7300 train 5.453214 (lr=8.3436e-06) (hash(x)=145486999)
+7400 val loss 5.6758
+7400 val perplexity 291.7281
+7400 train 5.647672 (lr=6.3266e-06) (hash(x)=155325873)
+7400 val loss 5.6790
+7400 val perplexity 292.6544
+7400 train 5.649244 (lr=7.9082e-06) (hash(x)=155325873)
+7500 val loss 5.6724
+7500 val perplexity 290.7249
+7500 train 5.639364 (lr=6.0010e-06) (hash(x)=145131256)
+7500 val loss 5.6756
+7500 val perplexity 291.6504
+7500 train 5.643639 (lr=7.5012e-06) (hash(x)=145131256)
+7600 val loss 5.6785
+7600 val perplexity 292.5182
+7600 train 5.605381 (lr=5.6986e-06) (hash(x)=144008365)
+7600 val loss 5.6795
+7600 val perplexity 292.7921
+7600 train 5.601858 (lr=7.1232e-06) (hash(x)=144008365)
+7700 val loss 5.6677
+7700 val perplexity 289.3816
+7700 train 5.762498 (lr=5.4198e-06) (hash(x)=148848532)
+7700 val loss 5.6705
+7700 val perplexity 290.1856
+7700 train 5.761738 (lr=6.7747e-06) (hash(x)=148848532)
+7800 val loss 5.6638
+7800 val perplexity 288.2536
+7800 train 5.649303 (lr=5.1650e-06) (hash(x)=150391642)
+7800 val loss 5.6665
+7800 val perplexity 289.0083
+7800 train 5.650315 (lr=6.4563e-06) (hash(x)=150391642)
+7900 val loss 5.6640
+7900 val perplexity 288.3105
+7900 train 5.417850 (lr=4.9347e-06) (hash(x)=152191414)
+7900 val loss 5.6675
+7900 val perplexity 289.3123
+7900 train 5.423675 (lr=6.1684e-06) (hash(x)=152191414)
+8000 val loss 5.6595
+8000 val perplexity 286.9916
+8000 train 5.745750 (lr=4.7291e-06) (hash(x)=159755587)
+8000 val loss 5.6615
+8000 val perplexity 287.5693
+8000 train 5.745170 (lr=5.9114e-06) (hash(x)=159755587)
+8100 val loss 5.6570
+8100 val perplexity 286.2865
+8100 train 5.684660 (lr=4.5486e-06) (hash(x)=156664468)
+8100 val loss 5.6583
+8100 val perplexity 286.6665
+8100 train 5.680454 (lr=5.6857e-06) (hash(x)=156664468)
+8200 val loss 5.6558
+8200 val perplexity 285.9560
+8200 train 5.501270 (lr=4.3933e-06) (hash(x)=139457379)
+8200 val loss 5.6573
+8200 val perplexity 286.3621
+8200 train 5.498520 (lr=5.4917e-06) (hash(x)=139457379)
+8300 val loss 5.6587
+8300 val perplexity 286.7625
+8300 train 5.425822 (lr=4.2636e-06) (hash(x)=145478564)
+8300 val loss 5.6614
+8300 val perplexity 287.5524
+8300 train 5.426951 (lr=5.3295e-06) (hash(x)=145478564)
+8400 val loss 5.6507
+8400 val perplexity 284.4789
+8400 train 5.595287 (lr=4.1596e-06) (hash(x)=154982769)
+8400 val loss 5.6524
+8400 val perplexity 284.9632
+8400 train 5.590334 (lr=5.1995e-06) (hash(x)=154982769)
+8500 val loss 5.6489
+8500 val perplexity 283.9763
+8500 train 5.613891 (lr=4.0815e-06) (hash(x)=145798118)
+8500 val loss 5.6504
+8500 val perplexity 284.4180
+8500 train 5.609663 (lr=5.1019e-06) (hash(x)=145798118)
+8600 val loss 5.6482
+8600 val perplexity 283.7690
+8600 train 5.556049 (lr=4.0294e-06) (hash(x)=143231551)
+8600 val loss 5.6502
+8600 val perplexity 284.3499
+8600 train 5.561720 (lr=5.0367e-06) (hash(x)=143231551)
+8700 val loss 5.6516
+8700 val perplexity 284.7452
+8700 train 5.549951 (lr=4.0033e-06) (hash(x)=154780112)
+8700 val loss 5.6544
+8700 val perplexity 285.5377
+8700 train 5.550443 (lr=5.0041e-06) (hash(x)=154780112)
+8749 val loss 5.6444
+8749 val perplexity 282.7081
+8749 val loss 5.6461
+8749 val perplexity 283.1779

attention_kindselective_n_heads4_seed1340/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d8e2b0468ad004e4a3dca37b6af504c947a49de6a4cfea81d9bb18d2376aedc4
+oid sha256:82070b445c5fb7b87a2432f2c9be19e36f59e420ad35c91cb27d3ee26906d943
 size 92843394

attention_kindselective_n_heads4_seed1340/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1d9d9ebb3d1513f4cd781465ebb779d8b4ced4d2890de951f3517c540179754
+oid sha256:1b8395f1511ab273302cd1051c83adc8f135bf5420c81cb3242101616c4abaf5
 size 179406214