Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4.5e-5_61440_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,267 +1,525 @@
 max_steps: 8750
+100 val loss 10.0225
+100 val perplexity 22528.5176
+100 train 10.075231 (lr=8.0800e-06) (hash(x)=166780046)
 0 val loss 11.2646
-0 val perplexity 78014.2266
-0 train 11.267744 (lr=9.0000e-08) (hash(x)=150327452)
-100 val loss 9.9869
-100 val perplexity 21740.5742
-100 train 10.040390 (lr=9.0900e-06) (hash(x)=166780046)
-200 val loss 9.3625
-200 val perplexity 11643.5410
-200 train 9.319342 (lr=1.8090e-05) (hash(x)=155040610)
-300 val loss 8.4028
-300 val perplexity 4459.5981
-300 train 8.416439 (lr=2.7090e-05) (hash(x)=155504036)
-400 val loss 7.7030
-400 val perplexity 2215.0740
-400 train 7.579350 (lr=3.6090e-05) (hash(x)=143823248)
-500 val loss 7.4875
-500 val perplexity 1785.6263
-500 train 7.457825 (lr=4.5000e-05) (hash(x)=143734685)
-600 val loss 7.4083
-600 val perplexity 1649.5944
-600 train 7.185647 (lr=4.4985e-05) (hash(x)=150678249)
-700 val loss 7.3038
-700 val perplexity 1485.9933
-700 train 7.454690 (lr=4.4941e-05) (hash(x)=175802021)
-800 val loss 7.1949
-800 val perplexity 1332.6075
-800 train 7.207471 (lr=4.4868e-05) (hash(x)=158681215)
-900 val loss 7.1159
-900 val perplexity 1231.3386
-900 train 7.008085 (lr=4.4766e-05) (hash(x)=146108145)
-1000 val loss 7.0572
-1000 val perplexity 1161.1853
-1000 train 6.899328 (lr=4.4634e-05) (hash(x)=154996086)
-1100 val loss 6.9827
-1100 val perplexity 1077.7751
-1100 train 6.962049 (lr=4.4474e-05) (hash(x)=153885445)
-1200 val loss 6.9096
-1200 val perplexity 1001.8237
-1200 train 6.864575 (lr=4.4285e-05) (hash(x)=142353087)
-1300 val loss 6.8560
-1300 val perplexity 949.5422
-1300 train 6.762306 (lr=4.4068e-05) (hash(x)=150750353)
-1400 val loss 6.7982
-1400 val perplexity 896.2370
-1400 train 6.618343 (lr=4.3822e-05) (hash(x)=152767913)
-1500 val loss 6.7233
-1500 val perplexity 831.5230
-1500 train 6.760575 (lr=4.3549e-05) (hash(x)=151562048)
-1600 val loss 6.6423
-1600 val perplexity 766.8309
-1600 train 6.643299 (lr=4.3249e-05) (hash(x)=166486165)
-1700 val loss 6.5837
-1700 val perplexity 723.2342
-1700 train 6.229164 (lr=4.2922e-05) (hash(x)=130835396)
-1800 val loss 6.5306
-1800 val perplexity 685.8266
-1800 train 6.576537 (lr=4.2569e-05) (hash(x)=158851816)
-1900 val loss 6.4929
-1900 val perplexity 660.4493
-1900 train 6.526217 (lr=4.2190e-05) (hash(x)=153313879)
-2000 val loss 6.4303
-2000 val perplexity 620.3297
-2000 train 6.358914 (lr=4.1785e-05) (hash(x)=158245023)
-2100 val loss 6.3892
-2100 val perplexity 595.4015
-2100 train 6.213952 (lr=4.1356e-05) (hash(x)=157204896)
-2200 val loss 6.3634
-2200 val perplexity 580.2315
-2200 train 6.200081 (lr=4.0903e-05) (hash(x)=137541932)
-2300 val loss 6.3336
-2300 val perplexity 563.1638
-2300 train 6.286159 (lr=4.0426e-05) (hash(x)=150149692)
-2400 val loss 6.2952
-2400 val perplexity 541.9569
-2400 train 6.276896 (lr=3.9927e-05) (hash(x)=151730720)
-2500 val loss 6.2657
-2500 val perplexity 526.2218
-2500 train 6.048392 (lr=3.9406e-05) (hash(x)=143406752)
-2600 val loss 6.2478
-2600 val perplexity 516.8922
-2600 train 6.079948 (lr=3.8863e-05) (hash(x)=157272496)
-2700 val loss 6.2204
-2700 val perplexity 502.9050
-2700 train 6.206706 (lr=3.8300e-05) (hash(x)=155342327)
-2800 val loss 6.1862
-2800 val perplexity 485.9829
-2800 train 6.062219 (lr=3.7717e-05) (hash(x)=140626679)
-2900 val loss 6.1618
-2900 val perplexity 474.2803
-2900 train 6.014542 (lr=3.7116e-05) (hash(x)=144953350)
-3000 val loss 6.1459
-3000 val perplexity 466.8199
-3000 train 6.044561 (lr=3.6496e-05) (hash(x)=172449837)
-3100 val loss 6.1350
-3100 val perplexity 461.7465
-3100 train 5.924545 (lr=3.5860e-05) (hash(x)=141710086)
-3200 val loss 6.0963
-3200 val perplexity 444.2307
-3200 train 6.025388 (lr=3.5207e-05) (hash(x)=151299772)
-3300 val loss 6.0834
-3300 val perplexity 438.5322
-3300 train 5.949057 (lr=3.4539e-05) (hash(x)=146473110)
-3400 val loss 6.0566
-3400 val perplexity 426.9290
-3400 train 6.142395 (lr=3.3857e-05) (hash(x)=153954157)
-3500 val loss 6.0423
-3500 val perplexity 420.8712
-3500 train 5.983976 (lr=3.3162e-05) (hash(x)=153717336)
-3600 val loss 6.0249
-3600 val perplexity 413.6099
-3600 train 5.780908 (lr=3.2455e-05) (hash(x)=144965161)
-3700 val loss 6.0154
-3700 val perplexity 409.6885
-3700 train 5.792235 (lr=3.1736e-05) (hash(x)=125969741)
-3800 val loss 5.9919
-3800 val perplexity 400.1568
-3800 train 5.857428 (lr=3.1008e-05) (hash(x)=155070487)
-3900 val loss 5.9737
-3900 val perplexity 392.9507
-3900 train 5.845443 (lr=3.0270e-05) (hash(x)=149444644)
-4000 val loss 5.9613
-4000 val perplexity 388.1236
-4000 train 5.812981 (lr=2.9524e-05) (hash(x)=151663033)
-4100 val loss 5.9495
-4100 val perplexity 383.5775
-4100 train 5.907077 (lr=2.8771e-05) (hash(x)=143688282)
-4200 val loss 5.9299
-4200 val perplexity 376.1001
-4200 train 5.928873 (lr=2.8013e-05) (hash(x)=163361651)
-4300 val loss 5.9129
-4300 val perplexity 369.7745
-4300 train 5.957870 (lr=2.7250e-05) (hash(x)=153619361)
-4400 val loss 5.9052
-4400 val perplexity 366.9546
-4400 train 6.219810 (lr=2.6483e-05) (hash(x)=168527064)
-4500 val loss 5.8973
-4500 val perplexity 364.0627
-4500 train 5.713622 (lr=2.5714e-05) (hash(x)=125588037)
-4600 val loss 5.8746
-4600 val perplexity 355.8658
-4600 train 5.831522 (lr=2.4943e-05) (hash(x)=143710941)
-4700 val loss 5.8623
-4700 val perplexity 351.5312
-4700 train 5.698743 (lr=2.4172e-05) (hash(x)=150952742)
-4800 val loss 5.8534
-4800 val perplexity 348.4177
-4800 train 5.727831 (lr=2.3402e-05) (hash(x)=145323659)
-4900 val loss 5.8470
-4900 val perplexity 346.1969
-4900 train 5.861619 (lr=2.2633e-05) (hash(x)=153151397)
-5000 val loss 5.8346
-5000 val perplexity 341.9235
-5000 train 5.740033 (lr=2.1868e-05) (hash(x)=143182059)
-5100 val loss 5.8234
-5100 val perplexity 338.1063
-5100 train 5.859928 (lr=2.1107e-05) (hash(x)=170083586)
-5200 val loss 5.8080
-5200 val perplexity 332.9380
-5200 train 5.692333 (lr=2.0351e-05) (hash(x)=149363919)
-5300 val loss 5.8036
-5300 val perplexity 331.4817
-5300 train 5.690423 (lr=1.9602e-05) (hash(x)=152033784)
-5400 val loss 5.7884
-5400 val perplexity 326.4917
-5400 train 5.901883 (lr=1.8860e-05) (hash(x)=154614289)
-5500 val loss 5.7798
-5500 val perplexity 323.7035
-5500 train 5.871509 (lr=1.8127e-05) (hash(x)=157745174)
-5600 val loss 5.7756
-5600 val perplexity 322.3418
-5600 train 5.672544 (lr=1.7403e-05) (hash(x)=147693222)
-5700 val loss 5.7623
-5700 val perplexity 318.0876
-5700 train 5.693845 (lr=1.6690e-05) (hash(x)=149784627)
-5800 val loss 5.7625
-5800 val perplexity 318.1317
-5800 train 5.686205 (lr=1.5989e-05) (hash(x)=158620729)
-5900 val loss 5.7499
-5900 val perplexity 314.1568
-5900 train 5.698482 (lr=1.5300e-05) (hash(x)=159763910)
-6000 val loss 5.7406
-6000 val perplexity 311.2519
-6000 train 5.675223 (lr=1.4625e-05) (hash(x)=147640561)
-6100 val loss 5.7337
-6100 val perplexity 309.1020
-6100 train 5.713614 (lr=1.3965e-05) (hash(x)=156613394)
-6200 val loss 5.7276
-6200 val perplexity 307.2375
-6200 train 5.664627 (lr=1.3320e-05) (hash(x)=186221290)
-6300 val loss 5.7168
-6300 val perplexity 303.9236
-6300 train 5.591850 (lr=1.2692e-05) (hash(x)=152081419)
-6400 val loss 5.7170
-6400 val perplexity 303.9820
-6400 train 5.669156 (lr=1.2081e-05) (hash(x)=154808349)
-6500 val loss 5.7096
-6500 val perplexity 301.7650
-6500 train 5.673279 (lr=1.1489e-05) (hash(x)=159437208)
-6600 val loss 5.7078
-6600 val perplexity 301.1956
-6600 train 5.518106 (lr=1.0916e-05) (hash(x)=157933074)
-6700 val loss 5.7002
-6700 val perplexity 298.9182
-6700 train 5.680590 (lr=1.0363e-05) (hash(x)=161560240)
-6800 val loss 5.6961
-6800 val perplexity 297.7002
-6800 train 5.674346 (lr=9.8310e-06) (hash(x)=155424292)
-6900 val loss 5.6936
-6900 val perplexity 296.9641
-6900 train 5.687900 (lr=9.3205e-06) (hash(x)=148561470)
-7000 val loss 5.6879
-7000 val perplexity 295.2784
-7000 train 5.582413 (lr=8.8324e-06) (hash(x)=141527450)
-7100 val loss 5.6817
-7100 val perplexity 293.4472
-7100 train 5.636769 (lr=8.3674e-06) (hash(x)=151066339)
-7200 val loss 5.6829
-7200 val perplexity 293.8135
-7200 train 5.478912 (lr=7.9261e-06) (hash(x)=155231264)
-7300 val loss 5.6733
-7300 val perplexity 290.9905
-7300 train 5.695995 (lr=7.5093e-06) (hash(x)=150281149)
-7400 val loss 5.6712
-7400 val perplexity 290.3902
-7400 train 5.553436 (lr=7.1174e-06) (hash(x)=148421717)
-7500 val loss 5.6716
-7500 val perplexity 290.4991
-7500 train 5.431908 (lr=6.7511e-06) (hash(x)=146921118)
-7600 val loss 5.6636
-7600 val perplexity 288.1749
-7600 train 5.711033 (lr=6.4109e-06) (hash(x)=150660048)
-7700 val loss 5.6626
-7700 val perplexity 287.9061
-7700 train 5.536907 (lr=6.0972e-06) (hash(x)=148059852)
-7800 val loss 5.6614
-7800 val perplexity 287.5586
-7800 train 5.351384 (lr=5.8107e-06) (hash(x)=148331002)
-7900 val loss 5.6558
-7900 val perplexity 285.9372
-7900 train 5.604271 (lr=5.5515e-06) (hash(x)=164923883)
-8000 val loss 5.6541
-8000 val perplexity 285.4702
-8000 train 5.535724 (lr=5.3203e-06) (hash(x)=143545384)
-8100 val loss 5.6531
-8100 val perplexity 285.1748
-8100 train 5.466304 (lr=5.1172e-06) (hash(x)=160686959)
-8200 val loss 5.6509
-8200 val perplexity 284.5504
-8200 train 5.601032 (lr=4.9425e-06) (hash(x)=156501889)
-8300 val loss 5.6488
-8300 val perplexity 283.9383
-8300 train 5.486124 (lr=4.7966e-06) (hash(x)=142716875)
-8400 val loss 5.6439
-8400 val perplexity 282.5641
-8400 train 5.561882 (lr=4.6796e-06) (hash(x)=154436684)
-8500 val loss 5.6430
-8500 val perplexity 282.2974
-8500 train 5.726328 (lr=4.5917e-06) (hash(x)=147965839)
-8600 val loss 5.6415
-8600 val perplexity 281.8958
-8600 train 5.439447 (lr=4.5330e-06) (hash(x)=145228097)
-8700 val loss 5.6370
-8700 val perplexity 280.6302
-8700 train 5.829304 (lr=4.5037e-06) (hash(x)=152910357)
-8749 val loss 5.6369
-8749 val perplexity 280.5848
+0 val perplexity 78014.3047
+200 val loss 9.5031
+200 val perplexity 13401.6328
+200 train 9.467043 (lr=1.6080e-05) (hash(x)=155040610)
+0 train 11.267765 (lr=1.0000e-07) (hash(x)=150327452)
+300 val loss 8.5047
+300 val perplexity 4938.0015
+300 train 8.524590 (lr=2.4080e-05) (hash(x)=155504036)
+100 val loss 9.9652
+100 val perplexity 21274.1113
+100 train 10.017369 (lr=1.0100e-05) (hash(x)=166780046)
+400 val loss 7.7785
+400 val perplexity 2388.7463
+400 train 7.663756 (lr=3.2080e-05) (hash(x)=143823248)
+200 val loss 9.2958
+200 val perplexity 10892.5625
+200 train 9.248415 (lr=2.0100e-05) (hash(x)=155040610)
+500 val loss 7.5217
+500 val perplexity 1847.7620
+500 train 7.493058 (lr=4.0000e-05) (hash(x)=143734685)
+300 val loss 8.1663
+300 val perplexity 3520.3948
+300 train 8.168233 (lr=3.0100e-05) (hash(x)=155504036)
+600 val loss 7.4353
+600 val perplexity 1694.7335
+600 train 7.215819 (lr=3.9987e-05) (hash(x)=150678249)
+400 val loss 7.6121
+400 val perplexity 2022.5510
+400 train 7.478056 (lr=4.0100e-05) (hash(x)=143823248)
+700 val loss 7.3528
+700 val perplexity 1560.5144
+700 train 7.493054 (lr=3.9948e-05) (hash(x)=175802021)
+500 val loss 7.4302
+500 val perplexity 1686.1023
+500 train 7.398177 (lr=5.0000e-05) (hash(x)=143734685)
+800 val loss 7.2439
+800 val perplexity 1399.5740
+800 train 7.254493 (lr=3.9883e-05) (hash(x)=158681215)
+600 val loss 7.3581
+600 val perplexity 1568.7780
+600 train 7.127360 (lr=4.9984e-05) (hash(x)=150678249)
+900 val loss 7.1674
+900 val perplexity 1296.4656
+900 train 7.060616 (lr=3.9792e-05) (hash(x)=146108145)
+700 val loss 7.2431
+700 val perplexity 1398.4459
+700 train 7.374388 (lr=4.9935e-05) (hash(x)=175802021)
+1000 val loss 7.1057
+1000 val perplexity 1218.8782
+1000 train 6.948586 (lr=3.9675e-05) (hash(x)=154996086)
+800 val loss 7.1141
+800 val perplexity 1229.1447
+800 train 7.129083 (lr=4.9853e-05) (hash(x)=158681215)
+1100 val loss 7.0173
+1100 val perplexity 1115.7611
+1100 train 7.000314 (lr=3.9532e-05) (hash(x)=153885445)
+900 val loss 7.0241
+900 val perplexity 1123.3754
+900 train 6.911726 (lr=4.9739e-05) (hash(x)=146108145)
+1200 val loss 6.9212
+1200 val perplexity 1013.5053
+1200 train 6.879501 (lr=3.9364e-05) (hash(x)=142353087)
+1000 val loss 6.9540
+1000 val perplexity 1047.3751
+1000 train 6.793063 (lr=4.9593e-05) (hash(x)=154996086)
+1300 val loss 6.8506
+1300 val perplexity 944.4738
+1300 train 6.759327 (lr=3.9171e-05) (hash(x)=150750353)
+1100 val loss 6.8664
+1100 val perplexity 959.4811
+1100 train 6.845954 (lr=4.9415e-05) (hash(x)=153885445)
+1400 val loss 6.7955
+1400 val perplexity 893.8453
+1400 train 6.617004 (lr=3.8953e-05) (hash(x)=152767913)
+1200 val loss 6.7910
+1200 val perplexity 889.7973
+1200 train 6.746166 (lr=4.9205e-05) (hash(x)=142353087)
+1500 val loss 6.7221
+1500 val perplexity 830.5728
+1500 train 6.764291 (lr=3.8711e-05) (hash(x)=151562048)
+1300 val loss 6.7221
+1300 val perplexity 830.5680
+1300 train 6.625785 (lr=4.8964e-05) (hash(x)=150750353)
+1600 val loss 6.6517
+1600 val perplexity 774.0981
+1600 train 6.655982 (lr=3.8444e-05) (hash(x)=166486165)
+1400 val loss 6.6699
+1400 val perplexity 788.3265
+1400 train 6.484465 (lr=4.8691e-05) (hash(x)=152767913)
+1700 val loss 6.5988
+1700 val perplexity 734.1826
+1700 train 6.246137 (lr=3.8153e-05) (hash(x)=130835396)
+1500 val loss 6.6045
+1500 val perplexity 738.4147
+1500 train 6.649715 (lr=4.8388e-05) (hash(x)=151562048)
+1800 val loss 6.5471
+1800 val perplexity 697.1981
+1800 train 6.590258 (lr=3.7839e-05) (hash(x)=158851816)
+1600 val loss 6.5291
+1600 val perplexity 684.7996
+1600 train 6.542716 (lr=4.8055e-05) (hash(x)=166486165)
+1900 val loss 6.5106
+1900 val perplexity 672.2518
+1900 train 6.546779 (lr=3.7502e-05) (hash(x)=153313879)
+1700 val loss 6.4875
+1700 val perplexity 656.9081
+1700 train 6.130574 (lr=4.7691e-05) (hash(x)=130835396)
+2000 val loss 6.4463
+2000 val perplexity 630.3575
+2000 train 6.380337 (lr=3.7143e-05) (hash(x)=158245023)
+1800 val loss 6.4404
+1800 val perplexity 626.6369
+1800 train 6.488020 (lr=4.7299e-05) (hash(x)=158851816)
+2100 val loss 6.4013
+2100 val perplexity 602.6064
+2100 train 6.222491 (lr=3.6761e-05) (hash(x)=157204896)
+1900 val loss 6.4053
+1900 val perplexity 605.0572
+1900 train 6.435752 (lr=4.6878e-05) (hash(x)=153313879)
+2200 val loss 6.3789
+2200 val perplexity 589.2851
+2200 train 6.219698 (lr=3.6358e-05) (hash(x)=137541932)
+2000 val loss 6.3458
+2000 val perplexity 570.0729
+2000 train 6.278543 (lr=4.6428e-05) (hash(x)=158245023)
+2300 val loss 6.3499
+2300 val perplexity 572.4460
+2300 train 6.300820 (lr=3.5935e-05) (hash(x)=150149692)
+2100 val loss 6.3149
+2100 val perplexity 552.7206
+2100 train 6.137981 (lr=4.5951e-05) (hash(x)=157204896)
+2400 val loss 6.3041
+2400 val perplexity 546.8093
+2400 train 6.291619 (lr=3.5491e-05) (hash(x)=151730720)
+2200 val loss 6.2885
+2200 val perplexity 538.3621
+2200 train 6.130053 (lr=4.5448e-05) (hash(x)=137541932)
+2500 val loss 6.2745
+2500 val perplexity 530.8547
+2500 train 6.058253 (lr=3.5027e-05) (hash(x)=143406752)
+2300 val loss 6.2654
+2300 val perplexity 526.0432
+2300 train 6.222541 (lr=4.4918e-05) (hash(x)=150149692)
+2600 val loss 6.2600
+2600 val perplexity 523.2106
+2600 train 6.092404 (lr=3.4545e-05) (hash(x)=157272496)
+2400 val loss 6.2239
+2400 val perplexity 504.6877
+2400 train 6.207308 (lr=4.4363e-05) (hash(x)=151730720)
+2700 val loss 6.2352
+2700 val perplexity 510.4092
+2700 train 6.229642 (lr=3.4044e-05) (hash(x)=155342327)
+2500 val loss 6.1932
+2500 val perplexity 489.4210
+2500 train 5.986238 (lr=4.3784e-05) (hash(x)=143406752)
+2800 val loss 6.2031
+2800 val perplexity 494.2682
+2800 train 6.078806 (lr=3.3526e-05) (hash(x)=140626679)
+2600 val loss 6.1778
+2600 val perplexity 481.9189
+2600 train 6.010714 (lr=4.3181e-05) (hash(x)=157272496)
+2900 val loss 6.1813
+2900 val perplexity 483.6212
+2900 train 6.041678 (lr=3.2992e-05) (hash(x)=144953350)
+2700 val loss 6.1551
+2700 val perplexity 471.0950
+2700 train 6.130734 (lr=4.2555e-05) (hash(x)=155342327)
+3000 val loss 6.1637
+3000 val perplexity 475.1647
+3000 train 6.065508 (lr=3.2441e-05) (hash(x)=172449837)
+2800 val loss 6.1203
+2800 val perplexity 454.9970
+2800 train 6.002769 (lr=4.1908e-05) (hash(x)=140626679)
+3100 val loss 6.1536
+3100 val perplexity 470.3969
+3100 train 5.937268 (lr=3.1875e-05) (hash(x)=141710086)
+2900 val loss 6.1003
+2900 val perplexity 446.0030
+2900 train 5.960584 (lr=4.1240e-05) (hash(x)=144953350)
+3200 val loss 6.1185
+3200 val perplexity 454.1793
+3200 train 6.050138 (lr=3.1295e-05) (hash(x)=151299772)
+3000 val loss 6.0852
+3000 val perplexity 439.3019
+3000 train 5.978558 (lr=4.0551e-05) (hash(x)=172449837)
+3300 val loss 6.1050
+3300 val perplexity 448.0775
+3300 train 5.970407 (lr=3.0702e-05) (hash(x)=146473110)
+3100 val loss 6.0773
+3100 val perplexity 435.8608
+3100 train 5.868666 (lr=3.9844e-05) (hash(x)=141710086)
+3400 val loss 6.0857
+3400 val perplexity 439.5253
+3400 train 6.168725 (lr=3.0095e-05) (hash(x)=153954157)
+3200 val loss 6.0406
+3200 val perplexity 420.1267
+3200 train 5.970029 (lr=3.9119e-05) (hash(x)=151299772)
+3500 val loss 6.0671
+3500 val perplexity 431.4481
+3500 train 6.002892 (lr=2.9477e-05) (hash(x)=153717336)
+3300 val loss 6.0289
+3300 val perplexity 415.2450
+3300 train 5.899902 (lr=3.8377e-05) (hash(x)=146473110)
+3600 val loss 6.0520
+3600 val perplexity 424.9601
+3600 train 5.805218 (lr=2.8849e-05) (hash(x)=144965161)
+3400 val loss 6.0029
+3400 val perplexity 404.5874
+3400 train 6.088784 (lr=3.7619e-05) (hash(x)=153954157)
+3700 val loss 6.0448
+3700 val perplexity 421.9280
+3700 train 5.821227 (lr=2.8210e-05) (hash(x)=125969741)
+3500 val loss 5.9897
+3500 val perplexity 399.2789
+3500 train 5.933339 (lr=3.6847e-05) (hash(x)=153717336)
+3800 val loss 6.0203
+3800 val perplexity 411.6863
+3800 train 5.885331 (lr=2.7562e-05) (hash(x)=155070487)
+3600 val loss 5.9693
+3600 val perplexity 391.2270
+3600 train 5.724619 (lr=3.6061e-05) (hash(x)=144965161)
+3900 val loss 6.0036
+3900 val perplexity 404.8695
+3900 train 5.874807 (lr=2.6907e-05) (hash(x)=149444644)
+3700 val loss 5.9651
+3700 val perplexity 389.6051
+3700 train 5.751326 (lr=3.5263e-05) (hash(x)=125969741)
+4000 val loss 5.9952
+4000 val perplexity 401.4825
+4000 train 5.852266 (lr=2.6244e-05) (hash(x)=151663033)
+3800 val loss 5.9416
+3800 val perplexity 380.5251
+3800 train 5.806542 (lr=3.4453e-05) (hash(x)=155070487)
+4100 val loss 5.9847
+4100 val perplexity 397.3002
+4100 train 5.937428 (lr=2.5575e-05) (hash(x)=143688282)
+3900 val loss 5.9216
+3900 val perplexity 372.9961
+3900 train 5.789830 (lr=3.3633e-05) (hash(x)=149444644)
+4200 val loss 5.9638
+4200 val perplexity 389.0882
+4200 train 5.963494 (lr=2.4900e-05) (hash(x)=163361651)
+4000 val loss 5.9130
+4000 val perplexity 369.8036
+4000 train 5.765071 (lr=3.2805e-05) (hash(x)=151663033)
+4300 val loss 5.9510
+4300 val perplexity 384.1372
+4300 train 5.998785 (lr=2.4222e-05) (hash(x)=153619361)
+4100 val loss 5.9076
+4100 val perplexity 367.8302
+4100 train 5.866171 (lr=3.1968e-05) (hash(x)=143688282)
+4400 val loss 5.9426
+4400 val perplexity 380.9430
+4400 train 6.251171 (lr=2.3540e-05) (hash(x)=168527064)
+4200 val loss 5.8817
+4200 val perplexity 358.4278
+4200 train 5.884516 (lr=3.1126e-05) (hash(x)=163361651)
+4500 val loss 5.9357
+4500 val perplexity 378.3056
+4500 train 5.746198 (lr=2.2856e-05) (hash(x)=125588037)
+4300 val loss 5.8654
+4300 val perplexity 352.6216
+4300 train 5.909609 (lr=3.0277e-05) (hash(x)=153619361)
+4600 val loss 5.9151
+4600 val perplexity 370.5818
+4600 train 5.879491 (lr=2.2171e-05) (hash(x)=143710941)
+4400 val loss 5.8604
+4400 val perplexity 350.8679
+4400 train 6.193501 (lr=2.9425e-05) (hash(x)=168527064)
+4700 val loss 5.9033
+4700 val perplexity 366.2306
+4700 train 5.731603 (lr=2.1486e-05) (hash(x)=150952742)
+4500 val loss 5.8487
+4500 val perplexity 346.7679
+4500 train 5.672557 (lr=2.8571e-05) (hash(x)=125588037)
+4800 val loss 5.8950
+4800 val perplexity 363.2156
+4800 train 5.766671 (lr=2.0801e-05) (hash(x)=145323659)
+4600 val loss 5.8316
+4600 val perplexity 340.9111
+4600 train 5.789861 (lr=2.7714e-05) (hash(x)=143710941)
+4900 val loss 5.8908
+4900 val perplexity 361.6895
+4900 train 5.910984 (lr=2.0118e-05) (hash(x)=153151397)
+4700 val loss 5.8176
+4700 val perplexity 336.1733
+4700 train 5.653608 (lr=2.6857e-05) (hash(x)=150952742)
+5000 val loss 5.8805
+5000 val perplexity 357.9796
+5000 train 5.776249 (lr=1.9438e-05) (hash(x)=143182059)
+4800 val loss 5.8069
+4800 val perplexity 332.5922
+4800 train 5.684433 (lr=2.6002e-05) (hash(x)=145323659)
+5100 val loss 5.8728
+5100 val perplexity 355.2327
+5100 train 5.917749 (lr=1.8762e-05) (hash(x)=170083586)
+4900 val loss 5.8035
+4900 val perplexity 331.4719
+4900 train 5.826119 (lr=2.5148e-05) (hash(x)=153151397)
+5200 val loss 5.8569
+5200 val perplexity 349.6357
+5200 train 5.746037 (lr=1.8090e-05) (hash(x)=149363919)
+5000 val loss 5.7909
+5000 val perplexity 327.2966
+5000 train 5.696330 (lr=2.4298e-05) (hash(x)=143182059)
+5300 val loss 5.8524
+5300 val perplexity 348.0753
+5300 train 5.746236 (lr=1.7424e-05) (hash(x)=152033784)
+5100 val loss 5.7796
+5100 val perplexity 323.6216
+5100 train 5.819366 (lr=2.3452e-05) (hash(x)=170083586)
+5400 val loss 5.8388
+5400 val perplexity 343.3778
+5400 train 5.942615 (lr=1.6765e-05) (hash(x)=154614289)
+5200 val loss 5.7686
+5200 val perplexity 320.0969
+5200 train 5.652661 (lr=2.2613e-05) (hash(x)=149363919)
+5500 val loss 5.8314
+5500 val perplexity 340.8504
+5500 train 5.922790 (lr=1.6113e-05) (hash(x)=157745174)
+5300 val loss 5.7625
+5300 val perplexity 318.1494
+5300 train 5.647845 (lr=2.1780e-05) (hash(x)=152033784)
+5600 val loss 5.8262
+5600 val perplexity 339.0699
+5600 train 5.719628 (lr=1.5469e-05) (hash(x)=147693222)
+5400 val loss 5.7464
+5400 val perplexity 313.0708
+5400 train 5.859529 (lr=2.0956e-05) (hash(x)=154614289)
+5700 val loss 5.8140
+5700 val perplexity 334.9571
+5700 train 5.746548 (lr=1.4836e-05) (hash(x)=149784627)
+5500 val loss 5.7393
+5500 val perplexity 310.8358
+5500 train 5.840951 (lr=2.0141e-05) (hash(x)=157745174)
+5800 val loss 5.8125
+5800 val perplexity 334.4635
+5800 train 5.733287 (lr=1.4212e-05) (hash(x)=158620729)
+5600 val loss 5.7353
+5600 val perplexity 309.6171
+5600 train 5.634243 (lr=1.9337e-05) (hash(x)=147693222)
+5900 val loss 5.8028
+5900 val perplexity 331.2259
+5900 train 5.752628 (lr=1.3600e-05) (hash(x)=159763910)
+5700 val loss 5.7216
+5700 val perplexity 305.3927
+5700 train 5.658313 (lr=1.8545e-05) (hash(x)=149784627)
+6000 val loss 5.7939
+6000 val perplexity 328.2917
+6000 train 5.729095 (lr=1.3000e-05) (hash(x)=147640561)
+5800 val loss 5.7199
+5800 val perplexity 304.8785
+5800 train 5.642802 (lr=1.7765e-05) (hash(x)=158620729)
+6100 val loss 5.7863
+6100 val perplexity 325.8080
+6100 train 5.767511 (lr=1.2413e-05) (hash(x)=156613394)
+5900 val loss 5.7106
+5900 val perplexity 302.0421
+5900 train 5.663085 (lr=1.7000e-05) (hash(x)=159763910)
+6200 val loss 5.7810
+6200 val perplexity 324.0978
+6200 train 5.732399 (lr=1.1840e-05) (hash(x)=186221290)
+6000 val loss 5.6995
+6000 val perplexity 298.7092
+6000 train 5.637892 (lr=1.6250e-05) (hash(x)=147640561)
+6300 val loss 5.7715
+6300 val perplexity 321.0345
+6300 train 5.650871 (lr=1.1282e-05) (hash(x)=152081419)
+6100 val loss 5.6932
+6100 val perplexity 296.8378
+6100 train 5.677419 (lr=1.5516e-05) (hash(x)=156613394)
+6400 val loss 5.7706
+6400 val perplexity 320.7337
+6400 train 5.718038 (lr=1.0739e-05) (hash(x)=154808349)
+6200 val loss 5.6879
+6200 val perplexity 295.2686
+6200 train 5.623254 (lr=1.4800e-05) (hash(x)=186221290)
+6500 val loss 5.7654
+6500 val perplexity 319.0538
+6500 train 5.731769 (lr=1.0213e-05) (hash(x)=159437208)
+6300 val loss 5.6766
+6300 val perplexity 291.9467
+6300 train 5.555213 (lr=1.4102e-05) (hash(x)=152081419)
+6600 val loss 5.7626
+6600 val perplexity 318.1778
+6600 train 5.570604 (lr=9.7032e-06) (hash(x)=157933074)
+6400 val loss 5.6760
+6400 val perplexity 291.7793
+6400 train 5.630403 (lr=1.3424e-05) (hash(x)=154808349)
+6700 val loss 5.7543
+6700 val perplexity 315.5476
+6500 val loss 5.6699
+6500 val perplexity 290.0038
+6700 train 5.733328 (lr=9.2116e-06) (hash(x)=161560240)
+6500 train 5.638351 (lr=1.2766e-05) (hash(x)=159437208)
+6800 val loss 5.7510
+6800 val perplexity 314.5105
+6800 train 5.735670 (lr=8.7387e-06) (hash(x)=155424292)
+6600 val loss 5.6674
+6600 val perplexity 289.2891
+6600 train 5.483891 (lr=1.2129e-05) (hash(x)=157933074)
+6900 val loss 5.7498
+6900 val perplexity 314.1256
+6900 train 5.741641 (lr=8.2849e-06) (hash(x)=148561470)
+6700 val loss 5.6597
+6700 val perplexity 287.0517
+6700 train 5.636087 (lr=1.1515e-05) (hash(x)=161560240)
+7000 val loss 5.7422
+7000 val perplexity 311.7635
+7000 train 5.635467 (lr=7.8510e-06) (hash(x)=141527450)
+6800 val loss 5.6551
+6800 val perplexity 285.7415
+6800 train 5.644596 (lr=1.0923e-05) (hash(x)=155424292)
+7100 val loss 5.7372
+7100 val perplexity 310.2070
+7100 train 5.695245 (lr=7.4377e-06) (hash(x)=151066339)
+6900 val loss 5.6537
+6900 val perplexity 285.3379
+6900 train 5.654519 (lr=1.0356e-05) (hash(x)=148561470)
+7200 val loss 5.7389
+7200 val perplexity 310.7103
+7200 train 5.537840 (lr=7.0455e-06) (hash(x)=155231264)
+7000 val loss 5.6465
+7000 val perplexity 283.2892
+7000 train 5.545733 (lr=9.8138e-06) (hash(x)=141527450)
+7300 val loss 5.7299
+7300 val perplexity 307.9386
+7300 train 5.751905 (lr=6.6749e-06) (hash(x)=150281149)
+7100 val loss 5.6420
+7100 val perplexity 282.0365
+7100 train 5.597219 (lr=9.2971e-06) (hash(x)=151066339)
+7400 val loss 5.7280
+7400 val perplexity 307.3646
+7400 train 5.612872 (lr=6.3266e-06) (hash(x)=148421717)
+7200 val loss 5.6433
+7200 val perplexity 282.3935
+7200 train 5.440510 (lr=8.8068e-06) (hash(x)=155231264)
+7500 val loss 5.7281
+7500 val perplexity 307.3760
+7500 train 5.486419 (lr=6.0010e-06) (hash(x)=146921118)
+7300 val loss 5.6343
+7300 val perplexity 279.8769
+7300 train 5.658390 (lr=8.3436e-06) (hash(x)=150281149)
+7600 val loss 5.7210
+7600 val perplexity 305.2163
+7600 train 5.768423 (lr=5.6986e-06) (hash(x)=150660048)
+7400 val loss 5.6316
+7400 val perplexity 279.1068
+7400 train 5.521311 (lr=7.9082e-06) (hash(x)=148421717)
+7700 val loss 5.7194
+7700 val perplexity 304.7192
+7700 train 5.598227 (lr=5.4198e-06) (hash(x)=148059852)
+7500 val loss 5.6322
+7500 val perplexity 279.2650
+7500 train 5.399149 (lr=7.5012e-06) (hash(x)=146921118)
+7800 val loss 5.7185
+7800 val perplexity 304.4613
+7800 train 5.413217 (lr=5.1650e-06) (hash(x)=148331002)
+7600 val loss 5.6242
+7600 val perplexity 277.0608
+7600 train 5.672753 (lr=7.1232e-06) (hash(x)=150660048)
+7900 val loss 5.7133
+7900 val perplexity 302.8731
+7900 train 5.659461 (lr=4.9347e-06) (hash(x)=164923883)
+7700 val loss 5.6228
+7700 val perplexity 276.6698
+7700 train 5.499656 (lr=6.7747e-06) (hash(x)=148059852)
+8000 val loss 5.7111
+8000 val perplexity 302.1974
+8000 train 5.595223 (lr=4.7291e-06) (hash(x)=143545384)
+7800 val loss 5.6219
+7800 val perplexity 276.4274
+7800 train 5.311404 (lr=6.4563e-06) (hash(x)=148331002)
+8100 val loss 5.7107
+8100 val perplexity 302.0790
+8100 train 5.530027 (lr=4.5486e-06) (hash(x)=160686959)
+7900 val loss 5.6168
+7900 val perplexity 275.0202
+7900 train 5.559655 (lr=6.1684e-06) (hash(x)=164923883)
+8000 val loss 5.6148
+8000 val perplexity 274.4717
+8000 train 5.498442 (lr=5.9114e-06) (hash(x)=143545384)
+8200 val loss 5.7087
+8200 val perplexity 301.4907
+8200 train 5.666525 (lr=4.3933e-06) (hash(x)=156501889)
+8100 val loss 5.6141
+8100 val perplexity 274.2537
+8100 train 5.425257 (lr=5.6857e-06) (hash(x)=160686959)
+8300 val loss 5.7069
+8300 val perplexity 300.9297
+8300 train 5.548706 (lr=4.2636e-06) (hash(x)=142716875)
+8400 val loss 5.7022
+8400 val perplexity 299.5299
+8200 val loss 5.6117
+8200 val perplexity 273.6158
+8400 train 5.616063 (lr=4.1596e-06) (hash(x)=154436684)
+8200 train 5.565293 (lr=5.4917e-06) (hash(x)=156501889)
+8300 val loss 5.6098
+8300 val perplexity 273.0905
+8500 val loss 5.7006
+8500 val perplexity 299.0606
+8300 train 5.455184 (lr=5.3295e-06) (hash(x)=142716875)
+8500 train 5.777520 (lr=4.0815e-06) (hash(x)=147965839)
+8400 val loss 5.6050
+8400 val perplexity 271.7937
+8600 val loss 5.6996
+8600 val perplexity 298.7383
+8400 train 5.522124 (lr=5.1995e-06) (hash(x)=154436684)
+8600 train 5.497610 (lr=4.0294e-06) (hash(x)=145228097)
+8500 val loss 5.6036
+8500 val perplexity 271.4066
+8700 val loss 5.6953
+8700 val perplexity 297.4666
+8500 train 5.683967 (lr=5.1019e-06) (hash(x)=147965839)
+8700 train 5.885415 (lr=4.0033e-06) (hash(x)=152910357)
+8749 val loss 5.6961
+8749 val perplexity 297.7067
+8600 val loss 5.6034
+8600 val perplexity 271.3527
+8600 train 5.399725 (lr=5.0367e-06) (hash(x)=145228097)

attention_kindselective_n_heads4_seed1338/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1194215ae1764e27f99d25868ee71ac63c74d1b625dda045798ebb1b8c89462
+oid sha256:8a027a07472d227bd517865e15803294bf1cba3e05b7cb35e0461461ed15ff0a
 size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:719546c6e08256f8acaae03ca5284013062efd1e6207724fcf8920f416f23bd3
+oid sha256:5938aeb6ef1944ddc12159568227c3c4dea45324a2282b520d0faccd7bea83b8
 size 179406214