Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5.5e-5_61440_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4.5e-5_61440_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,267 +1,267 @@
 max_steps: 8750
 0 val loss 11.2646
-0 val perplexity 78014.3047
-0 train 11.267765 (lr=1.1000e-07) (hash(x)=150327452)
-100 val loss 9.9457
-100 val perplexity 20862.5527
-100 train 9.998234 (lr=1.1110e-05) (hash(x)=166780046)
-200 val loss 9.1876
-200 val perplexity 9775.3037
-200 train 9.133502 (lr=2.2110e-05) (hash(x)=155040610)
-300 val loss 8.2995
-300 val perplexity 4022.0583
-300 train 8.305970 (lr=3.3110e-05) (hash(x)=155504036)
-400 val loss 7.7026
-400 val perplexity 2214.0981
-400 train 7.576903 (lr=4.4110e-05) (hash(x)=143823248)
-500 val loss 7.5018
-500 val perplexity 1811.3724
-500 train 7.471624 (lr=5.5000e-05) (hash(x)=143734685)
-600 val loss 7.4303
-600 val perplexity 1686.3419
-600 train 7.209157 (lr=5.4982e-05) (hash(x)=150678249)
-700 val loss 7.3549
-700 val perplexity 1563.7889
-700 train 7.492709 (lr=5.4928e-05) (hash(x)=175802021)
-800 val loss 7.2668
-800 val perplexity 1431.9143
-800 train 7.272984 (lr=5.4839e-05) (hash(x)=158681215)
-900 val loss 7.2057
-900 val perplexity 1347.0262
-900 train 7.102057 (lr=5.4713e-05) (hash(x)=146108145)
-1000 val loss 7.1370
-1000 val perplexity 1257.6271
-1000 train 6.979178 (lr=5.4553e-05) (hash(x)=154996086)
-1100 val loss 7.0562
-1100 val perplexity 1160.0608
-1100 train 7.036383 (lr=5.4357e-05) (hash(x)=153885445)
-1200 val loss 6.9739
-1200 val perplexity 1068.3485
-1200 train 6.932178 (lr=5.4126e-05) (hash(x)=142353087)
-1300 val loss 6.9191
-1300 val perplexity 1011.4178
-1300 train 6.830047 (lr=5.3860e-05) (hash(x)=150750353)
-1400 val loss 6.8690
-1400 val perplexity 961.9484
-1400 train 6.692386 (lr=5.3561e-05) (hash(x)=152767913)
-1500 val loss 6.7984
-1500 val perplexity 896.3729
-1500 train 6.837613 (lr=5.3227e-05) (hash(x)=151562048)
-1600 val loss 6.7206
-1600 val perplexity 829.3372
-1600 train 6.723626 (lr=5.2860e-05) (hash(x)=166486165)
-1700 val loss 6.6838
-1700 val perplexity 799.3312
-1700 train 6.325279 (lr=5.2461e-05) (hash(x)=130835396)
-1800 val loss 6.6342
-1800 val perplexity 760.6326
-1800 train 6.681156 (lr=5.2029e-05) (hash(x)=158851816)
-1900 val loss 6.5986
-1900 val perplexity 734.0478
-1900 train 6.630829 (lr=5.1565e-05) (hash(x)=153313879)
-2000 val loss 6.5331
-2000 val perplexity 687.5047
-2000 train 6.463948 (lr=5.1071e-05) (hash(x)=158245023)
-2100 val loss 6.4869
-2100 val perplexity 656.4935
-2100 train 6.311565 (lr=5.0547e-05) (hash(x)=157204896)
-2200 val loss 6.4565
-2200 val perplexity 636.8082
-2200 train 6.298766 (lr=4.9993e-05) (hash(x)=137541932)
-2300 val loss 6.4247
-2300 val perplexity 616.9203
-2300 train 6.379897 (lr=4.9410e-05) (hash(x)=150149692)
-2400 val loss 6.3730
-2400 val perplexity 585.8328
-2400 train 6.363482 (lr=4.8800e-05) (hash(x)=151730720)
-2500 val loss 6.3399
-2500 val perplexity 566.7245
-2500 train 6.111226 (lr=4.8162e-05) (hash(x)=143406752)
-2600 val loss 6.3211
-2600 val perplexity 556.1671
-2600 train 6.154155 (lr=4.7499e-05) (hash(x)=157272496)
-2700 val loss 6.2973
-2700 val perplexity 543.0967
-2700 train 6.291989 (lr=4.6811e-05) (hash(x)=155342327)
-2800 val loss 6.2554
-2800 val perplexity 520.8175
-2800 train 6.133845 (lr=4.6099e-05) (hash(x)=140626679)
-2900 val loss 6.2355
-2900 val perplexity 510.5799
-2900 train 6.095793 (lr=4.5364e-05) (hash(x)=144953350)
-3000 val loss 6.2132
-3000 val perplexity 499.2762
-3000 train 6.112416 (lr=4.4606e-05) (hash(x)=172449837)
-3100 val loss 6.2019
-3100 val perplexity 493.6857
-3100 train 5.997821 (lr=4.3828e-05) (hash(x)=141710086)
-3200 val loss 6.1629
-3200 val perplexity 474.8234
-3200 train 6.093801 (lr=4.3031e-05) (hash(x)=151299772)
-3300 val loss 6.1521
-3300 val perplexity 469.6945
-3300 train 6.020457 (lr=4.2215e-05) (hash(x)=146473110)
-3400 val loss 6.1303
-3400 val perplexity 459.5718
-3400 train 6.210660 (lr=4.1381e-05) (hash(x)=153954157)
-3500 val loss 6.1080
-3500 val perplexity 449.4359
-3500 train 6.042493 (lr=4.0532e-05) (hash(x)=153717336)
-3600 val loss 6.0886
-3600 val perplexity 440.8094
-3600 train 5.844422 (lr=3.9667e-05) (hash(x)=144965161)
-3700 val loss 6.0836
-3700 val perplexity 438.5945
-3700 train 5.863056 (lr=3.8789e-05) (hash(x)=125969741)
-3800 val loss 6.0621
-3800 val perplexity 429.2794
-3800 train 5.925856 (lr=3.7898e-05) (hash(x)=155070487)
-3900 val loss 6.0450
-3900 val perplexity 421.9990
-3900 train 5.920032 (lr=3.6996e-05) (hash(x)=149444644)
-4000 val loss 6.0358
-4000 val perplexity 418.1515
-4000 train 5.892958 (lr=3.6085e-05) (hash(x)=151663033)
-4100 val loss 6.0280
-4100 val perplexity 414.8808
-4100 train 5.983856 (lr=3.5165e-05) (hash(x)=143688282)
-4200 val loss 6.0011
-4200 val perplexity 403.8761
-4200 train 6.000255 (lr=3.4238e-05) (hash(x)=163361651)
-4300 val loss 5.9869
-4300 val perplexity 398.1633
-4300 train 6.021540 (lr=3.3305e-05) (hash(x)=153619361)
-4400 val loss 5.9853
-4400 val perplexity 397.5516
-4400 train 6.296560 (lr=3.2368e-05) (hash(x)=168527064)
-4500 val loss 5.9708
-4500 val perplexity 391.8027
-4500 train 5.778735 (lr=3.1428e-05) (hash(x)=125588037)
-4600 val loss 5.9559
-4600 val perplexity 386.0312
-4600 train 5.915372 (lr=3.0486e-05) (hash(x)=143710941)
-4700 val loss 5.9398
-4700 val perplexity 379.8761
-4700 train 5.773179 (lr=2.9543e-05) (hash(x)=150952742)
-4800 val loss 5.9353
-4800 val perplexity 378.1379
-4800 train 5.802930 (lr=2.8602e-05) (hash(x)=145323659)
-4900 val loss 5.9280
-4900 val perplexity 375.4010
-4900 train 5.946818 (lr=2.7663e-05) (hash(x)=153151397)
-5000 val loss 5.9167
-5000 val perplexity 371.1711
-5000 train 5.814476 (lr=2.6728e-05) (hash(x)=143182059)
-5100 val loss 5.9067
-5100 val perplexity 367.5034
-5100 train 5.954238 (lr=2.5798e-05) (hash(x)=170083586)
-5200 val loss 5.8899
-5200 val perplexity 361.3606
-5200 train 5.788404 (lr=2.4874e-05) (hash(x)=149363919)
-5300 val loss 5.8822
-5300 val perplexity 358.5796
-5300 train 5.769784 (lr=2.3958e-05) (hash(x)=152033784)
-5400 val loss 5.8687
-5400 val perplexity 353.7852
-5400 train 5.975514 (lr=2.3051e-05) (hash(x)=154614289)
-5500 val loss 5.8615
-5500 val perplexity 351.2599
-5500 train 5.953315 (lr=2.2155e-05) (hash(x)=157745174)
-5600 val loss 5.8554
-5600 val perplexity 349.1059
-5600 train 5.747090 (lr=2.1271e-05) (hash(x)=147693222)
-5700 val loss 5.8457
-5700 val perplexity 345.7462
-5700 train 5.784711 (lr=2.0399e-05) (hash(x)=149784627)
-5800 val loss 5.8401
-5800 val perplexity 343.8195
-5800 train 5.752527 (lr=1.9542e-05) (hash(x)=158620729)
-5900 val loss 5.8340
-5900 val perplexity 341.7084
-5900 train 5.779388 (lr=1.8700e-05) (hash(x)=159763910)
-6000 val loss 5.8213
-6000 val perplexity 337.4167
-6000 train 5.758483 (lr=1.7875e-05) (hash(x)=147640561)
-6100 val loss 5.8147
-6100 val perplexity 335.1861
-6100 train 5.795366 (lr=1.7068e-05) (hash(x)=156613394)
-6200 val loss 5.8099
-6200 val perplexity 333.6008
-6200 train 5.747422 (lr=1.6280e-05) (hash(x)=186221290)
-6300 val loss 5.7978
-6300 val perplexity 329.5759
-6300 train 5.679901 (lr=1.5512e-05) (hash(x)=152081419)
-6400 val loss 5.8018
-6400 val perplexity 330.8954
-6400 train 5.755892 (lr=1.4766e-05) (hash(x)=154808349)
-6500 val loss 5.7942
-6500 val perplexity 328.4028
-6500 train 5.763316 (lr=1.4042e-05) (hash(x)=159437208)
-6600 val loss 5.7910
-6600 val perplexity 327.3300
-6600 train 5.604477 (lr=1.3342e-05) (hash(x)=157933074)
-6700 val loss 5.7805
-6700 val perplexity 323.9347
-6700 train 5.757157 (lr=1.2666e-05) (hash(x)=161560240)
-6800 val loss 5.7777
-6800 val perplexity 323.0242
-6800 train 5.759155 (lr=1.2016e-05) (hash(x)=155424292)
-6900 val loss 5.7755
-6900 val perplexity 322.2903
-6900 train 5.768492 (lr=1.1392e-05) (hash(x)=148561470)
-7000 val loss 5.7678
-7000 val perplexity 319.8350
-7000 train 5.664386 (lr=1.0795e-05) (hash(x)=141527450)
-7100 val loss 5.7617
-7100 val perplexity 317.8992
-7100 train 5.717986 (lr=1.0227e-05) (hash(x)=151066339)
-7200 val loss 5.7629
-7200 val perplexity 318.2719
-7200 train 5.559781 (lr=9.6875e-06) (hash(x)=155231264)
-7300 val loss 5.7547
-7300 val perplexity 315.6577
-7300 train 5.772898 (lr=9.1780e-06) (hash(x)=150281149)
-7400 val loss 5.7514
-7400 val perplexity 314.6422
-7400 train 5.640527 (lr=8.6990e-06) (hash(x)=148421717)
-7500 val loss 5.7514
-7500 val perplexity 314.6400
-7500 train 5.513526 (lr=8.2513e-06) (hash(x)=146921118)
-7600 val loss 5.7441
-7600 val perplexity 312.3449
-7600 train 5.794578 (lr=7.8355e-06) (hash(x)=150660048)
-7700 val loss 5.7411
-7700 val perplexity 311.4137
-7700 train 5.619473 (lr=7.4522e-06) (hash(x)=148059852)
-7800 val loss 5.7406
-7800 val perplexity 311.2433
-7800 train 5.437676 (lr=7.1019e-06) (hash(x)=148331002)
-7900 val loss 5.7347
-7900 val perplexity 309.4308
-7900 train 5.687338 (lr=6.7852e-06) (hash(x)=164923883)
-8000 val loss 5.7323
-8000 val perplexity 308.6885
-8000 train 5.618847 (lr=6.5025e-06) (hash(x)=143545384)
-8100 val loss 5.7315
-8100 val perplexity 308.4400
-8100 train 5.545605 (lr=6.2543e-06) (hash(x)=160686959)
-8200 val loss 5.7291
-8200 val perplexity 307.6797
-8200 train 5.678798 (lr=6.0408e-06) (hash(x)=156501889)
-8300 val loss 5.7280
-8300 val perplexity 307.3556
-8300 train 5.569695 (lr=5.8625e-06) (hash(x)=142716875)
-8400 val loss 5.7214
-8400 val perplexity 305.3186
-8400 train 5.635721 (lr=5.7195e-06) (hash(x)=154436684)
-8500 val loss 5.7204
-8500 val perplexity 305.0224
-8500 train 5.795388 (lr=5.6121e-06) (hash(x)=147965839)
-8600 val loss 5.7193
-8600 val perplexity 304.6931
-8600 train 5.511634 (lr=5.5404e-06) (hash(x)=145228097)
-8700 val loss 5.7141
-8700 val perplexity 303.1104
-8700 train 5.900846 (lr=5.5045e-06) (hash(x)=152910357)
-8749 val loss 5.7145
-8749 val perplexity 303.2289
+0 val perplexity 78014.2266
+0 train 11.267744 (lr=9.0000e-08) (hash(x)=150327452)
+100 val loss 9.9869
+100 val perplexity 21740.5742
+100 train 10.040390 (lr=9.0900e-06) (hash(x)=166780046)
+200 val loss 9.3625
+200 val perplexity 11643.5410
+200 train 9.319342 (lr=1.8090e-05) (hash(x)=155040610)
+300 val loss 8.4028
+300 val perplexity 4459.5981
+300 train 8.416439 (lr=2.7090e-05) (hash(x)=155504036)
+400 val loss 7.7030
+400 val perplexity 2215.0740
+400 train 7.579350 (lr=3.6090e-05) (hash(x)=143823248)
+500 val loss 7.4875
+500 val perplexity 1785.6263
+500 train 7.457825 (lr=4.5000e-05) (hash(x)=143734685)
+600 val loss 7.4083
+600 val perplexity 1649.5944
+600 train 7.185647 (lr=4.4985e-05) (hash(x)=150678249)
+700 val loss 7.3038
+700 val perplexity 1485.9933
+700 train 7.454690 (lr=4.4941e-05) (hash(x)=175802021)
+800 val loss 7.1949
+800 val perplexity 1332.6075
+800 train 7.207471 (lr=4.4868e-05) (hash(x)=158681215)
+900 val loss 7.1159
+900 val perplexity 1231.3386
+900 train 7.008085 (lr=4.4766e-05) (hash(x)=146108145)
+1000 val loss 7.0572
+1000 val perplexity 1161.1853
+1000 train 6.899328 (lr=4.4634e-05) (hash(x)=154996086)
+1100 val loss 6.9827
+1100 val perplexity 1077.7751
+1100 train 6.962049 (lr=4.4474e-05) (hash(x)=153885445)
+1200 val loss 6.9096
+1200 val perplexity 1001.8237
+1200 train 6.864575 (lr=4.4285e-05) (hash(x)=142353087)
+1300 val loss 6.8560
+1300 val perplexity 949.5422
+1300 train 6.762306 (lr=4.4068e-05) (hash(x)=150750353)
+1400 val loss 6.7982
+1400 val perplexity 896.2370
+1400 train 6.618343 (lr=4.3822e-05) (hash(x)=152767913)
+1500 val loss 6.7233
+1500 val perplexity 831.5230
+1500 train 6.760575 (lr=4.3549e-05) (hash(x)=151562048)
+1600 val loss 6.6423
+1600 val perplexity 766.8309
+1600 train 6.643299 (lr=4.3249e-05) (hash(x)=166486165)
+1700 val loss 6.5837
+1700 val perplexity 723.2342
+1700 train 6.229164 (lr=4.2922e-05) (hash(x)=130835396)
+1800 val loss 6.5306
+1800 val perplexity 685.8266
+1800 train 6.576537 (lr=4.2569e-05) (hash(x)=158851816)
+1900 val loss 6.4929
+1900 val perplexity 660.4493
+1900 train 6.526217 (lr=4.2190e-05) (hash(x)=153313879)
+2000 val loss 6.4303
+2000 val perplexity 620.3297
+2000 train 6.358914 (lr=4.1785e-05) (hash(x)=158245023)
+2100 val loss 6.3892
+2100 val perplexity 595.4015
+2100 train 6.213952 (lr=4.1356e-05) (hash(x)=157204896)
+2200 val loss 6.3634
+2200 val perplexity 580.2315
+2200 train 6.200081 (lr=4.0903e-05) (hash(x)=137541932)
+2300 val loss 6.3336
+2300 val perplexity 563.1638
+2300 train 6.286159 (lr=4.0426e-05) (hash(x)=150149692)
+2400 val loss 6.2952
+2400 val perplexity 541.9569
+2400 train 6.276896 (lr=3.9927e-05) (hash(x)=151730720)
+2500 val loss 6.2657
+2500 val perplexity 526.2218
+2500 train 6.048392 (lr=3.9406e-05) (hash(x)=143406752)
+2600 val loss 6.2478
+2600 val perplexity 516.8922
+2600 train 6.079948 (lr=3.8863e-05) (hash(x)=157272496)
+2700 val loss 6.2204
+2700 val perplexity 502.9050
+2700 train 6.206706 (lr=3.8300e-05) (hash(x)=155342327)
+2800 val loss 6.1862
+2800 val perplexity 485.9829
+2800 train 6.062219 (lr=3.7717e-05) (hash(x)=140626679)
+2900 val loss 6.1618
+2900 val perplexity 474.2803
+2900 train 6.014542 (lr=3.7116e-05) (hash(x)=144953350)
+3000 val loss 6.1459
+3000 val perplexity 466.8199
+3000 train 6.044561 (lr=3.6496e-05) (hash(x)=172449837)
+3100 val loss 6.1350
+3100 val perplexity 461.7465
+3100 train 5.924545 (lr=3.5860e-05) (hash(x)=141710086)
+3200 val loss 6.0963
+3200 val perplexity 444.2307
+3200 train 6.025388 (lr=3.5207e-05) (hash(x)=151299772)
+3300 val loss 6.0834
+3300 val perplexity 438.5322
+3300 train 5.949057 (lr=3.4539e-05) (hash(x)=146473110)
+3400 val loss 6.0566
+3400 val perplexity 426.9290
+3400 train 6.142395 (lr=3.3857e-05) (hash(x)=153954157)
+3500 val loss 6.0423
+3500 val perplexity 420.8712
+3500 train 5.983976 (lr=3.3162e-05) (hash(x)=153717336)
+3600 val loss 6.0249
+3600 val perplexity 413.6099
+3600 train 5.780908 (lr=3.2455e-05) (hash(x)=144965161)
+3700 val loss 6.0154
+3700 val perplexity 409.6885
+3700 train 5.792235 (lr=3.1736e-05) (hash(x)=125969741)
+3800 val loss 5.9919
+3800 val perplexity 400.1568
+3800 train 5.857428 (lr=3.1008e-05) (hash(x)=155070487)
+3900 val loss 5.9737
+3900 val perplexity 392.9507
+3900 train 5.845443 (lr=3.0270e-05) (hash(x)=149444644)
+4000 val loss 5.9613
+4000 val perplexity 388.1236
+4000 train 5.812981 (lr=2.9524e-05) (hash(x)=151663033)
+4100 val loss 5.9495
+4100 val perplexity 383.5775
+4100 train 5.907077 (lr=2.8771e-05) (hash(x)=143688282)
+4200 val loss 5.9299
+4200 val perplexity 376.1001
+4200 train 5.928873 (lr=2.8013e-05) (hash(x)=163361651)
+4300 val loss 5.9129
+4300 val perplexity 369.7745
+4300 train 5.957870 (lr=2.7250e-05) (hash(x)=153619361)
+4400 val loss 5.9052
+4400 val perplexity 366.9546
+4400 train 6.219810 (lr=2.6483e-05) (hash(x)=168527064)
+4500 val loss 5.8973
+4500 val perplexity 364.0627
+4500 train 5.713622 (lr=2.5714e-05) (hash(x)=125588037)
+4600 val loss 5.8746
+4600 val perplexity 355.8658
+4600 train 5.831522 (lr=2.4943e-05) (hash(x)=143710941)
+4700 val loss 5.8623
+4700 val perplexity 351.5312
+4700 train 5.698743 (lr=2.4172e-05) (hash(x)=150952742)
+4800 val loss 5.8534
+4800 val perplexity 348.4177
+4800 train 5.727831 (lr=2.3402e-05) (hash(x)=145323659)
+4900 val loss 5.8470
+4900 val perplexity 346.1969
+4900 train 5.861619 (lr=2.2633e-05) (hash(x)=153151397)
+5000 val loss 5.8346
+5000 val perplexity 341.9235
+5000 train 5.740033 (lr=2.1868e-05) (hash(x)=143182059)
+5100 val loss 5.8234
+5100 val perplexity 338.1063
+5100 train 5.859928 (lr=2.1107e-05) (hash(x)=170083586)
+5200 val loss 5.8080
+5200 val perplexity 332.9380
+5200 train 5.692333 (lr=2.0351e-05) (hash(x)=149363919)
+5300 val loss 5.8036
+5300 val perplexity 331.4817
+5300 train 5.690423 (lr=1.9602e-05) (hash(x)=152033784)
+5400 val loss 5.7884
+5400 val perplexity 326.4917
+5400 train 5.901883 (lr=1.8860e-05) (hash(x)=154614289)
+5500 val loss 5.7798
+5500 val perplexity 323.7035
+5500 train 5.871509 (lr=1.8127e-05) (hash(x)=157745174)
+5600 val loss 5.7756
+5600 val perplexity 322.3418
+5600 train 5.672544 (lr=1.7403e-05) (hash(x)=147693222)
+5700 val loss 5.7623
+5700 val perplexity 318.0876
+5700 train 5.693845 (lr=1.6690e-05) (hash(x)=149784627)
+5800 val loss 5.7625
+5800 val perplexity 318.1317
+5800 train 5.686205 (lr=1.5989e-05) (hash(x)=158620729)
+5900 val loss 5.7499
+5900 val perplexity 314.1568
+5900 train 5.698482 (lr=1.5300e-05) (hash(x)=159763910)
+6000 val loss 5.7406
+6000 val perplexity 311.2519
+6000 train 5.675223 (lr=1.4625e-05) (hash(x)=147640561)
+6100 val loss 5.7337
+6100 val perplexity 309.1020
+6100 train 5.713614 (lr=1.3965e-05) (hash(x)=156613394)
+6200 val loss 5.7276
+6200 val perplexity 307.2375
+6200 train 5.664627 (lr=1.3320e-05) (hash(x)=186221290)
+6300 val loss 5.7168
+6300 val perplexity 303.9236
+6300 train 5.591850 (lr=1.2692e-05) (hash(x)=152081419)
+6400 val loss 5.7170
+6400 val perplexity 303.9820
+6400 train 5.669156 (lr=1.2081e-05) (hash(x)=154808349)
+6500 val loss 5.7096
+6500 val perplexity 301.7650
+6500 train 5.673279 (lr=1.1489e-05) (hash(x)=159437208)
+6600 val loss 5.7078
+6600 val perplexity 301.1956
+6600 train 5.518106 (lr=1.0916e-05) (hash(x)=157933074)
+6700 val loss 5.7002
+6700 val perplexity 298.9182
+6700 train 5.680590 (lr=1.0363e-05) (hash(x)=161560240)
+6800 val loss 5.6961
+6800 val perplexity 297.7002
+6800 train 5.674346 (lr=9.8310e-06) (hash(x)=155424292)
+6900 val loss 5.6936
+6900 val perplexity 296.9641
+6900 train 5.687900 (lr=9.3205e-06) (hash(x)=148561470)
+7000 val loss 5.6879
+7000 val perplexity 295.2784
+7000 train 5.582413 (lr=8.8324e-06) (hash(x)=141527450)
+7100 val loss 5.6817
+7100 val perplexity 293.4472
+7100 train 5.636769 (lr=8.3674e-06) (hash(x)=151066339)
+7200 val loss 5.6829
+7200 val perplexity 293.8135
+7200 train 5.478912 (lr=7.9261e-06) (hash(x)=155231264)
+7300 val loss 5.6733
+7300 val perplexity 290.9905
+7300 train 5.695995 (lr=7.5093e-06) (hash(x)=150281149)
+7400 val loss 5.6712
+7400 val perplexity 290.3902
+7400 train 5.553436 (lr=7.1174e-06) (hash(x)=148421717)
+7500 val loss 5.6716
+7500 val perplexity 290.4991
+7500 train 5.431908 (lr=6.7511e-06) (hash(x)=146921118)
+7600 val loss 5.6636
+7600 val perplexity 288.1749
+7600 train 5.711033 (lr=6.4109e-06) (hash(x)=150660048)
+7700 val loss 5.6626
+7700 val perplexity 287.9061
+7700 train 5.536907 (lr=6.0972e-06) (hash(x)=148059852)
+7800 val loss 5.6614
+7800 val perplexity 287.5586
+7800 train 5.351384 (lr=5.8107e-06) (hash(x)=148331002)
+7900 val loss 5.6558
+7900 val perplexity 285.9372
+7900 train 5.604271 (lr=5.5515e-06) (hash(x)=164923883)
+8000 val loss 5.6541
+8000 val perplexity 285.4702
+8000 train 5.535724 (lr=5.3203e-06) (hash(x)=143545384)
+8100 val loss 5.6531
+8100 val perplexity 285.1748
+8100 train 5.466304 (lr=5.1172e-06) (hash(x)=160686959)
+8200 val loss 5.6509
+8200 val perplexity 284.5504
+8200 train 5.601032 (lr=4.9425e-06) (hash(x)=156501889)
+8300 val loss 5.6488
+8300 val perplexity 283.9383
+8300 train 5.486124 (lr=4.7966e-06) (hash(x)=142716875)
+8400 val loss 5.6439
+8400 val perplexity 282.5641
+8400 train 5.561882 (lr=4.6796e-06) (hash(x)=154436684)
+8500 val loss 5.6430
+8500 val perplexity 282.2974
+8500 train 5.726328 (lr=4.5917e-06) (hash(x)=147965839)
+8600 val loss 5.6415
+8600 val perplexity 281.8958
+8600 train 5.439447 (lr=4.5330e-06) (hash(x)=145228097)
+8700 val loss 5.6370
+8700 val perplexity 280.6302
+8700 train 5.829304 (lr=4.5037e-06) (hash(x)=152910357)
+8749 val loss 5.6369
+8749 val perplexity 280.5848

attention_kindselective_n_heads4_seed1338/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a131c48954674e396c86cde25dd3f7fc64149914014049b28ac85daeaa29f8e4
+oid sha256:b1194215ae1764e27f99d25868ee71ac63c74d1b625dda045798ebb1b8c89462
 size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc5c4c2a91810fed0268ecc62c25629b3dbf30047fa030d380998253c008ba02
+oid sha256:719546c6e08256f8acaae03ca5284013062efd1e6207724fcf8920f416f23bd3
 size 179406214