Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_61440_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5.5e-5_61440_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,533 +1,267 @@
 max_steps: 8750
 0 val loss 11.2646
 0 val perplexity 78014.3047
-0 val loss 11.2646
-0 val perplexity 78014.3047
-0 train 11.267765 (lr=6.0000e-08) (hash(x)=150327452)
-0 train 11.267765 (lr=7.0000e-08) (hash(x)=150327452)
-100 val loss 10.0785
-100 val perplexity 23826.3750
-100 val loss 10.1480
-100 val perplexity 25539.9902
-100 train 10.130300 (lr=7.0700e-06) (hash(x)=166780046)
-100 train 10.197817 (lr=6.0600e-06) (hash(x)=166780046)
-200 val loss 9.5828
-200 val perplexity 14512.7627
-200 train 9.549955 (lr=1.4070e-05) (hash(x)=155040610)
-200 val loss 9.6799
-200 val perplexity 15992.7021
-200 train 9.651791 (lr=1.2060e-05) (hash(x)=155040610)
-300 val loss 8.7068
-300 val perplexity 6043.8276
-300 train 8.732620 (lr=2.1070e-05) (hash(x)=155504036)
-300 val loss 8.8988
-300 val perplexity 7323.0049
-300 train 8.930824 (lr=1.8060e-05) (hash(x)=155504036)
-400 val loss 7.9358
-400 val perplexity 2795.6145
-400 train 7.833813 (lr=2.8070e-05) (hash(x)=143823248)
-400 val loss 8.0672
-400 val perplexity 3188.0217
-400 train 7.977346 (lr=2.4060e-05) (hash(x)=143823248)
-500 val loss 7.6097
-500 val perplexity 2017.7271
-500 train 7.582412 (lr=3.5000e-05) (hash(x)=143734685)
-500 val loss 7.6500
-500 val perplexity 2100.6929
-500 train 7.625039 (lr=3.0000e-05) (hash(x)=143734685)
-600 val loss 7.4992
-600 val perplexity 1806.5713
-600 train 7.291920 (lr=3.4989e-05) (hash(x)=150678249)
-600 val loss 7.5211
-600 val perplexity 1846.5060
-600 train 7.316416 (lr=2.9990e-05) (hash(x)=150678249)
-700 val loss 7.4154
-700 val perplexity 1661.3982
-700 train 7.564420 (lr=3.4954e-05) (hash(x)=175802021)
-700 val loss 7.4274
-700 val perplexity 1681.4119
-700 train 7.571382 (lr=2.9961e-05) (hash(x)=175802021)
-800 val loss 7.3271
-800 val perplexity 1520.9635
-800 train 7.331609 (lr=3.4897e-05) (hash(x)=158681215)
-800 val loss 7.3369
-800 val perplexity 1535.9385
-800 train 7.345709 (lr=2.9912e-05) (hash(x)=158681215)
-900 val loss 7.2689
-900 val perplexity 1434.9026
-900 train 7.165131 (lr=3.4818e-05) (hash(x)=146108145)
-900 val loss 7.2753
-900 val perplexity 1444.1128
-900 train 7.172338 (lr=2.9844e-05) (hash(x)=146108145)
-1000 val loss 7.2140
-1000 val perplexity 1358.2812
-1000 train 7.060000 (lr=3.4715e-05) (hash(x)=154996086)
-1000 val loss 7.2104
-1000 val perplexity 1353.4595
-1000 train 7.056768 (lr=2.9756e-05) (hash(x)=154996086)
-1100 val loss 7.1419
-1100 val perplexity 1263.8793
-1100 train 7.124806 (lr=3.4591e-05) (hash(x)=153885445)
-1100 val loss 7.1242
-1100 val perplexity 1241.7123
-1100 train 7.107671 (lr=2.9649e-05) (hash(x)=153885445)
-1200 val loss 7.0668
-1200 val perplexity 1172.3687
-1200 train 7.020203 (lr=3.4444e-05) (hash(x)=142353087)
-1200 val loss 7.0451
-1200 val perplexity 1147.1823
-1200 train 6.998048 (lr=2.9523e-05) (hash(x)=142353087)
-1300 val loss 7.0154
-1300 val perplexity 1113.6349
-1300 train 6.930557 (lr=3.4275e-05) (hash(x)=150750353)
-1300 val loss 6.9831
-1300 val perplexity 1078.2115
-1300 train 6.896412 (lr=2.9378e-05) (hash(x)=150750353)
-1400 val loss 6.9752
-1400 val perplexity 1069.7871
-1400 train 6.799040 (lr=3.4084e-05) (hash(x)=152767913)
-1400 val loss 6.9322
-1400 val perplexity 1024.7649
-1400 train 6.756323 (lr=2.9215e-05) (hash(x)=152767913)
-1500 val loss 6.9097
-1500 val perplexity 1001.9031
-1500 train 6.945250 (lr=3.3872e-05) (hash(x)=151562048)
-1500 val loss 6.8590
-1500 val perplexity 952.3980
-1500 train 6.893571 (lr=2.9033e-05) (hash(x)=151562048)
-1600 val loss 6.8390
-1600 val perplexity 933.5735
-1600 train 6.832361 (lr=3.3638e-05) (hash(x)=166486165)
-1600 val loss 6.7795
-1600 val perplexity 879.6003
-1600 train 6.775093 (lr=2.8833e-05) (hash(x)=166486165)
-1700 val loss 6.7954
-1700 val perplexity 893.7272
-1700 train 6.446530 (lr=3.3384e-05) (hash(x)=130835396)
-1700 val loss 6.7200
-1700 val perplexity 828.8434
-1700 train 6.368818 (lr=2.8615e-05) (hash(x)=130835396)
-1800 val loss 6.7493
-1800 val perplexity 853.4929
-1800 train 6.801020 (lr=3.3109e-05) (hash(x)=158851816)
-1800 val loss 6.6636
-1800 val perplexity 783.3477
-1800 train 6.714691 (lr=2.8379e-05) (hash(x)=158851816)
-1900 val loss 6.7007
-1900 val perplexity 812.9660
-1900 train 6.740700 (lr=3.2814e-05) (hash(x)=153313879)
-1900 val loss 6.6132
-1900 val perplexity 744.8441
-1900 train 6.650075 (lr=2.8127e-05) (hash(x)=153313879)
-2000 val loss 6.6387
-2000 val perplexity 764.0649
-2000 train 6.572113 (lr=3.2500e-05) (hash(x)=158245023)
-2000 val loss 6.5498
-2000 val perplexity 699.1063
-2000 train 6.483957 (lr=2.7857e-05) (hash(x)=158245023)
-2100 val loss 6.5899
-2100 val perplexity 727.7206
-2100 train 6.421700 (lr=3.2166e-05) (hash(x)=157204896)
-2100 val loss 6.5033
-2100 val perplexity 667.3728
-2100 train 6.327139 (lr=2.7571e-05) (hash(x)=157204896)
-2200 val loss 6.5629
-2200 val perplexity 708.3510
-2200 train 6.407813 (lr=3.1813e-05) (hash(x)=137541932)
-2200 val loss 6.4726
-2200 val perplexity 647.1771
-2200 train 6.316986 (lr=2.7269e-05) (hash(x)=137541932)
-2300 val loss 6.5292
-2300 val perplexity 684.8694
-2300 train 6.481870 (lr=3.1443e-05) (hash(x)=150149692)
-2300 val loss 6.4415
-2300 val perplexity 627.3774
-2300 train 6.394355 (lr=2.6951e-05) (hash(x)=150149692)
-2400 val loss 6.4797
-2400 val perplexity 651.7456
-2400 train 6.469861 (lr=3.1054e-05) (hash(x)=151730720)
-2400 val loss 6.3974
-2400 val perplexity 600.2633
-2400 train 6.381145 (lr=2.6618e-05) (hash(x)=151730720)
-2500 val loss 6.4501
-2500 val perplexity 632.7767
-2500 train 6.218892 (lr=3.0649e-05) (hash(x)=143406752)
-2500 val loss 6.3688
-2500 val perplexity 583.3505
-2500 train 6.147860 (lr=2.6270e-05) (hash(x)=143406752)
-2600 val loss 6.4203
-2600 val perplexity 614.1707
-2600 train 6.254517 (lr=3.0227e-05) (hash(x)=157272496)
-2600 val loss 6.3524
-2600 val perplexity 573.8466
-2600 train 6.186916 (lr=2.5909e-05) (hash(x)=157272496)
-2700 val loss 6.3945
-2700 val perplexity 598.5561
-2700 train 6.410750 (lr=2.9789e-05) (hash(x)=155342327)
-2700 val loss 6.3262
-2700 val perplexity 559.0328
-2700 train 6.334075 (lr=2.5533e-05) (hash(x)=155342327)
-2800 val loss 6.3544
-2800 val perplexity 575.0129
-2800 train 6.225688 (lr=2.9336e-05) (hash(x)=140626679)
-2800 val loss 6.2908
-2800 val perplexity 539.5964
-2800 train 6.168956 (lr=2.5145e-05) (hash(x)=140626679)
-2900 val loss 6.3277
-2900 val perplexity 559.8731
-2900 train 6.188353 (lr=2.8868e-05) (hash(x)=144953350)
-2900 val loss 6.2653
-2900 val perplexity 526.0153
-2900 train 6.125029 (lr=2.4744e-05) (hash(x)=144953350)
-3000 val loss 6.3080
-3000 val perplexity 548.9646
-3000 train 6.216789 (lr=2.8386e-05) (hash(x)=172449837)
-3000 val loss 6.2493
-3000 val perplexity 517.6444
-3000 train 6.155919 (lr=2.4331e-05) (hash(x)=172449837)
-3100 val loss 6.2954
-3100 val perplexity 542.0637
-3100 train 6.085942 (lr=2.7891e-05) (hash(x)=141710086)
-3100 val loss 6.2444
-3100 val perplexity 515.1201
-3100 train 6.041224 (lr=2.3906e-05) (hash(x)=141710086)
-3200 val loss 6.2588
-3200 val perplexity 522.5837
-3200 train 6.189919 (lr=2.7383e-05) (hash(x)=151299772)
-3200 val loss 6.2087
-3200 val perplexity 497.0476
-3200 train 6.142445 (lr=2.3471e-05) (hash(x)=151299772)
-3300 val loss 6.2412
-3300 val perplexity 513.4838
-3300 train 6.105106 (lr=2.6864e-05) (hash(x)=146473110)
-3300 val loss 6.1987
-3300 val perplexity 492.0873
-3300 train 6.058009 (lr=2.3026e-05) (hash(x)=146473110)
-3400 val loss 6.2167
-3400 val perplexity 501.0693
-3400 train 6.293976 (lr=2.6333e-05) (hash(x)=153954157)
-3400 val loss 6.1735
-3400 val perplexity 479.8751
-3400 train 6.249593 (lr=2.2572e-05) (hash(x)=153954157)
-3500 val loss 6.2010
-3500 val perplexity 493.2191
-3500 train 6.127666 (lr=2.5793e-05) (hash(x)=153717336)
-3500 val loss 6.1570
-3500 val perplexity 471.9926
-3500 train 6.084462 (lr=2.2108e-05) (hash(x)=153717336)
-3600 val loss 6.1805
-3600 val perplexity 483.2234
-3600 train 5.934934 (lr=2.5243e-05) (hash(x)=144965161)
-3600 val loss 6.1449
-3600 val perplexity 466.3374
-3600 train 5.896233 (lr=2.1637e-05) (hash(x)=144965161)
-3700 val loss 6.1731
-3700 val perplexity 479.6843
-3700 train 5.946197 (lr=2.4684e-05) (hash(x)=125969741)
-3700 val loss 6.1403
-3700 val perplexity 464.2038
-3700 train 5.915022 (lr=2.1158e-05) (hash(x)=125969741)
-3800 val loss 6.1465
-3800 val perplexity 467.0876
-3800 train 6.011291 (lr=2.4117e-05) (hash(x)=155070487)
-3800 val loss 6.1142
-3800 val perplexity 452.2469
-3800 train 5.975008 (lr=2.0672e-05) (hash(x)=155070487)
-3900 val loss 6.1278
-3900 val perplexity 458.4420
-3900 train 6.005039 (lr=2.3543e-05) (hash(x)=149444644)
-3900 val loss 6.1017
-3900 val perplexity 446.5967
-3900 train 5.975531 (lr=2.0180e-05) (hash(x)=149444644)
-4000 val loss 6.1155
-4000 val perplexity 452.8252
-4000 train 5.972252 (lr=2.2963e-05) (hash(x)=151663033)
-4000 val loss 6.0910
-4000 val perplexity 441.8456
-4000 train 5.942688 (lr=1.9683e-05) (hash(x)=151663033)
-4100 val loss 6.1077
-4100 val perplexity 449.3011
-4100 train 6.060684 (lr=2.2378e-05) (hash(x)=143688282)
-4100 val loss 6.0840
-4100 val perplexity 438.7943
-4100 train 6.035672 (lr=1.9181e-05) (hash(x)=143688282)
-4200 val loss 6.0846
-4200 val perplexity 439.0362
-4200 train 6.078417 (lr=2.1788e-05) (hash(x)=163361651)
-4200 val loss 6.0626
-4200 val perplexity 429.4702
-4200 train 6.058801 (lr=1.8675e-05) (hash(x)=163361651)
-4300 val loss 6.0690
-4300 val perplexity 432.2372
-4300 train 6.133219 (lr=2.1194e-05) (hash(x)=153619361)
-4300 val loss 6.0485
-4300 val perplexity 423.4790
-4300 train 6.109252 (lr=1.8166e-05) (hash(x)=153619361)
-4400 val loss 6.0636
-4400 val perplexity 429.9185
-4400 train 6.355730 (lr=2.0598e-05) (hash(x)=168527064)
-4400 val loss 6.0480
-4400 val perplexity 423.2458
-4400 train 6.335233 (lr=1.7655e-05) (hash(x)=168527064)
-4500 val loss 6.0544
-4500 val perplexity 425.9846
-4500 train 5.854112 (lr=1.9999e-05) (hash(x)=125588037)
-4500 val loss 6.0369
-4500 val perplexity 418.6092
-4500 train 5.830371 (lr=1.7142e-05) (hash(x)=125588037)
-4600 val loss 6.0342
-4600 val perplexity 417.4467
-4600 train 5.998290 (lr=1.9400e-05) (hash(x)=143710941)
-4600 val loss 6.0200
-4600 val perplexity 411.5641
-4600 train 5.977354 (lr=1.6629e-05) (hash(x)=143710941)
-4700 val loss 6.0192
-4700 val perplexity 411.2676
-4700 train 5.844175 (lr=1.8800e-05) (hash(x)=150952742)
-4700 val loss 6.0082
-4700 val perplexity 406.7620
-4700 train 5.829061 (lr=1.6114e-05) (hash(x)=150952742)
-4800 val loss 6.0118
-4800 val perplexity 408.2310
-4800 train 5.876356 (lr=1.8201e-05) (hash(x)=145323659)
-4800 val loss 6.0027
-4800 val perplexity 404.5156
-4800 train 5.861185 (lr=1.5601e-05) (hash(x)=145323659)
-4900 val loss 6.0072
-4900 val perplexity 406.3518
-4900 train 6.022637 (lr=1.7604e-05) (hash(x)=153151397)
-4900 val loss 5.9937
-4900 val perplexity 400.8849
-4900 train 6.009444 (lr=1.5089e-05) (hash(x)=153151397)
-5000 val loss 5.9942
-5000 val perplexity 401.1065
-5000 train 5.895233 (lr=1.7009e-05) (hash(x)=143182059)
-5000 val loss 5.9870
-5000 val perplexity 398.2089
-5000 train 5.890842 (lr=1.4579e-05) (hash(x)=143182059)
-5100 val loss 5.9844
-5100 val perplexity 397.1822
-5100 train 6.032875 (lr=1.6417e-05) (hash(x)=170083586)
-5100 val loss 5.9762
-5100 val perplexity 393.9346
-5100 train 6.027209 (lr=1.4071e-05) (hash(x)=170083586)
-5200 val loss 5.9702
-5200 val perplexity 391.5924
-5200 train 5.865292 (lr=1.5829e-05) (hash(x)=149363919)
-5200 val loss 5.9649
-5200 val perplexity 389.5172
-5200 train 5.861600 (lr=1.3568e-05) (hash(x)=149363919)
-5300 val loss 5.9669
-5300 val perplexity 390.3069
-5300 train 5.877049 (lr=1.5246e-05) (hash(x)=152033784)
-5300 val loss 5.9612
-5300 val perplexity 388.0795
-5300 train 5.867223 (lr=1.3068e-05) (hash(x)=152033784)
-5400 val loss 5.9501
-5400 val perplexity 383.8001
-5400 train 6.054345 (lr=1.4669e-05) (hash(x)=154614289)
-5400 val loss 5.9471
-5400 val perplexity 382.6569
-5400 train 6.046638 (lr=1.2573e-05) (hash(x)=154614289)
-5500 val loss 5.9431
-5500 val perplexity 381.0967
-5500 train 6.028569 (lr=1.4099e-05) (hash(x)=157745174)
-5500 val loss 5.9410
-5500 val perplexity 380.3125
-5500 train 6.025530 (lr=1.2085e-05) (hash(x)=157745174)
-5600 val loss 5.9373
-5600 val perplexity 378.9089
-5600 train 5.829021 (lr=1.3536e-05) (hash(x)=147693222)
-5600 val loss 5.9377
-5600 val perplexity 379.0604
-5600 train 5.827706 (lr=1.1602e-05) (hash(x)=147693222)
-5700 val loss 5.9251
-5700 val perplexity 374.3267
-5700 train 5.866521 (lr=1.2981e-05) (hash(x)=149784627)
-5700 val loss 5.9253
-5700 val perplexity 374.3747
-5700 train 5.866199 (lr=1.1127e-05) (hash(x)=149784627)
-5800 val loss 5.9247
-5800 val perplexity 374.1822
-5800 train 5.834700 (lr=1.2436e-05) (hash(x)=158620729)
-5800 val loss 5.9247
-5800 val perplexity 374.1743
-5800 train 5.826473 (lr=1.0659e-05) (hash(x)=158620729)
-5900 val loss 5.9146
-5900 val perplexity 370.3900
-5900 train 5.858362 (lr=1.1900e-05) (hash(x)=159763910)
-5900 val loss 5.9180
-5900 val perplexity 371.6826
-5900 train 5.861243 (lr=1.0200e-05) (hash(x)=159763910)
-6000 val loss 5.9043
-6000 val perplexity 366.6024
-6000 train 5.836517 (lr=1.1375e-05) (hash(x)=147640561)
-6000 val loss 5.9081
-6000 val perplexity 367.9886
-6000 train 5.841051 (lr=9.7500e-06) (hash(x)=147640561)
-6100 val loss 5.8966
-6100 val perplexity 363.7880
-6100 train 5.878016 (lr=1.0861e-05) (hash(x)=156613394)
-6100 val loss 5.9015
-6100 val perplexity 365.5672
-6100 train 5.885981 (lr=9.3098e-06) (hash(x)=156613394)
-6200 val loss 5.8921
-6200 val perplexity 362.1524
-6200 train 5.853373 (lr=1.0360e-05) (hash(x)=186221290)
-6200 val loss 5.8976
-6200 val perplexity 364.1776
-6200 train 5.857404 (lr=8.8800e-06) (hash(x)=186221290)
-6300 val loss 5.8814
-6300 val perplexity 358.3256
-6300 train 5.760697 (lr=9.8715e-06) (hash(x)=152081419)
-6300 val loss 5.8885
-6300 val perplexity 360.8471
-6300 train 5.763207 (lr=8.4613e-06) (hash(x)=152081419)
-6400 val loss 5.8804
-6400 val perplexity 357.9507
-6400 train 5.825439 (lr=9.3966e-06) (hash(x)=154808349)
-6400 val loss 5.8885
-6400 val perplexity 360.8786
-6400 train 5.834856 (lr=8.0542e-06) (hash(x)=154808349)
-6500 val loss 5.8733
-6500 val perplexity 355.4034
-6500 train 5.840936 (lr=8.9359e-06) (hash(x)=159437208)
-6500 val loss 5.8832
-6500 val perplexity 358.9690
-6500 train 5.854115 (lr=7.6594e-06) (hash(x)=159437208)
-6600 val loss 5.8716
-6600 val perplexity 354.8281
-6600 train 5.681755 (lr=8.4903e-06) (hash(x)=157933074)
-6600 val loss 5.8805
-6600 val perplexity 358.0042
-6600 train 5.683859 (lr=7.2774e-06) (hash(x)=157933074)
-6700 val loss 5.8631
-6700 val perplexity 351.8213
-6700 train 5.846561 (lr=8.0602e-06) (hash(x)=161560240)
-6700 val loss 5.8725
-6700 val perplexity 355.1426
-6700 train 5.855772 (lr=6.9087e-06) (hash(x)=161560240)
-6800 val loss 5.8610
-6800 val perplexity 351.0881
-6800 train 5.835786 (lr=7.6463e-06) (hash(x)=155424292)
-6800 val loss 5.8702
-6800 val perplexity 354.3057
-6800 train 5.845886 (lr=6.5540e-06) (hash(x)=155424292)
-6900 val loss 5.8587
-6900 val perplexity 350.2836
-6900 train 5.851926 (lr=7.2493e-06) (hash(x)=148561470)
-6900 val loss 5.8693
-6900 val perplexity 354.0038
-6900 train 5.859293 (lr=6.2137e-06) (hash(x)=148561470)
-7000 val loss 5.8522
-7000 val perplexity 348.0006
-7000 train 5.749753 (lr=6.8697e-06) (hash(x)=141527450)
-7000 val loss 5.8612
-7000 val perplexity 351.1305
-7000 train 5.753252 (lr=5.8883e-06) (hash(x)=141527450)
-7100 val loss 5.8464
-7100 val perplexity 345.9703
-7100 train 5.800970 (lr=6.5080e-06) (hash(x)=151066339)
-7100 val loss 5.8581
-7100 val perplexity 350.0529
-7100 train 5.809224 (lr=5.5783e-06) (hash(x)=151066339)
-7200 val loss 5.8468
-7200 val perplexity 346.1266
-7200 train 5.652260 (lr=6.1648e-06) (hash(x)=155231264)
-7200 val loss 5.8590
-7200 val perplexity 350.3668
-7200 train 5.663008 (lr=5.2841e-06) (hash(x)=155231264)
-7300 val loss 5.8382
-7300 val perplexity 343.1572
-7300 train 5.854438 (lr=5.8405e-06) (hash(x)=150281149)
-7300 val loss 5.8514
-7300 val perplexity 347.7130
-7300 train 5.861528 (lr=5.0062e-06) (hash(x)=150281149)
-7400 val loss 5.8352
-7400 val perplexity 342.1290
-7400 train 5.717260 (lr=5.5357e-06) (hash(x)=148421717)
-7400 val loss 5.8498
-7400 val perplexity 347.1809
-7400 train 5.727182 (lr=4.7449e-06) (hash(x)=148421717)
-7500 val loss 5.8349
-7500 val perplexity 342.0240
-7500 train 5.600042 (lr=5.2508e-06) (hash(x)=146921118)
-7500 val loss 5.8495
-7500 val perplexity 347.0692
-7500 train 5.604131 (lr=4.5007e-06) (hash(x)=146921118)
-7600 val loss 5.8285
-7600 val perplexity 339.8405
-7600 train 5.879191 (lr=4.9862e-06) (hash(x)=150660048)
-7600 val loss 5.8426
-7600 val perplexity 344.6835
-7600 train 5.886945 (lr=4.2739e-06) (hash(x)=150660048)
-7700 val loss 5.8274
-7700 val perplexity 339.4743
-7700 train 5.699795 (lr=4.7423e-06) (hash(x)=148059852)
-7700 val loss 5.8417
-7700 val perplexity 344.3644
-7700 train 5.711916 (lr=4.0648e-06) (hash(x)=148059852)
-7800 val loss 5.8256
-7800 val perplexity 338.8620
-7800 train 5.527908 (lr=4.5194e-06) (hash(x)=148331002)
-7800 val loss 5.8412
-7800 val perplexity 344.1924
-7800 train 5.541048 (lr=3.8738e-06) (hash(x)=148331002)
-7900 val loss 5.8206
-7900 val perplexity 337.1732
-7900 train 5.777098 (lr=4.3179e-06) (hash(x)=164923883)
-7900 val loss 5.8355
-7900 val perplexity 342.2498
-7900 train 5.794849 (lr=3.7010e-06) (hash(x)=164923883)
-8000 val loss 5.8183
-8000 val perplexity 336.3914
-8000 train 5.699238 (lr=4.1380e-06) (hash(x)=143545384)
-8000 val loss 5.8344
-8000 val perplexity 341.8687
-8000 train 5.713335 (lr=3.5468e-06) (hash(x)=143545384)
-8100 val loss 5.8173
-8100 val perplexity 336.0612
-8100 train 5.634319 (lr=3.9800e-06) (hash(x)=160686959)
-8100 val loss 5.8336
-8100 val perplexity 341.5852
-8100 train 5.655682 (lr=3.4114e-06) (hash(x)=160686959)
-8200 val loss 5.8150
-8200 val perplexity 335.3045
-8200 train 5.773064 (lr=3.8442e-06) (hash(x)=156501889)
-8200 val loss 5.8326
-8200 val perplexity 341.2524
-8200 train 5.789817 (lr=3.2950e-06) (hash(x)=156501889)
-8300 val loss 5.8130
-8300 val perplexity 334.6131
-8300 train 5.656518 (lr=3.7307e-06) (hash(x)=142716875)
-8300 val loss 5.8295
-8300 val perplexity 340.2042
-8300 train 5.667881 (lr=3.1977e-06) (hash(x)=142716875)
-8400 val loss 5.8083
-8400 val perplexity 333.0655
-8400 train 5.715852 (lr=3.6397e-06) (hash(x)=154436684)
-8400 val loss 5.8256
-8400 val perplexity 338.8665
-8400 train 5.732801 (lr=3.1197e-06) (hash(x)=154436684)
-8500 val loss 5.8068
-8500 val perplexity 332.5389
-8500 train 5.878377 (lr=3.5713e-06) (hash(x)=147965839)
-8500 val loss 5.8247
-8500 val perplexity 338.5674
-8500 train 5.897286 (lr=3.0611e-06) (hash(x)=147965839)
-8600 val loss 5.8056
-8600 val perplexity 332.1485
-8600 train 5.608908 (lr=3.5257e-06) (hash(x)=145228097)
-8600 val loss 5.8239
-8600 val perplexity 338.2750
-8600 train 5.626842 (lr=3.0220e-06) (hash(x)=145228097)
-8700 val loss 5.8016
-8700 val perplexity 330.8403
-8700 train 5.983572 (lr=3.5029e-06) (hash(x)=152910357)
-8700 val loss 5.8198
-8700 val perplexity 336.9131
-8700 train 5.998275 (lr=3.0024e-06) (hash(x)=152910357)
-8749 val loss 5.8021
-8749 val perplexity 330.9829
-8749 val loss 5.8203
-8749 val perplexity 337.0700
+0 train 11.267765 (lr=1.1000e-07) (hash(x)=150327452)
+100 val loss 9.9457
+100 val perplexity 20862.5527
+100 train 9.998234 (lr=1.1110e-05) (hash(x)=166780046)
+200 val loss 9.1876
+200 val perplexity 9775.3037
+200 train 9.133502 (lr=2.2110e-05) (hash(x)=155040610)
+300 val loss 8.2995
+300 val perplexity 4022.0583
+300 train 8.305970 (lr=3.3110e-05) (hash(x)=155504036)
+400 val loss 7.7026
+400 val perplexity 2214.0981
+400 train 7.576903 (lr=4.4110e-05) (hash(x)=143823248)
+500 val loss 7.5018
+500 val perplexity 1811.3724
+500 train 7.471624 (lr=5.5000e-05) (hash(x)=143734685)
+600 val loss 7.4303
+600 val perplexity 1686.3419
+600 train 7.209157 (lr=5.4982e-05) (hash(x)=150678249)
+700 val loss 7.3549
+700 val perplexity 1563.7889
+700 train 7.492709 (lr=5.4928e-05) (hash(x)=175802021)
+800 val loss 7.2668
+800 val perplexity 1431.9143
+800 train 7.272984 (lr=5.4839e-05) (hash(x)=158681215)
+900 val loss 7.2057
+900 val perplexity 1347.0262
+900 train 7.102057 (lr=5.4713e-05) (hash(x)=146108145)
+1000 val loss 7.1370
+1000 val perplexity 1257.6271
+1000 train 6.979178 (lr=5.4553e-05) (hash(x)=154996086)
+1100 val loss 7.0562
+1100 val perplexity 1160.0608
+1100 train 7.036383 (lr=5.4357e-05) (hash(x)=153885445)
+1200 val loss 6.9739
+1200 val perplexity 1068.3485
+1200 train 6.932178 (lr=5.4126e-05) (hash(x)=142353087)
+1300 val loss 6.9191
+1300 val perplexity 1011.4178
+1300 train 6.830047 (lr=5.3860e-05) (hash(x)=150750353)
+1400 val loss 6.8690
+1400 val perplexity 961.9484
+1400 train 6.692386 (lr=5.3561e-05) (hash(x)=152767913)
+1500 val loss 6.7984
+1500 val perplexity 896.3729
+1500 train 6.837613 (lr=5.3227e-05) (hash(x)=151562048)
+1600 val loss 6.7206
+1600 val perplexity 829.3372
+1600 train 6.723626 (lr=5.2860e-05) (hash(x)=166486165)
+1700 val loss 6.6838
+1700 val perplexity 799.3312
+1700 train 6.325279 (lr=5.2461e-05) (hash(x)=130835396)
+1800 val loss 6.6342
+1800 val perplexity 760.6326
+1800 train 6.681156 (lr=5.2029e-05) (hash(x)=158851816)
+1900 val loss 6.5986
+1900 val perplexity 734.0478
+1900 train 6.630829 (lr=5.1565e-05) (hash(x)=153313879)
+2000 val loss 6.5331
+2000 val perplexity 687.5047
+2000 train 6.463948 (lr=5.1071e-05) (hash(x)=158245023)
+2100 val loss 6.4869
+2100 val perplexity 656.4935
+2100 train 6.311565 (lr=5.0547e-05) (hash(x)=157204896)
+2200 val loss 6.4565
+2200 val perplexity 636.8082
+2200 train 6.298766 (lr=4.9993e-05) (hash(x)=137541932)
+2300 val loss 6.4247
+2300 val perplexity 616.9203
+2300 train 6.379897 (lr=4.9410e-05) (hash(x)=150149692)
+2400 val loss 6.3730
+2400 val perplexity 585.8328
+2400 train 6.363482 (lr=4.8800e-05) (hash(x)=151730720)
+2500 val loss 6.3399
+2500 val perplexity 566.7245
+2500 train 6.111226 (lr=4.8162e-05) (hash(x)=143406752)
+2600 val loss 6.3211
+2600 val perplexity 556.1671
+2600 train 6.154155 (lr=4.7499e-05) (hash(x)=157272496)
+2700 val loss 6.2973
+2700 val perplexity 543.0967
+2700 train 6.291989 (lr=4.6811e-05) (hash(x)=155342327)
+2800 val loss 6.2554
+2800 val perplexity 520.8175
+2800 train 6.133845 (lr=4.6099e-05) (hash(x)=140626679)
+2900 val loss 6.2355
+2900 val perplexity 510.5799
+2900 train 6.095793 (lr=4.5364e-05) (hash(x)=144953350)
+3000 val loss 6.2132
+3000 val perplexity 499.2762
+3000 train 6.112416 (lr=4.4606e-05) (hash(x)=172449837)
+3100 val loss 6.2019
+3100 val perplexity 493.6857
+3100 train 5.997821 (lr=4.3828e-05) (hash(x)=141710086)
+3200 val loss 6.1629
+3200 val perplexity 474.8234
+3200 train 6.093801 (lr=4.3031e-05) (hash(x)=151299772)
+3300 val loss 6.1521
+3300 val perplexity 469.6945
+3300 train 6.020457 (lr=4.2215e-05) (hash(x)=146473110)
+3400 val loss 6.1303
+3400 val perplexity 459.5718
+3400 train 6.210660 (lr=4.1381e-05) (hash(x)=153954157)
+3500 val loss 6.1080
+3500 val perplexity 449.4359
+3500 train 6.042493 (lr=4.0532e-05) (hash(x)=153717336)
+3600 val loss 6.0886
+3600 val perplexity 440.8094
+3600 train 5.844422 (lr=3.9667e-05) (hash(x)=144965161)
+3700 val loss 6.0836
+3700 val perplexity 438.5945
+3700 train 5.863056 (lr=3.8789e-05) (hash(x)=125969741)
+3800 val loss 6.0621
+3800 val perplexity 429.2794
+3800 train 5.925856 (lr=3.7898e-05) (hash(x)=155070487)
+3900 val loss 6.0450
+3900 val perplexity 421.9990
+3900 train 5.920032 (lr=3.6996e-05) (hash(x)=149444644)
+4000 val loss 6.0358
+4000 val perplexity 418.1515
+4000 train 5.892958 (lr=3.6085e-05) (hash(x)=151663033)
+4100 val loss 6.0280
+4100 val perplexity 414.8808
+4100 train 5.983856 (lr=3.5165e-05) (hash(x)=143688282)
+4200 val loss 6.0011
+4200 val perplexity 403.8761
+4200 train 6.000255 (lr=3.4238e-05) (hash(x)=163361651)
+4300 val loss 5.9869
+4300 val perplexity 398.1633
+4300 train 6.021540 (lr=3.3305e-05) (hash(x)=153619361)
+4400 val loss 5.9853
+4400 val perplexity 397.5516
+4400 train 6.296560 (lr=3.2368e-05) (hash(x)=168527064)
+4500 val loss 5.9708
+4500 val perplexity 391.8027
+4500 train 5.778735 (lr=3.1428e-05) (hash(x)=125588037)
+4600 val loss 5.9559
+4600 val perplexity 386.0312
+4600 train 5.915372 (lr=3.0486e-05) (hash(x)=143710941)
+4700 val loss 5.9398
+4700 val perplexity 379.8761
+4700 train 5.773179 (lr=2.9543e-05) (hash(x)=150952742)
+4800 val loss 5.9353
+4800 val perplexity 378.1379
+4800 train 5.802930 (lr=2.8602e-05) (hash(x)=145323659)
+4900 val loss 5.9280
+4900 val perplexity 375.4010
+4900 train 5.946818 (lr=2.7663e-05) (hash(x)=153151397)
+5000 val loss 5.9167
+5000 val perplexity 371.1711
+5000 train 5.814476 (lr=2.6728e-05) (hash(x)=143182059)
+5100 val loss 5.9067
+5100 val perplexity 367.5034
+5100 train 5.954238 (lr=2.5798e-05) (hash(x)=170083586)
+5200 val loss 5.8899
+5200 val perplexity 361.3606
+5200 train 5.788404 (lr=2.4874e-05) (hash(x)=149363919)
+5300 val loss 5.8822
+5300 val perplexity 358.5796
+5300 train 5.769784 (lr=2.3958e-05) (hash(x)=152033784)
+5400 val loss 5.8687
+5400 val perplexity 353.7852
+5400 train 5.975514 (lr=2.3051e-05) (hash(x)=154614289)
+5500 val loss 5.8615
+5500 val perplexity 351.2599
+5500 train 5.953315 (lr=2.2155e-05) (hash(x)=157745174)
+5600 val loss 5.8554
+5600 val perplexity 349.1059
+5600 train 5.747090 (lr=2.1271e-05) (hash(x)=147693222)
+5700 val loss 5.8457
+5700 val perplexity 345.7462
+5700 train 5.784711 (lr=2.0399e-05) (hash(x)=149784627)
+5800 val loss 5.8401
+5800 val perplexity 343.8195
+5800 train 5.752527 (lr=1.9542e-05) (hash(x)=158620729)
+5900 val loss 5.8340
+5900 val perplexity 341.7084
+5900 train 5.779388 (lr=1.8700e-05) (hash(x)=159763910)
+6000 val loss 5.8213
+6000 val perplexity 337.4167
+6000 train 5.758483 (lr=1.7875e-05) (hash(x)=147640561)
+6100 val loss 5.8147
+6100 val perplexity 335.1861
+6100 train 5.795366 (lr=1.7068e-05) (hash(x)=156613394)
+6200 val loss 5.8099
+6200 val perplexity 333.6008
+6200 train 5.747422 (lr=1.6280e-05) (hash(x)=186221290)
+6300 val loss 5.7978
+6300 val perplexity 329.5759
+6300 train 5.679901 (lr=1.5512e-05) (hash(x)=152081419)
+6400 val loss 5.8018
+6400 val perplexity 330.8954
+6400 train 5.755892 (lr=1.4766e-05) (hash(x)=154808349)
+6500 val loss 5.7942
+6500 val perplexity 328.4028
+6500 train 5.763316 (lr=1.4042e-05) (hash(x)=159437208)
+6600 val loss 5.7910
+6600 val perplexity 327.3300
+6600 train 5.604477 (lr=1.3342e-05) (hash(x)=157933074)
+6700 val loss 5.7805
+6700 val perplexity 323.9347
+6700 train 5.757157 (lr=1.2666e-05) (hash(x)=161560240)
+6800 val loss 5.7777
+6800 val perplexity 323.0242
+6800 train 5.759155 (lr=1.2016e-05) (hash(x)=155424292)
+6900 val loss 5.7755
+6900 val perplexity 322.2903
+6900 train 5.768492 (lr=1.1392e-05) (hash(x)=148561470)
+7000 val loss 5.7678
+7000 val perplexity 319.8350
+7000 train 5.664386 (lr=1.0795e-05) (hash(x)=141527450)
+7100 val loss 5.7617
+7100 val perplexity 317.8992
+7100 train 5.717986 (lr=1.0227e-05) (hash(x)=151066339)
+7200 val loss 5.7629
+7200 val perplexity 318.2719
+7200 train 5.559781 (lr=9.6875e-06) (hash(x)=155231264)
+7300 val loss 5.7547
+7300 val perplexity 315.6577
+7300 train 5.772898 (lr=9.1780e-06) (hash(x)=150281149)
+7400 val loss 5.7514
+7400 val perplexity 314.6422
+7400 train 5.640527 (lr=8.6990e-06) (hash(x)=148421717)
+7500 val loss 5.7514
+7500 val perplexity 314.6400
+7500 train 5.513526 (lr=8.2513e-06) (hash(x)=146921118)
+7600 val loss 5.7441
+7600 val perplexity 312.3449
+7600 train 5.794578 (lr=7.8355e-06) (hash(x)=150660048)
+7700 val loss 5.7411
+7700 val perplexity 311.4137
+7700 train 5.619473 (lr=7.4522e-06) (hash(x)=148059852)
+7800 val loss 5.7406
+7800 val perplexity 311.2433
+7800 train 5.437676 (lr=7.1019e-06) (hash(x)=148331002)
+7900 val loss 5.7347
+7900 val perplexity 309.4308
+7900 train 5.687338 (lr=6.7852e-06) (hash(x)=164923883)
+8000 val loss 5.7323
+8000 val perplexity 308.6885
+8000 train 5.618847 (lr=6.5025e-06) (hash(x)=143545384)
+8100 val loss 5.7315
+8100 val perplexity 308.4400
+8100 train 5.545605 (lr=6.2543e-06) (hash(x)=160686959)
+8200 val loss 5.7291
+8200 val perplexity 307.6797
+8200 train 5.678798 (lr=6.0408e-06) (hash(x)=156501889)
+8300 val loss 5.7280
+8300 val perplexity 307.3556
+8300 train 5.569695 (lr=5.8625e-06) (hash(x)=142716875)
+8400 val loss 5.7214
+8400 val perplexity 305.3186
+8400 train 5.635721 (lr=5.7195e-06) (hash(x)=154436684)
+8500 val loss 5.7204
+8500 val perplexity 305.0224
+8500 train 5.795388 (lr=5.6121e-06) (hash(x)=147965839)
+8600 val loss 5.7193
+8600 val perplexity 304.6931
+8600 train 5.511634 (lr=5.5404e-06) (hash(x)=145228097)
+8700 val loss 5.7141
+8700 val perplexity 303.1104
+8700 train 5.900846 (lr=5.5045e-06) (hash(x)=152910357)
+8749 val loss 5.7145
+8749 val perplexity 303.2289

attention_kindselective_n_heads4_seed1338/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0ca05b3344aa66408b045c0838e43612e6fde6f89e7dfea8f7fe75e490896e0
+oid sha256:a131c48954674e396c86cde25dd3f7fc64149914014049b28ac85daeaa29f8e4
 size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70de7c3fffdc7ca40ec0743f8447c426842a43ad5a3f91f2f8dfdeb161f89769
+oid sha256:cc5c4c2a91810fed0268ecc62c25629b3dbf30047fa030d380998253c008ba02
 size 179406214