Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_61440_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5.5e-5_61440_4_1341", "n_embd": 256}

attention_kindselective_n_heads4_seed1341/log2.txt

@@ -1,534 +1,267 @@
 max_steps: 8750
-max_steps: 8750
-0 val loss 11.3095
-0 val perplexity 81594.2500
 0 val loss 11.3095
 0 val perplexity 81594.2500
-0 train 11.313038 (lr=7.0000e-08) (hash(x)=145079536)
-0 train 11.313038 (lr=6.0000e-08) (hash(x)=145079536)
-100 val loss 10.1630
-100 val perplexity 25926.1094
-100 val loss 10.2343
-100 val perplexity 27841.4590
-100 train 10.156528 (lr=7.0700e-06) (hash(x)=155800595)
-100 train 10.225678 (lr=6.0600e-06) (hash(x)=155800595)
-200 val loss 9.7494
-200 val perplexity 17144.4492
-200 train 9.706524 (lr=1.4070e-05) (hash(x)=145606733)
-200 val loss 9.7756
-200 val perplexity 17599.2559
-200 train 9.732876 (lr=1.2060e-05) (hash(x)=145606733)
-300 val loss 8.8013
-300 val perplexity 6643.1152
-300 train 8.798988 (lr=2.1070e-05) (hash(x)=150367139)
-300 val loss 8.9184
-300 val perplexity 7468.2007
-300 train 8.917364 (lr=1.8060e-05) (hash(x)=150367139)
-400 val loss 7.9646
-400 val perplexity 2877.3264
-400 train 8.234877 (lr=2.8070e-05) (hash(x)=155747374)
-400 val loss 7.9782
-400 val perplexity 2916.8208
-400 train 8.244305 (lr=2.4060e-05) (hash(x)=155747374)
-500 val loss 7.6135
-500 val perplexity 2025.2765
-500 train 7.510717 (lr=3.5000e-05) (hash(x)=140604760)
-500 val loss 7.5787
-500 val perplexity 1955.9950
-500 train 7.475321 (lr=3.0000e-05) (hash(x)=140604760)
-600 val loss 7.4823
-600 val perplexity 1776.2891
-600 train 7.433376 (lr=3.4989e-05) (hash(x)=148404734)
-600 val loss 7.4607
-600 val perplexity 1738.3960
-600 train 7.414741 (lr=2.9990e-05) (hash(x)=148404734)
-700 val loss 7.4270
-700 val perplexity 1680.7137
-700 train 7.384828 (lr=3.4954e-05) (hash(x)=148115934)
-700 val loss 7.3964
-700 val perplexity 1630.0658
-700 train 7.354086 (lr=2.9961e-05) (hash(x)=148115934)
-800 val loss 7.3625
-800 val perplexity 1575.7120
-800 train 7.258959 (lr=3.4897e-05) (hash(x)=137464699)
-800 val loss 7.3292
-800 val perplexity 1524.1631
-800 train 7.219800 (lr=2.9912e-05) (hash(x)=137464699)
-900 val loss 7.3188
-900 val perplexity 1508.4202
-900 train 7.139495 (lr=3.4818e-05) (hash(x)=143886042)
-900 val loss 7.2721
-900 val perplexity 1439.5078
-900 train 7.090878 (lr=2.9844e-05) (hash(x)=143886042)
-1000 val loss 7.2571
-1000 val perplexity 1418.1678
-1000 train 7.259868 (lr=3.4715e-05) (hash(x)=163799796)
-1000 val loss 7.2204
-1000 val perplexity 1367.0923
-1000 train 7.224873 (lr=2.9756e-05) (hash(x)=163799796)
-1100 val loss 7.2004
-1100 val perplexity 1339.9773
-1100 train 7.151253 (lr=3.4591e-05) (hash(x)=144592844)
-1100 val loss 7.1528
-1100 val perplexity 1277.6487
-1100 train 7.103980 (lr=2.9649e-05) (hash(x)=144592844)
-1200 val loss 7.1496
-1200 val perplexity 1273.5430
-1200 train 7.510290 (lr=3.4444e-05) (hash(x)=204706354)
-1200 val loss 7.0939
-1200 val perplexity 1204.5594
-1200 train 7.442509 (lr=2.9523e-05) (hash(x)=204706354)
-1300 val loss 7.0922
-1300 val perplexity 1202.5422
-1300 train 7.020781 (lr=3.4275e-05) (hash(x)=150862210)
-1300 val loss 7.0370
-1300 val perplexity 1138.0204
-1300 train 6.966258 (lr=2.9378e-05) (hash(x)=150862210)
-1400 val loss 7.0167
-1400 val perplexity 1115.1079
-1400 train 6.995105 (lr=3.4084e-05) (hash(x)=147766811)
-1400 val loss 6.9707
-1400 val perplexity 1064.9824
-1400 train 6.951100 (lr=2.9215e-05) (hash(x)=147766811)
-1500 val loss 6.9622
-1500 val perplexity 1056.0016
-1500 train 6.773920 (lr=3.3872e-05) (hash(x)=135925327)
-1500 val loss 6.9144
-1500 val perplexity 1006.6741
-1500 train 6.724929 (lr=2.9033e-05) (hash(x)=135925327)
-1600 val loss 6.9194
-1600 val perplexity 1011.7564
-1600 train 6.889316 (lr=3.3638e-05) (hash(x)=160440642)
-1600 val loss 6.8620
-1600 val perplexity 955.2529
-1600 train 6.835199 (lr=2.8833e-05) (hash(x)=160440642)
-1700 val loss 6.8600
-1700 val perplexity 953.3604
-1700 train 6.953665 (lr=3.3384e-05) (hash(x)=151184106)
-1700 val loss 6.7968
-1700 val perplexity 894.9904
-1700 train 6.890760 (lr=2.8615e-05) (hash(x)=151184106)
-1800 val loss 6.8067
-1800 val perplexity 903.8531
-1800 train 6.697411 (lr=3.3109e-05) (hash(x)=148100580)
-1800 val loss 6.7426
-1800 val perplexity 847.7204
-1800 train 6.634793 (lr=2.8379e-05) (hash(x)=148100580)
-1900 val loss 6.7725
-1900 val perplexity 873.4525
-1900 train 6.604763 (lr=3.2814e-05) (hash(x)=149434659)
-1900 val loss 6.6984
-1900 val perplexity 811.1066
-1900 train 6.530216 (lr=2.8127e-05) (hash(x)=149434659)
-2000 val loss 6.7098
-2000 val perplexity 820.4353
-2000 train 6.604181 (lr=3.2500e-05) (hash(x)=152285486)
-2000 val loss 6.6355
-2000 val perplexity 761.6394
-2000 train 6.533551 (lr=2.7857e-05) (hash(x)=152285486)
-2100 val loss 6.6650
-2100 val perplexity 784.4327
-2100 train 6.413668 (lr=3.2166e-05) (hash(x)=144294295)
-2100 val loss 6.5839
-2100 val perplexity 723.3694
-2100 train 6.331763 (lr=2.7571e-05) (hash(x)=144294295)
-2200 val loss 6.6268
-2200 val perplexity 755.0834
-2200 train 6.659666 (lr=3.1813e-05) (hash(x)=175030215)
-2200 val loss 6.5444
-2200 val perplexity 695.3740
-2200 train 6.575781 (lr=2.7269e-05) (hash(x)=175030215)
-2300 val loss 6.5935
-2300 val perplexity 730.3160
-2300 train 6.395243 (lr=3.1443e-05) (hash(x)=150831428)
-2300 val loss 6.5118
-2300 val perplexity 673.0463
-2300 train 6.314830 (lr=2.6951e-05) (hash(x)=150831428)
-2400 val loss 6.5340
-2400 val perplexity 688.1121
-2400 train 6.689769 (lr=3.1054e-05) (hash(x)=140808297)
-2400 val loss 6.4649
-2400 val perplexity 642.2308
-2400 train 6.624811 (lr=2.6618e-05) (hash(x)=140808297)
-2500 val loss 6.4948
-2500 val perplexity 661.7064
-2500 train 6.484925 (lr=3.0649e-05) (hash(x)=153160275)
-2500 val loss 6.4345
-2500 val perplexity 622.9711
-2500 train 6.418693 (lr=2.6270e-05) (hash(x)=153160275)
-2600 val loss 6.4673
-2600 val perplexity 643.7227
-2600 train 6.227062 (lr=3.0227e-05) (hash(x)=133990623)
-2600 val loss 6.4079
-2600 val perplexity 606.6450
-2600 train 6.165543 (lr=2.5909e-05) (hash(x)=133990623)
-2700 val loss 6.4393
-2700 val perplexity 625.9974
-2700 train 6.414563 (lr=2.9789e-05) (hash(x)=142860944)
-2700 val loss 6.3846
-2700 val perplexity 592.6437
-2700 train 6.360803 (lr=2.5533e-05) (hash(x)=142860944)
-2800 val loss 6.4029
-2800 val perplexity 603.5896
-2800 train 6.432872 (lr=2.9336e-05) (hash(x)=137959511)
-2800 val loss 6.3519
-2800 val perplexity 573.5657
-2800 train 6.378989 (lr=2.5145e-05) (hash(x)=137959511)
-2900 val loss 6.3718
-2900 val perplexity 585.1338
-2900 train 6.311280 (lr=2.8868e-05) (hash(x)=147009873)
-2900 val loss 6.3265
-2900 val perplexity 559.1986
-2900 train 6.269898 (lr=2.4744e-05) (hash(x)=147009873)
-3000 val loss 6.3476
-3000 val perplexity 571.1144
-3000 train 6.218025 (lr=2.8386e-05) (hash(x)=158264841)
-3000 val loss 6.3063
-3000 val perplexity 548.0392
-3000 train 6.173168 (lr=2.4331e-05) (hash(x)=158264841)
-3100 val loss 6.3292
-3100 val perplexity 560.6805
-3100 train 6.188198 (lr=2.7891e-05) (hash(x)=139232251)
-3100 val loss 6.2925
-3100 val perplexity 540.5228
-3100 train 6.155460 (lr=2.3906e-05) (hash(x)=139232251)
-3200 val loss 6.2932
-3200 val perplexity 540.8548
-3200 train 6.341791 (lr=2.7383e-05) (hash(x)=153436104)
-3200 val loss 6.2591
-3200 val perplexity 522.7567
-3200 train 6.305866 (lr=2.3471e-05) (hash(x)=153436104)
-3300 val loss 6.2764
-3300 val perplexity 531.8832
-3300 train 6.092842 (lr=2.6864e-05) (hash(x)=149681831)
-3300 val loss 6.2423
-3300 val perplexity 514.0164
-3300 train 6.057350 (lr=2.3026e-05) (hash(x)=149681831)
-3400 val loss 6.2633
-3400 val perplexity 524.9594
-3400 train 6.166925 (lr=2.6333e-05) (hash(x)=168425516)
-3400 val loss 6.2299
-3400 val perplexity 507.6933
-3400 train 6.136177 (lr=2.2572e-05) (hash(x)=168425516)
-3500 val loss 6.2348
-3500 val perplexity 510.2036
-3500 train 6.263444 (lr=2.5793e-05) (hash(x)=163104338)
-3500 val loss 6.2078
-3500 val perplexity 496.5847
-3500 train 6.231250 (lr=2.2108e-05) (hash(x)=163104338)
-3600 val loss 6.2157
-3600 val perplexity 500.5353
-3600 train 6.190797 (lr=2.5243e-05) (hash(x)=165109772)
-3600 val loss 6.1890
-3600 val perplexity 487.3521
-3600 train 6.160804 (lr=2.1637e-05) (hash(x)=165109772)
-3700 val loss 6.2076
-3700 val perplexity 496.5286
-3700 train 5.940827 (lr=2.4684e-05) (hash(x)=153420306)
-3700 val loss 6.1816
-3700 val perplexity 483.7829
-3700 train 5.910549 (lr=2.1158e-05) (hash(x)=153420306)
-3800 val loss 6.1786
-3800 val perplexity 482.2998
-3800 train 6.204558 (lr=2.4117e-05) (hash(x)=160168863)
-3800 val loss 6.1568
-3800 val perplexity 471.9337
-3800 train 6.183776 (lr=2.0672e-05) (hash(x)=160168863)
-3900 val loss 6.1666
-3900 val perplexity 476.5572
-3900 train 6.095548 (lr=2.3543e-05) (hash(x)=153906073)
-3900 val loss 6.1428
-3900 val perplexity 465.3320
-3900 train 6.070639 (lr=2.0180e-05) (hash(x)=153906073)
-4000 val loss 6.1527
-4000 val perplexity 469.9752
-4000 train 6.081241 (lr=2.2963e-05) (hash(x)=151055067)
-4000 val loss 6.1338
-4000 val perplexity 461.1803
-4000 train 6.064647 (lr=1.9683e-05) (hash(x)=151055067)
-4100 val loss 6.1350
-4100 val perplexity 461.7586
-4100 train 6.116509 (lr=2.2378e-05) (hash(x)=149629830)
-4100 val loss 6.1163
-4100 val perplexity 453.1846
-4100 train 6.090425 (lr=1.9181e-05) (hash(x)=149629830)
-4200 val loss 6.1249
-4200 val perplexity 457.1209
-4200 train 6.003567 (lr=2.1788e-05) (hash(x)=143101381)
-4200 val loss 6.1033
-4200 val perplexity 447.3311
-4200 train 5.984151 (lr=1.8675e-05) (hash(x)=143101381)
-4300 val loss 6.1115
-4300 val perplexity 451.0039
-4300 train 6.010469 (lr=2.1194e-05) (hash(x)=149712044)
-4300 val loss 6.0949
-4300 val perplexity 443.5709
-4300 train 5.993714 (lr=1.8166e-05) (hash(x)=149712044)
-4400 val loss 6.0941
-4400 val perplexity 443.2506
-4400 train 6.020243 (lr=2.0598e-05) (hash(x)=153446449)
-4400 val loss 6.0809
-4400 val perplexity 437.4159
-4400 train 6.007243 (lr=1.7655e-05) (hash(x)=153446449)
-4500 val loss 6.0824
-4500 val perplexity 438.0891
-4500 train 6.027428 (lr=1.9999e-05) (hash(x)=146086947)
-4500 val loss 6.0681
-4500 val perplexity 431.8431
-4500 train 6.013597 (lr=1.7142e-05) (hash(x)=146086947)
-4600 val loss 6.0720
-4600 val perplexity 433.5492
-4600 train 5.910559 (lr=1.9400e-05) (hash(x)=153800173)
-4600 val loss 6.0581
-4600 val perplexity 427.5742
-4600 train 5.898770 (lr=1.6629e-05) (hash(x)=153800173)
-4700 val loss 6.0597
-4700 val perplexity 428.2572
-4700 train 6.110185 (lr=1.8800e-05) (hash(x)=155962726)
-4700 val loss 6.0479
-4700 val perplexity 423.2194
-4700 train 6.100060 (lr=1.6114e-05) (hash(x)=155962726)
-4800 val loss 6.0515
-4800 val perplexity 424.7603
-4800 train 6.275963 (lr=1.8201e-05) (hash(x)=142045616)
-4800 val loss 6.0388
-4800 val perplexity 419.4092
-4800 train 6.270424 (lr=1.5601e-05) (hash(x)=142045616)
-4900 val loss 6.0377
-4900 val perplexity 418.9119
-4900 train 6.005858 (lr=1.7604e-05) (hash(x)=143418248)
-4900 val loss 6.0267
-4900 val perplexity 414.3510
-4900 train 5.989192 (lr=1.5089e-05) (hash(x)=143418248)
-5000 val loss 6.0290
-5000 val perplexity 415.2997
-5000 train 5.954850 (lr=1.7009e-05) (hash(x)=145789790)
-5000 val loss 6.0175
-5000 val perplexity 410.5454
-5000 train 5.938465 (lr=1.4579e-05) (hash(x)=145789790)
-5100 val loss 6.0238
-5100 val perplexity 413.1402
-5100 train 5.980156 (lr=1.6417e-05) (hash(x)=137795633)
-5100 val loss 6.0147
-5100 val perplexity 409.3998
-5100 train 5.970935 (lr=1.4071e-05) (hash(x)=137795633)
-5200 val loss 6.0137
-5200 val perplexity 408.9803
-5200 train 6.178001 (lr=1.5829e-05) (hash(x)=148907132)
-5200 val loss 6.0025
-5200 val perplexity 404.4547
-5200 train 6.166307 (lr=1.3568e-05) (hash(x)=148907132)
-5300 val loss 6.0028
-5300 val perplexity 404.5762
-5300 train 5.882075 (lr=1.5246e-05) (hash(x)=152343580)
-5300 val loss 5.9958
-5300 val perplexity 401.7314
-5300 train 5.877028 (lr=1.3068e-05) (hash(x)=152343580)
-5400 val loss 5.9968
-5400 val perplexity 402.1420
-5400 train 5.843998 (lr=1.4669e-05) (hash(x)=148578264)
-5400 val loss 5.9889
-5400 val perplexity 398.9933
-5400 train 5.840799 (lr=1.2573e-05) (hash(x)=148578264)
-5500 val loss 5.9842
-5500 val perplexity 397.1019
-5500 train 6.141342 (lr=1.4099e-05) (hash(x)=145635833)
-5500 val loss 5.9769
-5500 val perplexity 394.1999
-5500 train 6.139304 (lr=1.2085e-05) (hash(x)=145635833)
-5600 val loss 5.9796
-5600 val perplexity 395.2935
-5600 train 5.955497 (lr=1.3536e-05) (hash(x)=156337844)
-5600 val loss 5.9735
-5600 val perplexity 392.8972
-5600 train 5.955674 (lr=1.1602e-05) (hash(x)=156337844)
-5700 val loss 5.9731
-5700 val perplexity 392.7069
-5700 train 5.856874 (lr=1.2981e-05) (hash(x)=147168506)
-5700 val loss 5.9688
-5700 val perplexity 391.0504
-5700 train 5.857741 (lr=1.1127e-05) (hash(x)=147168506)
-5800 val loss 5.9644
-5800 val perplexity 389.3232
-5800 train 6.032385 (lr=1.2436e-05) (hash(x)=159566920)
-5800 val loss 5.9585
-5800 val perplexity 387.0450
-5800 train 6.023722 (lr=1.0659e-05) (hash(x)=159566920)
-5900 val loss 5.9578
-5900 val perplexity 386.7631
-5900 train 6.025391 (lr=1.1900e-05) (hash(x)=158273929)
-5900 val loss 5.9536
-5900 val perplexity 385.1432
-5900 train 6.027411 (lr=1.0200e-05) (hash(x)=158273929)
-6000 val loss 5.9532
-6000 val perplexity 384.9910
-6000 train 5.921682 (lr=1.1375e-05) (hash(x)=156649749)
-6000 val loss 5.9499
-6000 val perplexity 383.6991
-6000 train 5.916996 (lr=9.7500e-06) (hash(x)=156649749)
-6100 val loss 5.9437
-6100 val perplexity 381.3258
-6100 train 5.860867 (lr=1.0861e-05) (hash(x)=146812388)
-6100 val loss 5.9397
-6100 val perplexity 379.8224
-6100 train 5.858076 (lr=9.3098e-06) (hash(x)=146812388)
-6200 val loss 5.9380
-6200 val perplexity 379.1947
-6200 train 5.760242 (lr=1.0360e-05) (hash(x)=143522146)
-6200 val loss 5.9360
-6200 val perplexity 378.4272
-6200 train 5.755511 (lr=8.8800e-06) (hash(x)=143522146)
-6300 val loss 5.9354
-6300 val perplexity 378.1945
-6300 train 5.772961 (lr=9.8715e-06) (hash(x)=150124474)
-6300 val loss 5.9314
-6300 val perplexity 376.6789
-6300 train 5.768746 (lr=8.4613e-06) (hash(x)=150124474)
-6400 val loss 5.9273
-6400 val perplexity 375.1227
-6400 train 5.950421 (lr=9.3966e-06) (hash(x)=141242117)
-6400 val loss 5.9284
-6400 val perplexity 375.5501
-6400 train 5.955993 (lr=8.0542e-06) (hash(x)=141242117)
-6500 val loss 5.9207
-6500 val perplexity 372.6563
-6500 train 5.801312 (lr=8.9359e-06) (hash(x)=143529762)
-6500 val loss 5.9189
-6500 val perplexity 371.9922
-6500 train 5.801332 (lr=7.6594e-06) (hash(x)=143529762)
-6600 val loss 5.9155
-6600 val perplexity 370.7257
-6600 train 5.697849 (lr=8.4903e-06) (hash(x)=136948374)
-6600 val loss 5.9148
-6600 val perplexity 370.4690
-6600 train 5.696297 (lr=7.2774e-06) (hash(x)=136948374)
-6700 val loss 5.9141
-6700 val perplexity 370.2074
-6700 train 5.733417 (lr=8.0602e-06) (hash(x)=146268592)
-6700 val loss 5.9131
-6700 val perplexity 369.8498
-6700 train 5.730753 (lr=6.9087e-06) (hash(x)=146268592)
-6800 val loss 5.9055
-6800 val perplexity 367.0687
-6800 train 5.868842 (lr=7.6463e-06) (hash(x)=152676836)
-6800 val loss 5.9058
-6800 val perplexity 367.1629
-6800 train 5.863817 (lr=6.5540e-06) (hash(x)=152676836)
-6900 val loss 5.8984
-6900 val perplexity 364.4714
-6900 train 5.812114 (lr=7.2493e-06) (hash(x)=134657776)
-6900 val loss 5.8995
-6900 val perplexity 364.8668
-6900 train 5.811509 (lr=6.2137e-06) (hash(x)=134657776)
-7000 val loss 5.8971
-7000 val perplexity 363.9915
-7000 train 5.862196 (lr=6.8697e-06) (hash(x)=166721861)
-7000 val loss 5.8977
-7000 val perplexity 364.2080
-7000 train 5.862389 (lr=5.8883e-06) (hash(x)=166721861)
-7100 val loss 5.8936
-7100 val perplexity 362.6912
-7100 train 5.697666 (lr=6.5080e-06) (hash(x)=135496702)
-7100 val loss 5.8958
-7100 val perplexity 363.5053
-7100 train 5.697424 (lr=5.5783e-06) (hash(x)=135496702)
-7200 val loss 5.8906
-7200 val perplexity 361.6083
-7200 train 5.993858 (lr=6.1648e-06) (hash(x)=155567461)
-7200 val loss 5.8928
-7200 val perplexity 362.4025
-7200 train 5.994301 (lr=5.2841e-06) (hash(x)=155567461)
-7300 val loss 5.8846
-7300 val perplexity 359.4441
-7300 train 5.676643 (lr=5.8405e-06) (hash(x)=142803829)
-7300 val loss 5.8867
-7300 val perplexity 360.1984
-7300 train 5.685822 (lr=5.0062e-06) (hash(x)=142803829)
-7400 val loss 5.8824
-7400 val perplexity 358.6774
-7400 train 5.699541 (lr=5.5357e-06) (hash(x)=145294178)
-7400 val loss 5.8848
-7400 val perplexity 359.5374
-7400 train 5.707546 (lr=4.7449e-06) (hash(x)=145294178)
-7500 val loss 5.8797
-7500 val perplexity 357.6940
-7500 train 5.642534 (lr=5.2508e-06) (hash(x)=150573713)
-7500 val loss 5.8828
-7500 val perplexity 358.8115
-7500 train 5.639037 (lr=4.5007e-06) (hash(x)=150573713)
-7600 val loss 5.8773
-7600 val perplexity 356.8476
-7600 train 5.904793 (lr=4.9862e-06) (hash(x)=142771511)
-7600 val loss 5.8806
-7600 val perplexity 358.0397
-7600 train 5.905676 (lr=4.2739e-06) (hash(x)=142771511)
-7700 val loss 5.8716
-7700 val perplexity 354.8214
-7700 train 5.766538 (lr=4.7423e-06) (hash(x)=143602175)
-7700 val loss 5.8750
-7700 val perplexity 356.0248
-7700 train 5.774282 (lr=4.0648e-06) (hash(x)=143602175)
-7800 val loss 5.8698
-7800 val perplexity 354.1783
-7800 train 5.892739 (lr=4.5194e-06) (hash(x)=152379862)
-7800 val loss 5.8722
-7800 val perplexity 355.0424
-7800 train 5.895572 (lr=3.8738e-06) (hash(x)=152379862)
-7900 val loss 5.8686
-7900 val perplexity 353.7581
-7900 train 5.703426 (lr=4.3179e-06) (hash(x)=146655921)
-7900 val loss 5.8714
-7900 val perplexity 354.7439
-7900 train 5.703874 (lr=3.7010e-06) (hash(x)=146655921)
-8000 val loss 5.8693
-8000 val perplexity 353.9875
-8000 train 5.982882 (lr=4.1380e-06) (hash(x)=148262482)
-8000 val loss 5.8727
-8000 val perplexity 355.2155
-8000 train 5.985026 (lr=3.5468e-06) (hash(x)=148262482)
-8100 val loss 5.8610
-8100 val perplexity 351.0891
-8100 train 5.792439 (lr=3.9800e-06) (hash(x)=147683655)
-8100 val loss 5.8643
-8100 val perplexity 352.2198
-8100 train 5.793932 (lr=3.4114e-06) (hash(x)=147683655)
-8200 val loss 5.8610
-8200 val perplexity 351.0704
-8200 train 5.954844 (lr=3.8442e-06) (hash(x)=157312987)
-8200 val loss 5.8644
-8200 val perplexity 352.2767
-8200 train 5.959354 (lr=3.2950e-06) (hash(x)=157312987)
-8300 val loss 5.8584
-8300 val perplexity 350.1542
-8300 train 5.827306 (lr=3.7307e-06) (hash(x)=141107543)
-8300 val loss 5.8617
-8300 val perplexity 351.3268
-8300 train 5.833569 (lr=3.1977e-06) (hash(x)=141107543)
-8400 val loss 5.8546
-8400 val perplexity 348.8233
-8400 train 5.829935 (lr=3.6397e-06) (hash(x)=141323024)
-8400 val loss 5.8581
-8400 val perplexity 350.0630
-8400 train 5.836152 (lr=3.1197e-06) (hash(x)=141323024)
-8500 val loss 5.8523
-8500 val perplexity 348.0277
-8500 train 5.828456 (lr=3.5713e-06) (hash(x)=150696521)
-8500 val loss 5.8561
-8500 val perplexity 349.3654
-8500 train 5.834352 (lr=3.0611e-06) (hash(x)=150696521)
-8600 val loss 5.8522
-8600 val perplexity 348.0011
-8600 train 5.811702 (lr=3.5257e-06) (hash(x)=162288191)
-8600 val loss 5.8554
-8600 val perplexity 349.1052
-8600 train 5.809893 (lr=3.0220e-06) (hash(x)=162288191)
-8700 val loss 5.8482
-8700 val perplexity 346.6077
-8700 train 5.718023 (lr=3.5029e-06) (hash(x)=152860941)
-8700 val loss 5.8523
-8700 val perplexity 348.0176
-8700 train 5.726460 (lr=3.0024e-06) (hash(x)=152860941)
-8749 val loss 5.8467
-8749 val perplexity 346.0782
-8749 val loss 5.8518
-8749 val perplexity 347.8583
+0 train 11.313040 (lr=1.1000e-07) (hash(x)=145079536)
+100 val loss 10.0285
+100 val perplexity 22663.5449
+100 train 10.022821 (lr=1.1110e-05) (hash(x)=155800595)
+200 val loss 9.3391
+200 val perplexity 11373.7100
+200 train 9.288088 (lr=2.2110e-05) (hash(x)=145606733)
+300 val loss 8.0597
+300 val perplexity 3164.1838
+300 train 8.040195 (lr=3.3110e-05) (hash(x)=150367139)
+400 val loss 7.5562
+400 val perplexity 1912.4974
+400 train 7.863613 (lr=4.4110e-05) (hash(x)=155747374)
+500 val loss 7.4240
+500 val perplexity 1675.6675
+500 train 7.315066 (lr=5.5000e-05) (hash(x)=140604760)
+600 val loss 7.3587
+600 val perplexity 1569.7710
+600 train 7.302488 (lr=5.4982e-05) (hash(x)=148404734)
+700 val loss 7.3047
+700 val perplexity 1487.2728
+700 train 7.262489 (lr=5.4928e-05) (hash(x)=148115934)
+800 val loss 7.2321
+800 val perplexity 1383.0587
+800 train 7.115292 (lr=5.4839e-05) (hash(x)=137464699)
+900 val loss 7.1679
+900 val perplexity 1297.1080
+900 train 6.982425 (lr=5.4713e-05) (hash(x)=143886042)
+1000 val loss 7.1077
+1000 val perplexity 1221.3217
+1000 train 7.107786 (lr=5.4553e-05) (hash(x)=163799796)
+1100 val loss 7.0422
+1100 val perplexity 1143.8781
+1100 train 6.998779 (lr=5.4357e-05) (hash(x)=144592844)
+1200 val loss 6.9882
+1200 val perplexity 1083.7599
+1200 train 7.327436 (lr=5.4126e-05) (hash(x)=204706354)
+1300 val loss 6.9446
+1300 val perplexity 1037.5281
+1300 train 6.878752 (lr=5.3860e-05) (hash(x)=150862210)
+1400 val loss 6.8865
+1400 val perplexity 978.9376
+1400 train 6.871954 (lr=5.3561e-05) (hash(x)=147766811)
+1500 val loss 6.8288
+1500 val perplexity 924.0453
+1500 train 6.639542 (lr=5.3227e-05) (hash(x)=135925327)
+1600 val loss 6.7740
+1600 val perplexity 874.8393
+1600 train 6.746453 (lr=5.2860e-05) (hash(x)=160440642)
+1700 val loss 6.7164
+1700 val perplexity 825.8420
+1700 train 6.815380 (lr=5.2461e-05) (hash(x)=151184106)
+1800 val loss 6.6585
+1800 val perplexity 779.3540
+1800 train 6.557573 (lr=5.2029e-05) (hash(x)=148100580)
+1900 val loss 6.6249
+1900 val perplexity 753.6406
+1900 train 6.457758 (lr=5.1565e-05) (hash(x)=149434659)
+2000 val loss 6.5724
+2000 val perplexity 715.0803
+2000 train 6.471416 (lr=5.1071e-05) (hash(x)=152285486)
+2100 val loss 6.5395
+2100 val perplexity 691.9631
+2100 train 6.288988 (lr=5.0547e-05) (hash(x)=144294295)
+2200 val loss 6.5062
+2200 val perplexity 669.3031
+2200 train 6.534094 (lr=4.9993e-05) (hash(x)=175030215)
+2300 val loss 6.4785
+2300 val perplexity 651.0085
+2300 train 6.280591 (lr=4.9410e-05) (hash(x)=150831428)
+2400 val loss 6.4330
+2400 val perplexity 622.0258
+2400 train 6.595540 (lr=4.8800e-05) (hash(x)=140808297)
+2500 val loss 6.4295
+2500 val perplexity 619.8776
+2500 train 6.419309 (lr=4.8162e-05) (hash(x)=153160275)
+2600 val loss 6.4029
+2600 val perplexity 603.5640
+2600 train 6.176099 (lr=4.7499e-05) (hash(x)=133990623)
+2700 val loss 6.3730
+2700 val perplexity 585.8351
+2700 train 6.347503 (lr=4.6811e-05) (hash(x)=142860944)
+2800 val loss 6.3438
+2800 val perplexity 568.9605
+2800 train 6.365329 (lr=4.6099e-05) (hash(x)=137959511)
+2900 val loss 6.3173
+2900 val perplexity 554.1024
+2900 train 6.263561 (lr=4.5364e-05) (hash(x)=147009873)
+3000 val loss 6.2962
+3000 val perplexity 542.5182
+3000 train 6.152777 (lr=4.4606e-05) (hash(x)=158264841)
+3100 val loss 6.2885
+3100 val perplexity 538.3243
+3100 train 6.133510 (lr=4.3828e-05) (hash(x)=139232251)
+3200 val loss 6.2569
+3200 val perplexity 521.6201
+3200 train 6.301063 (lr=4.3031e-05) (hash(x)=153436104)
+3300 val loss 6.2290
+3300 val perplexity 507.2550
+3300 train 6.049569 (lr=4.2215e-05) (hash(x)=149681831)
+3400 val loss 6.2229
+3400 val perplexity 504.1629
+3400 train 6.117395 (lr=4.1381e-05) (hash(x)=168425516)
+3500 val loss 6.1881
+3500 val perplexity 486.9261
+3500 train 6.212366 (lr=4.0532e-05) (hash(x)=163104338)
+3600 val loss 6.1671
+3600 val perplexity 476.8029
+3600 train 6.143349 (lr=3.9667e-05) (hash(x)=165109772)
+3700 val loss 6.1626
+3700 val perplexity 474.6586
+3700 train 5.890706 (lr=3.8789e-05) (hash(x)=153420306)
+3800 val loss 6.1309
+3800 val perplexity 459.8583
+3800 train 6.154771 (lr=3.7898e-05) (hash(x)=160168863)
+3900 val loss 6.1152
+3900 val perplexity 452.6844
+3900 train 6.032316 (lr=3.6996e-05) (hash(x)=153906073)
+4000 val loss 6.1059
+4000 val perplexity 448.4742
+4000 train 6.034504 (lr=3.6085e-05) (hash(x)=151055067)
+4100 val loss 6.0870
+4100 val perplexity 440.1180
+4100 train 6.061316 (lr=3.5165e-05) (hash(x)=149629830)
+4200 val loss 6.0718
+4200 val perplexity 433.4450
+4200 train 5.946362 (lr=3.4238e-05) (hash(x)=143101381)
+4300 val loss 6.0579
+4300 val perplexity 427.4625
+4300 train 5.960875 (lr=3.3305e-05) (hash(x)=149712044)
+4400 val loss 6.0403
+4400 val perplexity 419.9999
+4400 train 5.964753 (lr=3.2368e-05) (hash(x)=153446449)
+4500 val loss 6.0242
+4500 val perplexity 413.3047
+4500 train 5.972738 (lr=3.1428e-05) (hash(x)=146086947)
+4600 val loss 6.0203
+4600 val perplexity 411.6975
+4600 train 5.859571 (lr=3.0486e-05) (hash(x)=153800173)
+4700 val loss 6.0057
+4700 val perplexity 405.7249
+4700 train 6.063126 (lr=2.9543e-05) (hash(x)=155962726)
+4800 val loss 5.9915
+4800 val perplexity 400.0210
+4800 train 6.226341 (lr=2.8602e-05) (hash(x)=142045616)
+4900 val loss 5.9758
+4900 val perplexity 393.7772
+4900 train 5.946213 (lr=2.7663e-05) (hash(x)=143418248)
+5000 val loss 5.9665
+5000 val perplexity 390.1279
+5000 train 5.884785 (lr=2.6728e-05) (hash(x)=145789790)
+5100 val loss 5.9587
+5100 val perplexity 387.1258
+5100 train 5.927770 (lr=2.5798e-05) (hash(x)=137795633)
+5200 val loss 5.9446
+5200 val perplexity 381.7025
+5200 train 6.104008 (lr=2.4874e-05) (hash(x)=148907132)
+5300 val loss 5.9428
+5300 val perplexity 380.9910
+5300 train 5.824908 (lr=2.3958e-05) (hash(x)=152343580)
+5400 val loss 5.9278
+5400 val perplexity 375.3416
+5400 train 5.779144 (lr=2.3051e-05) (hash(x)=148578264)
+5500 val loss 5.9159
+5500 val perplexity 370.8882
+5500 train 6.081324 (lr=2.2155e-05) (hash(x)=145635833)
+5600 val loss 5.9097
+5600 val perplexity 368.5929
+5600 train 5.884354 (lr=2.1271e-05) (hash(x)=156337844)
+5700 val loss 5.9002
+5700 val perplexity 365.1134
+5700 train 5.783232 (lr=2.0399e-05) (hash(x)=147168506)
+5800 val loss 5.8906
+5800 val perplexity 361.6292
+5800 train 5.954978 (lr=1.9542e-05) (hash(x)=159566920)
+5900 val loss 5.8862
+5900 val perplexity 360.0167
+5900 train 5.948883 (lr=1.8700e-05) (hash(x)=158273929)
+6000 val loss 5.8794
+6000 val perplexity 357.5957
+6000 train 5.850380 (lr=1.7875e-05) (hash(x)=156649749)
+6100 val loss 5.8706
+6100 val perplexity 354.4674
+6100 train 5.786806 (lr=1.7068e-05) (hash(x)=146812388)
+6200 val loss 5.8642
+6200 val perplexity 352.1928
+6200 train 5.684584 (lr=1.6280e-05) (hash(x)=143522146)
+6300 val loss 5.8598
+6300 val perplexity 350.6573
+6300 train 5.697289 (lr=1.5512e-05) (hash(x)=150124474)
+6400 val loss 5.8556
+6400 val perplexity 349.1890
+6400 train 5.881581 (lr=1.4766e-05) (hash(x)=141242117)
+6500 val loss 5.8434
+6500 val perplexity 344.9429
+6500 train 5.724736 (lr=1.4042e-05) (hash(x)=143529762)
+6600 val loss 5.8384
+6600 val perplexity 343.2130
+6600 train 5.621650 (lr=1.3342e-05) (hash(x)=136948374)
+6700 val loss 5.8354
+6700 val perplexity 342.2034
+6700 train 5.649374 (lr=1.2666e-05) (hash(x)=146268592)
+6800 val loss 5.8289
+6800 val perplexity 339.9703
+6800 train 5.787588 (lr=1.2016e-05) (hash(x)=152676836)
+6900 val loss 5.8204
+6900 val perplexity 337.0951
+6900 train 5.736982 (lr=1.1392e-05) (hash(x)=134657776)
+7000 val loss 5.8177
+7000 val perplexity 336.1875
+7000 train 5.775075 (lr=1.0795e-05) (hash(x)=166721861)
+7100 val loss 5.8143
+7100 val perplexity 335.0538
+7100 train 5.619752 (lr=1.0227e-05) (hash(x)=135496702)
+7200 val loss 5.8106
+7200 val perplexity 333.8330
+7200 train 5.914602 (lr=9.6875e-06) (hash(x)=155567461)
+7300 val loss 5.8045
+7300 val perplexity 331.7833
+7300 train 5.601978 (lr=9.1780e-06) (hash(x)=142803829)
+7400 val loss 5.8025
+7400 val perplexity 331.1190
+7400 train 5.623451 (lr=8.6990e-06) (hash(x)=145294178)
+7500 val loss 5.7976
+7500 val perplexity 329.5175
+7500 train 5.558932 (lr=8.2513e-06) (hash(x)=150573713)
+7600 val loss 5.7957
+7600 val perplexity 328.8690
+7600 train 5.817007 (lr=7.8355e-06) (hash(x)=142771511)
+7700 val loss 5.7898
+7700 val perplexity 326.9609
+7700 train 5.689264 (lr=7.4522e-06) (hash(x)=143602175)
+7800 val loss 5.7861
+7800 val perplexity 325.7462
+7800 train 5.816852 (lr=7.1019e-06) (hash(x)=152379862)
+7900 val loss 5.7852
+7900 val perplexity 325.4419
+7900 train 5.620042 (lr=6.7852e-06) (hash(x)=146655921)
+8000 val loss 5.7856
+8000 val perplexity 325.5628
+8000 train 5.900315 (lr=6.5025e-06) (hash(x)=148262482)
+8100 val loss 5.7776
+8100 val perplexity 322.9937
+8100 train 5.713513 (lr=6.2543e-06) (hash(x)=147683655)
+8200 val loss 5.7774
+8200 val perplexity 322.9288
+8200 train 5.876583 (lr=6.0408e-06) (hash(x)=157312987)
+8300 val loss 5.7743
+8300 val perplexity 321.9338
+8300 train 5.736997 (lr=5.8625e-06) (hash(x)=141107543)
+8400 val loss 5.7698
+8400 val perplexity 320.4694
+8400 train 5.747867 (lr=5.7195e-06) (hash(x)=141323024)
+8500 val loss 5.7673
+8500 val perplexity 319.6715
+8500 train 5.737051 (lr=5.6121e-06) (hash(x)=150696521)
+8600 val loss 5.7659
+8600 val perplexity 319.2322
+8600 train 5.718998 (lr=5.5404e-06) (hash(x)=162288191)
+8700 val loss 5.7624
+8700 val perplexity 318.1103
+8700 train 5.633211 (lr=5.5045e-06) (hash(x)=152860941)
+8749 val loss 5.7618
+8749 val perplexity 317.9061

attention_kindselective_n_heads4_seed1341/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:395f6caf00fc6931495f915a58e51ee5930a2f4ca2b3f84a36b624a6ae25a106
+oid sha256:8b7fd67aa3ad05060e49d991c1a5ce788f8dd22e7a1852efc3ad664cfb3e9e6a
 size 92843394

attention_kindselective_n_heads4_seed1341/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9059975da2d811a2ec619a9cd6f63d985d15ff163e8a4becb0434b514ed8062b
+oid sha256:e26d19de11e659cfb1c05e7f92e8c844a6fa530d7e243d5fc97a27e28b66d8ec
 size 179406214