Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5.5e-5_61440_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_61440_4_1339", "n_embd": 256}

attention_kindselective_n_heads4_seed1339/log2.txt

@@ -1,267 +1,502 @@
 max_steps: 8750
+max_steps: 8750
 0 val loss 11.2808
 0 val perplexity 79287.6797
-0 train 11.289297 (lr=1.1000e-07) (hash(x)=150724848)
-100 val loss 9.9926
-100 val perplexity 21863.0391
-100 train 10.007814 (lr=1.1110e-05) (hash(x)=149910534)
-200 val loss 9.1223
-200 val perplexity 9157.1182
-200 train 9.148949 (lr=2.2110e-05) (hash(x)=148123706)
-300 val loss 7.9004
-300 val perplexity 2698.2358
-300 train 7.756021 (lr=3.3110e-05) (hash(x)=146678221)
-400 val loss 7.5196
-400 val perplexity 1843.8901
-400 train 7.404154 (lr=4.4110e-05) (hash(x)=151700982)
-500 val loss 7.3670
-500 val perplexity 1582.9152
-500 train 7.450415 (lr=5.5000e-05) (hash(x)=156182087)
-600 val loss 7.2367
-600 val perplexity 1389.4574
-600 train 7.212444 (lr=5.4982e-05) (hash(x)=149318660)
-700 val loss 7.1474
-700 val perplexity 1270.7859
-700 train 7.189300 (lr=5.4928e-05) (hash(x)=150482428)
-800 val loss 7.0296
-800 val perplexity 1129.5702
-800 train 6.903787 (lr=5.4839e-05) (hash(x)=143268605)
-900 val loss 6.9340
-900 val perplexity 1026.5736
-900 train 6.897569 (lr=5.4713e-05) (hash(x)=152322423)
-1000 val loss 6.8391
-1000 val perplexity 933.6412
-1000 train 6.705533 (lr=5.4553e-05) (hash(x)=147904298)
-1100 val loss 6.7474
-1100 val perplexity 851.8519
-1100 train 6.950415 (lr=5.4357e-05) (hash(x)=154343147)
-1200 val loss 6.6458
-1200 val perplexity 769.5319
-1200 train 6.633022 (lr=5.4126e-05) (hash(x)=141843115)
-1300 val loss 6.5715
-1300 val perplexity 714.4750
-1300 train 6.459829 (lr=5.3860e-05) (hash(x)=145279030)
-1400 val loss 6.5204
-1400 val perplexity 678.8162
-1400 train 6.464448 (lr=5.3561e-05) (hash(x)=152507639)
-1500 val loss 6.4720
-1500 val perplexity 646.7606
-1500 train 6.506417 (lr=5.3227e-05) (hash(x)=148473774)
-1600 val loss 6.4094
-1600 val perplexity 607.5082
-1600 train 6.450625 (lr=5.2860e-05) (hash(x)=151117002)
-1700 val loss 6.3770
-1700 val perplexity 588.1442
-1700 train 6.323629 (lr=5.2461e-05) (hash(x)=138011335)
-1800 val loss 6.3356
-1800 val perplexity 564.3019
-1800 train 6.464371 (lr=5.2029e-05) (hash(x)=171180926)
-1900 val loss 6.2924
-1900 val perplexity 540.4733
-1900 train 6.228439 (lr=5.1565e-05) (hash(x)=141769419)
-2000 val loss 6.2590
-2000 val perplexity 522.6740
-2000 train 6.211834 (lr=5.1071e-05) (hash(x)=151963443)
-2100 val loss 6.2432
-2100 val perplexity 514.5130
-2100 train 6.295941 (lr=5.0547e-05) (hash(x)=162947470)
-2200 val loss 6.1986
-2200 val perplexity 492.0483
-2200 train 6.424762 (lr=4.9993e-05) (hash(x)=154954810)
-2300 val loss 6.1768
-2300 val perplexity 481.4570
-2300 train 6.154170 (lr=4.9410e-05) (hash(x)=151878111)
-2400 val loss 6.1559
-2400 val perplexity 471.4818
-2400 train 6.050483 (lr=4.8800e-05) (hash(x)=158661057)
-2500 val loss 6.1194
-2500 val perplexity 454.5914
-2500 train 6.141537 (lr=4.8162e-05) (hash(x)=150925584)
-2600 val loss 6.0965
-2600 val perplexity 444.3190
-2600 train 6.004150 (lr=4.7499e-05) (hash(x)=144515755)
-2700 val loss 6.0754
-2700 val perplexity 435.0298
-2700 train 6.059896 (lr=4.6811e-05) (hash(x)=153109144)
-2800 val loss 6.0466
-2800 val perplexity 422.6543
-2800 train 5.952908 (lr=4.6099e-05) (hash(x)=151152897)
-2900 val loss 6.0262
-2900 val perplexity 414.1284
-2900 train 5.969955 (lr=4.5364e-05) (hash(x)=145800210)
-3000 val loss 6.0130
-3000 val perplexity 408.7055
-3000 train 5.818487 (lr=4.4606e-05) (hash(x)=141997485)
-3100 val loss 5.9902
-3100 val perplexity 399.5090
-3100 train 5.910625 (lr=4.3828e-05) (hash(x)=154049740)
-3200 val loss 5.9697
-3200 val perplexity 391.3949
-3200 train 5.898226 (lr=4.3031e-05) (hash(x)=150471842)
-3300 val loss 5.9616
-3300 val perplexity 388.2483
-3300 train 5.906397 (lr=4.2215e-05) (hash(x)=149048126)
-3400 val loss 5.9343
-3400 val perplexity 377.7688
-3400 train 6.056663 (lr=4.1381e-05) (hash(x)=161261339)
-3500 val loss 5.9248
-3500 val perplexity 374.2032
-3500 train 5.890368 (lr=4.0532e-05) (hash(x)=157495564)
-3600 val loss 5.9106
-3600 val perplexity 368.9154
-3600 train 5.813707 (lr=3.9667e-05) (hash(x)=144352932)
-3700 val loss 5.8902
-3700 val perplexity 361.4862
-3700 train 5.860925 (lr=3.8789e-05) (hash(x)=149389012)
-3800 val loss 5.8681
-3800 val perplexity 353.5889
-3800 train 5.809790 (lr=3.7898e-05) (hash(x)=146607620)
-3900 val loss 5.8539
-3900 val perplexity 348.6085
-3900 train 5.772669 (lr=3.6996e-05) (hash(x)=143754617)
-4000 val loss 5.8410
-4000 val perplexity 344.1350
-4000 train 5.813826 (lr=3.6085e-05) (hash(x)=156930722)
-4100 val loss 5.8236
-4100 val perplexity 338.1763
-4100 train 5.653216 (lr=3.5165e-05) (hash(x)=147500519)
-4200 val loss 5.8148
-4200 val perplexity 335.2222
-4200 train 5.652660 (lr=3.4238e-05) (hash(x)=143232237)
-4300 val loss 5.8024
-4300 val perplexity 331.0877
-4300 train 5.709325 (lr=3.3305e-05) (hash(x)=146811670)
-4400 val loss 5.7871
-4400 val perplexity 326.0585
-4400 train 5.821728 (lr=3.2368e-05) (hash(x)=158418746)
-4500 val loss 5.7772
-4500 val perplexity 322.8669
-4500 train 5.743765 (lr=3.1428e-05) (hash(x)=156695778)
-4600 val loss 5.7656
-4600 val perplexity 319.1193
-4600 train 5.785645 (lr=3.0486e-05) (hash(x)=147791497)
-4700 val loss 5.7494
-4700 val perplexity 314.0002
-4700 train 5.637969 (lr=2.9543e-05) (hash(x)=155533088)
-4800 val loss 5.7457
-4800 val perplexity 312.8485
-4800 train 5.597758 (lr=2.8602e-05) (hash(x)=138350044)
-4900 val loss 5.7309
-4900 val perplexity 308.2330
-4900 train 5.726795 (lr=2.7663e-05) (hash(x)=143735284)
-5000 val loss 5.7179
-5000 val perplexity 304.2539
-5000 train 5.672445 (lr=2.6728e-05) (hash(x)=154976463)
-5100 val loss 5.7154
-5100 val perplexity 303.5133
-5100 train 5.600068 (lr=2.5798e-05) (hash(x)=149894982)
-5200 val loss 5.7096
-5200 val perplexity 301.7632
-5200 train 5.481579 (lr=2.4874e-05) (hash(x)=159326689)
-5300 val loss 5.6972
-5300 val perplexity 298.0291
-5300 train 5.708793 (lr=2.3958e-05) (hash(x)=159484800)
-5400 val loss 5.6821
-5400 val perplexity 293.5782
-5400 train 5.585687 (lr=2.3051e-05) (hash(x)=140385615)
-5500 val loss 5.6753
-5500 val perplexity 291.5740
-5500 train 5.498550 (lr=2.2155e-05) (hash(x)=148498335)
-5600 val loss 5.6751
-5600 val perplexity 291.5265
-5600 train 5.412326 (lr=2.1271e-05) (hash(x)=151907614)
-5700 val loss 5.6657
-5700 val perplexity 288.7810
-5700 train 5.637896 (lr=2.0399e-05) (hash(x)=155192267)
-5800 val loss 5.6533
-5800 val perplexity 285.2272
-5800 train 5.701861 (lr=1.9542e-05) (hash(x)=153132158)
-5900 val loss 5.6519
-5900 val perplexity 284.8352
-5900 train 5.619544 (lr=1.8700e-05) (hash(x)=161446764)
-6000 val loss 5.6450
-6000 val perplexity 282.8669
-6000 train 5.439802 (lr=1.7875e-05) (hash(x)=151512446)
-6100 val loss 5.6340
-6100 val perplexity 279.7804
-6100 train 5.747126 (lr=1.7068e-05) (hash(x)=188094053)
-6200 val loss 5.6256
-6200 val perplexity 277.4352
-6200 train 5.508698 (lr=1.6280e-05) (hash(x)=149389789)
-6300 val loss 5.6217
-6300 val perplexity 276.3475
-6300 train 5.438232 (lr=1.5512e-05) (hash(x)=138212820)
-6400 val loss 5.6208
-6400 val perplexity 276.1091
-6400 train 5.405768 (lr=1.4766e-05) (hash(x)=146535423)
-6500 val loss 5.6109
-6500 val perplexity 273.3779
-6500 train 5.479463 (lr=1.4042e-05) (hash(x)=145950843)
-6600 val loss 5.6075
-6600 val perplexity 272.4713
-6600 train 5.507493 (lr=1.3342e-05) (hash(x)=141162902)
-6700 val loss 5.6048
-6700 val perplexity 271.7363
-6700 train 5.515503 (lr=1.2666e-05) (hash(x)=153018737)
-6800 val loss 5.5980
-6800 val perplexity 269.8820
-6800 train 5.513815 (lr=1.2016e-05) (hash(x)=155640155)
-6900 val loss 5.5941
-6900 val perplexity 268.8358
-6900 train 5.567899 (lr=1.1392e-05) (hash(x)=153722115)
-7000 val loss 5.5878
-7000 val perplexity 267.1516
-7000 train 5.493548 (lr=1.0795e-05) (hash(x)=146953450)
-7100 val loss 5.5874
-7100 val perplexity 267.0288
-7100 train 5.427116 (lr=1.0227e-05) (hash(x)=137663885)
-7200 val loss 5.5790
-7200 val perplexity 264.7992
-7200 train 5.650048 (lr=9.6875e-06) (hash(x)=146172950)
-7300 val loss 5.5747
-7300 val perplexity 263.6632
-7300 train 5.460082 (lr=9.1780e-06) (hash(x)=150018163)
-7400 val loss 5.5715
-7400 val perplexity 262.8262
-7400 train 5.576471 (lr=8.6990e-06) (hash(x)=145351166)
-7500 val loss 5.5722
-7500 val perplexity 263.0211
-7500 train 5.347385 (lr=8.2513e-06) (hash(x)=145292116)
-7600 val loss 5.5646
-7600 val perplexity 261.0114
-7600 train 5.485201 (lr=7.8355e-06) (hash(x)=150235132)
-7700 val loss 5.5624
-7700 val perplexity 260.4434
-7700 train 5.463830 (lr=7.4522e-06) (hash(x)=154543455)
-7800 val loss 5.5601
-7800 val perplexity 259.8475
-7800 train 5.462539 (lr=7.1019e-06) (hash(x)=142456852)
-7900 val loss 5.5605
-7900 val perplexity 259.9470
-7900 train 5.337718 (lr=6.7852e-06) (hash(x)=147363479)
-8000 val loss 5.5541
-8000 val perplexity 258.2981
-8000 train 5.583329 (lr=6.5025e-06) (hash(x)=156122973)
-8100 val loss 5.5513
-8100 val perplexity 257.5828
-8100 train 5.597711 (lr=6.2543e-06) (hash(x)=156153179)
-8200 val loss 5.5499
-8200 val perplexity 257.2050
-8200 train 5.596151 (lr=6.0408e-06) (hash(x)=146430698)
-8300 val loss 5.5482
-8300 val perplexity 256.7670
-8300 train 5.411896 (lr=5.8625e-06) (hash(x)=143507257)
-8400 val loss 5.5431
-8400 val perplexity 255.4713
-8400 train 5.553337 (lr=5.7195e-06) (hash(x)=166272643)
-8500 val loss 5.5421
-8500 val perplexity 255.2123
-8500 train 5.447274 (lr=5.6121e-06) (hash(x)=143887848)
-8600 val loss 5.5419
-8600 val perplexity 255.1584
-8600 train 5.574839 (lr=5.5404e-06) (hash(x)=156900341)
-8700 val loss 5.5367
-8700 val perplexity 253.8504
-8700 train 5.774979 (lr=5.5045e-06) (hash(x)=146417632)
-8749 val loss 5.5365
-8749 val perplexity 253.7859
+0 val loss 11.2808
+0 val perplexity 79287.5234
+0 train 11.289339 (lr=6.0000e-08) (hash(x)=150724848)
+0 train 11.289339 (lr=7.0000e-08) (hash(x)=150724848)
+100 val loss 10.1249
+100 val perplexity 24955.8281
+100 train 10.136632 (lr=7.0700e-06) (hash(x)=149910534)
+100 val loss 10.1868
+100 val perplexity 26549.2832
+100 train 10.199195 (lr=6.0600e-06) (hash(x)=149910534)
+200 val loss 9.5679
+200 val perplexity 14297.8906
+200 train 9.597668 (lr=1.4070e-05) (hash(x)=148123706)
+200 val loss 9.6580
+200 val perplexity 15646.8330
+200 train 9.688334 (lr=1.2060e-05) (hash(x)=148123706)
+300 val loss 8.4851
+300 val perplexity 4841.9927
+300 train 8.403298 (lr=2.1070e-05) (hash(x)=146678221)
+300 val loss 8.7199
+300 val perplexity 6123.2988
+300 train 8.652300 (lr=1.8060e-05) (hash(x)=146678221)
+400 val loss 7.7157
+400 val perplexity 2243.2341
+400 train 7.622499 (lr=2.8070e-05) (hash(x)=151700982)
+400 val loss 7.8706
+400 val perplexity 2619.0610
+400 train 7.789970 (lr=2.4060e-05) (hash(x)=151700982)
+500 val loss 7.4807
+500 val perplexity 1773.4902
+500 train 7.566009 (lr=3.5000e-05) (hash(x)=156182087)
+500 val loss 7.5420
+500 val perplexity 1885.5305
+500 train 7.629263 (lr=3.0000e-05) (hash(x)=156182087)
+600 val loss 7.3659
+600 val perplexity 1581.0912
+600 train 7.345613 (lr=3.4989e-05) (hash(x)=149318660)
+600 val loss 7.4137
+600 val perplexity 1658.5233
+600 train 7.403398 (lr=2.9990e-05) (hash(x)=149318660)
+700 val loss 7.2842
+700 val perplexity 1457.1127
+700 train 7.324867 (lr=3.4954e-05) (hash(x)=150482428)
+800 val loss 7.1606
+800 val perplexity 1287.6595
+800 train 7.034153 (lr=3.4897e-05) (hash(x)=143268605)
+700 val loss 7.3358
+700 val perplexity 1534.3062
+700 train 7.378386 (lr=2.9961e-05) (hash(x)=150482428)
+900 val loss 7.0731
+900 val perplexity 1179.7480
+900 train 7.036097 (lr=3.4818e-05) (hash(x)=152322423)
+800 val loss 7.2311
+800 val perplexity 1381.7266
+800 train 7.106897 (lr=2.9912e-05) (hash(x)=143268605)
+1000 val loss 7.0052
+1000 val perplexity 1102.3225
+1000 train 6.872244 (lr=3.4715e-05) (hash(x)=147904298)
+900 val loss 7.1499
+900 val perplexity 1273.9365
+900 train 7.112676 (lr=2.9844e-05) (hash(x)=152322423)
+1100 val loss 6.9231
+1100 val perplexity 1015.4374
+1100 train 7.109494 (lr=3.4591e-05) (hash(x)=154343147)
+1000 val loss 7.0856
+1000 val perplexity 1194.6191
+1000 train 6.954570 (lr=2.9756e-05) (hash(x)=147904298)
+1200 val loss 6.8335
+1200 val perplexity 928.4275
+1200 train 6.822800 (lr=3.4444e-05) (hash(x)=141843115)
+1100 val loss 7.0176
+1100 val perplexity 1116.0782
+1100 train 7.204097 (lr=2.9649e-05) (hash(x)=154343147)
+1300 val loss 6.7614
+1300 val perplexity 863.8369
+1300 train 6.647630 (lr=3.4275e-05) (hash(x)=145279030)
+1200 val loss 6.9380
+1200 val perplexity 1030.7321
+1200 train 6.928824 (lr=2.9523e-05) (hash(x)=141843115)
+1400 val loss 6.7027
+1400 val perplexity 814.5911
+1400 train 6.650844 (lr=3.4084e-05) (hash(x)=152507639)
+1300 val loss 6.8701
+1300 val perplexity 963.0128
+1300 train 6.756791 (lr=2.9378e-05) (hash(x)=145279030)
+1500 val loss 6.6486
+1500 val perplexity 771.6702
+1500 train 6.681236 (lr=3.3872e-05) (hash(x)=148473774)
+1600 val loss 6.5658
+1600 val perplexity 710.3974
+1600 train 6.604722 (lr=3.3638e-05) (hash(x)=151117002)
+1400 val loss 6.8108
+1400 val perplexity 907.5634
+1400 train 6.767691 (lr=2.9215e-05) (hash(x)=152507639)
+1700 val loss 6.5219
+1700 val perplexity 679.8840
+1700 train 6.456680 (lr=3.3384e-05) (hash(x)=138011335)
+1500 val loss 6.7521
+1500 val perplexity 855.8281
+1500 train 6.789307 (lr=2.9033e-05) (hash(x)=148473774)
+1800 val loss 6.4774
+1800 val perplexity 650.2515
+1800 train 6.617409 (lr=3.3109e-05) (hash(x)=171180926)
+1600 val loss 6.6831
+1600 val perplexity 798.8001
+1600 train 6.716653 (lr=2.8833e-05) (hash(x)=151117002)
+1900 val loss 6.4318
+1900 val perplexity 621.3072
+1900 train 6.355664 (lr=3.2814e-05) (hash(x)=141769419)
+1700 val loss 6.6285
+1700 val perplexity 756.3746
+1700 train 6.554468 (lr=2.8615e-05) (hash(x)=138011335)
+2000 val loss 6.3922
+2000 val perplexity 597.1834
+2000 train 6.329804 (lr=3.2500e-05) (hash(x)=151963443)
+1800 val loss 6.5782
+1800 val perplexity 719.2757
+1800 train 6.725643 (lr=2.8379e-05) (hash(x)=171180926)
+2100 val loss 6.3737
+2100 val perplexity 586.1983
+2100 train 6.427935 (lr=3.2166e-05) (hash(x)=162947470)
+1900 val loss 6.5278
+1900 val perplexity 683.8735
+1900 train 6.439614 (lr=2.8127e-05) (hash(x)=141769419)
+2200 val loss 6.3265
+2200 val perplexity 559.2034
+2200 train 6.572281 (lr=3.1813e-05) (hash(x)=154954810)
+2000 val loss 6.4865
+2000 val perplexity 656.2449
+2000 train 6.420126 (lr=2.7857e-05) (hash(x)=151963443)
+2300 val loss 6.3022
+2300 val perplexity 545.7751
+2300 train 6.293201 (lr=3.1443e-05) (hash(x)=151878111)
+2400 val loss 6.2787
+2400 val perplexity 533.1205
+2400 train 6.169630 (lr=3.1054e-05) (hash(x)=158661057)
+2100 val loss 6.4588
+2100 val perplexity 638.2729
+2100 train 6.514755 (lr=2.7571e-05) (hash(x)=162947470)
+2500 val loss 6.2441
+2500 val perplexity 514.9452
+2500 train 6.266413 (lr=3.0649e-05) (hash(x)=150925584)
+2200 val loss 6.4110
+2200 val perplexity 608.5148
+2200 train 6.664954 (lr=2.7269e-05) (hash(x)=154954810)
+2600 val loss 6.2241
+2600 val perplexity 504.7674
+2600 train 6.132840 (lr=3.0227e-05) (hash(x)=144515755)
+2300 val loss 6.3852
+2300 val perplexity 593.0313
+2300 train 6.376594 (lr=2.6951e-05) (hash(x)=151878111)
+2700 val loss 6.2047
+2700 val perplexity 495.0883
+2700 train 6.193225 (lr=2.9789e-05) (hash(x)=153109144)
+2400 val loss 6.3573
+2400 val perplexity 576.6909
+2400 train 6.246971 (lr=2.6618e-05) (hash(x)=158661057)
+2800 val loss 6.1741
+2800 val perplexity 480.1690
+2800 train 6.078516 (lr=2.9336e-05) (hash(x)=151152897)
+2500 val loss 6.3226
+2500 val perplexity 557.0036
+2500 train 6.344519 (lr=2.6270e-05) (hash(x)=150925584)
+2900 val loss 6.1540
+2900 val perplexity 470.5836
+2900 train 6.096991 (lr=2.8868e-05) (hash(x)=145800210)
+2600 val loss 6.3012
+2600 val perplexity 545.2101
+2600 train 6.207327 (lr=2.5909e-05) (hash(x)=144515755)
+3000 val loss 6.1405
+3000 val perplexity 464.3056
+3000 train 5.938540 (lr=2.8386e-05) (hash(x)=141997485)
+2700 val loss 6.2843
+2700 val perplexity 536.0944
+2700 train 6.276767 (lr=2.5533e-05) (hash(x)=153109144)
+3100 val loss 6.1141
+3100 val perplexity 452.2033
+3100 train 6.023589 (lr=2.7891e-05) (hash(x)=154049740)
+3200 val loss 6.0990
+3200 val perplexity 445.3922
+3200 train 6.033494 (lr=2.7383e-05) (hash(x)=150471842)
+2800 val loss 6.2540
+2800 val perplexity 520.0851
+2800 train 6.155468 (lr=2.5145e-05) (hash(x)=151152897)
+3300 val loss 6.0854
+3300 val perplexity 439.4073
+3300 train 6.027749 (lr=2.6864e-05) (hash(x)=149048126)
+2900 val loss 6.2297
+2900 val perplexity 507.5872
+2900 train 6.172598 (lr=2.4744e-05) (hash(x)=145800210)
+3400 val loss 6.0636
+3400 val perplexity 429.9261
+3400 train 6.185919 (lr=2.6333e-05) (hash(x)=161261339)
+3000 val loss 6.2173
+3000 val perplexity 501.3599
+3000 train 6.008087 (lr=2.4331e-05) (hash(x)=141997485)
+3500 val loss 6.0527
+3500 val perplexity 425.2686
+3500 train 6.020202 (lr=2.5793e-05) (hash(x)=157495564)
+3100 val loss 6.1885
+3100 val perplexity 487.1272
+3100 train 6.090694 (lr=2.3906e-05) (hash(x)=154049740)
+3600 val loss 6.0389
+3600 val perplexity 419.4500
+3600 train 5.936598 (lr=2.5243e-05) (hash(x)=144352932)
+3200 val loss 6.1727
+3200 val perplexity 479.4807
+3200 train 6.118541 (lr=2.3471e-05) (hash(x)=150471842)
+3700 val loss 6.0198
+3700 val perplexity 411.4905
+3700 train 6.000595 (lr=2.4684e-05) (hash(x)=149389012)
+3300 val loss 6.1624
+3300 val perplexity 474.5893
+3300 train 6.099824 (lr=2.3026e-05) (hash(x)=149048126)
+3800 val loss 6.0058
+3800 val perplexity 405.7742
+3800 train 5.947519 (lr=2.4117e-05) (hash(x)=146607620)
+3400 val loss 6.1404
+3400 val perplexity 464.2498
+3400 train 6.267547 (lr=2.2572e-05) (hash(x)=161261339)
+3900 val loss 5.9940
+3900 val perplexity 401.0047
+3900 train 5.909438 (lr=2.3543e-05) (hash(x)=143754617)
+3500 val loss 6.1301
+3500 val perplexity 459.4873
+3500 train 6.102514 (lr=2.2108e-05) (hash(x)=157495564)
+4000 val loss 5.9769
+4000 val perplexity 394.2093
+4000 train 5.956694 (lr=2.2963e-05) (hash(x)=156930722)
+4100 val loss 5.9657
+4100 val perplexity 389.8218
+4100 train 5.784899 (lr=2.2378e-05) (hash(x)=147500519)
+3600 val loss 6.1144
+3600 val perplexity 452.3314
+3600 train 6.008862 (lr=2.1637e-05) (hash(x)=144352932)
+4200 val loss 5.9562
+4200 val perplexity 386.1222
+4200 train 5.796459 (lr=2.1788e-05) (hash(x)=143232237)
+3700 val loss 6.0986
+3700 val perplexity 445.2134
+3700 train 6.078876 (lr=2.1158e-05) (hash(x)=149389012)
+4300 val loss 5.9433
+4300 val perplexity 381.2018
+4300 train 5.855104 (lr=2.1194e-05) (hash(x)=146811670)
+3800 val loss 6.0828
+3800 val perplexity 438.2579
+3800 train 6.024745 (lr=2.0672e-05) (hash(x)=146607620)
+4400 val loss 5.9278
+4400 val perplexity 375.3206
+4400 train 5.963883 (lr=2.0598e-05) (hash(x)=158418746)
+3900 val loss 6.0744
+3900 val perplexity 434.6102
+3900 train 5.987369 (lr=2.0180e-05) (hash(x)=143754617)
+4500 val loss 5.9224
+4500 val perplexity 373.2968
+4500 train 5.888974 (lr=1.9999e-05) (hash(x)=156695778)
+4000 val loss 6.0584
+4000 val perplexity 427.6898
+4000 train 6.038101 (lr=1.9683e-05) (hash(x)=156930722)
+4600 val loss 5.9099
+4600 val perplexity 368.6750
+4600 train 5.934970 (lr=1.9400e-05) (hash(x)=147791497)
+4100 val loss 6.0466
+4100 val perplexity 422.6849
+4100 train 5.858241 (lr=1.9181e-05) (hash(x)=147500519)
+4700 val loss 5.8958
+4700 val perplexity 363.5157
+4700 train 5.775061 (lr=1.8800e-05) (hash(x)=155533088)
+4200 val loss 6.0418
+4200 val perplexity 420.6553
+4200 train 5.880073 (lr=1.8675e-05) (hash(x)=143232237)
+4800 val loss 5.8900
+4800 val perplexity 361.4218
+4800 train 5.741070 (lr=1.8201e-05) (hash(x)=138350044)
+4300 val loss 6.0258
+4300 val perplexity 413.9633
+4300 train 5.939585 (lr=1.8166e-05) (hash(x)=146811670)
+4900 val loss 5.8761
+4900 val perplexity 356.4004
+4900 train 5.877145 (lr=1.7604e-05) (hash(x)=143735284)
+5000 val loss 5.8655
+5000 val perplexity 352.6599
+5000 train 5.829748 (lr=1.7009e-05) (hash(x)=154976463)
+4400 val loss 6.0129
+4400 val perplexity 408.6630
+4400 train 6.047963 (lr=1.7655e-05) (hash(x)=158418746)
+5100 val loss 5.8599
+5100 val perplexity 350.6802
+5100 train 5.741970 (lr=1.6417e-05) (hash(x)=149894982)
+4500 val loss 6.0092
+4500 val perplexity 407.1400
+4500 train 5.970369 (lr=1.7142e-05) (hash(x)=156695778)
+5200 val loss 5.8555
+5200 val perplexity 349.1558
+5200 train 5.647627 (lr=1.5829e-05) (hash(x)=159326689)
+4600 val loss 5.9965
+4600 val perplexity 402.0312
+4600 train 6.023885 (lr=1.6629e-05) (hash(x)=147791497)
+5300 val loss 5.8430
+5300 val perplexity 344.8275
+5300 train 5.846123 (lr=1.5246e-05) (hash(x)=159484800)
+4700 val loss 5.9832
+4700 val perplexity 396.7080
+4700 train 5.854864 (lr=1.6114e-05) (hash(x)=155533088)
+5400 val loss 5.8314
+5400 val perplexity 340.8238
+5400 train 5.731322 (lr=1.4669e-05) (hash(x)=140385615)
+4800 val loss 5.9774
+4800 val perplexity 394.3975
+4800 train 5.825758 (lr=1.5601e-05) (hash(x)=138350044)
+5500 val loss 5.8240
+5500 val perplexity 338.3344
+5500 train 5.657810 (lr=1.4099e-05) (hash(x)=148498335)
+4900 val loss 5.9659
+4900 val perplexity 389.9207
+4900 train 5.970723 (lr=1.5089e-05) (hash(x)=143735284)
+5600 val loss 5.8222
+5600 val perplexity 337.7233
+5600 train 5.556942 (lr=1.3536e-05) (hash(x)=151907614)
+5000 val loss 5.9558
+5000 val perplexity 385.9966
+5000 train 5.919913 (lr=1.4579e-05) (hash(x)=154976463)
+5700 val loss 5.8141
+5700 val perplexity 334.9905
+5700 train 5.779860 (lr=1.2981e-05) (hash(x)=155192267)
+5800 val loss 5.8026
+5800 val perplexity 331.1440
+5800 train 5.849533 (lr=1.2436e-05) (hash(x)=153132158)
+5100 val loss 5.9527
+5100 val perplexity 384.7841
+5100 train 5.830037 (lr=1.4071e-05) (hash(x)=149894982)
+5900 val loss 5.7982
+5900 val perplexity 329.7009
+5900 train 5.772695 (lr=1.1900e-05) (hash(x)=161446764)
+5200 val loss 5.9438
+5200 val perplexity 381.3849
+5200 train 5.748489 (lr=1.3568e-05) (hash(x)=159326689)
+6000 val loss 5.7969
+6000 val perplexity 329.2916
+6000 train 5.613126 (lr=1.1375e-05) (hash(x)=151512446)
+5300 val loss 5.9373
+5300 val perplexity 378.8938
+5300 train 5.933059 (lr=1.3068e-05) (hash(x)=159484800)
+6100 val loss 5.7852
+6100 val perplexity 325.4444
+6100 train 5.915392 (lr=1.0861e-05) (hash(x)=188094053)
+5400 val loss 5.9255
+5400 val perplexity 374.4638
+5400 train 5.812152 (lr=1.2573e-05) (hash(x)=140385615)
+6200 val loss 5.7787
+6200 val perplexity 323.3293
+6200 train 5.666321 (lr=1.0360e-05) (hash(x)=149389789)
+5500 val loss 5.9199
+5500 val perplexity 372.3792
+5500 train 5.751431 (lr=1.2085e-05) (hash(x)=148498335)
+6300 val loss 5.7735
+6300 val perplexity 321.6672
+6300 train 5.593270 (lr=9.8715e-06) (hash(x)=138212820)
+5600 val loss 5.9173
+5600 val perplexity 371.4073
+5600 train 5.648520 (lr=1.1602e-05) (hash(x)=151907614)
+6400 val loss 5.7708
+6400 val perplexity 320.7969
+6400 train 5.569121 (lr=9.3966e-06) (hash(x)=146535423)
+5700 val loss 5.9118
+5700 val perplexity 369.3690
+5700 train 5.872494 (lr=1.1127e-05) (hash(x)=155192267)
+6500 val loss 5.7620
+6500 val perplexity 317.9776
+6500 train 5.634068 (lr=8.9359e-06) (hash(x)=145950843)
+6600 val loss 5.7580
+6600 val perplexity 316.7046
+6600 train 5.652801 (lr=8.4903e-06) (hash(x)=141162902)
+5800 val loss 5.9006
+5800 val perplexity 365.2532
+5800 train 5.951333 (lr=1.0659e-05) (hash(x)=153132158)
+6700 val loss 5.7561
+6700 val perplexity 316.1148
+6700 train 5.670326 (lr=8.0602e-06) (hash(x)=153018737)
+5900 val loss 5.8966
+5900 val perplexity 363.7935
+5900 train 5.868530 (lr=1.0200e-05) (hash(x)=161446764)
+6800 val loss 5.7493
+6800 val perplexity 313.9696
+6800 train 5.676184 (lr=7.6463e-06) (hash(x)=155640155)
+6000 val loss 5.8915
+6000 val perplexity 361.9300
+6000 train 5.715796 (lr=9.7500e-06) (hash(x)=151512446)
+6900 val loss 5.7452
+6900 val perplexity 312.6940
+6900 train 5.712743 (lr=7.2493e-06) (hash(x)=153722115)
+6100 val loss 5.8828
+6100 val perplexity 358.8126
+6100 train 6.019364 (lr=9.3098e-06) (hash(x)=188094053)
+7000 val loss 5.7401
+7000 val perplexity 311.0814
+7000 train 5.646478 (lr=6.8697e-06) (hash(x)=146953450)
+6200 val loss 5.8757
+6200 val perplexity 356.2753
+6200 train 5.769423 (lr=8.8800e-06) (hash(x)=149389789)
+7100 val loss 5.7406
+7100 val perplexity 311.2574
+7100 train 5.574067 (lr=6.5080e-06) (hash(x)=137663885)
+6300 val loss 5.8715
+6300 val perplexity 354.7726
+6300 train 5.685650 (lr=8.4613e-06) (hash(x)=138212820)
+7200 val loss 5.7313
+7200 val perplexity 308.3727
+7200 train 5.794419 (lr=6.1648e-06) (hash(x)=146172950)
+6400 val loss 5.8699
+6400 val perplexity 354.1973
+6400 train 5.674243 (lr=8.0542e-06) (hash(x)=146535423)
+7300 val loss 5.7278
+7300 val perplexity 307.2869
+7300 train 5.623385 (lr=5.8405e-06) (hash(x)=150018163)
+6500 val loss 5.8620
+6500 val perplexity 351.4325
+6500 train 5.728920 (lr=7.6594e-06) (hash(x)=145950843)
+7400 val loss 5.7251
+7400 val perplexity 306.4714
+7400 train 5.720573 (lr=5.5357e-06) (hash(x)=145351166)
+7500 val loss 5.7261
+7500 val perplexity 306.7808
+7500 train 5.509435 (lr=5.2508e-06) (hash(x)=145292116)
+6600 val loss 5.8585
+6600 val perplexity 350.2053
+6600 train 5.750317 (lr=7.2774e-06) (hash(x)=141162902)
+7600 val loss 5.7189
+7600 val perplexity 304.5720
+7600 train 5.639998 (lr=4.9862e-06) (hash(x)=150235132)
+6700 val loss 5.8564
+6700 val perplexity 349.4663
+6700 train 5.760009 (lr=6.9087e-06) (hash(x)=153018737)
+7700 val loss 5.7165
+7700 val perplexity 303.8320
+7700 train 5.625147 (lr=4.7423e-06) (hash(x)=154543455)
+6800 val loss 5.8492
+6800 val perplexity 346.9674
+6800 train 5.780306 (lr=6.5540e-06) (hash(x)=155640155)
+7800 val loss 5.7141
+7800 val perplexity 303.1216
+7800 train 5.606357 (lr=4.5194e-06) (hash(x)=142456852)
+6900 val loss 5.8453
+6900 val perplexity 345.5936
+6900 train 5.812240 (lr=6.2137e-06) (hash(x)=153722115)
+7900 val loss 5.7148
+7900 val perplexity 303.3319
+7900 train 5.495040 (lr=4.3179e-06) (hash(x)=147363479)
+7000 val loss 5.8411
+7000 val perplexity 344.1611
+7000 train 5.742386 (lr=5.8883e-06) (hash(x)=146953450)
+8000 val loss 5.7078
+8000 val perplexity 301.2154
+8000 train 5.742621 (lr=4.1380e-06) (hash(x)=156122973)
+7100 val loss 5.8403
+7100 val perplexity 343.8915
+7100 train 5.669028 (lr=5.5783e-06) (hash(x)=137663885)
+8100 val loss 5.7063
+8100 val perplexity 300.7574
+8100 train 5.747419 (lr=3.9800e-06) (hash(x)=156153179)
+7200 val loss 5.8320
+7200 val perplexity 341.0402
+7200 train 5.886730 (lr=5.2841e-06) (hash(x)=146172950)
+8200 val loss 5.7053
+8200 val perplexity 300.4537
+8200 train 5.748656 (lr=3.8442e-06) (hash(x)=146430698)
+8300 val loss 5.7037
+8300 val perplexity 299.9784
+8300 train 5.569663 (lr=3.7307e-06) (hash(x)=143507257)
+7300 val loss 5.8294
+7300 val perplexity 340.1391
+7300 train 5.728073 (lr=5.0062e-06) (hash(x)=150018163)
+8400 val loss 5.6993
+8400 val perplexity 298.6586
+8400 train 5.759250 (lr=3.6397e-06) (hash(x)=166272643)
+7400 val loss 5.8271
+7400 val perplexity 339.3649
+7400 train 5.802515 (lr=4.7449e-06) (hash(x)=145351166)
+8500 val loss 5.6977
+8500 val perplexity 298.1721
+8500 train 5.598984 (lr=3.5713e-06) (hash(x)=143887848)
+7500 val loss 5.8267
+7500 val perplexity 339.2248
+7500 train 5.605633 (lr=4.5007e-06) (hash(x)=145292116)
+8600 val loss 5.6975
+8600 val perplexity 298.1066
+8600 train 5.740758 (lr=3.5257e-06) (hash(x)=156900341)
+7600 val loss 5.8205
+7600 val perplexity 337.1429
+7600 train 5.732724 (lr=4.2739e-06) (hash(x)=150235132)
+8700 val loss 5.6928
+8700 val perplexity 296.7115
+8700 train 5.918503 (lr=3.5029e-06) (hash(x)=146417632)
+8749 val loss 5.6925
+8749 val perplexity 296.6473
+7700 val loss 5.8191
+7700 val perplexity 336.6639
+7700 train 5.747572 (lr=4.0648e-06) (hash(x)=154543455)

attention_kindselective_n_heads4_seed1339/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0799e271014bd3a93b4c15f8a90dc002dc00df4c2d676c29130f65bddb5be45b
+oid sha256:5f9f086e5bc9016de004cf97e0e1fe7e0a7ef5565d49ea076873db395a988fe1
 size 92843394

attention_kindselective_n_heads4_seed1339/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5efbfc12e221b8e452d6c7051c9f70c4a071a6f63d0982e0f3688b6b2e0170f2
+oid sha256:22c2cc84339b65454afc89bf4f0daaa707e2a4500376081f0b13ded666f36934
 size 179406214