Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_61440_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1339", "n_embd": 256}

attention_kindselective_n_heads4_seed1339/log2.txt

@@ -1,502 +1,49 @@
-max_steps: 8750
+8000 train 5.839575 (lr=3.5468e-06) (hash(x)=156122973)
 max_steps: 8750
 0 val loss 11.2808
 0 val perplexity 79287.6797
-0 val loss 11.2808
-0 val perplexity 79287.5234
-0 train 11.289339 (lr=6.0000e-08) (hash(x)=150724848)
-0 train 11.289339 (lr=7.0000e-08) (hash(x)=150724848)
-100 val loss 10.1249
-100 val perplexity 24955.8281
-100 train 10.136632 (lr=7.0700e-06) (hash(x)=149910534)
-100 val loss 10.1868
-100 val perplexity 26549.2832
-100 train 10.199195 (lr=6.0600e-06) (hash(x)=149910534)
-200 val loss 9.5679
-200 val perplexity 14297.8906
-200 train 9.597668 (lr=1.4070e-05) (hash(x)=148123706)
-200 val loss 9.6580
-200 val perplexity 15646.8330
-200 train 9.688334 (lr=1.2060e-05) (hash(x)=148123706)
-300 val loss 8.4851
-300 val perplexity 4841.9927
-300 train 8.403298 (lr=2.1070e-05) (hash(x)=146678221)
-300 val loss 8.7199
-300 val perplexity 6123.2988
-300 train 8.652300 (lr=1.8060e-05) (hash(x)=146678221)
-400 val loss 7.7157
-400 val perplexity 2243.2341
-400 train 7.622499 (lr=2.8070e-05) (hash(x)=151700982)
-400 val loss 7.8706
-400 val perplexity 2619.0610
-400 train 7.789970 (lr=2.4060e-05) (hash(x)=151700982)
-500 val loss 7.4807
-500 val perplexity 1773.4902
-500 train 7.566009 (lr=3.5000e-05) (hash(x)=156182087)
-500 val loss 7.5420
-500 val perplexity 1885.5305
-500 train 7.629263 (lr=3.0000e-05) (hash(x)=156182087)
-600 val loss 7.3659
-600 val perplexity 1581.0912
-600 train 7.345613 (lr=3.4989e-05) (hash(x)=149318660)
-600 val loss 7.4137
-600 val perplexity 1658.5233
-600 train 7.403398 (lr=2.9990e-05) (hash(x)=149318660)
-700 val loss 7.2842
-700 val perplexity 1457.1127
-700 train 7.324867 (lr=3.4954e-05) (hash(x)=150482428)
-800 val loss 7.1606
-800 val perplexity 1287.6595
-800 train 7.034153 (lr=3.4897e-05) (hash(x)=143268605)
-700 val loss 7.3358
-700 val perplexity 1534.3062
-700 train 7.378386 (lr=2.9961e-05) (hash(x)=150482428)
-900 val loss 7.0731
-900 val perplexity 1179.7480
-900 train 7.036097 (lr=3.4818e-05) (hash(x)=152322423)
-800 val loss 7.2311
-800 val perplexity 1381.7266
-800 train 7.106897 (lr=2.9912e-05) (hash(x)=143268605)
-1000 val loss 7.0052
-1000 val perplexity 1102.3225
-1000 train 6.872244 (lr=3.4715e-05) (hash(x)=147904298)
-900 val loss 7.1499
-900 val perplexity 1273.9365
-900 train 7.112676 (lr=2.9844e-05) (hash(x)=152322423)
-1100 val loss 6.9231
-1100 val perplexity 1015.4374
-1100 train 7.109494 (lr=3.4591e-05) (hash(x)=154343147)
-1000 val loss 7.0856
-1000 val perplexity 1194.6191
-1000 train 6.954570 (lr=2.9756e-05) (hash(x)=147904298)
-1200 val loss 6.8335
-1200 val perplexity 928.4275
-1200 train 6.822800 (lr=3.4444e-05) (hash(x)=141843115)
-1100 val loss 7.0176
-1100 val perplexity 1116.0782
-1100 train 7.204097 (lr=2.9649e-05) (hash(x)=154343147)
-1300 val loss 6.7614
-1300 val perplexity 863.8369
-1300 train 6.647630 (lr=3.4275e-05) (hash(x)=145279030)
-1200 val loss 6.9380
-1200 val perplexity 1030.7321
-1200 train 6.928824 (lr=2.9523e-05) (hash(x)=141843115)
-1400 val loss 6.7027
-1400 val perplexity 814.5911
-1400 train 6.650844 (lr=3.4084e-05) (hash(x)=152507639)
-1300 val loss 6.8701
-1300 val perplexity 963.0128
-1300 train 6.756791 (lr=2.9378e-05) (hash(x)=145279030)
-1500 val loss 6.6486
-1500 val perplexity 771.6702
-1500 train 6.681236 (lr=3.3872e-05) (hash(x)=148473774)
-1600 val loss 6.5658
-1600 val perplexity 710.3974
-1600 train 6.604722 (lr=3.3638e-05) (hash(x)=151117002)
-1400 val loss 6.8108
-1400 val perplexity 907.5634
-1400 train 6.767691 (lr=2.9215e-05) (hash(x)=152507639)
-1700 val loss 6.5219
-1700 val perplexity 679.8840
-1700 train 6.456680 (lr=3.3384e-05) (hash(x)=138011335)
-1500 val loss 6.7521
-1500 val perplexity 855.8281
-1500 train 6.789307 (lr=2.9033e-05) (hash(x)=148473774)
-1800 val loss 6.4774
-1800 val perplexity 650.2515
-1800 train 6.617409 (lr=3.3109e-05) (hash(x)=171180926)
-1600 val loss 6.6831
-1600 val perplexity 798.8001
-1600 train 6.716653 (lr=2.8833e-05) (hash(x)=151117002)
-1900 val loss 6.4318
-1900 val perplexity 621.3072
-1900 train 6.355664 (lr=3.2814e-05) (hash(x)=141769419)
-1700 val loss 6.6285
-1700 val perplexity 756.3746
-1700 train 6.554468 (lr=2.8615e-05) (hash(x)=138011335)
-2000 val loss 6.3922
-2000 val perplexity 597.1834
-2000 train 6.329804 (lr=3.2500e-05) (hash(x)=151963443)
-1800 val loss 6.5782
-1800 val perplexity 719.2757
-1800 train 6.725643 (lr=2.8379e-05) (hash(x)=171180926)
-2100 val loss 6.3737
-2100 val perplexity 586.1983
-2100 train 6.427935 (lr=3.2166e-05) (hash(x)=162947470)
-1900 val loss 6.5278
-1900 val perplexity 683.8735
-1900 train 6.439614 (lr=2.8127e-05) (hash(x)=141769419)
-2200 val loss 6.3265
-2200 val perplexity 559.2034
-2200 train 6.572281 (lr=3.1813e-05) (hash(x)=154954810)
-2000 val loss 6.4865
-2000 val perplexity 656.2449
-2000 train 6.420126 (lr=2.7857e-05) (hash(x)=151963443)
-2300 val loss 6.3022
-2300 val perplexity 545.7751
-2300 train 6.293201 (lr=3.1443e-05) (hash(x)=151878111)
-2400 val loss 6.2787
-2400 val perplexity 533.1205
-2400 train 6.169630 (lr=3.1054e-05) (hash(x)=158661057)
-2100 val loss 6.4588
-2100 val perplexity 638.2729
-2100 train 6.514755 (lr=2.7571e-05) (hash(x)=162947470)
-2500 val loss 6.2441
-2500 val perplexity 514.9452
-2500 train 6.266413 (lr=3.0649e-05) (hash(x)=150925584)
-2200 val loss 6.4110
-2200 val perplexity 608.5148
-2200 train 6.664954 (lr=2.7269e-05) (hash(x)=154954810)
-2600 val loss 6.2241
-2600 val perplexity 504.7674
-2600 train 6.132840 (lr=3.0227e-05) (hash(x)=144515755)
-2300 val loss 6.3852
-2300 val perplexity 593.0313
-2300 train 6.376594 (lr=2.6951e-05) (hash(x)=151878111)
-2700 val loss 6.2047
-2700 val perplexity 495.0883
-2700 train 6.193225 (lr=2.9789e-05) (hash(x)=153109144)
-2400 val loss 6.3573
-2400 val perplexity 576.6909
-2400 train 6.246971 (lr=2.6618e-05) (hash(x)=158661057)
-2800 val loss 6.1741
-2800 val perplexity 480.1690
-2800 train 6.078516 (lr=2.9336e-05) (hash(x)=151152897)
-2500 val loss 6.3226
-2500 val perplexity 557.0036
-2500 train 6.344519 (lr=2.6270e-05) (hash(x)=150925584)
-2900 val loss 6.1540
-2900 val perplexity 470.5836
-2900 train 6.096991 (lr=2.8868e-05) (hash(x)=145800210)
-2600 val loss 6.3012
-2600 val perplexity 545.2101
-2600 train 6.207327 (lr=2.5909e-05) (hash(x)=144515755)
-3000 val loss 6.1405
-3000 val perplexity 464.3056
-3000 train 5.938540 (lr=2.8386e-05) (hash(x)=141997485)
-2700 val loss 6.2843
-2700 val perplexity 536.0944
-2700 train 6.276767 (lr=2.5533e-05) (hash(x)=153109144)
-3100 val loss 6.1141
-3100 val perplexity 452.2033
-3100 train 6.023589 (lr=2.7891e-05) (hash(x)=154049740)
-3200 val loss 6.0990
-3200 val perplexity 445.3922
-3200 train 6.033494 (lr=2.7383e-05) (hash(x)=150471842)
-2800 val loss 6.2540
-2800 val perplexity 520.0851
-2800 train 6.155468 (lr=2.5145e-05) (hash(x)=151152897)
-3300 val loss 6.0854
-3300 val perplexity 439.4073
-3300 train 6.027749 (lr=2.6864e-05) (hash(x)=149048126)
-2900 val loss 6.2297
-2900 val perplexity 507.5872
-2900 train 6.172598 (lr=2.4744e-05) (hash(x)=145800210)
-3400 val loss 6.0636
-3400 val perplexity 429.9261
-3400 train 6.185919 (lr=2.6333e-05) (hash(x)=161261339)
-3000 val loss 6.2173
-3000 val perplexity 501.3599
-3000 train 6.008087 (lr=2.4331e-05) (hash(x)=141997485)
-3500 val loss 6.0527
-3500 val perplexity 425.2686
-3500 train 6.020202 (lr=2.5793e-05) (hash(x)=157495564)
-3100 val loss 6.1885
-3100 val perplexity 487.1272
-3100 train 6.090694 (lr=2.3906e-05) (hash(x)=154049740)
-3600 val loss 6.0389
-3600 val perplexity 419.4500
-3600 train 5.936598 (lr=2.5243e-05) (hash(x)=144352932)
-3200 val loss 6.1727
-3200 val perplexity 479.4807
-3200 train 6.118541 (lr=2.3471e-05) (hash(x)=150471842)
-3700 val loss 6.0198
-3700 val perplexity 411.4905
-3700 train 6.000595 (lr=2.4684e-05) (hash(x)=149389012)
-3300 val loss 6.1624
-3300 val perplexity 474.5893
-3300 train 6.099824 (lr=2.3026e-05) (hash(x)=149048126)
-3800 val loss 6.0058
-3800 val perplexity 405.7742
-3800 train 5.947519 (lr=2.4117e-05) (hash(x)=146607620)
-3400 val loss 6.1404
-3400 val perplexity 464.2498
-3400 train 6.267547 (lr=2.2572e-05) (hash(x)=161261339)
-3900 val loss 5.9940
-3900 val perplexity 401.0047
-3900 train 5.909438 (lr=2.3543e-05) (hash(x)=143754617)
-3500 val loss 6.1301
-3500 val perplexity 459.4873
-3500 train 6.102514 (lr=2.2108e-05) (hash(x)=157495564)
-4000 val loss 5.9769
-4000 val perplexity 394.2093
-4000 train 5.956694 (lr=2.2963e-05) (hash(x)=156930722)
-4100 val loss 5.9657
-4100 val perplexity 389.8218
-4100 train 5.784899 (lr=2.2378e-05) (hash(x)=147500519)
-3600 val loss 6.1144
-3600 val perplexity 452.3314
-3600 train 6.008862 (lr=2.1637e-05) (hash(x)=144352932)
-4200 val loss 5.9562
-4200 val perplexity 386.1222
-4200 train 5.796459 (lr=2.1788e-05) (hash(x)=143232237)
-3700 val loss 6.0986
-3700 val perplexity 445.2134
-3700 train 6.078876 (lr=2.1158e-05) (hash(x)=149389012)
-4300 val loss 5.9433
-4300 val perplexity 381.2018
-4300 train 5.855104 (lr=2.1194e-05) (hash(x)=146811670)
-3800 val loss 6.0828
-3800 val perplexity 438.2579
-3800 train 6.024745 (lr=2.0672e-05) (hash(x)=146607620)
-4400 val loss 5.9278
-4400 val perplexity 375.3206
-4400 train 5.963883 (lr=2.0598e-05) (hash(x)=158418746)
-3900 val loss 6.0744
-3900 val perplexity 434.6102
-3900 train 5.987369 (lr=2.0180e-05) (hash(x)=143754617)
-4500 val loss 5.9224
-4500 val perplexity 373.2968
-4500 train 5.888974 (lr=1.9999e-05) (hash(x)=156695778)
-4000 val loss 6.0584
-4000 val perplexity 427.6898
-4000 train 6.038101 (lr=1.9683e-05) (hash(x)=156930722)
-4600 val loss 5.9099
-4600 val perplexity 368.6750
-4600 train 5.934970 (lr=1.9400e-05) (hash(x)=147791497)
-4100 val loss 6.0466
-4100 val perplexity 422.6849
-4100 train 5.858241 (lr=1.9181e-05) (hash(x)=147500519)
-4700 val loss 5.8958
-4700 val perplexity 363.5157
-4700 train 5.775061 (lr=1.8800e-05) (hash(x)=155533088)
-4200 val loss 6.0418
-4200 val perplexity 420.6553
-4200 train 5.880073 (lr=1.8675e-05) (hash(x)=143232237)
-4800 val loss 5.8900
-4800 val perplexity 361.4218
-4800 train 5.741070 (lr=1.8201e-05) (hash(x)=138350044)
-4300 val loss 6.0258
-4300 val perplexity 413.9633
-4300 train 5.939585 (lr=1.8166e-05) (hash(x)=146811670)
-4900 val loss 5.8761
-4900 val perplexity 356.4004
-4900 train 5.877145 (lr=1.7604e-05) (hash(x)=143735284)
-5000 val loss 5.8655
-5000 val perplexity 352.6599
-5000 train 5.829748 (lr=1.7009e-05) (hash(x)=154976463)
-4400 val loss 6.0129
-4400 val perplexity 408.6630
-4400 train 6.047963 (lr=1.7655e-05) (hash(x)=158418746)
-5100 val loss 5.8599
-5100 val perplexity 350.6802
-5100 train 5.741970 (lr=1.6417e-05) (hash(x)=149894982)
-4500 val loss 6.0092
-4500 val perplexity 407.1400
-4500 train 5.970369 (lr=1.7142e-05) (hash(x)=156695778)
-5200 val loss 5.8555
-5200 val perplexity 349.1558
-5200 train 5.647627 (lr=1.5829e-05) (hash(x)=159326689)
-4600 val loss 5.9965
-4600 val perplexity 402.0312
-4600 train 6.023885 (lr=1.6629e-05) (hash(x)=147791497)
-5300 val loss 5.8430
-5300 val perplexity 344.8275
-5300 train 5.846123 (lr=1.5246e-05) (hash(x)=159484800)
-4700 val loss 5.9832
-4700 val perplexity 396.7080
-4700 train 5.854864 (lr=1.6114e-05) (hash(x)=155533088)
-5400 val loss 5.8314
-5400 val perplexity 340.8238
-5400 train 5.731322 (lr=1.4669e-05) (hash(x)=140385615)
-4800 val loss 5.9774
-4800 val perplexity 394.3975
-4800 train 5.825758 (lr=1.5601e-05) (hash(x)=138350044)
-5500 val loss 5.8240
-5500 val perplexity 338.3344
-5500 train 5.657810 (lr=1.4099e-05) (hash(x)=148498335)
-4900 val loss 5.9659
-4900 val perplexity 389.9207
-4900 train 5.970723 (lr=1.5089e-05) (hash(x)=143735284)
-5600 val loss 5.8222
-5600 val perplexity 337.7233
-5600 train 5.556942 (lr=1.3536e-05) (hash(x)=151907614)
-5000 val loss 5.9558
-5000 val perplexity 385.9966
-5000 train 5.919913 (lr=1.4579e-05) (hash(x)=154976463)
-5700 val loss 5.8141
-5700 val perplexity 334.9905
-5700 train 5.779860 (lr=1.2981e-05) (hash(x)=155192267)
-5800 val loss 5.8026
-5800 val perplexity 331.1440
-5800 train 5.849533 (lr=1.2436e-05) (hash(x)=153132158)
-5100 val loss 5.9527
-5100 val perplexity 384.7841
-5100 train 5.830037 (lr=1.4071e-05) (hash(x)=149894982)
-5900 val loss 5.7982
-5900 val perplexity 329.7009
-5900 train 5.772695 (lr=1.1900e-05) (hash(x)=161446764)
-5200 val loss 5.9438
-5200 val perplexity 381.3849
-5200 train 5.748489 (lr=1.3568e-05) (hash(x)=159326689)
-6000 val loss 5.7969
-6000 val perplexity 329.2916
-6000 train 5.613126 (lr=1.1375e-05) (hash(x)=151512446)
-5300 val loss 5.9373
-5300 val perplexity 378.8938
-5300 train 5.933059 (lr=1.3068e-05) (hash(x)=159484800)
-6100 val loss 5.7852
-6100 val perplexity 325.4444
-6100 train 5.915392 (lr=1.0861e-05) (hash(x)=188094053)
-5400 val loss 5.9255
-5400 val perplexity 374.4638
-5400 train 5.812152 (lr=1.2573e-05) (hash(x)=140385615)
-6200 val loss 5.7787
-6200 val perplexity 323.3293
-6200 train 5.666321 (lr=1.0360e-05) (hash(x)=149389789)
-5500 val loss 5.9199
-5500 val perplexity 372.3792
-5500 train 5.751431 (lr=1.2085e-05) (hash(x)=148498335)
-6300 val loss 5.7735
-6300 val perplexity 321.6672
-6300 train 5.593270 (lr=9.8715e-06) (hash(x)=138212820)
-5600 val loss 5.9173
-5600 val perplexity 371.4073
-5600 train 5.648520 (lr=1.1602e-05) (hash(x)=151907614)
-6400 val loss 5.7708
-6400 val perplexity 320.7969
-6400 train 5.569121 (lr=9.3966e-06) (hash(x)=146535423)
-5700 val loss 5.9118
-5700 val perplexity 369.3690
-5700 train 5.872494 (lr=1.1127e-05) (hash(x)=155192267)
-6500 val loss 5.7620
-6500 val perplexity 317.9776
-6500 train 5.634068 (lr=8.9359e-06) (hash(x)=145950843)
-6600 val loss 5.7580
-6600 val perplexity 316.7046
-6600 train 5.652801 (lr=8.4903e-06) (hash(x)=141162902)
-5800 val loss 5.9006
-5800 val perplexity 365.2532
-5800 train 5.951333 (lr=1.0659e-05) (hash(x)=153132158)
-6700 val loss 5.7561
-6700 val perplexity 316.1148
-6700 train 5.670326 (lr=8.0602e-06) (hash(x)=153018737)
-5900 val loss 5.8966
-5900 val perplexity 363.7935
-5900 train 5.868530 (lr=1.0200e-05) (hash(x)=161446764)
-6800 val loss 5.7493
-6800 val perplexity 313.9696
-6800 train 5.676184 (lr=7.6463e-06) (hash(x)=155640155)
-6000 val loss 5.8915
-6000 val perplexity 361.9300
-6000 train 5.715796 (lr=9.7500e-06) (hash(x)=151512446)
-6900 val loss 5.7452
-6900 val perplexity 312.6940
-6900 train 5.712743 (lr=7.2493e-06) (hash(x)=153722115)
-6100 val loss 5.8828
-6100 val perplexity 358.8126
-6100 train 6.019364 (lr=9.3098e-06) (hash(x)=188094053)
-7000 val loss 5.7401
-7000 val perplexity 311.0814
-7000 train 5.646478 (lr=6.8697e-06) (hash(x)=146953450)
-6200 val loss 5.8757
-6200 val perplexity 356.2753
-6200 train 5.769423 (lr=8.8800e-06) (hash(x)=149389789)
-7100 val loss 5.7406
-7100 val perplexity 311.2574
-7100 train 5.574067 (lr=6.5080e-06) (hash(x)=137663885)
-6300 val loss 5.8715
-6300 val perplexity 354.7726
-6300 train 5.685650 (lr=8.4613e-06) (hash(x)=138212820)
-7200 val loss 5.7313
-7200 val perplexity 308.3727
-7200 train 5.794419 (lr=6.1648e-06) (hash(x)=146172950)
-6400 val loss 5.8699
-6400 val perplexity 354.1973
-6400 train 5.674243 (lr=8.0542e-06) (hash(x)=146535423)
-7300 val loss 5.7278
-7300 val perplexity 307.2869
-7300 train 5.623385 (lr=5.8405e-06) (hash(x)=150018163)
-6500 val loss 5.8620
-6500 val perplexity 351.4325
-6500 train 5.728920 (lr=7.6594e-06) (hash(x)=145950843)
-7400 val loss 5.7251
-7400 val perplexity 306.4714
-7400 train 5.720573 (lr=5.5357e-06) (hash(x)=145351166)
-7500 val loss 5.7261
-7500 val perplexity 306.7808
-7500 train 5.509435 (lr=5.2508e-06) (hash(x)=145292116)
-6600 val loss 5.8585
-6600 val perplexity 350.2053
-6600 train 5.750317 (lr=7.2774e-06) (hash(x)=141162902)
-7600 val loss 5.7189
-7600 val perplexity 304.5720
-7600 train 5.639998 (lr=4.9862e-06) (hash(x)=150235132)
-6700 val loss 5.8564
-6700 val perplexity 349.4663
-6700 train 5.760009 (lr=6.9087e-06) (hash(x)=153018737)
-7700 val loss 5.7165
-7700 val perplexity 303.8320
-7700 train 5.625147 (lr=4.7423e-06) (hash(x)=154543455)
-6800 val loss 5.8492
-6800 val perplexity 346.9674
-6800 train 5.780306 (lr=6.5540e-06) (hash(x)=155640155)
-7800 val loss 5.7141
-7800 val perplexity 303.1216
-7800 train 5.606357 (lr=4.5194e-06) (hash(x)=142456852)
-6900 val loss 5.8453
-6900 val perplexity 345.5936
-6900 train 5.812240 (lr=6.2137e-06) (hash(x)=153722115)
-7900 val loss 5.7148
-7900 val perplexity 303.3319
-7900 train 5.495040 (lr=4.3179e-06) (hash(x)=147363479)
-7000 val loss 5.8411
-7000 val perplexity 344.1611
-7000 train 5.742386 (lr=5.8883e-06) (hash(x)=146953450)
-8000 val loss 5.7078
-8000 val perplexity 301.2154
-8000 train 5.742621 (lr=4.1380e-06) (hash(x)=156122973)
-7100 val loss 5.8403
-7100 val perplexity 343.8915
-7100 train 5.669028 (lr=5.5783e-06) (hash(x)=137663885)
-8100 val loss 5.7063
-8100 val perplexity 300.7574
-8100 train 5.747419 (lr=3.9800e-06) (hash(x)=156153179)
-7200 val loss 5.8320
-7200 val perplexity 341.0402
-7200 train 5.886730 (lr=5.2841e-06) (hash(x)=146172950)
-8200 val loss 5.7053
-8200 val perplexity 300.4537
-8200 train 5.748656 (lr=3.8442e-06) (hash(x)=146430698)
-8300 val loss 5.7037
-8300 val perplexity 299.9784
-8300 train 5.569663 (lr=3.7307e-06) (hash(x)=143507257)
-7300 val loss 5.8294
-7300 val perplexity 340.1391
-7300 train 5.728073 (lr=5.0062e-06) (hash(x)=150018163)
-8400 val loss 5.6993
-8400 val perplexity 298.6586
-8400 train 5.759250 (lr=3.6397e-06) (hash(x)=166272643)
-7400 val loss 5.8271
-7400 val perplexity 339.3649
-7400 train 5.802515 (lr=4.7449e-06) (hash(x)=145351166)
-8500 val loss 5.6977
-8500 val perplexity 298.1721
-8500 train 5.598984 (lr=3.5713e-06) (hash(x)=143887848)
-7500 val loss 5.8267
-7500 val perplexity 339.2248
-7500 train 5.605633 (lr=4.5007e-06) (hash(x)=145292116)
-8600 val loss 5.6975
-8600 val perplexity 298.1066
-8600 train 5.740758 (lr=3.5257e-06) (hash(x)=156900341)
-7600 val loss 5.8205
-7600 val perplexity 337.1429
-7600 train 5.732724 (lr=4.2739e-06) (hash(x)=150235132)
-8700 val loss 5.6928
-8700 val perplexity 296.7115
-8700 train 5.918503 (lr=3.5029e-06) (hash(x)=146417632)
-8749 val loss 5.6925
-8749 val perplexity 296.6473
-7700 val loss 5.8191
-7700 val perplexity 336.6639
-7700 train 5.747572 (lr=4.0648e-06) (hash(x)=154543455)
+8100 val loss 5.8084
+8100 val perplexity 333.0969
+8100 train 5.844770 (lr=3.4114e-06) (hash(x)=156153179)
+0 train 11.289339 (lr=1.0000e-07) (hash(x)=150724848)
+100 val loss 10.0102
+100 val perplexity 22253.1465
+100 train 10.025309 (lr=1.0100e-05) (hash(x)=149910534)
+8200 val loss 5.8078
+8200 val perplexity 332.8854
+8200 train 5.844475 (lr=3.2950e-06) (hash(x)=146430698)
+200 val loss 9.2383
+200 val perplexity 10283.3877
+200 train 9.265762 (lr=2.0100e-05) (hash(x)=148123706)
+8300 val loss 5.8067
+8300 val perplexity 332.5358
+8300 train 5.676077 (lr=3.1977e-06) (hash(x)=143507257)
+300 val loss 8.0321
+300 val perplexity 3078.1367
+300 train 7.902669 (lr=3.0100e-05) (hash(x)=146678221)
+8400 val loss 5.8027
+8400 val perplexity 331.1894
+8400 train 5.896213 (lr=3.1197e-06) (hash(x)=166272643)
+400 val loss 7.5622
+400 val perplexity 1924.0914
+400 train 7.451694 (lr=4.0100e-05) (hash(x)=151700982)
+8500 val loss 5.8010
+8500 val perplexity 330.6212
+8500 train 5.702298 (lr=3.0611e-06) (hash(x)=143887848)
+500 val loss 7.4032
+500 val perplexity 1641.3018
+500 train 7.482969 (lr=5.0000e-05) (hash(x)=156182087)
+8600 val loss 5.8008
+8600 val perplexity 330.5657
+8600 train 5.850898 (lr=3.0220e-06) (hash(x)=156900341)
+600 val loss 7.2893
+600 val perplexity 1464.6086
+600 train 7.262670 (lr=4.9984e-05) (hash(x)=149318660)
+8700 val loss 5.7967
+8700 val perplexity 329.2034
+8700 train 6.024566 (lr=3.0024e-06) (hash(x)=146417632)
+700 val loss 7.1942
+700 val perplexity 1331.6840
+700 train 7.235372 (lr=4.9935e-05) (hash(x)=150482428)
+8749 val loss 5.7958
+8749 val perplexity 328.9303

attention_kindselective_n_heads4_seed1339/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f9f086e5bc9016de004cf97e0e1fe7e0a7ef5565d49ea076873db395a988fe1
+oid sha256:e5e246a3164bacf50817c15a67bd6b599ff7acc11ae461b713148076d24a4299
 size 92843394

attention_kindselective_n_heads4_seed1339/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:22c2cc84339b65454afc89bf4f0daaa707e2a4500376081f0b13ded666f36934
+oid sha256:4ee398ac385f1f29570d770fc19076827ace193d5ebf660ef5f349bc78d93979
 size 179406214