Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1345/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_61440_4_1345", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4.5e-5_61440_4_1345", "n_embd": 256}

attention_kindselective_n_heads4_seed1345/log2.txt

@@ -1,534 +1,267 @@
 max_steps: 8750
-max_steps: 8750
-0 val loss 11.2669
-0 val perplexity 78187.7734
 0 val loss 11.2669
 0 val perplexity 78187.7734
-0 train 11.258430 (lr=7.0000e-08) (hash(x)=134751525)
-0 train 11.258430 (lr=6.0000e-08) (hash(x)=134751525)
-100 val loss 10.1750
-100 val perplexity 26239.3242
-100 val loss 10.2733
-100 val perplexity 28949.7637
-100 train 10.194183 (lr=7.0700e-06) (hash(x)=150358957)
-100 train 10.292044 (lr=6.0600e-06) (hash(x)=150358957)
-200 val loss 9.7819
-200 val perplexity 17709.6523
-200 val loss 9.6434
-200 val perplexity 15420.2432
-200 train 9.817169 (lr=1.2060e-05) (hash(x)=126319983)
-200 train 9.672717 (lr=1.4070e-05) (hash(x)=126319983)
-300 val loss 9.0849
-300 val perplexity 8821.1250
-300 train 9.013100 (lr=1.8060e-05) (hash(x)=159305874)
-300 val loss 8.6685
-300 val perplexity 5817.0503
-300 train 8.598497 (lr=2.1070e-05) (hash(x)=159305874)
-400 val loss 8.2047
-400 val perplexity 3658.1746
-400 train 8.218742 (lr=2.4060e-05) (hash(x)=149855631)
-400 val loss 7.8251
-400 val perplexity 2502.6160
-400 train 7.823033 (lr=2.8070e-05) (hash(x)=149855631)
-500 val loss 7.6848
-500 val perplexity 2175.0186
-500 train 7.707771 (lr=3.0000e-05) (hash(x)=150706760)
-500 val loss 7.5264
-500 val perplexity 1856.4717
-500 train 7.555436 (lr=3.5000e-05) (hash(x)=150706760)
-600 val loss 7.4936
-600 val perplexity 1796.5043
-600 train 7.436448 (lr=2.9990e-05) (hash(x)=146858163)
-600 val loss 7.4167
-600 val perplexity 1663.5187
-600 train 7.352398 (lr=3.4989e-05) (hash(x)=146858163)
-700 val loss 7.4029
-700 val perplexity 1640.6820
-700 train 7.286075 (lr=2.9961e-05) (hash(x)=144262072)
-700 val loss 7.3377
-700 val perplexity 1537.2355
-700 train 7.216905 (lr=3.4954e-05) (hash(x)=144262072)
-800 val loss 7.3213
-800 val perplexity 1512.1823
-800 train 7.288542 (lr=2.9912e-05) (hash(x)=135443880)
-800 val loss 7.2552
-800 val perplexity 1415.4203
-800 train 7.228300 (lr=3.4897e-05) (hash(x)=135443880)
-900 val loss 7.2423
-900 val perplexity 1397.3021
-900 train 7.185520 (lr=2.9844e-05) (hash(x)=153147388)
-900 val loss 7.1680
-900 val perplexity 1297.2410
-900 train 7.111306 (lr=3.4818e-05) (hash(x)=153147388)
-1000 val loss 7.1730
-1000 val perplexity 1303.8014
-1000 train 7.101679 (lr=2.9756e-05) (hash(x)=151448445)
-1000 val loss 7.1006
-1000 val perplexity 1212.7179
-1000 train 7.030866 (lr=3.4715e-05) (hash(x)=151448445)
-1100 val loss 7.1041
-1100 val perplexity 1216.9623
-1100 train 7.102648 (lr=2.9649e-05) (hash(x)=153161010)
-1100 val loss 7.0359
-1100 val perplexity 1136.6917
-1100 train 7.039263 (lr=3.4591e-05) (hash(x)=153161010)
-1200 val loss 7.0362
-1200 val perplexity 1137.0972
-1200 train 6.867702 (lr=2.9523e-05) (hash(x)=143227423)
-1200 val loss 6.9704
-1200 val perplexity 1064.5956
-1200 train 6.798764 (lr=3.4444e-05) (hash(x)=143227423)
-1300 val loss 6.9623
-1300 val perplexity 1056.0319
-1300 train 7.441185 (lr=2.9378e-05) (hash(x)=176373796)
-1300 val loss 6.9014
-1300 val perplexity 993.7034
-1300 train 7.385946 (lr=3.4275e-05) (hash(x)=176373796)
-1400 val loss 6.8869
-1400 val perplexity 979.3471
-1400 train 6.808840 (lr=2.9215e-05) (hash(x)=155989503)
-1400 val loss 6.8330
-1400 val perplexity 927.9659
-1400 train 6.754021 (lr=3.4084e-05) (hash(x)=155989503)
-1500 val loss 6.8189
-1500 val perplexity 914.9921
-1500 train 6.790832 (lr=2.9033e-05) (hash(x)=156507542)
-1500 val loss 6.7613
-1500 val perplexity 863.7879
-1500 train 6.730458 (lr=3.3872e-05) (hash(x)=156507542)
-1600 val loss 6.7582
-1600 val perplexity 861.0930
-1600 train 6.778030 (lr=2.8833e-05) (hash(x)=156078901)
-1600 val loss 6.6964
-1600 val perplexity 809.4827
-1600 train 6.710325 (lr=3.3638e-05) (hash(x)=156078901)
-1700 val loss 6.6933
-1700 val perplexity 807.0126
-1700 train 6.597588 (lr=2.8615e-05) (hash(x)=156889457)
-1700 val loss 6.6403
-1700 val perplexity 765.3299
-1700 train 6.545618 (lr=3.3384e-05) (hash(x)=156889457)
-1800 val loss 6.6458
-1800 val perplexity 769.5268
-1800 train 6.895873 (lr=2.8379e-05) (hash(x)=162477906)
-1800 val loss 6.5875
-1800 val perplexity 725.9478
-1800 train 6.827469 (lr=3.3109e-05) (hash(x)=162477906)
-1900 val loss 6.5819
-1900 val perplexity 721.9397
-1900 train 6.625430 (lr=2.8127e-05) (hash(x)=158648033)
-1900 val loss 6.5338
-1900 val perplexity 688.0337
-1900 train 6.575150 (lr=3.2814e-05) (hash(x)=158648033)
-2000 val loss 6.5412
-2000 val perplexity 693.0895
-2000 train 6.461535 (lr=2.7857e-05) (hash(x)=156732586)
-2000 val loss 6.4917
-2000 val perplexity 659.6571
-2000 train 6.414240 (lr=3.2500e-05) (hash(x)=156732586)
-2100 val loss 6.4977
-2100 val perplexity 663.5922
-2100 train 6.521001 (lr=2.7571e-05) (hash(x)=159721084)
-2100 val loss 6.4501
-2100 val perplexity 632.7357
-2100 train 6.469040 (lr=3.2166e-05) (hash(x)=159721084)
-2200 val loss 6.4529
-2200 val perplexity 634.5237
-2200 train 6.426077 (lr=2.7269e-05) (hash(x)=144907001)
-2200 val loss 6.4076
-2200 val perplexity 606.4177
-2200 train 6.382673 (lr=3.1813e-05) (hash(x)=144907001)
-2300 val loss 6.4275
-2300 val perplexity 618.5964
-2300 train 6.370883 (lr=2.6951e-05) (hash(x)=154541837)
-2300 val loss 6.3895
-2300 val perplexity 595.5529
-2300 train 6.327733 (lr=3.1443e-05) (hash(x)=154541837)
-2400 val loss 6.3938
-2400 val perplexity 598.1364
-2400 train 6.478797 (lr=2.6618e-05) (hash(x)=161706790)
-2400 val loss 6.3556
-2400 val perplexity 575.6793
-2400 train 6.445687 (lr=3.1054e-05) (hash(x)=161706790)
-2500 val loss 6.3581
-2500 val perplexity 577.1680
-2500 train 6.277150 (lr=2.6270e-05) (hash(x)=143556162)
-2500 val loss 6.3157
-2500 val perplexity 553.1826
-2500 train 6.235758 (lr=3.0649e-05) (hash(x)=143556162)
-2600 val loss 6.3334
-2600 val perplexity 563.0918
-2600 train 6.187194 (lr=2.5909e-05) (hash(x)=142425078)
-2600 val loss 6.2915
-2600 val perplexity 539.9864
-2600 train 6.143201 (lr=3.0227e-05) (hash(x)=142425078)
-2700 val loss 6.3056
-2700 val perplexity 547.6417
-2700 train 6.324026 (lr=2.5533e-05) (hash(x)=163195606)
-2700 val loss 6.2703
-2700 val perplexity 528.6313
-2700 train 6.290276 (lr=2.9789e-05) (hash(x)=163195606)
-2800 val loss 6.2788
-2800 val perplexity 533.1338
-2800 train 6.179240 (lr=2.5145e-05) (hash(x)=154078337)
-2800 val loss 6.2392
-2800 val perplexity 512.4443
-2800 train 6.143046 (lr=2.9336e-05) (hash(x)=154078337)
-2900 val loss 6.2577
-2900 val perplexity 522.0040
-2900 train 6.118522 (lr=2.4744e-05) (hash(x)=145186687)
-2900 val loss 6.2222
-2900 val perplexity 503.8161
-2900 train 6.084497 (lr=2.8868e-05) (hash(x)=145186687)
-3000 val loss 6.2310
-3000 val perplexity 508.2703
-3000 train 6.183014 (lr=2.4331e-05) (hash(x)=149726716)
-3000 val loss 6.1939
-3000 val perplexity 489.7742
-3000 train 6.147453 (lr=2.8386e-05) (hash(x)=149726716)
-3100 val loss 6.2127
-3100 val perplexity 499.0278
-3100 train 6.136366 (lr=2.3906e-05) (hash(x)=151122509)
-3100 val loss 6.1761
-3100 val perplexity 481.1284
-3100 train 6.095349 (lr=2.7891e-05) (hash(x)=151122509)
-3200 val loss 6.1962
-3200 val perplexity 490.8775
-3200 train 6.167598 (lr=2.3471e-05) (hash(x)=152172187)
-3200 val loss 6.1624
-3200 val perplexity 474.5640
-3200 train 6.133832 (lr=2.7383e-05) (hash(x)=152172187)
-3300 val loss 6.1759
-3300 val perplexity 481.0361
-3300 train 6.112210 (lr=2.3026e-05) (hash(x)=150581974)
-3300 val loss 6.1415
-3300 val perplexity 464.7546
-3300 train 6.078421 (lr=2.6864e-05) (hash(x)=150581974)
-3400 val loss 6.1620
-3400 val perplexity 474.3841
-3400 train 6.091798 (lr=2.2572e-05) (hash(x)=160061666)
-3400 val loss 6.1262
-3400 val perplexity 457.6865
-3400 train 6.052583 (lr=2.6333e-05) (hash(x)=160061666)
-3500 val loss 6.1390
-3500 val perplexity 463.6089
-3500 train 6.089871 (lr=2.2108e-05) (hash(x)=150556913)
-3500 val loss 6.1030
-3500 val perplexity 447.1866
-3500 train 6.052739 (lr=2.5793e-05) (hash(x)=150556913)
-3600 val loss 6.1277
-3600 val perplexity 458.3880
-3600 train 6.037222 (lr=2.1637e-05) (hash(x)=148311961)
-3600 val loss 6.0914
-3600 val perplexity 442.0209
-3600 train 5.994712 (lr=2.5243e-05) (hash(x)=148311961)
-3700 val loss 6.1109
-3700 val perplexity 450.7532
-3700 train 6.010225 (lr=2.1158e-05) (hash(x)=150305284)
-3700 val loss 6.0769
-3700 val perplexity 435.6559
-3700 train 5.972827 (lr=2.4684e-05) (hash(x)=150305284)
-3800 val loss 6.0935
-3800 val perplexity 442.9523
-3800 train 6.107690 (lr=2.0672e-05) (hash(x)=148428531)
-3800 val loss 6.0581
-3800 val perplexity 427.5569
-3800 train 6.070702 (lr=2.4117e-05) (hash(x)=148428531)
-3900 val loss 6.0789
-3900 val perplexity 436.5287
-3900 train 5.950237 (lr=2.0180e-05) (hash(x)=142448374)
-3900 val loss 6.0463
-3900 val perplexity 422.5590
-3900 train 5.920741 (lr=2.3543e-05) (hash(x)=142448374)
-4000 val loss 6.0688
-4000 val perplexity 432.1684
-4000 train 6.003921 (lr=1.9683e-05) (hash(x)=158002288)
-4000 val loss 6.0329
-4000 val perplexity 416.9308
-4000 train 5.960671 (lr=2.2963e-05) (hash(x)=158002288)
-4100 val loss 6.0503
-4100 val perplexity 424.2229
-4100 train 5.992175 (lr=1.9181e-05) (hash(x)=151388626)
-4100 val loss 6.0160
-4100 val perplexity 409.9351
-4100 train 5.953603 (lr=2.2378e-05) (hash(x)=151388626)
-4200 val loss 6.0385
-4200 val perplexity 419.2610
-4200 train 5.956691 (lr=1.8675e-05) (hash(x)=145123294)
-4200 val loss 6.0044
-4200 val perplexity 405.1890
-4200 train 5.921735 (lr=2.1788e-05) (hash(x)=145123294)
-4300 val loss 6.0281
-4300 val perplexity 414.9109
-4300 train 5.895347 (lr=1.8166e-05) (hash(x)=145869150)
-4300 val loss 5.9918
-4300 val perplexity 400.1511
-4300 train 5.864752 (lr=2.1194e-05) (hash(x)=145869150)
-4400 val loss 6.0184
-4400 val perplexity 410.9097
-4400 train 6.043102 (lr=1.7655e-05) (hash(x)=146923320)
-4400 val loss 5.9831
-4400 val perplexity 396.6707
-4400 train 6.006795 (lr=2.0598e-05) (hash(x)=146923320)
-4500 val loss 6.0034
-4500 val perplexity 404.8162
-4500 train 5.822377 (lr=1.7142e-05) (hash(x)=146720819)
-4500 val loss 5.9655
-4500 val perplexity 389.7443
-4500 train 5.786932 (lr=1.9999e-05) (hash(x)=146720819)
-4600 val loss 5.9952
-4600 val perplexity 401.5007
-4600 train 5.732059 (lr=1.6629e-05) (hash(x)=140342909)
-4600 val loss 5.9578
-4600 val perplexity 386.7601
-4600 train 5.701501 (lr=1.9400e-05) (hash(x)=140342909)
-4700 val loss 5.9897
-4700 val perplexity 399.2866
-4700 train 6.129215 (lr=1.6114e-05) (hash(x)=163866463)
-4700 val loss 5.9490
-4700 val perplexity 383.3703
-4700 train 6.082926 (lr=1.8800e-05) (hash(x)=163866463)
-4800 val loss 5.9734
-4800 val perplexity 392.8430
-4800 train 5.674410 (lr=1.5601e-05) (hash(x)=133459145)
-4800 val loss 5.9356
-4800 val perplexity 378.2767
-4800 train 5.632190 (lr=1.8201e-05) (hash(x)=133459145)
-4900 val loss 5.9669
-4900 val perplexity 390.2767
-4900 train 5.767529 (lr=1.5089e-05) (hash(x)=143144356)
-4900 val loss 5.9261
-4900 val perplexity 374.6805
-4900 train 5.728006 (lr=1.7604e-05) (hash(x)=143144356)
-5000 val loss 5.9762
-5000 val perplexity 393.9481
-5000 train 5.873638 (lr=1.4579e-05) (hash(x)=132636494)
-5000 val loss 5.9323
-5000 val perplexity 377.0263
-5000 train 5.834423 (lr=1.7009e-05) (hash(x)=132636494)
-5100 val loss 5.9468
-5100 val perplexity 382.5381
-5100 train 5.866047 (lr=1.4071e-05) (hash(x)=157278728)
-5100 val loss 5.9048
-5100 val perplexity 366.7968
-5100 train 5.822343 (lr=1.6417e-05) (hash(x)=157278728)
-5200 val loss 5.9387
-5200 val perplexity 379.4555
-5200 train 5.798069 (lr=1.3568e-05) (hash(x)=148198434)
-5200 val loss 5.8980
-5200 val perplexity 364.3023
-5200 train 5.757902 (lr=1.5829e-05) (hash(x)=148198434)
-5300 val loss 5.9282
-5300 val perplexity 375.4887
-5300 train 5.859479 (lr=1.3068e-05) (hash(x)=138380906)
-5300 val loss 5.8857
-5300 val perplexity 359.8555
-5300 train 5.814668 (lr=1.5246e-05) (hash(x)=138380906)
-5400 val loss 5.9193
-5400 val perplexity 372.1389
-5400 train 6.013505 (lr=1.2573e-05) (hash(x)=163441464)
-5400 val loss 5.8765
-5400 val perplexity 356.5496
-5400 train 5.965582 (lr=1.4669e-05) (hash(x)=163441464)
-5500 val loss 5.9173
-5500 val perplexity 371.4005
-5500 train 5.873360 (lr=1.2085e-05) (hash(x)=154347714)
-5500 val loss 5.8715
-5500 val perplexity 354.7765
-5500 train 5.825165 (lr=1.4099e-05) (hash(x)=154347714)
-5600 val loss 5.9056
-5600 val perplexity 367.1030
-5600 train 5.935726 (lr=1.1602e-05) (hash(x)=148449981)
-5600 val loss 5.8623
-5600 val perplexity 351.5228
-5600 train 5.894027 (lr=1.3536e-05) (hash(x)=148449981)
-5700 val loss 5.8968
-5700 val perplexity 363.8591
-5700 train 5.787348 (lr=1.1127e-05) (hash(x)=139300274)
-5700 val loss 5.8525
-5700 val perplexity 348.0916
-5700 train 5.742023 (lr=1.2981e-05) (hash(x)=139300274)
-5800 val loss 5.8947
-5800 val perplexity 363.1082
-5800 train 5.880175 (lr=1.0659e-05) (hash(x)=154857144)
-5800 val loss 5.8471
-5800 val perplexity 346.2293
-5800 train 5.833060 (lr=1.2436e-05) (hash(x)=154857144)
-5900 val loss 5.8848
-5900 val perplexity 359.5408
-5900 train 5.955789 (lr=1.0200e-05) (hash(x)=151756013)
-5900 val loss 5.8394
-5900 val perplexity 343.5686
-5900 train 5.909068 (lr=1.1900e-05) (hash(x)=151756013)
-6000 val loss 5.8803
-6000 val perplexity 357.8994
-6000 train 5.779049 (lr=9.7500e-06) (hash(x)=145414657)
-6000 val loss 5.8321
-6000 val perplexity 341.0894
-6000 train 5.732763 (lr=1.1375e-05) (hash(x)=145414657)
-6100 val loss 5.8751
-6100 val perplexity 356.0493
-6100 train 5.700992 (lr=9.3098e-06) (hash(x)=139501217)
-6100 val loss 5.8282
-6100 val perplexity 339.7401
-6100 train 5.652315 (lr=1.0861e-05) (hash(x)=139501217)
-6200 val loss 5.8671
-6200 val perplexity 353.2263
-6200 train 5.908251 (lr=8.8800e-06) (hash(x)=155783358)
-6200 val loss 5.8197
-6200 val perplexity 336.8552
-6200 train 5.859098 (lr=1.0360e-05) (hash(x)=155783358)
-6300 val loss 5.8604
-6300 val perplexity 350.8642
-6300 train 5.921547 (lr=8.4613e-06) (hash(x)=157033091)
-6300 val loss 5.8135
-6300 val perplexity 334.7826
-6300 train 5.880195 (lr=9.8715e-06) (hash(x)=157033091)
-6400 val loss 5.8582
-6400 val perplexity 350.0984
-6400 train 5.662329 (lr=8.0542e-06) (hash(x)=144475330)
-6400 val loss 5.8101
-6400 val perplexity 333.6645
-6400 train 5.613411 (lr=9.3966e-06) (hash(x)=144475330)
-6500 val loss 5.8514
-6500 val perplexity 347.7374
-6500 train 5.894106 (lr=7.6594e-06) (hash(x)=124041822)
-6500 val loss 5.8043
-6500 val perplexity 331.7292
-6500 train 5.851692 (lr=8.9359e-06) (hash(x)=124041822)
-6600 val loss 5.8488
-6600 val perplexity 346.8195
-6600 train 5.777397 (lr=7.2774e-06) (hash(x)=142131981)
-6600 val loss 5.8006
-6600 val perplexity 330.5139
-6600 train 5.729760 (lr=8.4903e-06) (hash(x)=142131981)
-6700 val loss 5.8397
-6700 val perplexity 343.6647
-6700 train 5.932328 (lr=6.9087e-06) (hash(x)=154916248)
-6700 val loss 5.7911
-6700 val perplexity 327.3885
-6700 train 5.880747 (lr=8.0602e-06) (hash(x)=154916248)
-6800 val loss 5.8399
-6800 val perplexity 343.7483
-6800 train 5.837266 (lr=6.5540e-06) (hash(x)=147996387)
-6800 val loss 5.7908
-6800 val perplexity 327.2812
-6800 train 5.789515 (lr=7.6463e-06) (hash(x)=147996387)
-6900 val loss 5.8349
-6900 val perplexity 342.0426
-6900 train 5.866730 (lr=6.2137e-06) (hash(x)=153422428)
-6900 val loss 5.7859
-6900 val perplexity 325.6611
-6900 train 5.821322 (lr=7.2493e-06) (hash(x)=153422428)
-7000 val loss 5.8287
-7000 val perplexity 339.9290
-7000 train 5.845691 (lr=5.8883e-06) (hash(x)=174135078)
-7000 val loss 5.7791
-7000 val perplexity 323.4625
-7000 train 5.799284 (lr=6.8697e-06) (hash(x)=174135078)
-7100 val loss 5.8261
-7100 val perplexity 339.0508
-7100 train 5.890498 (lr=5.5783e-06) (hash(x)=160382475)
-7100 val loss 5.7757
-7100 val perplexity 322.3593
-7100 train 5.846087 (lr=6.5080e-06) (hash(x)=160382475)
-7200 val loss 5.8238
-7200 val perplexity 338.2447
-7200 train 5.911463 (lr=5.2841e-06) (hash(x)=155310085)
-7200 val loss 5.7736
-7200 val perplexity 321.6892
-7200 train 5.862586 (lr=6.1648e-06) (hash(x)=155310085)
-7300 val loss 5.8176
-7300 val perplexity 336.1636
-7300 train 5.947505 (lr=5.0062e-06) (hash(x)=159329031)
-7300 val loss 5.7693
-7300 val perplexity 320.3102
-7300 train 5.899338 (lr=5.8405e-06) (hash(x)=159329031)
-7400 val loss 5.8165
-7400 val perplexity 335.7898
-7400 train 5.755105 (lr=4.7449e-06) (hash(x)=151577269)
-7400 val loss 5.7669
-7400 val perplexity 319.5350
-7400 train 5.706599 (lr=5.5357e-06) (hash(x)=151577269)
-7500 val loss 5.8145
-7500 val perplexity 335.1223
-7500 train 5.860515 (lr=4.5007e-06) (hash(x)=160620287)
-7500 val loss 5.7642
-7500 val perplexity 318.6920
-7500 train 5.815320 (lr=5.2508e-06) (hash(x)=160620287)
-7600 val loss 5.8093
-7600 val perplexity 333.3877
-7600 train 5.698301 (lr=4.2739e-06) (hash(x)=143711597)
-7600 val loss 5.7591
-7600 val perplexity 317.0681
-7600 train 5.647112 (lr=4.9862e-06) (hash(x)=143711597)
-7700 val loss 5.8077
-7700 val perplexity 332.8600
-7700 train 5.814168 (lr=4.0648e-06) (hash(x)=149243577)
-7700 val loss 5.7580
-7700 val perplexity 316.6993
-7700 train 5.764244 (lr=4.7423e-06) (hash(x)=149243577)
-7800 val loss 5.8074
-7800 val perplexity 332.7407
-7800 train 5.800000 (lr=3.8738e-06) (hash(x)=147677247)
-7800 val loss 5.7568
-7800 val perplexity 316.3229
-7800 train 5.748062 (lr=4.5194e-06) (hash(x)=147677247)
-7900 val loss 5.8000
-7900 val perplexity 330.2837
-7900 train 5.754653 (lr=3.7010e-06) (hash(x)=145801236)
-7900 val loss 5.7508
-7900 val perplexity 314.4332
-7900 train 5.701422 (lr=4.3179e-06) (hash(x)=145801236)
-8000 val loss 5.7995
-8000 val perplexity 330.1450
-8000 train 5.912129 (lr=3.5468e-06) (hash(x)=158007655)
-8000 val loss 5.7496
-8000 val perplexity 314.0717
-8000 train 5.863448 (lr=4.1380e-06) (hash(x)=158007655)
-8100 val loss 5.7986
-8100 val perplexity 329.8300
-8100 train 5.867946 (lr=3.4114e-06) (hash(x)=158030048)
-8100 val loss 5.7484
-8100 val perplexity 313.6838
-8100 train 5.815113 (lr=3.9800e-06) (hash(x)=158030048)
-8200 val loss 5.7953
-8200 val perplexity 328.7345
-8200 train 5.643704 (lr=3.2950e-06) (hash(x)=150674725)
-8200 val loss 5.7446
-8200 val perplexity 312.4954
-8200 train 5.588871 (lr=3.8442e-06) (hash(x)=150674725)
-8300 val loss 5.7946
-8300 val perplexity 328.5292
-8300 train 5.713874 (lr=3.1977e-06) (hash(x)=150805711)
-8300 val loss 5.7437
-8300 val perplexity 312.2120
-8300 train 5.659549 (lr=3.7307e-06) (hash(x)=150805711)
-8400 val loss 5.7908
-8400 val perplexity 327.2678
-8400 train 5.666281 (lr=3.1197e-06) (hash(x)=138396210)
-8400 val loss 5.7395
-8400 val perplexity 310.9045
-8400 train 5.610754 (lr=3.6397e-06) (hash(x)=138396210)
-8500 val loss 5.7905
-8500 val perplexity 327.1677
-8500 train 5.663531 (lr=3.0611e-06) (hash(x)=156172740)
-8500 val loss 5.7396
-8500 val perplexity 310.9279
-8500 train 5.616183 (lr=3.5713e-06) (hash(x)=156172740)
-8600 val loss 5.7883
-8600 val perplexity 326.4504
-8600 train 5.838070 (lr=3.0220e-06) (hash(x)=154911352)
-8600 val loss 5.7373
-8600 val perplexity 310.2140
-8600 train 5.787480 (lr=3.5257e-06) (hash(x)=154911352)
-8700 val loss 5.7860
-8700 val perplexity 325.7220
-8700 train 5.850677 (lr=3.0024e-06) (hash(x)=153446789)
-8749 val loss 5.7843
-8749 val perplexity 325.1580
-8700 val loss 5.7350
-8700 val perplexity 309.5193
-8700 train 5.797166 (lr=3.5029e-06) (hash(x)=153446789)
-8749 val loss 5.7334
-8749 val perplexity 309.0074
+0 train 11.258421 (lr=9.0000e-08) (hash(x)=134751525)
+100 val loss 10.0591
+100 val perplexity 23368.2949
+100 train 10.084015 (lr=9.0900e-06) (hash(x)=150358957)
+200 val loss 9.4623
+200 val perplexity 12865.2119
+200 train 9.482208 (lr=1.8090e-05) (hash(x)=126319983)
+300 val loss 8.2846
+300 val perplexity 3962.3010
+300 train 8.218966 (lr=2.7090e-05) (hash(x)=159305874)
+400 val loss 7.6156
+400 val perplexity 2029.5901
+400 train 7.596677 (lr=3.6090e-05) (hash(x)=149855631)
+500 val loss 7.4200
+500 val perplexity 1669.0735
+500 train 7.448377 (lr=4.5000e-05) (hash(x)=150706760)
+600 val loss 7.3022
+600 val perplexity 1483.5238
+600 train 7.233404 (lr=4.4985e-05) (hash(x)=146858163)
+700 val loss 7.1946
+700 val perplexity 1332.2479
+700 train 7.066607 (lr=4.4941e-05) (hash(x)=144262072)
+800 val loss 7.0810
+800 val perplexity 1189.1654
+800 train 7.053042 (lr=4.4868e-05) (hash(x)=135443880)
+900 val loss 6.9834
+900 val perplexity 1078.5498
+900 train 6.929296 (lr=4.4766e-05) (hash(x)=153147388)
+1000 val loss 6.9159
+1000 val perplexity 1008.2147
+1000 train 6.843293 (lr=4.4634e-05) (hash(x)=151448445)
+1100 val loss 6.8259
+1100 val perplexity 921.3812
+1100 train 6.837877 (lr=4.4474e-05) (hash(x)=153161010)
+1200 val loss 6.7506
+1200 val perplexity 854.6096
+1200 train 6.578292 (lr=4.4285e-05) (hash(x)=143227423)
+1300 val loss 6.6868
+1300 val perplexity 801.7579
+1300 train 7.172554 (lr=4.4068e-05) (hash(x)=176373796)
+1400 val loss 6.6160
+1400 val perplexity 746.9671
+1400 train 6.540329 (lr=4.3822e-05) (hash(x)=155989503)
+1500 val loss 6.5556
+1500 val perplexity 703.1462
+1500 train 6.532267 (lr=4.3549e-05) (hash(x)=156507542)
+1600 val loss 6.5053
+1600 val perplexity 668.6906
+1600 train 6.507467 (lr=4.3249e-05) (hash(x)=156078901)
+1700 val loss 6.4618
+1700 val perplexity 640.2064
+1700 train 6.365796 (lr=4.2922e-05) (hash(x)=156889457)
+1800 val loss 6.4338
+1800 val perplexity 622.5391
+1800 train 6.691582 (lr=4.2569e-05) (hash(x)=162477906)
+1900 val loss 6.3811
+1900 val perplexity 590.5602
+1900 train 6.422180 (lr=4.2190e-05) (hash(x)=158648033)
+2000 val loss 6.3547
+2000 val perplexity 575.2073
+2000 train 6.281474 (lr=4.1785e-05) (hash(x)=156732586)
+2100 val loss 6.3244
+2100 val perplexity 558.0005
+2100 train 6.340858 (lr=4.1356e-05) (hash(x)=159721084)
+2200 val loss 6.2931
+2200 val perplexity 540.8015
+2200 train 6.273417 (lr=4.0903e-05) (hash(x)=144907001)
+2300 val loss 6.2683
+2300 val perplexity 527.5813
+2300 train 6.208333 (lr=4.0426e-05) (hash(x)=154541837)
+2400 val loss 6.2421
+2400 val perplexity 513.9549
+2400 train 6.331748 (lr=3.9927e-05) (hash(x)=161706790)
+2500 val loss 6.2153
+2500 val perplexity 500.3525
+2500 train 6.141446 (lr=3.9406e-05) (hash(x)=143556162)
+2600 val loss 6.1963
+2600 val perplexity 490.9108
+2600 train 6.050023 (lr=3.8863e-05) (hash(x)=142425078)
+2700 val loss 6.1650
+2700 val perplexity 475.8134
+2700 train 6.189079 (lr=3.8300e-05) (hash(x)=163195606)
+2800 val loss 6.1348
+2800 val perplexity 461.6595
+2800 train 6.041891 (lr=3.7717e-05) (hash(x)=154078337)
+2900 val loss 6.1197
+2900 val perplexity 454.7504
+2900 train 5.981575 (lr=3.7116e-05) (hash(x)=145186687)
+3000 val loss 6.0966
+3000 val perplexity 444.3332
+3000 train 6.058323 (lr=3.6496e-05) (hash(x)=149726716)
+3100 val loss 6.0758
+3100 val perplexity 435.2021
+3100 train 6.000450 (lr=3.5860e-05) (hash(x)=151122509)
+3200 val loss 6.0591
+3200 val perplexity 427.9922
+3200 train 6.028750 (lr=3.5207e-05) (hash(x)=152172187)
+3300 val loss 6.0385
+3300 val perplexity 419.2482
+3300 train 5.972092 (lr=3.4539e-05) (hash(x)=150581974)
+3400 val loss 6.0271
+3400 val perplexity 414.4937
+3400 train 5.958289 (lr=3.3857e-05) (hash(x)=160061666)
+3500 val loss 6.0029
+3500 val perplexity 404.6082
+3500 train 5.951034 (lr=3.3162e-05) (hash(x)=150556913)
+3600 val loss 5.9885
+3600 val perplexity 398.8278
+3600 train 5.896544 (lr=3.2455e-05) (hash(x)=148311961)
+3700 val loss 5.9747
+3700 val perplexity 393.3694
+3700 train 5.866710 (lr=3.1736e-05) (hash(x)=150305284)
+3800 val loss 5.9561
+3800 val perplexity 386.0872
+3800 train 5.966131 (lr=3.1008e-05) (hash(x)=148428531)
+3900 val loss 5.9435
+3900 val perplexity 381.2489
+3900 train 5.814651 (lr=3.0270e-05) (hash(x)=142448374)
+4000 val loss 5.9283
+4000 val perplexity 375.5012
+4000 train 5.862759 (lr=2.9524e-05) (hash(x)=158002288)
+4100 val loss 5.9091
+4100 val perplexity 368.3666
+4100 train 5.850963 (lr=2.8771e-05) (hash(x)=151388626)
+4200 val loss 5.8978
+4200 val perplexity 364.2400
+4200 train 5.821980 (lr=2.8013e-05) (hash(x)=145123294)
+4300 val loss 5.8837
+4300 val perplexity 359.1506
+4300 train 5.765946 (lr=2.7250e-05) (hash(x)=145869150)
+4400 val loss 5.8761
+4400 val perplexity 356.4175
+4400 train 5.911475 (lr=2.6483e-05) (hash(x)=146923320)
+4500 val loss 5.8595
+4500 val perplexity 350.5503
+4500 train 5.688518 (lr=2.5714e-05) (hash(x)=146720819)
+4600 val loss 5.8489
+4600 val perplexity 346.8638
+4600 train 5.593976 (lr=2.4943e-05) (hash(x)=140342909)
+4700 val loss 5.8383
+4700 val perplexity 343.2022
+4700 train 5.974589 (lr=2.4172e-05) (hash(x)=163866463)
+4800 val loss 5.8217
+4800 val perplexity 337.5406
+4800 train 5.515379 (lr=2.3402e-05) (hash(x)=133459145)
+4900 val loss 5.8140
+4900 val perplexity 334.9486
+4900 train 5.611025 (lr=2.2633e-05) (hash(x)=143144356)
+5000 val loss 5.8290
+5000 val perplexity 340.0303
+5000 train 5.728374 (lr=2.1868e-05) (hash(x)=132636494)
+5100 val loss 5.7898
+5100 val perplexity 326.9575
+5100 train 5.715200 (lr=2.1107e-05) (hash(x)=157278728)
+5200 val loss 5.7817
+5200 val perplexity 324.3232
+5200 train 5.637352 (lr=2.0351e-05) (hash(x)=148198434)
+5300 val loss 5.7699
+5300 val perplexity 320.5101
+5300 train 5.696755 (lr=1.9602e-05) (hash(x)=138380906)
+5400 val loss 5.7576
+5400 val perplexity 316.5839
+5400 train 5.838938 (lr=1.8860e-05) (hash(x)=163441464)
+5500 val loss 5.7551
+5500 val perplexity 315.7874
+5500 train 5.701233 (lr=1.8127e-05) (hash(x)=154347714)
+5600 val loss 5.7429
+5600 val perplexity 311.9672
+5600 train 5.776368 (lr=1.7403e-05) (hash(x)=148449981)
+5700 val loss 5.7320
+5700 val perplexity 308.5851
+5700 train 5.623940 (lr=1.6690e-05) (hash(x)=139300274)
+5800 val loss 5.7271
+5800 val perplexity 307.0853
+5800 train 5.716210 (lr=1.5989e-05) (hash(x)=154857144)
+5900 val loss 5.7195
+5900 val perplexity 304.7662
+5900 train 5.779558 (lr=1.5300e-05) (hash(x)=151756013)
+6000 val loss 5.7133
+6000 val perplexity 302.8814
+6000 train 5.619116 (lr=1.4625e-05) (hash(x)=145414657)
+6100 val loss 5.7073
+6100 val perplexity 301.0667
+6100 train 5.534505 (lr=1.3965e-05) (hash(x)=139501217)
+6200 val loss 5.6980
+6200 val perplexity 298.2719
+6200 train 5.732094 (lr=1.3320e-05) (hash(x)=155783358)
+6300 val loss 5.6917
+6300 val perplexity 296.4115
+6300 train 5.751144 (lr=1.2692e-05) (hash(x)=157033091)
+6400 val loss 5.6877
+6400 val perplexity 295.2075
+6400 train 5.495005 (lr=1.2081e-05) (hash(x)=144475330)
+6500 val loss 5.6813
+6500 val perplexity 293.3268
+6500 train 5.727438 (lr=1.1489e-05) (hash(x)=124041822)
+6600 val loss 5.6776
+6600 val perplexity 292.2538
+6600 train 5.614210 (lr=1.0916e-05) (hash(x)=142131981)
+6700 val loss 5.6687
+6700 val perplexity 289.6444
+6700 train 5.769709 (lr=1.0363e-05) (hash(x)=154916248)
+6800 val loss 5.6677
+6800 val perplexity 289.3799
+6800 train 5.666794 (lr=9.8310e-06) (hash(x)=147996387)
+6900 val loss 5.6623
+6900 val perplexity 287.8063
+6900 train 5.690933 (lr=9.3205e-06) (hash(x)=153422428)
+7000 val loss 5.6558
+7000 val perplexity 285.9500
+7000 train 5.677885 (lr=8.8324e-06) (hash(x)=174135078)
+7100 val loss 5.6527
+7100 val perplexity 285.0608
+7100 train 5.732208 (lr=8.3674e-06) (hash(x)=160382475)
+7200 val loss 5.6492
+7200 val perplexity 284.0699
+7200 train 5.743230 (lr=7.9261e-06) (hash(x)=155310085)
+7300 val loss 5.6438
+7300 val perplexity 282.5283
+7300 train 5.782816 (lr=7.5093e-06) (hash(x)=159329031)
+7400 val loss 5.6415
+7400 val perplexity 281.8864
+7400 train 5.581890 (lr=7.1174e-06) (hash(x)=151577269)
+7500 val loss 5.6393
+7500 val perplexity 281.2724
+7500 train 5.690196 (lr=6.7511e-06) (hash(x)=160620287)
+7600 val loss 5.6345
+7600 val perplexity 279.9241
+7600 train 5.519984 (lr=6.4109e-06) (hash(x)=143711597)
+7700 val loss 5.6324
+7700 val perplexity 279.3272
+7700 train 5.632235 (lr=6.0972e-06) (hash(x)=149243577)
+7800 val loss 5.6312
+7800 val perplexity 278.9935
+7800 train 5.617033 (lr=5.8107e-06) (hash(x)=147677247)
+7900 val loss 5.6240
+7900 val perplexity 276.9847
+7900 train 5.574734 (lr=5.5515e-06) (hash(x)=145801236)
+8000 val loss 5.6234
+8000 val perplexity 276.8374
+8000 train 5.743947 (lr=5.3203e-06) (hash(x)=158007655)
+8100 val loss 5.6221
+8100 val perplexity 276.4651
+8100 train 5.681450 (lr=5.1172e-06) (hash(x)=158030048)
+8200 val loss 5.6184
+8200 val perplexity 275.4516
+8200 train 5.468052 (lr=4.9425e-06) (hash(x)=150674725)
+8300 val loss 5.6173
+8300 val perplexity 275.1565
+8300 train 5.523213 (lr=4.7966e-06) (hash(x)=150805711)
+8400 val loss 5.6133
+8400 val perplexity 274.0571
+8400 train 5.488098 (lr=4.6796e-06) (hash(x)=138396210)
+8500 val loss 5.6127
+8500 val perplexity 273.8857
+8500 train 5.489274 (lr=4.5917e-06) (hash(x)=156172740)
+8600 val loss 5.6100
+8600 val perplexity 273.1501
+8600 train 5.649384 (lr=4.5330e-06) (hash(x)=154911352)
+8700 val loss 5.6074
+8700 val perplexity 272.4272
+8700 train 5.664300 (lr=4.5037e-06) (hash(x)=153446789)
+8749 val loss 5.6057
+8749 val perplexity 271.9800

attention_kindselective_n_heads4_seed1345/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a375128a9f17c31f33b98216bcc6b05316b464fb3b587a6e156429f7a49eb32b
+oid sha256:63a5d75f3d50c1e38bd49d0b3ffde6667deba50fb9d33f7f876d2ce34040e7cc
 size 92843394

attention_kindselective_n_heads4_seed1345/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee5a8b46cb4807643d5e88ff5502c2e4b92b4746a4e2dd6d59ed8a639804c3bb
+oid sha256:48e14c02fa70197c42cfb86a6c654015643809abd3d0f2330ae92703b10052ff
 size 179406214