Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_4_1341", "n_embd": 256}

attention_kindselective_n_heads4_seed1341/log2.txt

@@ -1,530 +1,267 @@
 max_steps: 8750
-100 val loss 10.1171
-100 val perplexity 24761.9980
-100 train 10.112064 (lr=8.0800e-06) (hash(x)=155800595)
 0 val loss 11.3095
-0 val perplexity 81594.2500
-200 val loss 9.6651
-200 val perplexity 15758.2021
-200 train 9.620489 (lr=1.6080e-05) (hash(x)=145606733)
-0 train 11.313038 (lr=1.0000e-07) (hash(x)=145079536)
-300 val loss 8.5370
-300 val perplexity 5099.7910
-300 train 8.528650 (lr=2.4080e-05) (hash(x)=150367139)
-100 val loss 10.0532
-100 val perplexity 23230.9727
-100 train 10.047434 (lr=1.0100e-05) (hash(x)=155800595)
-400 val loss 7.7461
-400 val perplexity 2312.6008
-400 train 8.032513 (lr=3.2080e-05) (hash(x)=155747374)
-200 val loss 9.4102
-200 val perplexity 12211.7793
-200 train 9.361097 (lr=2.0100e-05) (hash(x)=145606733)
-500 val loss 7.5205
-500 val perplexity 1845.5077
-500 train 7.412711 (lr=4.0000e-05) (hash(x)=140604760)
-300 val loss 8.0795
-300 val perplexity 3227.5703
-300 train 8.047569 (lr=3.0100e-05) (hash(x)=150367139)
-600 val loss 7.4285
-600 val perplexity 1683.2040
-600 train 7.377125 (lr=3.9987e-05) (hash(x)=148404734)
-400 val loss 7.5665
-400 val perplexity 1932.2872
-400 train 7.873068 (lr=4.0100e-05) (hash(x)=155747374)
-700 val loss 7.3551
-700 val perplexity 1564.1499
-700 train 7.307829 (lr=3.9948e-05) (hash(x)=148115934)
-500 val loss 7.4186
-500 val perplexity 1666.7257
-500 train 7.307360 (lr=5.0000e-05) (hash(x)=140604760)
-800 val loss 7.2645
-800 val perplexity 1428.6420
-800 train 7.149124 (lr=3.9883e-05) (hash(x)=137464699)
-600 val loss 7.3078
-600 val perplexity 1491.8918
-600 train 7.244214 (lr=4.9984e-05) (hash(x)=148404734)
-900 val loss 7.1849
-900 val perplexity 1319.3766
-900 train 6.996758 (lr=3.9792e-05) (hash(x)=143886042)
-700 val loss 7.2002
-700 val perplexity 1339.6649
-700 train 7.151448 (lr=4.9935e-05) (hash(x)=148115934)
-1000 val loss 7.1057
-1000 val perplexity 1218.9403
-1000 train 7.108614 (lr=3.9675e-05) (hash(x)=163799796)
-800 val loss 7.0893
-800 val perplexity 1199.0146
-800 train 6.967818 (lr=4.9853e-05) (hash(x)=137464699)
-1100 val loss 7.0070
-1100 val perplexity 1104.3334
-1100 train 6.960635 (lr=3.9532e-05) (hash(x)=144592844)
-900 val loss 6.9989
-900 val perplexity 1095.4117
-900 train 6.811148 (lr=4.9739e-05) (hash(x)=143886042)
-1200 val loss 6.9284
-1200 val perplexity 1020.8506
-1200 train 7.269214 (lr=3.9364e-05) (hash(x)=204706354)
-1000 val loss 6.9037
-1000 val perplexity 995.9300
-1000 train 6.910965 (lr=4.9593e-05) (hash(x)=163799796)
-1300 val loss 6.8478
-1300 val perplexity 941.8114
-1300 train 6.778065 (lr=3.9171e-05) (hash(x)=150862210)
-1100 val loss 6.7992
-1100 val perplexity 897.1482
-1100 train 6.766705 (lr=4.9415e-05) (hash(x)=144592844)
-1400 val loss 6.7591
-1400 val perplexity 861.8267
-1400 train 6.746733 (lr=3.8953e-05) (hash(x)=147766811)
-1200 val loss 6.7084
-1200 val perplexity 819.2203
-1200 train 7.037606 (lr=4.9205e-05) (hash(x)=204706354)
-1500 val loss 6.6851
-1500 val perplexity 800.3522
-1500 train 6.497079 (lr=3.8711e-05) (hash(x)=135925327)
-1300 val loss 6.6284
-1300 val perplexity 756.2941
-1300 train 6.567288 (lr=4.8964e-05) (hash(x)=150862210)
-1600 val loss 6.6306
-1600 val perplexity 757.9744
-1600 train 6.597507 (lr=3.8444e-05) (hash(x)=160440642)
-1400 val loss 6.5553
-1400 val perplexity 702.9320
-1400 train 6.554690 (lr=4.8691e-05) (hash(x)=147766811)
-1700 val loss 6.5611
-1700 val perplexity 707.0446
-1700 train 6.665172 (lr=3.8153e-05) (hash(x)=151184106)
-1500 val loss 6.4862
-1500 val perplexity 655.9997
-1500 train 6.305926 (lr=4.8388e-05) (hash(x)=135925327)
-1800 val loss 6.5172
-1800 val perplexity 676.6623
-1800 train 6.413347 (lr=3.7839e-05) (hash(x)=148100580)
-1600 val loss 6.4483
-1600 val perplexity 631.6206
-1600 train 6.418903 (lr=4.8055e-05) (hash(x)=160440642)
-1900 val loss 6.4792
-1900 val perplexity 651.4436
-1900 train 6.312384 (lr=3.7502e-05) (hash(x)=149434659)
-1700 val loss 6.3908
-1700 val perplexity 596.3445
-1700 train 6.514476 (lr=4.7691e-05) (hash(x)=151184106)
-2000 val loss 6.4197
-2000 val perplexity 613.8058
-2000 train 6.322248 (lr=3.7143e-05) (hash(x)=152285486)
-1800 val loss 6.3551
-1800 val perplexity 575.4153
-1800 train 6.255002 (lr=4.7299e-05) (hash(x)=148100580)
-2100 val loss 6.3787
-2100 val perplexity 589.1384
-2100 train 6.119319 (lr=3.6761e-05) (hash(x)=144294295)
-1900 val loss 6.3350
-1900 val perplexity 563.9974
-1900 train 6.163347 (lr=4.6878e-05) (hash(x)=149434659)
-2200 val loss 6.3501
-2200 val perplexity 572.5634
-2200 train 6.368137 (lr=3.6358e-05) (hash(x)=175030215)
-2000 val loss 6.2730
-2000 val perplexity 530.0711
-2000 train 6.177517 (lr=4.6428e-05) (hash(x)=152285486)
-2300 val loss 6.3275
-2300 val perplexity 559.7407
-2300 train 6.127307 (lr=3.5935e-05) (hash(x)=150831428)
-2100 val loss 6.2385
-2100 val perplexity 512.0662
-2100 train 5.980282 (lr=4.5951e-05) (hash(x)=144294295)
-2400 val loss 6.2802
-2400 val perplexity 533.9189
-2400 train 6.456465 (lr=3.5491e-05) (hash(x)=140808297)
-2200 val loss 6.2135
-2200 val perplexity 499.4246
-2200 train 6.216037 (lr=4.5448e-05) (hash(x)=175030215)
-2500 val loss 6.2531
-2500 val perplexity 519.6355
-2500 train 6.235676 (lr=3.5027e-05) (hash(x)=153160275)
-2300 val loss 6.1925
-2300 val perplexity 489.0615
-2300 train 6.003661 (lr=4.4918e-05) (hash(x)=150831428)
-2600 val loss 6.2302
-2600 val perplexity 507.8468
-2600 train 6.007129 (lr=3.4545e-05) (hash(x)=133990623)
-2400 val loss 6.1525
-2400 val perplexity 469.8813
-2400 train 6.337286 (lr=4.4363e-05) (hash(x)=140808297)
-2700 val loss 6.2128
-2700 val perplexity 499.0825
-2700 train 6.184642 (lr=3.4044e-05) (hash(x)=142860944)
-2500 val loss 6.1242
-2500 val perplexity 456.7643
-2500 train 6.100560 (lr=4.3784e-05) (hash(x)=153160275)
-2800 val loss 6.1823
-2800 val perplexity 484.1247
-2800 train 6.199511 (lr=3.3526e-05) (hash(x)=137959511)
-2600 val loss 6.1124
-2600 val perplexity 451.4389
-2600 train 5.900442 (lr=4.3181e-05) (hash(x)=133990623)
-2900 val loss 6.1564
-2900 val perplexity 471.7105
-2900 train 6.107897 (lr=3.2992e-05) (hash(x)=147009873)
-2700 val loss 6.0887
-2700 val perplexity 440.8575
-2700 train 6.055900 (lr=4.2555e-05) (hash(x)=142860944)
-3000 val loss 6.1408
-3000 val perplexity 464.4283
-3000 train 5.998592 (lr=3.2441e-05) (hash(x)=158264841)
-2800 val loss 6.0603
-2800 val perplexity 428.4908
-2800 train 6.074576 (lr=4.1908e-05) (hash(x)=137959511)
-3100 val loss 6.1262
-3100 val perplexity 457.6978
-3100 train 5.968262 (lr=3.1875e-05) (hash(x)=139232251)
-2900 val loss 6.0398
-2900 val perplexity 419.8025
-2900 train 5.998567 (lr=4.1240e-05) (hash(x)=147009873)
-3200 val loss 6.0981
-3200 val perplexity 444.9918
-3200 train 6.139701 (lr=3.1295e-05) (hash(x)=153436104)
-3000 val loss 6.0207
-3000 val perplexity 411.8819
-3000 train 5.875710 (lr=4.0551e-05) (hash(x)=158264841)
-3300 val loss 6.0828
-3300 val perplexity 438.2633
-3300 train 5.901386 (lr=3.0702e-05) (hash(x)=149681831)
-3100 val loss 6.0151
-3100 val perplexity 409.5502
-3100 train 5.848797 (lr=3.9844e-05) (hash(x)=139232251)
-3400 val loss 6.0727
-3400 val perplexity 433.8536
-3400 train 5.962709 (lr=3.0095e-05) (hash(x)=168425516)
-3200 val loss 5.9804
-3200 val perplexity 395.6065
-3200 train 6.026180 (lr=3.9119e-05) (hash(x)=153436104)
-3500 val loss 6.0482
-3500 val perplexity 423.3637
-3500 train 6.077012 (lr=2.9477e-05) (hash(x)=163104338)
-3300 val loss 5.9644
-3300 val perplexity 389.3310
-3300 train 5.787261 (lr=3.8377e-05) (hash(x)=149681831)
-3600 val loss 6.0315
-3600 val perplexity 416.3579
-3600 train 6.009035 (lr=2.8849e-05) (hash(x)=165109772)
-3400 val loss 5.9549
-3400 val perplexity 385.6530
-3400 train 5.827873 (lr=3.7619e-05) (hash(x)=168425516)
-3700 val loss 6.0278
-3700 val perplexity 414.8033
-3700 train 5.760067 (lr=2.8210e-05) (hash(x)=153420306)
-3500 val loss 5.9309
-3500 val perplexity 376.5024
-3500 train 5.961608 (lr=3.6847e-05) (hash(x)=163104338)
-3800 val loss 5.9994
-3800 val perplexity 403.2057
-3800 train 6.023686 (lr=2.7562e-05) (hash(x)=160168863)
-3600 val loss 5.9163
-3600 val perplexity 371.0494
-3600 train 5.887290 (lr=3.6061e-05) (hash(x)=165109772)
-3900 val loss 5.9951
-3900 val perplexity 401.4536
-3900 train 5.910920 (lr=2.6907e-05) (hash(x)=153906073)
-3700 val loss 5.9126
-3700 val perplexity 369.6673
-3700 train 5.644318 (lr=3.5263e-05) (hash(x)=153420306)
-4000 val loss 5.9782
-4000 val perplexity 394.7390
-4000 train 5.915583 (lr=2.6244e-05) (hash(x)=151055067)
-3800 val loss 5.8868
-3800 val perplexity 360.2454
-3800 train 5.901347 (lr=3.4453e-05) (hash(x)=160168863)
-4100 val loss 5.9631
-4100 val perplexity 388.8174
-4100 train 5.942003 (lr=2.5575e-05) (hash(x)=149629830)
-3900 val loss 5.8704
-3900 val perplexity 354.3925
-3900 train 5.782591 (lr=3.3633e-05) (hash(x)=153906073)
-4200 val loss 5.9547
-4200 val perplexity 385.5438
-4200 train 5.834734 (lr=2.4900e-05) (hash(x)=143101381)
-4000 val loss 5.8594
-4000 val perplexity 350.5174
-4000 train 5.806424 (lr=3.2805e-05) (hash(x)=151055067)
-4300 val loss 5.9447
-4300 val perplexity 381.7111
-4300 train 5.853470 (lr=2.4222e-05) (hash(x)=149712044)
-4100 val loss 5.8430
-4100 val perplexity 344.7963
-4100 train 5.818872 (lr=3.1968e-05) (hash(x)=149629830)
-4400 val loss 5.9280
-4400 val perplexity 375.4156
-4400 train 5.852243 (lr=2.3540e-05) (hash(x)=153446449)
-4200 val loss 5.8300
-4200 val perplexity 340.3713
-4200 train 5.713513 (lr=3.1126e-05) (hash(x)=143101381)
-4500 val loss 5.9134
-4500 val perplexity 369.9484
-4500 train 5.863609 (lr=2.2856e-05) (hash(x)=146086947)
-4300 val loss 5.8179
-4300 val perplexity 336.2538
-4300 train 5.726902 (lr=3.0277e-05) (hash(x)=149712044)
-4600 val loss 5.9079
-4600 val perplexity 367.9209
-4600 train 5.746026 (lr=2.2171e-05) (hash(x)=153800173)
-4400 val loss 5.8030
-4400 val perplexity 331.2757
-4400 train 5.729753 (lr=2.9425e-05) (hash(x)=153446449)
-4700 val loss 5.8936
-4700 val perplexity 362.7248
-4700 train 5.950695 (lr=2.1486e-05) (hash(x)=155962726)
-4500 val loss 5.7892
-4500 val perplexity 326.7505
-4500 train 5.736334 (lr=2.8571e-05) (hash(x)=146086947)
-4800 val loss 5.8870
-4800 val perplexity 360.3289
-4800 train 6.127524 (lr=2.0801e-05) (hash(x)=142045616)
-4600 val loss 5.7816
-4600 val perplexity 324.2762
-4600 train 5.622052 (lr=2.7714e-05) (hash(x)=153800173)
-4900 val loss 5.8733
-4900 val perplexity 355.4223
-4900 train 5.846272 (lr=2.0118e-05) (hash(x)=143418248)
-4700 val loss 5.7701
-4700 val perplexity 320.5825
-4700 train 5.849279 (lr=2.6857e-05) (hash(x)=155962726)
-5000 val loss 5.8674
-5000 val perplexity 353.3279
-5000 train 5.789097 (lr=1.9438e-05) (hash(x)=145789790)
-4800 val loss 5.7609
-4800 val perplexity 317.6197
-4800 train 6.008756 (lr=2.6002e-05) (hash(x)=142045616)
-5100 val loss 5.8621
-5100 val perplexity 351.4626
-5100 train 5.837859 (lr=1.8762e-05) (hash(x)=137795633)
-4900 val loss 5.7479
-4900 val perplexity 313.5262
-4900 train 5.718339 (lr=2.5148e-05) (hash(x)=143418248)
-5200 val loss 5.8508
-5200 val perplexity 347.5151
-5200 train 6.012800 (lr=1.8090e-05) (hash(x)=148907132)
-5000 val loss 5.7359
-5000 val perplexity 309.7813
-5000 train 5.662224 (lr=2.4298e-05) (hash(x)=145789790)
-5300 val loss 5.8430
-5300 val perplexity 344.8073
-5300 train 5.720342 (lr=1.7424e-05) (hash(x)=152343580)
-5100 val loss 5.7307
-5100 val perplexity 308.1837
-5100 train 5.712019 (lr=2.3452e-05) (hash(x)=137795633)
-5400 val loss 5.8335
-5400 val perplexity 341.5411
-5400 train 5.687907 (lr=1.6765e-05) (hash(x)=148578264)
-5200 val loss 5.7172
-5200 val perplexity 304.0673
-5200 train 5.874541 (lr=2.2613e-05) (hash(x)=148907132)
-5500 val loss 5.8229
-5500 val perplexity 337.9455
-5500 train 5.987973 (lr=1.6113e-05) (hash(x)=145635833)
-5300 val loss 5.7104
-5300 val perplexity 301.9879
-5300 train 5.584001 (lr=2.1780e-05) (hash(x)=152343580)
-5600 val loss 5.8184
-5600 val perplexity 336.4306
-5600 train 5.804915 (lr=1.5469e-05) (hash(x)=156337844)
-5400 val loss 5.7041
-5400 val perplexity 300.1102
-5400 train 5.555911 (lr=2.0956e-05) (hash(x)=148578264)
-5700 val loss 5.8127
-5700 val perplexity 334.5068
-5700 train 5.692535 (lr=1.4836e-05) (hash(x)=147168506)
-5500 val loss 5.6908
-5500 val perplexity 296.1326
-5500 train 5.854535 (lr=2.0141e-05) (hash(x)=145635833)
-5800 val loss 5.8009
-5800 val perplexity 330.6078
-5800 train 5.861600 (lr=1.4212e-05) (hash(x)=159566920)
-5600 val loss 5.6859
-5600 val perplexity 294.6682
-5600 train 5.674375 (lr=1.9337e-05) (hash(x)=156337844)
-5900 val loss 5.7958
-5900 val perplexity 328.9098
-5900 train 5.869177 (lr=1.3600e-05) (hash(x)=158273929)
-5700 val loss 5.6814
-5700 val perplexity 293.3595
-5700 train 5.554213 (lr=1.8545e-05) (hash(x)=147168506)
-6000 val loss 5.7954
-6000 val perplexity 328.7870
-6000 train 5.764714 (lr=1.3000e-05) (hash(x)=156649749)
-5800 val loss 5.6695
-5800 val perplexity 289.8786
-5800 train 5.722130 (lr=1.7765e-05) (hash(x)=159566920)
-6100 val loss 5.7833
-6100 val perplexity 324.8376
-6100 train 5.699610 (lr=1.2413e-05) (hash(x)=146812388)
-5900 val loss 5.6619
-5900 val perplexity 287.7089
-5900 train 5.723249 (lr=1.7000e-05) (hash(x)=158273929)
-6200 val loss 5.7795
-6200 val perplexity 323.6072
-6200 train 5.603285 (lr=1.1840e-05) (hash(x)=143522146)
-6000 val loss 5.6624
-6000 val perplexity 287.8420
-6000 train 5.632993 (lr=1.6250e-05) (hash(x)=156649749)
-6300 val loss 5.7735
-6300 val perplexity 321.6751
-6300 train 5.599552 (lr=1.1282e-05) (hash(x)=150124474)
-6100 val loss 5.6500
-6100 val perplexity 284.3049
-6100 train 5.567279 (lr=1.5516e-05) (hash(x)=146812388)
-6400 val loss 5.7706
-6400 val perplexity 320.7302
-6400 train 5.803510 (lr=1.0739e-05) (hash(x)=141242117)
-6200 val loss 5.6470
-6200 val perplexity 283.4369
-6200 train 5.477498 (lr=1.4800e-05) (hash(x)=143522146)
-6500 val loss 5.7612
-6500 val perplexity 317.7372
-6500 train 5.646392 (lr=1.0213e-05) (hash(x)=143529762)
-6300 val loss 5.6428
-6300 val perplexity 282.2395
-6300 train 5.460387 (lr=1.4102e-05) (hash(x)=150124474)
-6600 val loss 5.7580
-6600 val perplexity 316.7123
-6600 train 5.543950 (lr=9.7032e-06) (hash(x)=136948374)
-6400 val loss 5.6350
-6400 val perplexity 280.0507
-6400 train 5.671438 (lr=1.3424e-05) (hash(x)=141242117)
-6700 val loss 5.7555
-6700 val perplexity 315.9332
-6700 train 5.561884 (lr=9.2116e-06) (hash(x)=146268592)
-6500 val loss 5.6280
-6500 val perplexity 278.1046
-6500 train 5.501300 (lr=1.2766e-05) (hash(x)=143529762)
-6800 val loss 5.7477
-6800 val perplexity 313.4810
-6800 train 5.701334 (lr=8.7387e-06) (hash(x)=152676836)
-6600 val loss 5.6238
-6600 val perplexity 276.9468
-6600 train 5.410007 (lr=1.2129e-05) (hash(x)=136948374)
-6900 val loss 5.7414
-6900 val perplexity 311.5153
-6900 train 5.658071 (lr=8.2849e-06) (hash(x)=134657776)
-6700 val loss 5.6229
-6700 val perplexity 276.6846
-6700 train 5.419105 (lr=1.1515e-05) (hash(x)=146268592)
-7000 val loss 5.7382
-7000 val perplexity 310.4917
-7000 train 5.696815 (lr=7.8510e-06) (hash(x)=166721861)
-6800 val loss 5.6152
-6800 val perplexity 274.5602
-6800 train 5.570204 (lr=1.0923e-05) (hash(x)=152676836)
-7100 val loss 5.7366
-7100 val perplexity 310.0130
-7100 train 5.535275 (lr=7.4377e-06) (hash(x)=135496702)
-6900 val loss 5.6066
-6900 val perplexity 272.2095
-6900 train 5.530090 (lr=1.0356e-05) (hash(x)=134657776)
-7200 val loss 5.7336
-7200 val perplexity 309.0950
-7200 train 5.841577 (lr=7.0455e-06) (hash(x)=155567461)
-7000 val loss 5.6050
-7000 val perplexity 271.7800
-7000 train 5.559492 (lr=9.8138e-06) (hash(x)=166721861)
-7300 val loss 5.7267
-7300 val perplexity 306.9496
-7300 train 5.524444 (lr=6.6749e-06) (hash(x)=142803829)
-7100 val loss 5.6054
-7100 val perplexity 271.8886
-7100 train 5.398874 (lr=9.2971e-06) (hash(x)=135496702)
-7400 val loss 5.7253
-7400 val perplexity 306.5243
-7400 train 5.550263 (lr=6.3266e-06) (hash(x)=145294178)
-7200 val loss 5.5987
-7200 val perplexity 270.0845
-7200 train 5.709658 (lr=8.8068e-06) (hash(x)=155567461)
-7500 val loss 5.7234
-7500 val perplexity 305.9535
-7500 train 5.481024 (lr=6.0010e-06) (hash(x)=150573713)
-7300 val loss 5.5917
-7300 val perplexity 268.2025
-7300 train 5.395301 (lr=8.3436e-06) (hash(x)=142803829)
-7600 val loss 5.7208
-7600 val perplexity 305.1357
-7600 train 5.742909 (lr=5.6986e-06) (hash(x)=142771511)
-7400 val loss 5.5920
-7400 val perplexity 268.2589
-7400 train 5.410421 (lr=7.9082e-06) (hash(x)=145294178)
-7700 val loss 5.7145
-7700 val perplexity 303.2360
-7700 train 5.618597 (lr=5.4198e-06) (hash(x)=143602175)
-7500 val loss 5.5897
-7500 val perplexity 267.6545
-7500 train 5.347271 (lr=7.5012e-06) (hash(x)=150573713)
-7800 val loss 5.7126
-7800 val perplexity 302.6505
-7800 train 5.756348 (lr=5.1650e-06) (hash(x)=152379862)
-7600 val loss 5.5874
-7600 val perplexity 267.0536
-7600 train 5.605257 (lr=7.1232e-06) (hash(x)=142771511)
-7900 val loss 5.7111
-7900 val perplexity 302.1945
-7900 train 5.537019 (lr=4.9347e-06) (hash(x)=146655921)
-7700 val loss 5.5809
-7700 val perplexity 265.2994
-7700 train 5.483287 (lr=6.7747e-06) (hash(x)=143602175)
-8000 val loss 5.7129
-8000 val perplexity 302.7426
-8000 train 5.840605 (lr=4.7291e-06) (hash(x)=148262482)
-7800 val loss 5.5780
-7800 val perplexity 264.5419
-7800 train 5.642337 (lr=6.4563e-06) (hash(x)=152379862)
-8100 val loss 5.7041
-8100 val perplexity 300.1092
-8100 train 5.641214 (lr=4.5486e-06) (hash(x)=147683655)
-7900 val loss 5.5774
-7900 val perplexity 264.3962
-7900 train 5.391592 (lr=6.1684e-06) (hash(x)=146655921)
-8200 val loss 5.7048
-8200 val perplexity 300.2959
-8200 train 5.807380 (lr=4.3933e-06) (hash(x)=157312987)
-8000 val loss 5.5782
-8000 val perplexity 264.5894
-8000 train 5.719192 (lr=5.9114e-06) (hash(x)=148262482)
-8300 val loss 5.7019
-8300 val perplexity 299.4422
-8300 train 5.664444 (lr=4.2636e-06) (hash(x)=141107543)
-8100 val loss 5.5697
-8100 val perplexity 262.3572
-8100 train 5.517353 (lr=5.6857e-06) (hash(x)=147683655)
-8400 val loss 5.6976
-8400 val perplexity 298.1539
-8400 train 5.683732 (lr=4.1596e-06) (hash(x)=141323024)
-8200 val loss 5.5705
-8200 val perplexity 262.5577
-8200 train 5.688238 (lr=5.4917e-06) (hash(x)=157312987)
-8500 val loss 5.6963
-8500 val perplexity 297.7502
-8500 train 5.670836 (lr=4.0815e-06) (hash(x)=150696521)
-8300 val loss 5.5666
-8300 val perplexity 261.5423
-8300 train 5.523445 (lr=5.3295e-06) (hash(x)=141107543)
-8600 val loss 5.6959
-8600 val perplexity 297.6495
-8600 train 5.643767 (lr=4.0294e-06) (hash(x)=162288191)
-8400 val loss 5.5627
-8400 val perplexity 260.5137
-8400 train 5.558888 (lr=5.1995e-06) (hash(x)=141323024)
-8700 val loss 5.6922
-8700 val perplexity 296.5599
-8700 train 5.560731 (lr=4.0033e-06) (hash(x)=152860941)
-8500 val loss 5.5615
-8500 val perplexity 260.2245
-8500 train 5.526323 (lr=5.1019e-06) (hash(x)=150696521)
-8749 val loss 5.6907
-8749 val perplexity 296.0869
-8600 val loss 5.5609
-8600 val perplexity 260.0624
-8600 train 5.497671 (lr=5.0367e-06) (hash(x)=162288191)
-8700 val loss 5.5574
-8700 val perplexity 259.1537
-8700 train 5.420652 (lr=5.0041e-06) (hash(x)=152860941)
-8749 val loss 5.5552
-8749 val perplexity 258.5751
+0 val perplexity 81594.3281
+0 train 11.313001 (lr=1.2000e-07) (hash(x)=145079536)
+100 val loss 10.0144
+100 val perplexity 22345.9102
+100 train 10.009398 (lr=1.2120e-05) (hash(x)=155800595)
+200 val loss 9.2647
+200 val perplexity 10559.1250
+200 train 9.213030 (lr=2.4120e-05) (hash(x)=145606733)
+300 val loss 8.0461
+300 val perplexity 3121.6345
+300 train 8.025386 (lr=3.6120e-05) (hash(x)=150367139)
+400 val loss 7.6229
+400 val perplexity 2044.4747
+400 train 7.925928 (lr=4.8120e-05) (hash(x)=155747374)
+500 val loss 7.4753
+500 val perplexity 1763.9541
+500 train 7.365836 (lr=6.0000e-05) (hash(x)=140604760)
+600 val loss 7.3951
+600 val perplexity 1628.0135
+600 train 7.341086 (lr=5.9980e-05) (hash(x)=148404734)
+700 val loss 7.3237
+700 val perplexity 1515.7427
+700 train 7.281212 (lr=5.9922e-05) (hash(x)=148115934)
+800 val loss 7.2559
+800 val perplexity 1416.4121
+800 train 7.138442 (lr=5.9824e-05) (hash(x)=137464699)
+900 val loss 7.1986
+900 val perplexity 1337.5930
+900 train 7.013222 (lr=5.9687e-05) (hash(x)=143886042)
+1000 val loss 7.1368
+1000 val perplexity 1257.3633
+1000 train 7.140777 (lr=5.9512e-05) (hash(x)=163799796)
+1100 val loss 7.0886
+1100 val perplexity 1198.1842
+1100 train 7.041957 (lr=5.9298e-05) (hash(x)=144592844)
+1200 val loss 7.0396
+1200 val perplexity 1140.9196
+1200 train 7.366237 (lr=5.9046e-05) (hash(x)=204706354)
+1300 val loss 6.9962
+1300 val perplexity 1092.5182
+1300 train 6.922908 (lr=5.8757e-05) (hash(x)=150862210)
+1400 val loss 6.9463
+1400 val perplexity 1039.2561
+1400 train 6.929276 (lr=5.8430e-05) (hash(x)=147766811)
+1500 val loss 6.9052
+1500 val perplexity 997.4670
+1500 train 6.713291 (lr=5.8066e-05) (hash(x)=135925327)
+1600 val loss 6.8914
+1600 val perplexity 983.7772
+1600 train 6.861294 (lr=5.7666e-05) (hash(x)=160440642)
+1700 val loss 6.8537
+1700 val perplexity 947.3388
+1700 train 6.945174 (lr=5.7230e-05) (hash(x)=151184106)
+1800 val loss 6.8147
+1800 val perplexity 911.1041
+1800 train 6.705882 (lr=5.6759e-05) (hash(x)=148100580)
+1900 val loss 6.7905
+1900 val perplexity 889.3358
+1900 train 6.627530 (lr=5.6253e-05) (hash(x)=149434659)
+2000 val loss 6.7372
+2000 val perplexity 843.1987
+2000 train 6.636229 (lr=5.5714e-05) (hash(x)=152285486)
+2100 val loss 6.7012
+2100 val perplexity 813.4182
+2100 train 6.454887 (lr=5.5142e-05) (hash(x)=144294295)
+2200 val loss 6.6600
+2200 val perplexity 780.5523
+2200 train 6.700250 (lr=5.4537e-05) (hash(x)=175030215)
+2300 val loss 6.6278
+2300 val perplexity 755.8517
+2300 train 6.426424 (lr=5.3902e-05) (hash(x)=150831428)
+2400 val loss 6.5708
+2400 val perplexity 713.9117
+2400 train 6.719954 (lr=5.3236e-05) (hash(x)=140808297)
+2500 val loss 6.5486
+2500 val perplexity 698.2894
+2500 train 6.535069 (lr=5.2541e-05) (hash(x)=153160275)
+2600 val loss 6.5124
+2600 val perplexity 673.4161
+2600 train 6.281240 (lr=5.1817e-05) (hash(x)=133990623)
+2700 val loss 6.4780
+2700 val perplexity 650.6824
+2700 train 6.445283 (lr=5.1067e-05) (hash(x)=142860944)
+2800 val loss 6.4419
+2800 val perplexity 627.5731
+2800 train 6.465282 (lr=5.0290e-05) (hash(x)=137959511)
+2900 val loss 6.4154
+2900 val perplexity 611.1779
+2900 train 6.356608 (lr=4.9487e-05) (hash(x)=147009873)
+3000 val loss 6.3901
+3000 val perplexity 595.8951
+3000 train 6.244250 (lr=4.8662e-05) (hash(x)=158264841)
+3100 val loss 6.3805
+3100 val perplexity 590.2316
+3100 train 6.229535 (lr=4.7813e-05) (hash(x)=139232251)
+3200 val loss 6.3363
+3200 val perplexity 564.6946
+3200 train 6.381537 (lr=4.6943e-05) (hash(x)=153436104)
+3300 val loss 6.3156
+3300 val perplexity 553.1343
+3300 train 6.131235 (lr=4.6052e-05) (hash(x)=149681831)
+3400 val loss 6.3028
+3400 val perplexity 546.1260
+3400 train 6.199091 (lr=4.5143e-05) (hash(x)=168425516)
+3500 val loss 6.2774
+3500 val perplexity 532.4174
+3500 train 6.302076 (lr=4.4216e-05) (hash(x)=163104338)
+3600 val loss 6.2523
+3600 val perplexity 519.2015
+3600 train 6.228388 (lr=4.3273e-05) (hash(x)=165109772)
+3700 val loss 6.2477
+3700 val perplexity 516.8152
+3700 train 5.971647 (lr=4.2315e-05) (hash(x)=153420306)
+3800 val loss 6.2143
+3800 val perplexity 499.8556
+3800 train 6.238448 (lr=4.1343e-05) (hash(x)=160168863)
+3900 val loss 6.1952
+3900 val perplexity 490.3670
+3900 train 6.114435 (lr=4.0360e-05) (hash(x)=153906073)
+4000 val loss 6.1810
+4000 val perplexity 483.4672
+4000 train 6.111329 (lr=3.9365e-05) (hash(x)=151055067)
+4100 val loss 6.1627
+4100 val perplexity 474.6907
+4100 train 6.139215 (lr=3.8362e-05) (hash(x)=149629830)
+4200 val loss 6.1469
+4200 val perplexity 467.2827
+4200 train 6.024417 (lr=3.7351e-05) (hash(x)=143101381)
+4300 val loss 6.1317
+4300 val perplexity 460.2216
+4300 train 6.032101 (lr=3.6333e-05) (hash(x)=149712044)
+4400 val loss 6.1080
+4400 val perplexity 449.4301
+4400 train 6.034675 (lr=3.5311e-05) (hash(x)=153446449)
+4500 val loss 6.0907
+4500 val perplexity 441.7390
+4500 train 6.038786 (lr=3.4285e-05) (hash(x)=146086947)
+4600 val loss 6.0822
+4600 val perplexity 438.0018
+4600 train 5.921267 (lr=3.3257e-05) (hash(x)=153800173)
+4700 val loss 6.0664
+4700 val perplexity 431.1287
+4700 train 6.112772 (lr=3.2229e-05) (hash(x)=155962726)
+4800 val loss 6.0477
+4800 val perplexity 423.1508
+4800 train 6.266189 (lr=3.1202e-05) (hash(x)=142045616)
+4900 val loss 6.0345
+4900 val perplexity 417.5972
+4900 train 5.999585 (lr=3.0178e-05) (hash(x)=143418248)
+5000 val loss 6.0216
+5000 val perplexity 412.2560
+5000 train 5.936325 (lr=2.9157e-05) (hash(x)=145789790)
+5100 val loss 6.0159
+5100 val perplexity 409.8862
+5100 train 5.977530 (lr=2.8143e-05) (hash(x)=137795633)
+5200 val loss 5.9990
+5200 val perplexity 403.0296
+5200 train 6.162383 (lr=2.7135e-05) (hash(x)=148907132)
+5300 val loss 5.9855
+5300 val perplexity 397.6393
+5300 train 5.870116 (lr=2.6136e-05) (hash(x)=152343580)
+5400 val loss 5.9759
+5400 val perplexity 393.8195
+5400 train 5.828068 (lr=2.5147e-05) (hash(x)=148578264)
+5500 val loss 5.9597
+5500 val perplexity 387.4826
+5500 train 6.117543 (lr=2.4169e-05) (hash(x)=145635833)
+5600 val loss 5.9549
+5600 val perplexity 385.6309
+5600 train 5.933925 (lr=2.3204e-05) (hash(x)=156337844)
+5700 val loss 5.9468
+5700 val perplexity 382.5128
+5700 train 5.829904 (lr=2.2253e-05) (hash(x)=147168506)
+5800 val loss 5.9339
+5800 val perplexity 377.6135
+5800 train 5.995249 (lr=2.1318e-05) (hash(x)=159566920)
+5900 val loss 5.9254
+5900 val perplexity 374.4188
+5900 train 5.996993 (lr=2.0400e-05) (hash(x)=158273929)
+6000 val loss 5.9167
+6000 val perplexity 371.1860
+6000 train 5.886044 (lr=1.9500e-05) (hash(x)=156649749)
+6100 val loss 5.9074
+6100 val perplexity 367.7481
+6100 train 5.825771 (lr=1.8620e-05) (hash(x)=146812388)
+6200 val loss 5.9018
+6200 val perplexity 365.6997
+6200 train 5.725801 (lr=1.7760e-05) (hash(x)=143522146)
+6300 val loss 5.8958
+6300 val perplexity 363.4925
+6300 train 5.731684 (lr=1.6923e-05) (hash(x)=150124474)
+6400 val loss 5.8883
+6400 val perplexity 360.7917
+6400 train 5.913698 (lr=1.6108e-05) (hash(x)=141242117)
+6500 val loss 5.8754
+6500 val perplexity 356.1613
+6500 train 5.755292 (lr=1.5319e-05) (hash(x)=143529762)
+6600 val loss 5.8709
+6600 val perplexity 354.5582
+6600 train 5.649996 (lr=1.4555e-05) (hash(x)=136948374)
+6700 val loss 5.8684
+6700 val perplexity 353.6667
+6700 train 5.685143 (lr=1.3817e-05) (hash(x)=146268592)
+6800 val loss 5.8575
+6800 val perplexity 349.8598
+6800 train 5.809295 (lr=1.3108e-05) (hash(x)=152676836)
+6900 val loss 5.8488
+6900 val perplexity 346.8253
+6900 train 5.759398 (lr=1.2427e-05) (hash(x)=134657776)
+7000 val loss 5.8450
+7000 val perplexity 345.4870
+7000 train 5.802629 (lr=1.1777e-05) (hash(x)=166721861)
+7100 val loss 5.8427
+7100 val perplexity 344.7226
+7100 train 5.648686 (lr=1.1157e-05) (hash(x)=135496702)
+7200 val loss 5.8379
+7200 val perplexity 343.0486
+7200 train 5.943250 (lr=1.0568e-05) (hash(x)=155567461)
+7300 val loss 5.8298
+7300 val perplexity 340.3032
+7300 train 5.633186 (lr=1.0012e-05) (hash(x)=142803829)
+7400 val loss 5.8284
+7400 val perplexity 339.8104
+7400 train 5.648532 (lr=9.4899e-06) (hash(x)=145294178)
+7500 val loss 5.8231
+7500 val perplexity 338.0238
+7500 train 5.583829 (lr=9.0014e-06) (hash(x)=150573713)
+7600 val loss 5.8209
+7600 val perplexity 337.2845
+7600 train 5.839932 (lr=8.5478e-06) (hash(x)=142771511)
+7700 val loss 5.8144
+7700 val perplexity 335.0940
+7700 train 5.717018 (lr=8.1297e-06) (hash(x)=143602175)
+7800 val loss 5.8101
+7800 val perplexity 333.6594
+7800 train 5.844014 (lr=7.7476e-06) (hash(x)=152379862)
+7900 val loss 5.8091
+7900 val perplexity 333.3321
+7900 train 5.640357 (lr=7.4021e-06) (hash(x)=146655921)
+8000 val loss 5.8085
+8000 val perplexity 333.1198
+8000 train 5.923010 (lr=7.0937e-06) (hash(x)=148262482)
+8100 val loss 5.7988
+8100 val perplexity 329.9184
+8100 train 5.736006 (lr=6.8229e-06) (hash(x)=147683655)
+8200 val loss 5.7991
+8200 val perplexity 329.9864
+8200 train 5.897091 (lr=6.5900e-06) (hash(x)=157312987)
+8300 val loss 5.7943
+8300 val perplexity 328.4193
+8300 train 5.762470 (lr=6.3954e-06) (hash(x)=141107543)
+8400 val loss 5.7906
+8400 val perplexity 327.2213
+8400 train 5.773040 (lr=6.2395e-06) (hash(x)=141323024)
+8500 val loss 5.7881
+8500 val perplexity 326.4009
+8500 train 5.758968 (lr=6.1223e-06) (hash(x)=150696521)
+8600 val loss 5.7865
+8600 val perplexity 325.8832
+8600 train 5.732920 (lr=6.0440e-06) (hash(x)=162288191)
+8700 val loss 5.7831
+8700 val perplexity 324.7614
+8700 train 5.651917 (lr=6.0049e-06) (hash(x)=152860941)
+8749 val loss 5.7818
+8749 val perplexity 324.3570

attention_kindselective_n_heads4_seed1341/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed5116034684e6d34af195f28f93503512a3e0e90722614c503ae4a0ca29f948
+oid sha256:1caaf89874c458915f6266397a9331b8f6e4fe202586b33af24ab793cfe9e1eb
 size 92843394

attention_kindselective_n_heads4_seed1341/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7fdebfb5ed81da0c541b370e2f0bb47a27698b1c422cf7e238ab56a1bbea0dc7
+oid sha256:23538647f05fdeff92357bb85d20edc6f8ec05480d6bb7d4ba0b0ba6964d9862
 size 179406214