Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4.5e-5_61440_4_1341", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1341", "n_embd": 256}

attention_kindselective_n_heads4_seed1341/log2.txt

@@ -1,267 +1,522 @@
 max_steps: 8750
+100 val loss 10.1171
+100 val perplexity 24761.9980
+100 train 10.112064 (lr=8.0800e-06) (hash(x)=155800595)
 0 val loss 11.3095
-0 val perplexity 81594.3281
-0 train 11.313040 (lr=9.0000e-08) (hash(x)=145079536)
-100 val loss 10.0761
-100 val perplexity 23767.8672
-100 train 10.071070 (lr=9.0900e-06) (hash(x)=155800595)
-200 val loss 9.5264
-200 val perplexity 13717.7168
-200 train 9.479525 (lr=1.8090e-05) (hash(x)=145606733)
-300 val loss 8.3244
-300 val perplexity 4123.3882
-300 train 8.313211 (lr=2.7090e-05) (hash(x)=150367139)
-400 val loss 7.6483
-400 val perplexity 2096.9778
-400 train 7.941428 (lr=3.6090e-05) (hash(x)=155747374)
-500 val loss 7.4758
-500 val perplexity 1764.8981
-500 train 7.367132 (lr=4.5000e-05) (hash(x)=140604760)
-600 val loss 7.3790
-600 val perplexity 1602.0277
-600 train 7.323732 (lr=4.4985e-05) (hash(x)=148404734)
-700 val loss 7.2870
-700 val perplexity 1461.2491
-700 train 7.240127 (lr=4.4941e-05) (hash(x)=148115934)
-800 val loss 7.1677
-800 val perplexity 1296.9070
-800 train 7.044980 (lr=4.4868e-05) (hash(x)=137464699)
-900 val loss 7.0626
-900 val perplexity 1167.4529
-900 train 6.872476 (lr=4.4766e-05) (hash(x)=143886042)
-1000 val loss 6.9679
-1000 val perplexity 1061.9860
-1000 train 6.971719 (lr=4.4634e-05) (hash(x)=163799796)
-1100 val loss 6.8610
-1100 val perplexity 954.3155
-1100 train 6.825153 (lr=4.4474e-05) (hash(x)=144592844)
-1200 val loss 6.7707
-1200 val perplexity 871.9357
-1200 train 7.097885 (lr=4.4285e-05) (hash(x)=204706354)
-1300 val loss 6.6947
-1300 val perplexity 808.1379
-1300 train 6.631471 (lr=4.4068e-05) (hash(x)=150862210)
-1400 val loss 6.6223
-1400 val perplexity 751.7076
-1400 train 6.619408 (lr=4.3822e-05) (hash(x)=147766811)
-1500 val loss 6.5493
-1500 val perplexity 698.7467
-1500 train 6.367006 (lr=4.3549e-05) (hash(x)=135925327)
-1600 val loss 6.5088
-1600 val perplexity 671.0128
-1600 train 6.479077 (lr=4.3249e-05) (hash(x)=160440642)
-1700 val loss 6.4531
-1700 val perplexity 634.6756
-1700 train 6.575705 (lr=4.2922e-05) (hash(x)=151184106)
-1800 val loss 6.4062
-1800 val perplexity 605.5698
-1800 train 6.302565 (lr=4.2569e-05) (hash(x)=148100580)
-1900 val loss 6.3862
-1900 val perplexity 593.6020
-1900 train 6.214580 (lr=4.2190e-05) (hash(x)=149434659)
-2000 val loss 6.3288
-2000 val perplexity 560.5059
-2000 train 6.230272 (lr=4.1785e-05) (hash(x)=152285486)
-2100 val loss 6.2902
-2100 val perplexity 539.2587
-2100 train 6.033951 (lr=4.1356e-05) (hash(x)=144294295)
-2200 val loss 6.2656
-2200 val perplexity 526.1706
-2200 train 6.274797 (lr=4.0903e-05) (hash(x)=175030215)
-2300 val loss 6.2425
-2300 val perplexity 514.1522
-2300 train 6.053737 (lr=4.0426e-05) (hash(x)=150831428)
-2400 val loss 6.2011
-2400 val perplexity 493.2859
-2400 train 6.386537 (lr=3.9927e-05) (hash(x)=140808297)
-2500 val loss 6.1761
-2500 val perplexity 481.1316
-2500 train 6.153784 (lr=3.9406e-05) (hash(x)=153160275)
-2600 val loss 6.1569
-2600 val perplexity 471.9454
-2600 train 5.944077 (lr=3.8863e-05) (hash(x)=133990623)
-2700 val loss 6.1428
-2700 val perplexity 465.3676
-2700 train 6.110357 (lr=3.8300e-05) (hash(x)=142860944)
-2800 val loss 6.1104
-2800 val perplexity 450.5392
-2800 train 6.122779 (lr=3.7717e-05) (hash(x)=137959511)
-2900 val loss 6.0858
-2900 val perplexity 439.5509
-2900 train 6.040658 (lr=3.7116e-05) (hash(x)=147009873)
-3000 val loss 6.0703
-3000 val perplexity 432.8103
-3000 train 5.927878 (lr=3.6496e-05) (hash(x)=158264841)
-3100 val loss 6.0638
-3100 val perplexity 430.0071
-3100 train 5.902818 (lr=3.5860e-05) (hash(x)=139232251)
-3200 val loss 6.0297
-3200 val perplexity 415.6103
-3200 train 6.069080 (lr=3.5207e-05) (hash(x)=153436104)
-3300 val loss 6.0157
-3300 val perplexity 409.8088
-3300 train 5.843759 (lr=3.4539e-05) (hash(x)=149681831)
-3400 val loss 6.0050
-3400 val perplexity 405.4450
-3400 train 5.885643 (lr=3.3857e-05) (hash(x)=168425516)
-3500 val loss 5.9896
-3500 val perplexity 399.2447
-3500 train 6.019372 (lr=3.3162e-05) (hash(x)=163104338)
-3600 val loss 5.9641
-3600 val perplexity 389.2151
-3600 train 5.938704 (lr=3.2455e-05) (hash(x)=165109772)
-3700 val loss 5.9637
-3700 val perplexity 389.0497
-3700 train 5.694476 (lr=3.1736e-05) (hash(x)=153420306)
-3800 val loss 5.9347
-3800 val perplexity 377.9158
-3800 train 5.948569 (lr=3.1008e-05) (hash(x)=160168863)
-3900 val loss 5.9212
-3900 val perplexity 372.8728
-3900 train 5.840442 (lr=3.0270e-05) (hash(x)=153906073)
-4000 val loss 5.9103
-4000 val perplexity 368.8176
-4000 train 5.857364 (lr=2.9524e-05) (hash(x)=151055067)
-4100 val loss 5.8939
-4100 val perplexity 362.8057
-4100 train 5.873942 (lr=2.8771e-05) (hash(x)=149629830)
-4200 val loss 5.8796
-4200 val perplexity 357.6752
-4200 train 5.760632 (lr=2.8013e-05) (hash(x)=143101381)
-4300 val loss 5.8715
-4300 val perplexity 354.7955
-4300 train 5.774401 (lr=2.7250e-05) (hash(x)=149712044)
-4400 val loss 5.8542
-4400 val perplexity 348.6813
-4400 train 5.778264 (lr=2.6483e-05) (hash(x)=153446449)
-4500 val loss 5.8414
-4500 val perplexity 344.2694
-4500 train 5.791447 (lr=2.5714e-05) (hash(x)=146086947)
-4600 val loss 5.8349
-4600 val perplexity 342.0447
-4600 train 5.677424 (lr=2.4943e-05) (hash(x)=153800173)
-4700 val loss 5.8201
-4700 val perplexity 336.9922
-4700 train 5.893148 (lr=2.4172e-05) (hash(x)=155962726)
-4800 val loss 5.8126
-4800 val perplexity 334.4903
-4800 train 6.055249 (lr=2.3402e-05) (hash(x)=142045616)
-4900 val loss 5.8005
-4900 val perplexity 330.4518
-4900 train 5.765253 (lr=2.2633e-05) (hash(x)=143418248)
-5000 val loss 5.7905
-5000 val perplexity 327.1776
-5000 train 5.720255 (lr=2.1868e-05) (hash(x)=145789790)
-5100 val loss 5.7847
-5100 val perplexity 325.2880
-5100 train 5.768248 (lr=2.1107e-05) (hash(x)=137795633)
-5200 val loss 5.7723
-5200 val perplexity 321.2600
-5200 train 5.932185 (lr=2.0351e-05) (hash(x)=148907132)
-5300 val loss 5.7665
-5300 val perplexity 319.4049
-5300 train 5.643368 (lr=1.9602e-05) (hash(x)=152343580)
-5400 val loss 5.7569
-5400 val perplexity 316.3512
-5400 train 5.610413 (lr=1.8860e-05) (hash(x)=148578264)
-5500 val loss 5.7440
-5500 val perplexity 312.2961
-5500 train 5.908919 (lr=1.8127e-05) (hash(x)=145635833)
-5600 val loss 5.7393
-5600 val perplexity 310.8486
-5600 train 5.729499 (lr=1.7403e-05) (hash(x)=156337844)
-5700 val loss 5.7345
-5700 val perplexity 309.3647
-5700 train 5.609031 (lr=1.6690e-05) (hash(x)=147168506)
-5800 val loss 5.7259
-5800 val perplexity 306.7088
-5800 train 5.782136 (lr=1.5989e-05) (hash(x)=159566920)
-5900 val loss 5.7173
-5900 val perplexity 304.0780
-5900 train 5.780312 (lr=1.5300e-05) (hash(x)=158273929)
-6000 val loss 5.7163
-6000 val perplexity 303.7896
-6000 train 5.688311 (lr=1.4625e-05) (hash(x)=156649749)
-6100 val loss 5.7060
-6100 val perplexity 300.6730
-6100 train 5.621117 (lr=1.3965e-05) (hash(x)=146812388)
-6200 val loss 5.7017
-6200 val perplexity 299.3790
-6200 train 5.531676 (lr=1.3320e-05) (hash(x)=143522146)
-6300 val loss 5.6966
-6300 val perplexity 297.8462
-6300 train 5.516797 (lr=1.2692e-05) (hash(x)=150124474)
-6400 val loss 5.6903
-6400 val perplexity 295.9842
-6400 train 5.728277 (lr=1.2081e-05) (hash(x)=141242117)
-6500 val loss 5.6830
-6500 val perplexity 293.8160
-6500 train 5.561332 (lr=1.1489e-05) (hash(x)=143529762)
-6600 val loss 5.6778
-6600 val perplexity 292.3132
-6600 train 5.466468 (lr=1.0916e-05) (hash(x)=136948374)
-6700 val loss 5.6787
-6700 val perplexity 292.5690
-6700 train 5.479150 (lr=1.0363e-05) (hash(x)=146268592)
-6800 val loss 5.6684
-6800 val perplexity 289.5688
-6800 train 5.621566 (lr=9.8310e-06) (hash(x)=152676836)
-6900 val loss 5.6620
-6900 val perplexity 287.7147
-6900 train 5.583413 (lr=9.3205e-06) (hash(x)=134657776)
-7000 val loss 5.6593
-7000 val perplexity 286.9359
-7000 train 5.614666 (lr=8.8324e-06) (hash(x)=166721861)
-7100 val loss 5.6580
-7100 val perplexity 286.5740
-7100 train 5.455072 (lr=8.3674e-06) (hash(x)=135496702)
-7200 val loss 5.6546
-7200 val perplexity 285.6103
-7200 train 5.764223 (lr=7.9261e-06) (hash(x)=155567461)
-7300 val loss 5.6465
-7300 val perplexity 283.3002
-7300 train 5.449112 (lr=7.5093e-06) (hash(x)=142803829)
-7400 val loss 5.6460
-7400 val perplexity 283.1502
-7400 train 5.468495 (lr=7.1174e-06) (hash(x)=145294178)
-7500 val loss 5.6438
-7500 val perplexity 282.5349
-7500 train 5.400206 (lr=6.7511e-06) (hash(x)=150573713)
-7600 val loss 5.6417
-7600 val perplexity 281.9352
-7600 train 5.657176 (lr=6.4109e-06) (hash(x)=142771511)
-7700 val loss 5.6357
-7700 val perplexity 280.2563
-7700 train 5.538167 (lr=6.0972e-06) (hash(x)=143602175)
-7800 val loss 5.6325
-7800 val perplexity 279.3481
-7800 train 5.691221 (lr=5.8107e-06) (hash(x)=152379862)
-7900 val loss 5.6315
-7900 val perplexity 279.0784
-7900 train 5.453447 (lr=5.5515e-06) (hash(x)=146655921)
-8000 val loss 5.6326
-8000 val perplexity 279.3739
-8000 train 5.770980 (lr=5.3203e-06) (hash(x)=148262482)
-8100 val loss 5.6243
-8100 val perplexity 277.0847
-8100 train 5.568313 (lr=5.1172e-06) (hash(x)=147683655)
-8200 val loss 5.6249
-8200 val perplexity 277.2312
-8200 train 5.739700 (lr=4.9425e-06) (hash(x)=157312987)
-8300 val loss 5.6217
-8300 val perplexity 276.3641
-8300 train 5.581598 (lr=4.7966e-06) (hash(x)=141107543)
-8400 val loss 5.6180
-8400 val perplexity 275.3497
-8400 train 5.610967 (lr=4.6796e-06) (hash(x)=141323024)
-8500 val loss 5.6163
-8500 val perplexity 274.8697
-8500 train 5.587502 (lr=4.5917e-06) (hash(x)=150696521)
-8600 val loss 5.6160
-8600 val perplexity 274.7957
-8600 train 5.565991 (lr=4.5330e-06) (hash(x)=162288191)
-8700 val loss 5.6119
-8700 val perplexity 273.6724
-8700 train 5.479275 (lr=4.5037e-06) (hash(x)=152860941)
-8749 val loss 5.6102
-8749 val perplexity 273.1960
+0 val perplexity 81594.2500
+200 val loss 9.6651
+200 val perplexity 15758.2021
+200 train 9.620489 (lr=1.6080e-05) (hash(x)=145606733)
+0 train 11.313038 (lr=1.0000e-07) (hash(x)=145079536)
+300 val loss 8.5370
+300 val perplexity 5099.7910
+300 train 8.528650 (lr=2.4080e-05) (hash(x)=150367139)
+100 val loss 10.0532
+100 val perplexity 23230.9727
+100 train 10.047434 (lr=1.0100e-05) (hash(x)=155800595)
+400 val loss 7.7461
+400 val perplexity 2312.6008
+400 train 8.032513 (lr=3.2080e-05) (hash(x)=155747374)
+200 val loss 9.4102
+200 val perplexity 12211.7793
+200 train 9.361097 (lr=2.0100e-05) (hash(x)=145606733)
+500 val loss 7.5205
+500 val perplexity 1845.5077
+500 train 7.412711 (lr=4.0000e-05) (hash(x)=140604760)
+300 val loss 8.0795
+300 val perplexity 3227.5703
+300 train 8.047569 (lr=3.0100e-05) (hash(x)=150367139)
+600 val loss 7.4285
+600 val perplexity 1683.2040
+600 train 7.377125 (lr=3.9987e-05) (hash(x)=148404734)
+400 val loss 7.5665
+400 val perplexity 1932.2872
+400 train 7.873068 (lr=4.0100e-05) (hash(x)=155747374)
+700 val loss 7.3551
+700 val perplexity 1564.1499
+700 train 7.307829 (lr=3.9948e-05) (hash(x)=148115934)
+500 val loss 7.4186
+500 val perplexity 1666.7257
+500 train 7.307360 (lr=5.0000e-05) (hash(x)=140604760)
+800 val loss 7.2645
+800 val perplexity 1428.6420
+800 train 7.149124 (lr=3.9883e-05) (hash(x)=137464699)
+600 val loss 7.3078
+600 val perplexity 1491.8918
+600 train 7.244214 (lr=4.9984e-05) (hash(x)=148404734)
+900 val loss 7.1849
+900 val perplexity 1319.3766
+900 train 6.996758 (lr=3.9792e-05) (hash(x)=143886042)
+700 val loss 7.2002
+700 val perplexity 1339.6649
+700 train 7.151448 (lr=4.9935e-05) (hash(x)=148115934)
+1000 val loss 7.1057
+1000 val perplexity 1218.9403
+1000 train 7.108614 (lr=3.9675e-05) (hash(x)=163799796)
+800 val loss 7.0893
+800 val perplexity 1199.0146
+800 train 6.967818 (lr=4.9853e-05) (hash(x)=137464699)
+1100 val loss 7.0070
+1100 val perplexity 1104.3334
+1100 train 6.960635 (lr=3.9532e-05) (hash(x)=144592844)
+900 val loss 6.9989
+900 val perplexity 1095.4117
+900 train 6.811148 (lr=4.9739e-05) (hash(x)=143886042)
+1200 val loss 6.9284
+1200 val perplexity 1020.8506
+1200 train 7.269214 (lr=3.9364e-05) (hash(x)=204706354)
+1000 val loss 6.9037
+1000 val perplexity 995.9300
+1000 train 6.910965 (lr=4.9593e-05) (hash(x)=163799796)
+1300 val loss 6.8478
+1300 val perplexity 941.8114
+1300 train 6.778065 (lr=3.9171e-05) (hash(x)=150862210)
+1100 val loss 6.7992
+1100 val perplexity 897.1482
+1100 train 6.766705 (lr=4.9415e-05) (hash(x)=144592844)
+1400 val loss 6.7591
+1400 val perplexity 861.8267
+1400 train 6.746733 (lr=3.8953e-05) (hash(x)=147766811)
+1200 val loss 6.7084
+1200 val perplexity 819.2203
+1200 train 7.037606 (lr=4.9205e-05) (hash(x)=204706354)
+1500 val loss 6.6851
+1500 val perplexity 800.3522
+1500 train 6.497079 (lr=3.8711e-05) (hash(x)=135925327)
+1300 val loss 6.6284
+1300 val perplexity 756.2941
+1300 train 6.567288 (lr=4.8964e-05) (hash(x)=150862210)
+1600 val loss 6.6306
+1600 val perplexity 757.9744
+1600 train 6.597507 (lr=3.8444e-05) (hash(x)=160440642)
+1400 val loss 6.5553
+1400 val perplexity 702.9320
+1400 train 6.554690 (lr=4.8691e-05) (hash(x)=147766811)
+1700 val loss 6.5611
+1700 val perplexity 707.0446
+1700 train 6.665172 (lr=3.8153e-05) (hash(x)=151184106)
+1500 val loss 6.4862
+1500 val perplexity 655.9997
+1500 train 6.305926 (lr=4.8388e-05) (hash(x)=135925327)
+1800 val loss 6.5172
+1800 val perplexity 676.6623
+1800 train 6.413347 (lr=3.7839e-05) (hash(x)=148100580)
+1600 val loss 6.4483
+1600 val perplexity 631.6206
+1600 train 6.418903 (lr=4.8055e-05) (hash(x)=160440642)
+1900 val loss 6.4792
+1900 val perplexity 651.4436
+1900 train 6.312384 (lr=3.7502e-05) (hash(x)=149434659)
+1700 val loss 6.3908
+1700 val perplexity 596.3445
+1700 train 6.514476 (lr=4.7691e-05) (hash(x)=151184106)
+2000 val loss 6.4197
+2000 val perplexity 613.8058
+2000 train 6.322248 (lr=3.7143e-05) (hash(x)=152285486)
+1800 val loss 6.3551
+1800 val perplexity 575.4153
+1800 train 6.255002 (lr=4.7299e-05) (hash(x)=148100580)
+2100 val loss 6.3787
+2100 val perplexity 589.1384
+2100 train 6.119319 (lr=3.6761e-05) (hash(x)=144294295)
+1900 val loss 6.3350
+1900 val perplexity 563.9974
+1900 train 6.163347 (lr=4.6878e-05) (hash(x)=149434659)
+2200 val loss 6.3501
+2200 val perplexity 572.5634
+2200 train 6.368137 (lr=3.6358e-05) (hash(x)=175030215)
+2000 val loss 6.2730
+2000 val perplexity 530.0711
+2000 train 6.177517 (lr=4.6428e-05) (hash(x)=152285486)
+2300 val loss 6.3275
+2300 val perplexity 559.7407
+2300 train 6.127307 (lr=3.5935e-05) (hash(x)=150831428)
+2100 val loss 6.2385
+2100 val perplexity 512.0662
+2100 train 5.980282 (lr=4.5951e-05) (hash(x)=144294295)
+2400 val loss 6.2802
+2400 val perplexity 533.9189
+2400 train 6.456465 (lr=3.5491e-05) (hash(x)=140808297)
+2200 val loss 6.2135
+2200 val perplexity 499.4246
+2200 train 6.216037 (lr=4.5448e-05) (hash(x)=175030215)
+2500 val loss 6.2531
+2500 val perplexity 519.6355
+2500 train 6.235676 (lr=3.5027e-05) (hash(x)=153160275)
+2300 val loss 6.1925
+2300 val perplexity 489.0615
+2300 train 6.003661 (lr=4.4918e-05) (hash(x)=150831428)
+2600 val loss 6.2302
+2600 val perplexity 507.8468
+2600 train 6.007129 (lr=3.4545e-05) (hash(x)=133990623)
+2400 val loss 6.1525
+2400 val perplexity 469.8813
+2400 train 6.337286 (lr=4.4363e-05) (hash(x)=140808297)
+2700 val loss 6.2128
+2700 val perplexity 499.0825
+2700 train 6.184642 (lr=3.4044e-05) (hash(x)=142860944)
+2500 val loss 6.1242
+2500 val perplexity 456.7643
+2500 train 6.100560 (lr=4.3784e-05) (hash(x)=153160275)
+2800 val loss 6.1823
+2800 val perplexity 484.1247
+2800 train 6.199511 (lr=3.3526e-05) (hash(x)=137959511)
+2600 val loss 6.1124
+2600 val perplexity 451.4389
+2600 train 5.900442 (lr=4.3181e-05) (hash(x)=133990623)
+2900 val loss 6.1564
+2900 val perplexity 471.7105
+2900 train 6.107897 (lr=3.2992e-05) (hash(x)=147009873)
+2700 val loss 6.0887
+2700 val perplexity 440.8575
+2700 train 6.055900 (lr=4.2555e-05) (hash(x)=142860944)
+3000 val loss 6.1408
+3000 val perplexity 464.4283
+3000 train 5.998592 (lr=3.2441e-05) (hash(x)=158264841)
+2800 val loss 6.0603
+2800 val perplexity 428.4908
+2800 train 6.074576 (lr=4.1908e-05) (hash(x)=137959511)
+3100 val loss 6.1262
+3100 val perplexity 457.6978
+3100 train 5.968262 (lr=3.1875e-05) (hash(x)=139232251)
+2900 val loss 6.0398
+2900 val perplexity 419.8025
+2900 train 5.998567 (lr=4.1240e-05) (hash(x)=147009873)
+3200 val loss 6.0981
+3200 val perplexity 444.9918
+3200 train 6.139701 (lr=3.1295e-05) (hash(x)=153436104)
+3000 val loss 6.0207
+3000 val perplexity 411.8819
+3000 train 5.875710 (lr=4.0551e-05) (hash(x)=158264841)
+3300 val loss 6.0828
+3300 val perplexity 438.2633
+3300 train 5.901386 (lr=3.0702e-05) (hash(x)=149681831)
+3100 val loss 6.0151
+3100 val perplexity 409.5502
+3100 train 5.848797 (lr=3.9844e-05) (hash(x)=139232251)
+3400 val loss 6.0727
+3400 val perplexity 433.8536
+3400 train 5.962709 (lr=3.0095e-05) (hash(x)=168425516)
+3200 val loss 5.9804
+3200 val perplexity 395.6065
+3200 train 6.026180 (lr=3.9119e-05) (hash(x)=153436104)
+3500 val loss 6.0482
+3500 val perplexity 423.3637
+3500 train 6.077012 (lr=2.9477e-05) (hash(x)=163104338)
+3300 val loss 5.9644
+3300 val perplexity 389.3310
+3300 train 5.787261 (lr=3.8377e-05) (hash(x)=149681831)
+3600 val loss 6.0315
+3600 val perplexity 416.3579
+3600 train 6.009035 (lr=2.8849e-05) (hash(x)=165109772)
+3400 val loss 5.9549
+3400 val perplexity 385.6530
+3400 train 5.827873 (lr=3.7619e-05) (hash(x)=168425516)
+3700 val loss 6.0278
+3700 val perplexity 414.8033
+3700 train 5.760067 (lr=2.8210e-05) (hash(x)=153420306)
+3500 val loss 5.9309
+3500 val perplexity 376.5024
+3500 train 5.961608 (lr=3.6847e-05) (hash(x)=163104338)
+3800 val loss 5.9994
+3800 val perplexity 403.2057
+3800 train 6.023686 (lr=2.7562e-05) (hash(x)=160168863)
+3600 val loss 5.9163
+3600 val perplexity 371.0494
+3600 train 5.887290 (lr=3.6061e-05) (hash(x)=165109772)
+3900 val loss 5.9951
+3900 val perplexity 401.4536
+3900 train 5.910920 (lr=2.6907e-05) (hash(x)=153906073)
+3700 val loss 5.9126
+3700 val perplexity 369.6673
+3700 train 5.644318 (lr=3.5263e-05) (hash(x)=153420306)
+4000 val loss 5.9782
+4000 val perplexity 394.7390
+4000 train 5.915583 (lr=2.6244e-05) (hash(x)=151055067)
+3800 val loss 5.8868
+3800 val perplexity 360.2454
+3800 train 5.901347 (lr=3.4453e-05) (hash(x)=160168863)
+4100 val loss 5.9631
+4100 val perplexity 388.8174
+4100 train 5.942003 (lr=2.5575e-05) (hash(x)=149629830)
+3900 val loss 5.8704
+3900 val perplexity 354.3925
+3900 train 5.782591 (lr=3.3633e-05) (hash(x)=153906073)
+4200 val loss 5.9547
+4200 val perplexity 385.5438
+4200 train 5.834734 (lr=2.4900e-05) (hash(x)=143101381)
+4000 val loss 5.8594
+4000 val perplexity 350.5174
+4000 train 5.806424 (lr=3.2805e-05) (hash(x)=151055067)
+4300 val loss 5.9447
+4300 val perplexity 381.7111
+4300 train 5.853470 (lr=2.4222e-05) (hash(x)=149712044)
+4100 val loss 5.8430
+4100 val perplexity 344.7963
+4100 train 5.818872 (lr=3.1968e-05) (hash(x)=149629830)
+4400 val loss 5.9280
+4400 val perplexity 375.4156
+4400 train 5.852243 (lr=2.3540e-05) (hash(x)=153446449)
+4200 val loss 5.8300
+4200 val perplexity 340.3713
+4200 train 5.713513 (lr=3.1126e-05) (hash(x)=143101381)
+4500 val loss 5.9134
+4500 val perplexity 369.9484
+4500 train 5.863609 (lr=2.2856e-05) (hash(x)=146086947)
+4300 val loss 5.8179
+4300 val perplexity 336.2538
+4300 train 5.726902 (lr=3.0277e-05) (hash(x)=149712044)
+4600 val loss 5.9079
+4600 val perplexity 367.9209
+4600 train 5.746026 (lr=2.2171e-05) (hash(x)=153800173)
+4400 val loss 5.8030
+4400 val perplexity 331.2757
+4400 train 5.729753 (lr=2.9425e-05) (hash(x)=153446449)
+4700 val loss 5.8936
+4700 val perplexity 362.7248
+4700 train 5.950695 (lr=2.1486e-05) (hash(x)=155962726)
+4500 val loss 5.7892
+4500 val perplexity 326.7505
+4500 train 5.736334 (lr=2.8571e-05) (hash(x)=146086947)
+4800 val loss 5.8870
+4800 val perplexity 360.3289
+4800 train 6.127524 (lr=2.0801e-05) (hash(x)=142045616)
+4600 val loss 5.7816
+4600 val perplexity 324.2762
+4600 train 5.622052 (lr=2.7714e-05) (hash(x)=153800173)
+4900 val loss 5.8733
+4900 val perplexity 355.4223
+4900 train 5.846272 (lr=2.0118e-05) (hash(x)=143418248)
+4700 val loss 5.7701
+4700 val perplexity 320.5825
+4700 train 5.849279 (lr=2.6857e-05) (hash(x)=155962726)
+5000 val loss 5.8674
+5000 val perplexity 353.3279
+5000 train 5.789097 (lr=1.9438e-05) (hash(x)=145789790)
+4800 val loss 5.7609
+4800 val perplexity 317.6197
+4800 train 6.008756 (lr=2.6002e-05) (hash(x)=142045616)
+5100 val loss 5.8621
+5100 val perplexity 351.4626
+5100 train 5.837859 (lr=1.8762e-05) (hash(x)=137795633)
+4900 val loss 5.7479
+4900 val perplexity 313.5262
+4900 train 5.718339 (lr=2.5148e-05) (hash(x)=143418248)
+5200 val loss 5.8508
+5200 val perplexity 347.5151
+5200 train 6.012800 (lr=1.8090e-05) (hash(x)=148907132)
+5000 val loss 5.7359
+5000 val perplexity 309.7813
+5000 train 5.662224 (lr=2.4298e-05) (hash(x)=145789790)
+5300 val loss 5.8430
+5300 val perplexity 344.8073
+5300 train 5.720342 (lr=1.7424e-05) (hash(x)=152343580)
+5100 val loss 5.7307
+5100 val perplexity 308.1837
+5100 train 5.712019 (lr=2.3452e-05) (hash(x)=137795633)
+5400 val loss 5.8335
+5400 val perplexity 341.5411
+5400 train 5.687907 (lr=1.6765e-05) (hash(x)=148578264)
+5200 val loss 5.7172
+5200 val perplexity 304.0673
+5200 train 5.874541 (lr=2.2613e-05) (hash(x)=148907132)
+5500 val loss 5.8229
+5500 val perplexity 337.9455
+5500 train 5.987973 (lr=1.6113e-05) (hash(x)=145635833)
+5300 val loss 5.7104
+5300 val perplexity 301.9879
+5300 train 5.584001 (lr=2.1780e-05) (hash(x)=152343580)
+5600 val loss 5.8184
+5600 val perplexity 336.4306
+5600 train 5.804915 (lr=1.5469e-05) (hash(x)=156337844)
+5400 val loss 5.7041
+5400 val perplexity 300.1102
+5400 train 5.555911 (lr=2.0956e-05) (hash(x)=148578264)
+5700 val loss 5.8127
+5700 val perplexity 334.5068
+5700 train 5.692535 (lr=1.4836e-05) (hash(x)=147168506)
+5500 val loss 5.6908
+5500 val perplexity 296.1326
+5500 train 5.854535 (lr=2.0141e-05) (hash(x)=145635833)
+5800 val loss 5.8009
+5800 val perplexity 330.6078
+5800 train 5.861600 (lr=1.4212e-05) (hash(x)=159566920)
+5600 val loss 5.6859
+5600 val perplexity 294.6682
+5600 train 5.674375 (lr=1.9337e-05) (hash(x)=156337844)
+5900 val loss 5.7958
+5900 val perplexity 328.9098
+5900 train 5.869177 (lr=1.3600e-05) (hash(x)=158273929)
+5700 val loss 5.6814
+5700 val perplexity 293.3595
+5700 train 5.554213 (lr=1.8545e-05) (hash(x)=147168506)
+6000 val loss 5.7954
+6000 val perplexity 328.7870
+6000 train 5.764714 (lr=1.3000e-05) (hash(x)=156649749)
+5800 val loss 5.6695
+5800 val perplexity 289.8786
+5800 train 5.722130 (lr=1.7765e-05) (hash(x)=159566920)
+6100 val loss 5.7833
+6100 val perplexity 324.8376
+6100 train 5.699610 (lr=1.2413e-05) (hash(x)=146812388)
+5900 val loss 5.6619
+5900 val perplexity 287.7089
+5900 train 5.723249 (lr=1.7000e-05) (hash(x)=158273929)
+6200 val loss 5.7795
+6200 val perplexity 323.6072
+6200 train 5.603285 (lr=1.1840e-05) (hash(x)=143522146)
+6000 val loss 5.6624
+6000 val perplexity 287.8420
+6000 train 5.632993 (lr=1.6250e-05) (hash(x)=156649749)
+6300 val loss 5.7735
+6300 val perplexity 321.6751
+6300 train 5.599552 (lr=1.1282e-05) (hash(x)=150124474)
+6100 val loss 5.6500
+6100 val perplexity 284.3049
+6100 train 5.567279 (lr=1.5516e-05) (hash(x)=146812388)
+6400 val loss 5.7706
+6400 val perplexity 320.7302
+6400 train 5.803510 (lr=1.0739e-05) (hash(x)=141242117)
+6200 val loss 5.6470
+6200 val perplexity 283.4369
+6200 train 5.477498 (lr=1.4800e-05) (hash(x)=143522146)
+6500 val loss 5.7612
+6500 val perplexity 317.7372
+6500 train 5.646392 (lr=1.0213e-05) (hash(x)=143529762)
+6300 val loss 5.6428
+6300 val perplexity 282.2395
+6300 train 5.460387 (lr=1.4102e-05) (hash(x)=150124474)
+6600 val loss 5.7580
+6600 val perplexity 316.7123
+6600 train 5.543950 (lr=9.7032e-06) (hash(x)=136948374)
+6400 val loss 5.6350
+6400 val perplexity 280.0507
+6400 train 5.671438 (lr=1.3424e-05) (hash(x)=141242117)
+6700 val loss 5.7555
+6700 val perplexity 315.9332
+6700 train 5.561884 (lr=9.2116e-06) (hash(x)=146268592)
+6500 val loss 5.6280
+6500 val perplexity 278.1046
+6500 train 5.501300 (lr=1.2766e-05) (hash(x)=143529762)
+6800 val loss 5.7477
+6800 val perplexity 313.4810
+6800 train 5.701334 (lr=8.7387e-06) (hash(x)=152676836)
+6600 val loss 5.6238
+6600 val perplexity 276.9468
+6600 train 5.410007 (lr=1.2129e-05) (hash(x)=136948374)
+6900 val loss 5.7414
+6900 val perplexity 311.5153
+6900 train 5.658071 (lr=8.2849e-06) (hash(x)=134657776)
+6700 val loss 5.6229
+6700 val perplexity 276.6846
+6700 train 5.419105 (lr=1.1515e-05) (hash(x)=146268592)
+7000 val loss 5.7382
+7000 val perplexity 310.4917
+7000 train 5.696815 (lr=7.8510e-06) (hash(x)=166721861)
+6800 val loss 5.6152
+6800 val perplexity 274.5602
+6800 train 5.570204 (lr=1.0923e-05) (hash(x)=152676836)
+7100 val loss 5.7366
+7100 val perplexity 310.0130
+7100 train 5.535275 (lr=7.4377e-06) (hash(x)=135496702)
+6900 val loss 5.6066
+6900 val perplexity 272.2095
+6900 train 5.530090 (lr=1.0356e-05) (hash(x)=134657776)
+7200 val loss 5.7336
+7200 val perplexity 309.0950
+7200 train 5.841577 (lr=7.0455e-06) (hash(x)=155567461)
+7000 val loss 5.6050
+7000 val perplexity 271.7800
+7000 train 5.559492 (lr=9.8138e-06) (hash(x)=166721861)
+7300 val loss 5.7267
+7300 val perplexity 306.9496
+7300 train 5.524444 (lr=6.6749e-06) (hash(x)=142803829)
+7100 val loss 5.6054
+7100 val perplexity 271.8886
+7100 train 5.398874 (lr=9.2971e-06) (hash(x)=135496702)
+7400 val loss 5.7253
+7400 val perplexity 306.5243
+7400 train 5.550263 (lr=6.3266e-06) (hash(x)=145294178)
+7200 val loss 5.5987
+7200 val perplexity 270.0845
+7200 train 5.709658 (lr=8.8068e-06) (hash(x)=155567461)
+7500 val loss 5.7234
+7500 val perplexity 305.9535
+7500 train 5.481024 (lr=6.0010e-06) (hash(x)=150573713)
+7300 val loss 5.5917
+7300 val perplexity 268.2025
+7300 train 5.395301 (lr=8.3436e-06) (hash(x)=142803829)
+7600 val loss 5.7208
+7600 val perplexity 305.1357
+7600 train 5.742909 (lr=5.6986e-06) (hash(x)=142771511)
+7400 val loss 5.5920
+7400 val perplexity 268.2589
+7400 train 5.410421 (lr=7.9082e-06) (hash(x)=145294178)
+7700 val loss 5.7145
+7700 val perplexity 303.2360
+7700 train 5.618597 (lr=5.4198e-06) (hash(x)=143602175)
+7500 val loss 5.5897
+7500 val perplexity 267.6545
+7500 train 5.347271 (lr=7.5012e-06) (hash(x)=150573713)
+7800 val loss 5.7126
+7800 val perplexity 302.6505
+7800 train 5.756348 (lr=5.1650e-06) (hash(x)=152379862)
+7600 val loss 5.5874
+7600 val perplexity 267.0536
+7600 train 5.605257 (lr=7.1232e-06) (hash(x)=142771511)
+7900 val loss 5.7111
+7900 val perplexity 302.1945
+7900 train 5.537019 (lr=4.9347e-06) (hash(x)=146655921)
+7700 val loss 5.5809
+7700 val perplexity 265.2994
+7700 train 5.483287 (lr=6.7747e-06) (hash(x)=143602175)
+8000 val loss 5.7129
+8000 val perplexity 302.7426
+8000 train 5.840605 (lr=4.7291e-06) (hash(x)=148262482)
+7800 val loss 5.5780
+7800 val perplexity 264.5419
+7800 train 5.642337 (lr=6.4563e-06) (hash(x)=152379862)
+8100 val loss 5.7041
+8100 val perplexity 300.1092
+8100 train 5.641214 (lr=4.5486e-06) (hash(x)=147683655)
+7900 val loss 5.5774
+7900 val perplexity 264.3962
+7900 train 5.391592 (lr=6.1684e-06) (hash(x)=146655921)
+8200 val loss 5.7048
+8200 val perplexity 300.2959
+8200 train 5.807380 (lr=4.3933e-06) (hash(x)=157312987)
+8000 val loss 5.5782
+8000 val perplexity 264.5894
+8000 train 5.719192 (lr=5.9114e-06) (hash(x)=148262482)
+8300 val loss 5.7019
+8300 val perplexity 299.4422
+8300 train 5.664444 (lr=4.2636e-06) (hash(x)=141107543)
+8100 val loss 5.5697
+8100 val perplexity 262.3572
+8100 train 5.517353 (lr=5.6857e-06) (hash(x)=147683655)
+8400 val loss 5.6976
+8400 val perplexity 298.1539
+8400 train 5.683732 (lr=4.1596e-06) (hash(x)=141323024)
+8200 val loss 5.5705
+8200 val perplexity 262.5577
+8200 train 5.688238 (lr=5.4917e-06) (hash(x)=157312987)
+8500 val loss 5.6963
+8500 val perplexity 297.7502
+8500 train 5.670836 (lr=4.0815e-06) (hash(x)=150696521)
+8300 val loss 5.5666
+8300 val perplexity 261.5423
+8300 train 5.523445 (lr=5.3295e-06) (hash(x)=141107543)
+8600 val loss 5.6959
+8600 val perplexity 297.6495
+8600 train 5.643767 (lr=4.0294e-06) (hash(x)=162288191)
+8400 val loss 5.5627
+8400 val perplexity 260.5137
+8400 train 5.558888 (lr=5.1995e-06) (hash(x)=141323024)
+8700 val loss 5.6922
+8700 val perplexity 296.5599
+8700 train 5.560731 (lr=4.0033e-06) (hash(x)=152860941)
+8500 val loss 5.5615
+8500 val perplexity 260.2245
+8500 train 5.526323 (lr=5.1019e-06) (hash(x)=150696521)
+8749 val loss 5.6907
+8749 val perplexity 296.0869

attention_kindselective_n_heads4_seed1341/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7b19bcdcc795560643bed6a5252e945b59403b36d9fb80a6c7c1978ec18d4c4d
+oid sha256:60f9795517c86381d63141ace51ce5da55dbc8de0aa3e2ee26b0ffd6d3c88332
 size 92843394

attention_kindselective_n_heads4_seed1341/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0642c7ecdf284e5d06d2b300e8988d6e95c5e528eeaf54c124ecd155f33aaac9
+oid sha256:43538ac4619bfa4754892879de2c96b45a186312926324b7e0fa80cc11d61f8b
 size 179406214