Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1342/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4.5e-5_61440_4_1342", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1342", "n_embd": 256}

attention_kindselective_n_heads4_seed1342/log2.txt

@@ -1,267 +1,522 @@
 max_steps: 8750
+200 val loss 9.4639
+200 val perplexity 12885.6191
+200 train 9.532709 (lr=1.6080e-05) (hash(x)=155934075)
 0 val loss 11.2898
 0 val perplexity 80004.5469
-0 train 11.289043 (lr=9.0000e-08) (hash(x)=150332693)
-100 val loss 10.0243
-100 val perplexity 22568.6875
-100 train 9.978326 (lr=9.0900e-06) (hash(x)=149277319)
-200 val loss 9.3785
-200 val perplexity 11831.2090
-200 train 9.445846 (lr=1.8090e-05) (hash(x)=155934075)
-300 val loss 8.2194
-300 val perplexity 3712.2971
-300 train 8.254610 (lr=2.7090e-05) (hash(x)=165402628)
-400 val loss 7.6163
-400 val perplexity 2031.0946
-400 train 7.421521 (lr=3.6090e-05) (hash(x)=139168939)
-500 val loss 7.4349
-500 val perplexity 1694.0548
-500 train 7.357471 (lr=4.5000e-05) (hash(x)=147134338)
-600 val loss 7.3305
-600 val perplexity 1526.1565
-600 train 7.243377 (lr=4.4985e-05) (hash(x)=159086459)
-700 val loss 7.2503
-700 val perplexity 1408.4863
-700 train 7.288620 (lr=4.4941e-05) (hash(x)=152399099)
-800 val loss 7.1363
-800 val perplexity 1256.7866
-800 train 7.140655 (lr=4.4868e-05) (hash(x)=156548843)
-900 val loss 7.0484
-900 val perplexity 1151.0243
-900 train 7.021039 (lr=4.4766e-05) (hash(x)=157889911)
-1000 val loss 6.9766
-1000 val perplexity 1071.3145
-1000 train 6.774483 (lr=4.4634e-05) (hash(x)=142943707)
-1100 val loss 6.9135
-1100 val perplexity 1005.7903
-1100 train 6.912020 (lr=4.4474e-05) (hash(x)=148938689)
-1200 val loss 6.8143
-1200 val perplexity 910.7801
-1200 train 6.954645 (lr=4.4285e-05) (hash(x)=161962903)
-1300 val loss 6.7411
-1300 val perplexity 846.4569
-1300 train 6.723616 (lr=4.4068e-05) (hash(x)=145979235)
-1400 val loss 6.6754
-1400 val perplexity 792.6756
-1400 train 6.625307 (lr=4.3822e-05) (hash(x)=159157507)
-1500 val loss 6.6153
-1500 val perplexity 746.3971
-1500 train 6.649820 (lr=4.3549e-05) (hash(x)=145943130)
-1600 val loss 6.5401
-1600 val perplexity 692.3466
-1600 train 6.341069 (lr=4.3249e-05) (hash(x)=143249173)
-1700 val loss 6.4872
-1700 val perplexity 656.6964
-1700 train 6.551693 (lr=4.2922e-05) (hash(x)=173907143)
-1800 val loss 6.4411
-1800 val perplexity 627.1124
-1800 train 6.368075 (lr=4.2569e-05) (hash(x)=138363755)
-1900 val loss 6.4086
-1900 val perplexity 607.0637
-1900 train 6.216325 (lr=4.2190e-05) (hash(x)=146341390)
-2000 val loss 6.3554
-2000 val perplexity 575.6113
-2000 train 6.384245 (lr=4.1785e-05) (hash(x)=154559671)
-2100 val loss 6.3281
-2100 val perplexity 560.0664
-2100 train 6.378413 (lr=4.1356e-05) (hash(x)=155607137)
-2200 val loss 6.2939
-2200 val perplexity 541.2359
-2200 train 6.150013 (lr=4.0903e-05) (hash(x)=153010221)
-2300 val loss 6.2609
-2300 val perplexity 523.7111
-2300 train 6.278090 (lr=4.0426e-05) (hash(x)=160097777)
-2400 val loss 6.2372
-2400 val perplexity 511.4395
-2400 train 6.214860 (lr=3.9927e-05) (hash(x)=135590094)
-2500 val loss 6.2120
-2500 val perplexity 498.6850
-2500 train 6.329730 (lr=3.9406e-05) (hash(x)=147042929)
-2600 val loss 6.1775
-2600 val perplexity 481.7953
-2600 train 6.138986 (lr=3.8863e-05) (hash(x)=163793881)
-2700 val loss 6.1626
-2700 val perplexity 474.6808
-2700 train 6.195170 (lr=3.8300e-05) (hash(x)=153598698)
-2800 val loss 6.1463
-2800 val perplexity 466.9887
-2800 train 5.944228 (lr=3.7717e-05) (hash(x)=133439803)
-2900 val loss 6.1207
-2900 val perplexity 455.1814
-2900 train 6.059956 (lr=3.7116e-05) (hash(x)=148239158)
-3000 val loss 6.0949
-3000 val perplexity 443.6075
-3000 train 6.099676 (lr=3.6496e-05) (hash(x)=150991971)
-3100 val loss 6.0827
-3100 val perplexity 438.2101
-3100 train 6.087112 (lr=3.5860e-05) (hash(x)=157791832)
-3200 val loss 6.0558
-3200 val perplexity 426.5774
-3200 train 6.124968 (lr=3.5207e-05) (hash(x)=163746305)
-3300 val loss 6.0481
-3300 val perplexity 423.3175
-3300 train 5.978697 (lr=3.4539e-05) (hash(x)=155120269)
-3400 val loss 6.0166
-3400 val perplexity 410.1862
-3400 train 5.958898 (lr=3.3857e-05) (hash(x)=152748451)
-3500 val loss 6.0043
-3500 val perplexity 405.1582
-3500 train 5.885794 (lr=3.3162e-05) (hash(x)=147395004)
-3600 val loss 5.9914
-3600 val perplexity 399.9552
-3600 train 6.009531 (lr=3.2455e-05) (hash(x)=157554250)
-3700 val loss 5.9740
-3700 val perplexity 393.0799
-3700 train 5.826887 (lr=3.1736e-05) (hash(x)=147953321)
-3800 val loss 5.9551
-3800 val perplexity 385.7162
-3800 train 5.805262 (lr=3.1008e-05) (hash(x)=146898609)
-3900 val loss 5.9564
-3900 val perplexity 386.2080
-3900 train 5.996837 (lr=3.0270e-05) (hash(x)=150115282)
-4000 val loss 5.9218
-4000 val perplexity 373.0663
-4000 train 5.790571 (lr=2.9524e-05) (hash(x)=142448619)
-4100 val loss 5.9223
-4100 val perplexity 373.2632
-4100 train 5.654961 (lr=2.8771e-05) (hash(x)=141244123)
-4200 val loss 5.9140
-4200 val perplexity 370.1866
-4200 train 5.927835 (lr=2.8013e-05) (hash(x)=165913661)
-4300 val loss 5.8861
-4300 val perplexity 360.0132
-4300 train 5.917614 (lr=2.7250e-05) (hash(x)=155744823)
-4400 val loss 5.8769
-4400 val perplexity 356.6874
-4400 train 5.872905 (lr=2.6483e-05) (hash(x)=160461243)
-4500 val loss 5.8657
-4500 val perplexity 352.7454
-4500 train 5.776689 (lr=2.5714e-05) (hash(x)=156073129)
-4600 val loss 5.8505
-4600 val perplexity 347.4088
-4600 train 5.897415 (lr=2.4943e-05) (hash(x)=152294132)
-4700 val loss 5.8393
-4700 val perplexity 343.5292
-4700 train 5.771152 (lr=2.4172e-05) (hash(x)=140904569)
-4800 val loss 5.8338
-4800 val perplexity 341.6528
-4800 train 5.739367 (lr=2.3402e-05) (hash(x)=156191635)
-4900 val loss 5.8181
-4900 val perplexity 336.3423
-4900 train 5.924228 (lr=2.2633e-05) (hash(x)=145063976)
-5000 val loss 5.8120
-5000 val perplexity 334.2776
-5000 train 5.754147 (lr=2.1868e-05) (hash(x)=160110619)
-5100 val loss 5.7954
-5100 val perplexity 328.7745
-5100 train 5.821685 (lr=2.1107e-05) (hash(x)=156270070)
-5200 val loss 5.7867
-5200 val perplexity 325.9519
-5200 train 5.674892 (lr=2.0351e-05) (hash(x)=138954242)
-5300 val loss 5.7796
-5300 val perplexity 323.6199
-5300 train 5.633004 (lr=1.9602e-05) (hash(x)=146472367)
-5400 val loss 5.7699
-5400 val perplexity 320.5030
-5400 train 5.702404 (lr=1.8860e-05) (hash(x)=146753405)
-5500 val loss 5.7575
-5500 val perplexity 316.5528
-5500 train 5.706319 (lr=1.8127e-05) (hash(x)=147757398)
-5600 val loss 5.7557
-5600 val perplexity 315.9973
-5600 train 5.624521 (lr=1.7403e-05) (hash(x)=151291757)
-5700 val loss 5.7424
-5700 val perplexity 311.8269
-5700 train 5.793109 (lr=1.6690e-05) (hash(x)=156274392)
-5800 val loss 5.7336
-5800 val perplexity 309.0707
-5800 train 5.702478 (lr=1.5989e-05) (hash(x)=152612266)
-5900 val loss 5.7301
-5900 val perplexity 307.9881
-5900 train 5.893963 (lr=1.5300e-05) (hash(x)=148973201)
-6000 val loss 5.7217
-6000 val perplexity 305.4335
-6000 train 5.687572 (lr=1.4625e-05) (hash(x)=158311490)
-6100 val loss 5.7135
-6100 val perplexity 302.9321
-6100 train 5.645096 (lr=1.3965e-05) (hash(x)=147541319)
-6200 val loss 5.7129
-6200 val perplexity 302.7402
-6200 train 5.777384 (lr=1.3320e-05) (hash(x)=154753980)
-6300 val loss 5.7037
-6300 val perplexity 299.9899
-6300 train 5.601553 (lr=1.2692e-05) (hash(x)=140798998)
-6400 val loss 5.6993
-6400 val perplexity 298.6710
-6400 train 5.588508 (lr=1.2081e-05) (hash(x)=149299516)
-6500 val loss 5.6953
-6500 val perplexity 297.4683
-6500 train 5.682362 (lr=1.1489e-05) (hash(x)=148495806)
-6600 val loss 5.6878
-6600 val perplexity 295.2325
-6600 train 5.665105 (lr=1.0916e-05) (hash(x)=153664387)
-6700 val loss 5.6849
-6700 val perplexity 294.3740
-6700 train 5.661620 (lr=1.0363e-05) (hash(x)=153333491)
-6800 val loss 5.6854
-6800 val perplexity 294.5452
-6800 train 5.438633 (lr=9.8310e-06) (hash(x)=150149382)
-6900 val loss 5.6780
-6900 val perplexity 292.3573
-6900 train 5.763326 (lr=9.3205e-06) (hash(x)=152379767)
-7000 val loss 5.6708
-7000 val perplexity 290.2598
-7000 train 5.719643 (lr=8.8324e-06) (hash(x)=150432537)
-7100 val loss 5.6688
-7100 val perplexity 289.6923
-7100 train 5.545722 (lr=8.3674e-06) (hash(x)=148277836)
-7200 val loss 5.6698
-7200 val perplexity 289.9790
-7200 train 5.664272 (lr=7.9261e-06) (hash(x)=168591659)
-7300 val loss 5.6615
-7300 val perplexity 287.5750
-7300 train 5.642782 (lr=7.5093e-06) (hash(x)=152764851)
-7400 val loss 5.6584
-7400 val perplexity 286.6996
-7400 train 5.621808 (lr=7.1174e-06) (hash(x)=142920131)
-7500 val loss 5.6559
-7500 val perplexity 285.9665
-7500 train 5.656323 (lr=6.7511e-06) (hash(x)=146954649)
-7600 val loss 5.6564
-7600 val perplexity 286.1283
-7600 train 5.434806 (lr=6.4109e-06) (hash(x)=144529566)
-7700 val loss 5.6503
-7700 val perplexity 284.3646
-7700 train 5.609683 (lr=6.0972e-06) (hash(x)=142669945)
-7800 val loss 5.6460
-7800 val perplexity 283.1537
-7800 train 5.704182 (lr=5.8107e-06) (hash(x)=161563764)
-7900 val loss 5.6443
-7900 val perplexity 282.6702
-7900 train 5.439746 (lr=5.5515e-06) (hash(x)=141516532)
-8000 val loss 5.6455
-8000 val perplexity 283.0064
-8000 train 5.515144 (lr=5.3203e-06) (hash(x)=152418457)
-8100 val loss 5.6398
-8100 val perplexity 281.4167
-8100 train 5.624706 (lr=5.1172e-06) (hash(x)=161109470)
-8200 val loss 5.6360
-8200 val perplexity 280.3446
-8200 train 5.758985 (lr=4.9425e-06) (hash(x)=154408822)
-8300 val loss 5.6342
-8300 val perplexity 279.8434
-8300 train 5.520042 (lr=4.7966e-06) (hash(x)=146431691)
-8400 val loss 5.6345
-8400 val perplexity 279.9092
-8400 train 5.412016 (lr=4.6796e-06) (hash(x)=150182527)
-8500 val loss 5.6286
-8500 val perplexity 278.2667
-8500 train 5.693805 (lr=4.5917e-06) (hash(x)=154828709)
-8600 val loss 5.6284
-8600 val perplexity 278.2239
-8600 train 5.539424 (lr=4.5330e-06) (hash(x)=154995428)
-8700 val loss 5.6285
-8700 val perplexity 278.2420
-8700 train 5.523919 (lr=4.5037e-06) (hash(x)=142111043)
-8749 val loss 5.6262
-8749 val perplexity 277.6001
+300 val loss 8.3583
+300 val perplexity 4265.4746
+300 train 8.394882 (lr=2.4080e-05) (hash(x)=165402628)
+0 train 11.289051 (lr=1.0000e-07) (hash(x)=150332693)
+400 val loss 7.6790
+400 val perplexity 2162.5322
+400 train 7.493039 (lr=3.2080e-05) (hash(x)=139168939)
+100 val loss 9.9962
+100 val perplexity 21943.4395
+100 train 9.948080 (lr=1.0100e-05) (hash(x)=149277319)
+500 val loss 7.4664
+500 val perplexity 1748.2917
+500 train 7.390862 (lr=4.0000e-05) (hash(x)=147134338)
+200 val loss 9.3283
+200 val perplexity 11251.4990
+200 train 9.395099 (lr=2.0100e-05) (hash(x)=155934075)
+600 val loss 7.3681
+600 val perplexity 1584.6302
+600 train 7.280435 (lr=3.9987e-05) (hash(x)=159086459)
+300 val loss 8.1346
+300 val perplexity 3410.3855
+300 train 8.168678 (lr=3.0100e-05) (hash(x)=165402628)
+700 val loss 7.2804
+700 val perplexity 1451.5247
+700 train 7.317653 (lr=3.9948e-05) (hash(x)=152399099)
+400 val loss 7.5968
+400 val perplexity 1991.7366
+400 train 7.398672 (lr=4.0100e-05) (hash(x)=139168939)
+800 val loss 7.1671
+800 val perplexity 1296.1224
+800 train 7.168324 (lr=3.9883e-05) (hash(x)=156548843)
+500 val loss 7.4448
+500 val perplexity 1711.0024
+500 train 7.364177 (lr=5.0000e-05) (hash(x)=147134338)
+900 val loss 7.0745
+900 val perplexity 1181.4358
+900 train 7.044015 (lr=3.9792e-05) (hash(x)=157889911)
+600 val loss 7.3553
+600 val perplexity 1564.4274
+600 train 7.269327 (lr=4.9984e-05) (hash(x)=159086459)
+1000 val loss 7.0000
+1000 val perplexity 1096.6268
+1000 train 6.798794 (lr=3.9675e-05) (hash(x)=142943707)
+700 val loss 7.2784
+700 val perplexity 1448.6952
+700 train 7.316185 (lr=4.9935e-05) (hash(x)=152399099)
+1100 val loss 6.9307
+1100 val perplexity 1023.2171
+1100 train 6.932874 (lr=3.9532e-05) (hash(x)=148938689)
+800 val loss 7.1807
+800 val perplexity 1313.8818
+800 train 7.183406 (lr=4.9853e-05) (hash(x)=156548843)
+1200 val loss 6.8365
+1200 val perplexity 931.1896
+1200 train 6.969981 (lr=3.9364e-05) (hash(x)=161962903)
+900 val loss 7.1040
+900 val perplexity 1216.8074
+900 train 7.077850 (lr=4.9739e-05) (hash(x)=157889911)
+1300 val loss 6.7581
+1300 val perplexity 861.0339
+1300 train 6.738302 (lr=3.9171e-05) (hash(x)=145979235)
+1000 val loss 7.0485
+1000 val perplexity 1151.1632
+1000 train 6.848722 (lr=4.9593e-05) (hash(x)=142943707)
+1400 val loss 6.6876
+1400 val perplexity 802.4104
+1400 train 6.635297 (lr=3.8953e-05) (hash(x)=159157507)
+1100 val loss 6.9970
+1100 val perplexity 1093.3792
+1100 train 7.001183 (lr=4.9415e-05) (hash(x)=148938689)
+1500 val loss 6.6290
+1500 val perplexity 756.6931
+1500 train 6.665385 (lr=3.8711e-05) (hash(x)=145943130)
+1200 val loss 6.9444
+1200 val perplexity 1037.3114
+1200 train 7.073627 (lr=4.9205e-05) (hash(x)=161962903)
+1600 val loss 6.5599
+1600 val perplexity 706.1689
+1600 train 6.357224 (lr=3.8444e-05) (hash(x)=143249173)
+1300 val loss 6.8886
+1300 val perplexity 981.0255
+1300 train 6.864804 (lr=4.8964e-05) (hash(x)=145979235)
+1700 val loss 6.5038
+1700 val perplexity 667.6825
+1700 train 6.566358 (lr=3.8153e-05) (hash(x)=173907143)
+1400 val loss 6.8403
+1400 val perplexity 934.7651
+1400 train 6.786602 (lr=4.8691e-05) (hash(x)=159157507)
+1800 val loss 6.4599
+1800 val perplexity 638.9742
+1800 train 6.385739 (lr=3.7839e-05) (hash(x)=138363755)
+1500 val loss 6.8001
+1500 val perplexity 897.9742
+1500 train 6.838896 (lr=4.8388e-05) (hash(x)=145943130)
+1900 val loss 6.4176
+1900 val perplexity 612.5451
+1900 train 6.228965 (lr=3.7502e-05) (hash(x)=146341390)
+1600 val loss 6.7494
+1600 val perplexity 853.5507
+1600 train 6.545481 (lr=4.8055e-05) (hash(x)=143249173)
+2000 val loss 6.3698
+2000 val perplexity 583.9255
+2000 train 6.398791 (lr=3.7143e-05) (hash(x)=154559671)
+1700 val loss 6.7104
+1700 val perplexity 820.8583
+1700 train 6.782251 (lr=4.7691e-05) (hash(x)=173907143)
+2100 val loss 6.3420
+2100 val perplexity 567.9370
+2100 train 6.400441 (lr=3.6761e-05) (hash(x)=155607137)
+1800 val loss 6.6697
+1800 val perplexity 788.1686
+1800 train 6.595098 (lr=4.7299e-05) (hash(x)=138363755)
+2200 val loss 6.3107
+2200 val perplexity 550.4556
+2200 train 6.170771 (lr=3.6358e-05) (hash(x)=153010221)
+1900 val loss 6.6298
+1900 val perplexity 757.3068
+1900 train 6.441909 (lr=4.6878e-05) (hash(x)=146341390)
+2300 val loss 6.2734
+2300 val perplexity 530.2939
+2300 train 6.293130 (lr=3.5935e-05) (hash(x)=160097777)
+2000 val loss 6.5908
+2000 val perplexity 728.3906
+2000 train 6.615541 (lr=4.6428e-05) (hash(x)=154559671)
+2400 val loss 6.2462
+2400 val perplexity 516.0425
+2400 train 6.216858 (lr=3.5491e-05) (hash(x)=135590094)
+2100 val loss 6.5619
+2100 val perplexity 707.5947
+2100 train 6.621873 (lr=4.5951e-05) (hash(x)=155607137)
+2500 val loss 6.2243
+2500 val perplexity 504.8625
+2500 train 6.341967 (lr=3.5027e-05) (hash(x)=147042929)
+2200 val loss 6.5254
+2200 val perplexity 682.2401
+2200 train 6.376645 (lr=4.5448e-05) (hash(x)=153010221)
+2600 val loss 6.1893
+2600 val perplexity 487.4810
+2600 train 6.154806 (lr=3.4545e-05) (hash(x)=163793881)
+2300 val loss 6.5038
+2300 val perplexity 667.6698
+2300 train 6.525729 (lr=4.4918e-05) (hash(x)=160097777)
+2700 val loss 6.1648
+2700 val perplexity 475.7272
+2700 train 6.199413 (lr=3.4044e-05) (hash(x)=153598698)
+2400 val loss 6.4631
+2400 val perplexity 641.0137
+2400 train 6.417774 (lr=4.4363e-05) (hash(x)=135590094)
+2800 val loss 6.1510
+2800 val perplexity 469.1915
+2800 train 5.944895 (lr=3.3526e-05) (hash(x)=133439803)
+2500 val loss 6.4394
+2500 val perplexity 626.0601
+2500 train 6.565438 (lr=4.3784e-05) (hash(x)=147042929)
+2900 val loss 6.1239
+2900 val perplexity 456.6519
+2900 train 6.061853 (lr=3.2992e-05) (hash(x)=148239158)
+2600 val loss 6.4139
+2600 val perplexity 610.2821
+2600 train 6.378555 (lr=4.3181e-05) (hash(x)=163793881)
+3000 val loss 6.1035
+3000 val perplexity 447.3994
+3000 train 6.109071 (lr=3.2441e-05) (hash(x)=150991971)
+2700 val loss 6.3924
+2700 val perplexity 597.3093
+2700 train 6.419984 (lr=4.2555e-05) (hash(x)=153598698)
+3100 val loss 6.0881
+3100 val perplexity 440.5812
+3100 train 6.093895 (lr=3.1875e-05) (hash(x)=157791832)
+2800 val loss 6.3788
+2800 val perplexity 589.2058
+2800 train 6.179019 (lr=4.1908e-05) (hash(x)=133439803)
+3200 val loss 6.0635
+3200 val perplexity 429.8743
+3200 train 6.134553 (lr=3.1295e-05) (hash(x)=163746305)
+2900 val loss 6.3535
+2900 val perplexity 574.4886
+2900 train 6.295610 (lr=4.1240e-05) (hash(x)=148239158)
+3300 val loss 6.0547
+3300 val perplexity 426.1033
+3300 train 5.977380 (lr=3.0702e-05) (hash(x)=155120269)
+3000 val loss 6.3357
+3000 val perplexity 564.3859
+3000 train 6.326377 (lr=4.0551e-05) (hash(x)=150991971)
+3400 val loss 6.0275
+3400 val perplexity 414.6939
+3400 train 5.970883 (lr=3.0095e-05) (hash(x)=152748451)
+3100 val loss 6.3179
+3100 val perplexity 554.4232
+3100 train 6.331873 (lr=3.9844e-05) (hash(x)=157791832)
+3500 val loss 6.0154
+3500 val perplexity 409.6898
+3500 train 5.895640 (lr=2.9477e-05) (hash(x)=147395004)
+3200 val loss 6.2993
+3200 val perplexity 544.2070
+3200 train 6.374971 (lr=3.9119e-05) (hash(x)=163746305)
+3600 val loss 6.0009
+3600 val perplexity 403.7821
+3600 train 6.022380 (lr=2.8849e-05) (hash(x)=157554250)
+3300 val loss 6.2950
+3300 val perplexity 541.8779
+3300 train 6.231050 (lr=3.8377e-05) (hash(x)=155120269)
+3700 val loss 5.9809
+3700 val perplexity 395.8124
+3700 train 5.834892 (lr=2.8210e-05) (hash(x)=147953321)
+3400 val loss 6.2608
+3400 val perplexity 523.6501
+3400 train 6.201460 (lr=3.7619e-05) (hash(x)=152748451)
+3800 val loss 5.9648
+3800 val perplexity 389.4606
+3800 train 5.812018 (lr=2.7562e-05) (hash(x)=146898609)
+3500 val loss 6.2453
+3500 val perplexity 515.5919
+3500 train 6.119865 (lr=3.6847e-05) (hash(x)=147395004)
+3900 val loss 5.9597
+3900 val perplexity 387.4993
+3900 train 6.004293 (lr=2.6907e-05) (hash(x)=150115282)
+3600 val loss 6.2349
+3600 val perplexity 510.2739
+3600 train 6.257258 (lr=3.6061e-05) (hash(x)=157554250)
+4000 val loss 5.9349
+4000 val perplexity 377.9908
+4000 train 5.803768 (lr=2.6244e-05) (hash(x)=142448619)
+3700 val loss 6.2165
+3700 val perplexity 500.9630
+3700 train 6.068434 (lr=3.5263e-05) (hash(x)=147953321)
+4100 val loss 5.9360
+4100 val perplexity 378.4156
+4100 train 5.667750 (lr=2.5575e-05) (hash(x)=141244123)
+3800 val loss 6.2068
+3800 val perplexity 496.1251
+3800 train 6.068558 (lr=3.4453e-05) (hash(x)=146898609)
+4200 val loss 5.9272
+4200 val perplexity 375.1158
+4200 train 5.949145 (lr=2.4900e-05) (hash(x)=165913661)
+3900 val loss 6.2001
+3900 val perplexity 492.8126
+3900 train 6.240398 (lr=3.3633e-05) (hash(x)=150115282)
+4300 val loss 5.9009
+4300 val perplexity 365.3578
+4300 train 5.933804 (lr=2.4222e-05) (hash(x)=155744823)
+4000 val loss 6.1741
+4000 val perplexity 480.1424
+4000 train 6.023779 (lr=3.2805e-05) (hash(x)=142448619)
+4400 val loss 5.8893
+4400 val perplexity 361.1411
+4400 train 5.882122 (lr=2.3540e-05) (hash(x)=160461243)
+4100 val loss 6.1704
+4100 val perplexity 478.3836
+4100 train 5.888917 (lr=3.1968e-05) (hash(x)=141244123)
+4500 val loss 5.8792
+4500 val perplexity 357.5369
+4500 train 5.789831 (lr=2.2856e-05) (hash(x)=156073129)
+4200 val loss 6.1580
+4200 val perplexity 472.5033
+4200 train 6.195307 (lr=3.1126e-05) (hash(x)=165913661)
+4600 val loss 5.8617
+4600 val perplexity 351.3350
+4600 train 5.907257 (lr=2.2171e-05) (hash(x)=152294132)
+4300 val loss 6.1397
+4300 val perplexity 463.9040
+4300 train 6.172644 (lr=3.0277e-05) (hash(x)=155744823)
+4700 val loss 5.8537
+4700 val perplexity 348.5111
+4700 train 5.785102 (lr=2.1486e-05) (hash(x)=140904569)
+4400 val loss 6.1511
+4400 val perplexity 469.2470
+4400 train 6.134822 (lr=2.9425e-05) (hash(x)=160461243)
+4800 val loss 5.8460
+4800 val perplexity 345.8560
+4800 train 5.754941 (lr=2.0801e-05) (hash(x)=156191635)
+4500 val loss 6.1273
+4500 val perplexity 458.1988
+4500 train 6.053799 (lr=2.8571e-05) (hash(x)=156073129)
+4900 val loss 5.8350
+4900 val perplexity 342.0713
+4900 train 5.941958 (lr=2.0118e-05) (hash(x)=145063976)
+4600 val loss 6.1165
+4600 val perplexity 453.2633
+4600 train 6.166283 (lr=2.7714e-05) (hash(x)=152294132)
+5000 val loss 5.8263
+5000 val perplexity 339.0927
+5000 train 5.771696 (lr=1.9438e-05) (hash(x)=160110619)
+4700 val loss 6.1035
+4700 val perplexity 447.4263
+4700 train 6.038318 (lr=2.6857e-05) (hash(x)=140904569)
+5100 val loss 5.8118
+5100 val perplexity 334.2325
+5100 train 5.841002 (lr=1.8762e-05) (hash(x)=156270070)
+4800 val loss 6.0977
+4800 val perplexity 444.8199
+4800 train 6.001534 (lr=2.6002e-05) (hash(x)=156191635)
+5200 val loss 5.8029
+5200 val perplexity 331.2700
+5200 train 5.687089 (lr=1.8090e-05) (hash(x)=138954242)
+4900 val loss 6.0911
+4900 val perplexity 441.9067
+4900 train 6.159082 (lr=2.5148e-05) (hash(x)=145063976)
+5300 val loss 5.7946
+5300 val perplexity 328.5219
+5300 train 5.646440 (lr=1.7424e-05) (hash(x)=146472367)
+5000 val loss 6.0757
+5000 val perplexity 435.1693
+5000 train 6.022645 (lr=2.4298e-05) (hash(x)=160110619)
+5400 val loss 5.7864
+5400 val perplexity 325.8291
+5400 train 5.721862 (lr=1.6765e-05) (hash(x)=146753405)
+5100 val loss 6.0679
+5100 val perplexity 431.7900
+5100 train 6.077213 (lr=2.3452e-05) (hash(x)=156270070)
+5500 val loss 5.7769
+5500 val perplexity 322.7703
+5500 train 5.724141 (lr=1.6113e-05) (hash(x)=147757398)
+5200 val loss 6.0523
+5200 val perplexity 425.0701
+5200 train 5.917948 (lr=2.2613e-05) (hash(x)=138954242)
+5600 val loss 5.7728
+5600 val perplexity 321.4291
+5600 train 5.645650 (lr=1.5469e-05) (hash(x)=151291757)
+5300 val loss 6.0469
+5300 val perplexity 422.7817
+5300 train 5.894984 (lr=2.1780e-05) (hash(x)=146472367)
+5700 val loss 5.7594
+5700 val perplexity 317.1720
+5700 train 5.807491 (lr=1.4836e-05) (hash(x)=156274392)
+5400 val loss 6.0328
+5400 val perplexity 416.8636
+5400 train 5.969725 (lr=2.0956e-05) (hash(x)=146753405)
+5800 val loss 5.7541
+5800 val perplexity 315.4712
+5800 train 5.722164 (lr=1.4212e-05) (hash(x)=152612266)
+5500 val loss 6.0239
+5500 val perplexity 413.1928
+5500 train 5.972818 (lr=2.0141e-05) (hash(x)=147757398)
+5900 val loss 5.7477
+5900 val perplexity 313.4535
+5900 train 5.910150 (lr=1.3600e-05) (hash(x)=148973201)
+5600 val loss 6.0236
+5600 val perplexity 413.0589
+5600 train 5.907842 (lr=1.9337e-05) (hash(x)=151291757)
+6000 val loss 5.7391
+6000 val perplexity 310.7723
+6000 train 5.699615 (lr=1.3000e-05) (hash(x)=158311490)
+5700 val loss 6.0070
+5700 val perplexity 406.2433
+5700 train 6.048605 (lr=1.8545e-05) (hash(x)=156274392)
+6100 val loss 5.7323
+6100 val perplexity 308.6890
+6100 train 5.659322 (lr=1.2413e-05) (hash(x)=147541319)
+5800 val loss 5.9992
+5800 val perplexity 403.1204
+5800 train 5.955314 (lr=1.7765e-05) (hash(x)=152612266)
+6200 val loss 5.7315
+6200 val perplexity 308.4253
+6200 train 5.799207 (lr=1.1840e-05) (hash(x)=154753980)
+5900 val loss 5.9945
+5900 val perplexity 401.2339
+5900 train 6.142016 (lr=1.7000e-05) (hash(x)=148973201)
+6300 val loss 5.7249
+6300 val perplexity 306.4074
+6300 train 5.620091 (lr=1.1282e-05) (hash(x)=140798998)
+6000 val loss 5.9841
+6000 val perplexity 397.0659
+6000 train 5.956523 (lr=1.6250e-05) (hash(x)=158311490)
+6400 val loss 5.7182
+6400 val perplexity 304.3530
+6400 train 5.600909 (lr=1.0739e-05) (hash(x)=149299516)
+6100 val loss 5.9760
+6100 val perplexity 393.8625
+6100 train 5.886071 (lr=1.5516e-05) (hash(x)=147541319)
+6500 val loss 5.7138
+6500 val perplexity 303.0228
+6500 train 5.701873 (lr=1.0213e-05) (hash(x)=148495806)
+6200 val loss 5.9756
+6200 val perplexity 393.7010
+6200 train 6.034393 (lr=1.4800e-05) (hash(x)=154753980)
+6600 val loss 5.7058
+6600 val perplexity 300.6016
+6600 train 5.683437 (lr=9.7032e-06) (hash(x)=153664387)
+6300 val loss 5.9669
+6300 val perplexity 390.2875
+6300 train 5.861510 (lr=1.4102e-05) (hash(x)=140798998)
+6700 val loss 5.7030
+6700 val perplexity 299.7645
+6700 train 5.687989 (lr=9.2116e-06) (hash(x)=153333491)
+6400 val loss 5.9614
+6400 val perplexity 388.1428
+6400 train 5.835191 (lr=1.3424e-05) (hash(x)=149299516)
+6800 val loss 5.7039
+6800 val perplexity 300.0225
+6800 train 5.457414 (lr=8.7387e-06) (hash(x)=150149382)
+6500 val loss 5.9534
+6500 val perplexity 385.0675
+6500 train 5.958037 (lr=1.2766e-05) (hash(x)=148495806)
+6900 val loss 5.6986
+6900 val perplexity 298.4626
+6900 train 5.785464 (lr=8.2849e-06) (hash(x)=152379767)
+6600 val loss 5.9466
+6600 val perplexity 382.4535
+6600 train 5.915279 (lr=1.2129e-05) (hash(x)=153664387)
+7000 val loss 5.6892
+7000 val perplexity 295.6606
+7000 train 5.736051 (lr=7.8510e-06) (hash(x)=150432537)
+6700 val loss 5.9436
+6700 val perplexity 381.3058
+6700 train 5.918467 (lr=1.1515e-05) (hash(x)=153333491)
+7100 val loss 5.6892
+7100 val perplexity 295.6718
+7100 train 5.562969 (lr=7.4377e-06) (hash(x)=148277836)
+6800 val loss 5.9436
+6800 val perplexity 381.3205
+6800 train 5.689354 (lr=1.0923e-05) (hash(x)=150149382)
+7200 val loss 5.6873
+7200 val perplexity 295.0999
+7200 train 5.689114 (lr=7.0455e-06) (hash(x)=168591659)
+6900 val loss 5.9350
+6900 val perplexity 378.0223
+6900 train 6.004079 (lr=1.0356e-05) (hash(x)=152379767)
+7300 val loss 5.6811
+7300 val perplexity 293.2657
+7300 train 5.662244 (lr=6.6749e-06) (hash(x)=152764851)
+7000 val loss 5.9268
+7000 val perplexity 374.9409
+7000 train 5.972569 (lr=9.8138e-06) (hash(x)=150432537)
+7400 val loss 5.6773
+7400 val perplexity 292.1537
+7400 train 5.639318 (lr=6.3266e-06) (hash(x)=142920131)
+7100 val loss 5.9259
+7100 val perplexity 374.5984
+7100 train 5.799407 (lr=9.2971e-06) (hash(x)=148277836)
+7500 val loss 5.6755
+7500 val perplexity 291.6258
+7500 train 5.671338 (lr=6.0010e-06) (hash(x)=146954649)
+7200 val loss 5.9228
+7200 val perplexity 373.4558
+7200 train 5.928644 (lr=8.8068e-06) (hash(x)=168591659)
+7600 val loss 5.6754
+7600 val perplexity 291.5921
+7600 train 5.457974 (lr=5.6986e-06) (hash(x)=144529566)
+7300 val loss 5.9162
+7300 val perplexity 371.0097
+7300 train 5.890861 (lr=8.3436e-06) (hash(x)=152764851)
+7700 val loss 5.6700
+7700 val perplexity 290.0484
+7700 train 5.628642 (lr=5.4198e-06) (hash(x)=142669945)
+7400 val loss 5.9115
+7400 val perplexity 369.2667
+7400 train 5.861132 (lr=7.9082e-06) (hash(x)=142920131)
+7800 val loss 5.6662
+7800 val perplexity 288.9460
+7800 train 5.725795 (lr=5.1650e-06) (hash(x)=161563764)
+7500 val loss 5.9086
+7500 val perplexity 368.1959
+7500 train 5.903454 (lr=7.5012e-06) (hash(x)=146954649)
+7900 val loss 5.6640
+7900 val perplexity 288.2950
+7900 train 5.460674 (lr=4.9347e-06) (hash(x)=141516532)
+7600 val loss 5.9086
+7600 val perplexity 368.1859
+7600 train 5.701641 (lr=7.1232e-06) (hash(x)=144529566)
+8000 val loss 5.6656
+8000 val perplexity 288.7531
+8000 train 5.532639 (lr=4.7291e-06) (hash(x)=152418457)
+7700 val loss 5.9026
+7700 val perplexity 365.9743
+7700 train 5.854231 (lr=6.7747e-06) (hash(x)=142669945)
+8100 val loss 5.6601
+8100 val perplexity 287.1883
+8100 train 5.657063 (lr=4.5486e-06) (hash(x)=161109470)
+7800 val loss 5.8985
+7800 val perplexity 364.4813
+7800 train 5.976606 (lr=6.4563e-06) (hash(x)=161563764)
+8200 val loss 5.6564
+8200 val perplexity 286.1299
+8200 train 5.783814 (lr=4.3933e-06) (hash(x)=154408822)
+7900 val loss 5.8950
+7900 val perplexity 363.2101
+7900 train 5.693729 (lr=6.1684e-06) (hash(x)=141516532)
+8300 val loss 5.6549
+8300 val perplexity 285.7017
+8300 train 5.537910 (lr=4.2636e-06) (hash(x)=146431691)
+8000 val loss 5.8972
+8000 val perplexity 364.0090
+8000 train 5.762141 (lr=5.9114e-06) (hash(x)=152418457)
+8400 val loss 5.6552
+8400 val perplexity 285.7867
+8400 train 5.427840 (lr=4.1596e-06) (hash(x)=150182527)
+8100 val loss 5.8903
+8100 val perplexity 361.5109
+8100 train 5.879162 (lr=5.6857e-06) (hash(x)=161109470)
+8500 val loss 5.6491
+8500 val perplexity 284.0453
+8500 train 5.715513 (lr=4.0815e-06) (hash(x)=154828709)
+8200 val loss 5.8859
+8200 val perplexity 359.9176
+8200 train 6.011838 (lr=5.4917e-06) (hash(x)=154408822)
+8600 val loss 5.6487
+8600 val perplexity 283.9250
+8600 train 5.561393 (lr=4.0294e-06) (hash(x)=154995428)
+8300 val loss 5.8841
+8300 val perplexity 359.2776
+8300 train 5.761460 (lr=5.3295e-06) (hash(x)=146431691)
+8700 val loss 5.6484
+8700 val perplexity 283.8460
+8700 train 5.544937 (lr=4.0033e-06) (hash(x)=142111043)
+8400 val loss 5.8840
+8400 val perplexity 359.2402
+8400 train 5.656861 (lr=5.1995e-06) (hash(x)=150182527)
+8749 val loss 5.6464
+8749 val perplexity 283.2623
+8500 val loss 5.8779
+8500 val perplexity 357.0568
+8500 train 5.951786 (lr=5.1019e-06) (hash(x)=154828709)
+8600 val loss 5.8766
+8600 val perplexity 356.5812
+8600 train 5.774024 (lr=5.0367e-06) (hash(x)=154995428)

attention_kindselective_n_heads4_seed1342/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b784ac0421732561a9feaba30a9e9fef056d1e13bb74eb961fa766d7412a975
+oid sha256:05f48028ce6ff170e45a1bc92fd2cb7055795a96c8a187c1a44aac35d2b8e75b
 size 92843394

attention_kindselective_n_heads4_seed1342/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a42419c28aa5e439c9bce28733a36c3eb5bf07c6395ea15a1c7f729048403a90
+oid sha256:f37ffc5be7cc22cb042d56756e571065c8a63e66b7db829970eb6c191299e319
 size 179406214