Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1342/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_61440_4_1342", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1342", "n_embd": 256}

attention_kindselective_n_heads4_seed1342/log2.txt

@@ -1,105 +1,411 @@
 max_steps: 8750
-7100 val loss 5.8251
-7100 val perplexity 338.6792
-7100 train 5.700815 (lr=6.5080e-06) (hash(x)=148277836)
+2000 val loss 6.3128
+2000 val perplexity 551.5859
+2000 train 6.337737 (lr=3.7143e-05) (hash(x)=154559671)
 0 val loss 11.2898
 0 val perplexity 80003.3281
-7200 val loss 5.8238
-7200 val perplexity 338.2640
-7200 train 5.831926 (lr=6.1648e-06) (hash(x)=168591659)
-0 train 11.289023 (lr=8.0000e-08) (hash(x)=150332693)
-7300 val loss 5.8168
-7300 val perplexity 335.9115
-7300 train 5.797405 (lr=5.8405e-06) (hash(x)=152764851)
-100 val loss 10.0512
-100 val perplexity 23184.4512
-100 train 10.007780 (lr=8.0800e-06) (hash(x)=149277319)
-7400 val loss 5.8128
-7400 val perplexity 334.5652
-7400 train 5.765549 (lr=5.5357e-06) (hash(x)=142920131)
-200 val loss 9.4244
-200 val perplexity 12386.6855
-200 train 9.491747 (lr=1.6080e-05) (hash(x)=155934075)
-7500 val loss 5.8108
-7500 val perplexity 333.8896
-7500 train 5.807702 (lr=5.2508e-06) (hash(x)=146954649)
-300 val loss 8.2609
-300 val perplexity 3869.7136
-300 train 8.296400 (lr=2.4080e-05) (hash(x)=165402628)
-7600 val loss 5.8105
-7600 val perplexity 333.8024
-7600 train 5.602143 (lr=4.9862e-06) (hash(x)=144529566)
-400 val loss 7.5974
-400 val perplexity 1992.9136
-400 train 7.401337 (lr=3.2080e-05) (hash(x)=139168939)
-7700 val loss 5.8047
-7700 val perplexity 331.8635
-7700 train 5.760702 (lr=4.7423e-06) (hash(x)=142669945)
-500 val loss 7.4160
-500 val perplexity 1662.4181
-500 train 7.336144 (lr=4.0000e-05) (hash(x)=147134338)
-7800 val loss 5.8011
-7800 val perplexity 330.6578
-7800 train 5.871948 (lr=4.5194e-06) (hash(x)=161563764)
-600 val loss 7.3080
-600 val perplexity 1492.2618
-600 train 7.216753 (lr=3.9987e-05) (hash(x)=159086459)
-7900 val loss 5.7989
-7900 val perplexity 329.9443
-7900 train 5.598620 (lr=4.3179e-06) (hash(x)=141516532)
-700 val loss 7.2178
-700 val perplexity 1363.4265
-700 train 7.253378 (lr=3.9948e-05) (hash(x)=152399099)
-8000 val loss 5.7998
-8000 val perplexity 330.2267
-8000 train 5.668744 (lr=4.1380e-06) (hash(x)=152418457)
-800 val loss 7.0971
-800 val perplexity 1208.4463
-800 train 7.100146 (lr=3.9883e-05) (hash(x)=156548843)
-8100 val loss 5.7940
-8100 val perplexity 328.3377
-8100 train 5.787144 (lr=3.9800e-06) (hash(x)=161109470)
-900 val loss 7.0009
-900 val perplexity 1097.5968
-900 train 6.970758 (lr=3.9792e-05) (hash(x)=157889911)
-8200 val loss 5.7904
-8200 val perplexity 327.1298
-8200 train 5.914674 (lr=3.8442e-06) (hash(x)=154408822)
-1000 val loss 6.9203
-1000 val perplexity 1012.6512
-1000 train 6.716169 (lr=3.9675e-05) (hash(x)=142943707)
-8300 val loss 5.7886
-8300 val perplexity 326.5483
-8300 train 5.672719 (lr=3.7307e-06) (hash(x)=146431691)
-1100 val loss 6.8508
-1100 val perplexity 944.6305
-1100 train 6.849274 (lr=3.9532e-05) (hash(x)=148938689)
-8400 val loss 5.7887
-8400 val perplexity 326.6004
-8400 train 5.564780 (lr=3.6397e-06) (hash(x)=150182527)
-1200 val loss 6.7516
-1200 val perplexity 855.4323
-1200 train 6.889379 (lr=3.9364e-05) (hash(x)=161962903)
-8500 val loss 5.7832
-8500 val perplexity 324.7987
-8500 train 5.846736 (lr=3.5713e-06) (hash(x)=154828709)
-1300 val loss 6.6709
-1300 val perplexity 789.0945
-1300 train 6.656214 (lr=3.9171e-05) (hash(x)=145979235)
-8600 val loss 5.7820
-8600 val perplexity 324.3983
-8600 train 5.691816 (lr=3.5257e-06) (hash(x)=154995428)
-1400 val loss 6.6137
-1400 val perplexity 745.2544
-1400 train 6.565486 (lr=3.8953e-05) (hash(x)=159157507)
-8700 val loss 5.7821
-8700 val perplexity 324.4330
-8700 train 5.676406 (lr=3.5029e-06) (hash(x)=142111043)
-1500 val loss 6.5521
-1500 val perplexity 700.7333
-1500 train 6.591178 (lr=3.8711e-05) (hash(x)=145943130)
-8749 val loss 5.7797
-8749 val perplexity 323.6483
-1600 val loss 6.4850
-1600 val perplexity 655.2203
-1600 train 6.287962 (lr=3.8444e-05) (hash(x)=143249173)
+2100 val loss 6.2877
+2100 val perplexity 537.9176
+2100 train 6.341491 (lr=3.6761e-05) (hash(x)=155607137)
+0 train 11.289023 (lr=1.0000e-07) (hash(x)=150332693)
+2200 val loss 6.2584
+2200 val perplexity 522.3959
+2200 train 6.118836 (lr=3.6358e-05) (hash(x)=153010221)
+100 val loss 9.9998
+100 val perplexity 22022.3066
+100 train 9.952392 (lr=1.0100e-05) (hash(x)=149277319)
+2300 val loss 6.2257
+2300 val perplexity 505.5900
+2300 train 6.241310 (lr=3.5935e-05) (hash(x)=160097777)
+200 val loss 9.2651
+200 val perplexity 10563.2148
+200 train 9.330362 (lr=2.0100e-05) (hash(x)=155934075)
+2400 val loss 6.2031
+2400 val perplexity 494.2805
+2400 train 6.178403 (lr=3.5491e-05) (hash(x)=135590094)
+300 val loss 8.0689
+300 val perplexity 3193.5994
+300 train 8.101431 (lr=3.0100e-05) (hash(x)=165402628)
+2500 val loss 6.1780
+2500 val perplexity 482.0299
+2500 train 6.297857 (lr=3.5027e-05) (hash(x)=147042929)
+400 val loss 7.5557
+400 val perplexity 1911.6412
+400 train 7.350784 (lr=4.0100e-05) (hash(x)=139168939)
+2600 val loss 6.1480
+2600 val perplexity 467.7656
+2600 train 6.110419 (lr=3.4545e-05) (hash(x)=163793881)
+500 val loss 7.3952
+500 val perplexity 1628.1936
+500 train 7.312452 (lr=5.0000e-05) (hash(x)=147134338)
+2700 val loss 6.1287
+2700 val perplexity 458.8197
+2700 train 6.155943 (lr=3.4044e-05) (hash(x)=153598698)
+600 val loss 7.2748
+600 val perplexity 1443.5325
+600 train 7.178120 (lr=4.9984e-05) (hash(x)=159086459)
+2800 val loss 6.1152
+2800 val perplexity 452.7000
+2800 train 5.913950 (lr=3.3526e-05) (hash(x)=133439803)
+700 val loss 7.1729
+700 val perplexity 1303.6285
+700 train 7.209314 (lr=4.9935e-05) (hash(x)=152399099)
+2900 val loss 6.0916
+2900 val perplexity 442.1120
+2900 train 6.026701 (lr=3.2992e-05) (hash(x)=148239158)
+800 val loss 7.0694
+800 val perplexity 1175.3920
+800 train 7.073838 (lr=4.9853e-05) (hash(x)=156548843)
+3000 val loss 6.0677
+3000 val perplexity 431.6776
+3000 train 6.077953 (lr=3.2441e-05) (hash(x)=150991971)
+900 val loss 6.9698
+900 val perplexity 1064.0211
+900 train 6.940661 (lr=4.9739e-05) (hash(x)=157889911)
+3100 val loss 6.0537
+3100 val perplexity 425.6675
+3100 train 6.060749 (lr=3.1875e-05) (hash(x)=157791832)
+1000 val loss 6.8910
+1000 val perplexity 983.3391
+1000 train 6.682793 (lr=4.9593e-05) (hash(x)=142943707)
+3200 val loss 6.0284
+3200 val perplexity 415.0627
+3200 train 6.095517 (lr=3.1295e-05) (hash(x)=163746305)
+1100 val loss 6.8231
+1100 val perplexity 918.8702
+1100 train 6.816766 (lr=4.9415e-05) (hash(x)=148938689)
+3300 val loss 6.0201
+3300 val perplexity 411.6396
+3300 train 5.941669 (lr=3.0702e-05) (hash(x)=155120269)
+1200 val loss 6.7355
+1200 val perplexity 841.7778
+1200 train 6.873916 (lr=4.9205e-05) (hash(x)=161962903)
+3400 val loss 5.9943
+3400 val perplexity 401.1503
+3400 train 5.938319 (lr=3.0095e-05) (hash(x)=152748451)
+1300 val loss 6.6620
+1300 val perplexity 782.0790
+1300 train 6.648552 (lr=4.8964e-05) (hash(x)=145979235)
+3500 val loss 5.9828
+3500 val perplexity 396.5558
+3500 train 5.865940 (lr=2.9477e-05) (hash(x)=147395004)
+1400 val loss 6.6040
+1400 val perplexity 738.0450
+1400 train 6.553430 (lr=4.8691e-05) (hash(x)=159157507)
+3600 val loss 5.9706
+3600 val perplexity 391.7508
+3600 train 5.986559 (lr=2.8849e-05) (hash(x)=157554250)
+1500 val loss 6.5556
+1500 val perplexity 703.1476
+1500 train 6.589256 (lr=4.8388e-05) (hash(x)=145943130)
+3700 val loss 5.9506
+3700 val perplexity 383.9760
+3700 train 5.804660 (lr=2.8210e-05) (hash(x)=147953321)
+1600 val loss 6.4946
+1600 val perplexity 661.5288
+1600 train 6.296082 (lr=4.8055e-05) (hash(x)=143249173)
+3800 val loss 5.9345
+3800 val perplexity 377.8583
+3800 train 5.786823 (lr=2.7562e-05) (hash(x)=146898609)
+1700 val loss 6.4479
+1700 val perplexity 631.3599
+1700 train 6.512136 (lr=4.7691e-05) (hash(x)=173907143)
+3900 val loss 5.9332
+3900 val perplexity 377.3427
+3900 train 5.972750 (lr=2.6907e-05) (hash(x)=150115282)
+1800 val loss 6.4099
+1800 val perplexity 607.8524
+1800 train 6.337204 (lr=4.7299e-05) (hash(x)=138363755)
+4000 val loss 5.9066
+4000 val perplexity 367.4610
+4000 train 5.782568 (lr=2.6244e-05) (hash(x)=142448619)
+1900 val loss 6.3790
+1900 val perplexity 589.3640
+1900 train 6.194283 (lr=4.6878e-05) (hash(x)=146341390)
+4100 val loss 5.8990
+4100 val perplexity 364.6619
+4100 train 5.634441 (lr=2.5575e-05) (hash(x)=141244123)
+2000 val loss 6.3352
+2000 val perplexity 564.0894
+2000 train 6.363589 (lr=4.6428e-05) (hash(x)=154559671)
+4200 val loss 5.8938
+4200 val perplexity 362.7773
+4200 train 5.912916 (lr=2.4900e-05) (hash(x)=165913661)
+2100 val loss 6.3062
+2100 val perplexity 547.9478
+2100 train 6.366211 (lr=4.5951e-05) (hash(x)=155607137)
+4300 val loss 5.8698
+4300 val perplexity 354.1734
+4300 train 5.900264 (lr=2.4222e-05) (hash(x)=155744823)
+2200 val loss 6.2716
+2200 val perplexity 529.3432
+2200 train 6.132524 (lr=4.5448e-05) (hash(x)=153010221)
+4400 val loss 5.8607
+4400 val perplexity 350.9864
+4400 train 5.853642 (lr=2.3540e-05) (hash(x)=160461243)
+2300 val loss 6.2417
+2300 val perplexity 513.7097
+2300 train 6.261227 (lr=4.4918e-05) (hash(x)=160097777)
+4500 val loss 5.8487
+4500 val perplexity 346.7758
+4500 train 5.761260 (lr=2.2856e-05) (hash(x)=156073129)
+2400 val loss 6.2164
+2400 val perplexity 500.9126
+2400 train 6.187909 (lr=4.4363e-05) (hash(x)=135590094)
+4600 val loss 5.8343
+4600 val perplexity 341.8205
+4600 train 5.881904 (lr=2.2171e-05) (hash(x)=152294132)
+2500 val loss 6.1885
+2500 val perplexity 487.1097
+2500 train 6.303681 (lr=4.3784e-05) (hash(x)=147042929)
+4700 val loss 5.8257
+4700 val perplexity 338.9147
+4700 train 5.762042 (lr=2.1486e-05) (hash(x)=140904569)
+2600 val loss 6.1625
+2600 val perplexity 474.6165
+2600 train 6.128020 (lr=4.3181e-05) (hash(x)=163793881)
+4800 val loss 5.8207
+4800 val perplexity 337.2106
+4800 train 5.733693 (lr=2.0801e-05) (hash(x)=156191635)
+2700 val loss 6.1385
+2700 val perplexity 463.3676
+2700 train 6.173907 (lr=4.2555e-05) (hash(x)=153598698)
+4900 val loss 5.8093
+4900 val perplexity 333.3725
+4900 train 5.915988 (lr=2.0118e-05) (hash(x)=145063976)
+2800 val loss 6.1290
+2800 val perplexity 458.9727
+2800 train 5.926794 (lr=4.1908e-05) (hash(x)=133439803)
+5000 val loss 5.7989
+5000 val perplexity 329.9442
+5000 train 5.750639 (lr=1.9438e-05) (hash(x)=160110619)
+2900 val loss 6.0952
+2900 val perplexity 443.7298
+2900 train 6.033276 (lr=4.1240e-05) (hash(x)=148239158)
+5100 val loss 5.7838
+5100 val perplexity 325.0067
+5100 train 5.813907 (lr=1.8762e-05) (hash(x)=156270070)
+3000 val loss 6.0713
+3000 val perplexity 433.2351
+3000 train 6.079644 (lr=4.0551e-05) (hash(x)=150991971)
+5200 val loss 5.7777
+5200 val perplexity 323.0109
+5200 train 5.671038 (lr=1.8090e-05) (hash(x)=138954242)
+3100 val loss 6.0554
+3100 val perplexity 426.4078
+3100 train 6.065526 (lr=3.9844e-05) (hash(x)=157791832)
+5300 val loss 5.7697
+5300 val perplexity 320.4575
+5300 train 5.616807 (lr=1.7424e-05) (hash(x)=146472367)
+3200 val loss 6.0314
+3200 val perplexity 416.3144
+3200 train 6.103322 (lr=3.9119e-05) (hash(x)=163746305)
+5400 val loss 5.7590
+5400 val perplexity 317.0278
+5400 train 5.699484 (lr=1.6765e-05) (hash(x)=146753405)
+3300 val loss 6.0258
+3300 val perplexity 413.9880
+3300 train 5.951910 (lr=3.8377e-05) (hash(x)=155120269)
+5500 val loss 5.7508
+5500 val perplexity 314.4398
+5500 train 5.703270 (lr=1.6113e-05) (hash(x)=147757398)
+3400 val loss 5.9908
+3400 val perplexity 399.7289
+3400 train 5.932531 (lr=3.7619e-05) (hash(x)=152748451)
+5600 val loss 5.7482
+5600 val perplexity 313.6392
+5600 train 5.621116 (lr=1.5469e-05) (hash(x)=151291757)
+3500 val loss 5.9803
+3500 val perplexity 395.5488
+3500 train 5.864267 (lr=3.6847e-05) (hash(x)=147395004)
+5700 val loss 5.7357
+5700 val perplexity 309.7417
+5700 train 5.784433 (lr=1.4836e-05) (hash(x)=156274392)
+3600 val loss 5.9625
+3600 val perplexity 388.5872
+3600 train 5.984228 (lr=3.6061e-05) (hash(x)=157554250)
+5800 val loss 5.7292
+5800 val perplexity 307.7190
+5800 train 5.697443 (lr=1.4212e-05) (hash(x)=152612266)
+3700 val loss 5.9419
+3700 val perplexity 380.6654
+3700 train 5.794821 (lr=3.5263e-05) (hash(x)=147953321)
+5900 val loss 5.7250
+5900 val perplexity 306.4240
+5900 train 5.896011 (lr=1.3600e-05) (hash(x)=148973201)
+3800 val loss 5.9262
+3800 val perplexity 374.7244
+3800 train 5.774441 (lr=3.4453e-05) (hash(x)=146898609)
+6000 val loss 5.7160
+6000 val perplexity 303.6807
+6000 train 5.680178 (lr=1.3000e-05) (hash(x)=158311490)
+3900 val loss 5.9226
+3900 val perplexity 373.3678
+3900 train 5.957313 (lr=3.3633e-05) (hash(x)=150115282)
+6100 val loss 5.7098
+6100 val perplexity 301.8034
+6100 train 5.641348 (lr=1.2413e-05) (hash(x)=147541319)
+4000 val loss 5.8940
+4000 val perplexity 362.8593
+4000 train 5.768799 (lr=3.2805e-05) (hash(x)=142448619)
+6200 val loss 5.7079
+6200 val perplexity 301.2393
+6200 train 5.772617 (lr=1.1840e-05) (hash(x)=154753980)
+4100 val loss 5.8918
+4100 val perplexity 362.0426
+4100 train 5.631465 (lr=3.1968e-05) (hash(x)=141244123)
+6300 val loss 5.7021
+6300 val perplexity 299.4808
+6300 train 5.594857 (lr=1.1282e-05) (hash(x)=140798998)
+4200 val loss 5.8810
+4200 val perplexity 358.1621
+4200 train 5.895331 (lr=3.1126e-05) (hash(x)=165913661)
+6400 val loss 5.6957
+6400 val perplexity 297.5710
+6400 train 5.581728 (lr=1.0739e-05) (hash(x)=149299516)
+4300 val loss 5.8533
+4300 val perplexity 348.3649
+4300 train 5.886961 (lr=3.0277e-05) (hash(x)=155744823)
+6500 val loss 5.6917
+6500 val perplexity 296.4005
+6500 train 5.678313 (lr=1.0213e-05) (hash(x)=148495806)
+4400 val loss 5.8436
+4400 val perplexity 345.0351
+4400 train 5.842594 (lr=2.9425e-05) (hash(x)=160461243)
+6600 val loss 5.6843
+6600 val perplexity 294.2187
+6600 train 5.663355 (lr=9.7032e-06) (hash(x)=153664387)
+4500 val loss 5.8311
+4500 val perplexity 340.7296
+4500 train 5.741596 (lr=2.8571e-05) (hash(x)=156073129)
+6700 val loss 5.6826
+6700 val perplexity 293.7068
+6700 train 5.665803 (lr=9.2116e-06) (hash(x)=153333491)
+4600 val loss 5.8152
+4600 val perplexity 335.3510
+4600 train 5.869062 (lr=2.7714e-05) (hash(x)=152294132)
+6800 val loss 5.6836
+6800 val perplexity 294.0042
+6800 train 5.433979 (lr=8.7387e-06) (hash(x)=150149382)
+4700 val loss 5.8103
+4700 val perplexity 333.7131
+4700 train 5.745833 (lr=2.6857e-05) (hash(x)=140904569)
+6900 val loss 5.6759
+6900 val perplexity 291.7373
+6900 train 5.762057 (lr=8.2849e-06) (hash(x)=152379767)
+4800 val loss 5.7961
+4800 val perplexity 329.0201
+4800 train 5.711601 (lr=2.6002e-05) (hash(x)=156191635)
+7000 val loss 5.6689
+7000 val perplexity 289.7231
+7000 train 5.715982 (lr=7.8510e-06) (hash(x)=150432537)
+4900 val loss 5.7850
+4900 val perplexity 325.3918
+4900 train 5.894067 (lr=2.5148e-05) (hash(x)=145063976)
+7100 val loss 5.6676
+7100 val perplexity 289.3501
+7100 train 5.539184 (lr=7.4377e-06) (hash(x)=148277836)
+5000 val loss 5.7780
+5000 val perplexity 323.1237
+5000 train 5.723316 (lr=2.4298e-05) (hash(x)=160110619)
+7200 val loss 5.6669
+7200 val perplexity 289.1384
+7200 train 5.674396 (lr=7.0455e-06) (hash(x)=168591659)
+5100 val loss 5.7605
+5100 val perplexity 317.5004
+5100 train 5.792020 (lr=2.3452e-05) (hash(x)=156270070)
+7300 val loss 5.6616
+7300 val perplexity 287.6112
+7300 train 5.641236 (lr=6.6749e-06) (hash(x)=152764851)
+5200 val loss 5.7509
+5200 val perplexity 314.4721
+5200 train 5.644578 (lr=2.2613e-05) (hash(x)=138954242)
+7400 val loss 5.6580
+7400 val perplexity 286.5637
+7400 train 5.623162 (lr=6.3266e-06) (hash(x)=142920131)
+5300 val loss 5.7429
+5300 val perplexity 311.9596
+5300 train 5.584954 (lr=2.1780e-05) (hash(x)=146472367)
+7500 val loss 5.6560
+7500 val perplexity 286.0032
+7500 train 5.658007 (lr=6.0010e-06) (hash(x)=146954649)
+5400 val loss 5.7299
+5400 val perplexity 307.9452
+5400 train 5.668500 (lr=2.0956e-05) (hash(x)=146753405)
+7600 val loss 5.6561
+7600 val perplexity 286.0308
+7600 train 5.441816 (lr=5.6986e-06) (hash(x)=144529566)
+5500 val loss 5.7215
+5500 val perplexity 305.3474
+5500 train 5.666595 (lr=2.0141e-05) (hash(x)=147757398)
+7700 val loss 5.6510
+7700 val perplexity 284.5873
+7700 train 5.610033 (lr=5.4198e-06) (hash(x)=142669945)
+5600 val loss 5.7181
+5600 val perplexity 304.3283
+5600 train 5.588433 (lr=1.9337e-05) (hash(x)=151291757)
+7800 val loss 5.6476
+7800 val perplexity 283.5984
+7800 train 5.704885 (lr=5.1650e-06) (hash(x)=161563764)
+5700 val loss 5.7061
+5700 val perplexity 300.6953
+5700 train 5.757588 (lr=1.8545e-05) (hash(x)=156274392)
+7900 val loss 5.6454
+7900 val perplexity 282.9980
+7900 train 5.442910 (lr=4.9347e-06) (hash(x)=141516532)
+5800 val loss 5.6988
+5800 val perplexity 298.5155
+5800 train 5.673255 (lr=1.7765e-05) (hash(x)=152612266)
+8000 val loss 5.6466
+8000 val perplexity 283.3248
+8000 train 5.512422 (lr=4.7291e-06) (hash(x)=152418457)
+5900 val loss 5.6917
+5900 val perplexity 296.3925
+5900 train 5.856750 (lr=1.7000e-05) (hash(x)=148973201)
+8100 val loss 5.6407
+8100 val perplexity 281.6575
+8100 train 5.635432 (lr=4.5486e-06) (hash(x)=161109470)
+6000 val loss 5.6834
+6000 val perplexity 293.9449
+6000 train 5.651141 (lr=1.6250e-05) (hash(x)=158311490)
+8200 val loss 5.6376
+8200 val perplexity 280.7789
+8200 train 5.764065 (lr=4.3933e-06) (hash(x)=154408822)
+6100 val loss 5.6763
+6100 val perplexity 291.8545
+6100 train 5.612087 (lr=1.5516e-05) (hash(x)=147541319)
+8300 val loss 5.6364
+8300 val perplexity 280.4599
+8300 train 5.520494 (lr=4.2636e-06) (hash(x)=146431691)
+6200 val loss 5.6737
+6200 val perplexity 291.1170
+6200 train 5.738555 (lr=1.4800e-05) (hash(x)=154753980)
+8400 val loss 5.6366
+8400 val perplexity 280.5117
+8400 train 5.412935 (lr=4.1596e-06) (hash(x)=150182527)
+6300 val loss 5.6660
+6300 val perplexity 288.8887
+6300 train 5.558257 (lr=1.4102e-05) (hash(x)=140798998)
+8500 val loss 5.6307
+8500 val perplexity 278.8655
+8500 train 5.696391 (lr=4.0815e-06) (hash(x)=154828709)
+6400 val loss 5.6596
+6400 val perplexity 287.0276
+6400 train 5.550190 (lr=1.3424e-05) (hash(x)=149299516)
+8600 val loss 5.6300
+8600 val perplexity 278.6720
+8600 train 5.542825 (lr=4.0294e-06) (hash(x)=154995428)
+6500 val loss 5.6557
+6500 val perplexity 285.9107
+6500 train 5.639792 (lr=1.2766e-05) (hash(x)=148495806)
+8700 val loss 5.6308
+8700 val perplexity 278.8865
+8700 train 5.526932 (lr=4.0033e-06) (hash(x)=142111043)
+6600 val loss 5.6476
+6600 val perplexity 283.6003
+6600 train 5.626984 (lr=1.2129e-05) (hash(x)=153664387)
+8749 val loss 5.6277
+8749 val perplexity 278.0269
+6700 val loss 5.6444
+6700 val perplexity 282.6929
+6700 train 5.626853 (lr=1.1515e-05) (hash(x)=153333491)

attention_kindselective_n_heads4_seed1342/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4554f9feef4cd1714de74f2d36dfb797afdd186752d5ae5a18a0c3ad58deb6d7
+oid sha256:cef91b2b163741e77bb5ae13fff8d391e25f32c47887020c7e0571411d419631
 size 92843394

attention_kindselective_n_heads4_seed1342/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:86936929383c09f29442147958ddf75c30a2f329230a59348238ed2c4e7f9596
+oid sha256:f701333ad8c644d69822c9a6d1f7a8a43585bd2f75b501d2d0042c47e6499892
 size 179406214