Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1343/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.5e-5_61440_4_1343", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1343", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1343, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4.5e-5_61440_4_1343", "n_embd": 256}

attention_kindselective_n_heads4_seed1343/log2.txt

@@ -1,424 +1,267 @@
 max_steps: 8750
-max_steps: 8750
-0 val loss 11.3014
-0 val perplexity 80932.9531
 0 val loss 11.3014
 0 val perplexity 80932.9531
-0 train 11.305982 (lr=7.0000e-08) (hash(x)=153418542)
-0 train 11.305982 (lr=6.0000e-08) (hash(x)=153418542)
-100 val loss 10.0438
-100 val perplexity 23011.9707
-100 train 10.050365 (lr=7.0700e-06) (hash(x)=143722001)
-100 val loss 10.1073
-100 val perplexity 24520.4238
-100 train 10.113642 (lr=6.0600e-06) (hash(x)=143722001)
-200 val loss 9.5448
-200 val perplexity 13971.1826
-200 train 9.549261 (lr=1.4070e-05) (hash(x)=152989689)
-300 val loss 8.6011
-300 val perplexity 5437.8970
-300 train 8.619393 (lr=2.1070e-05) (hash(x)=150071018)
-200 val loss 9.6223
-200 val perplexity 15098.4424
-200 train 9.629188 (lr=1.2060e-05) (hash(x)=152989689)
-400 val loss 7.8269
-400 val perplexity 2507.1106
-400 train 7.854829 (lr=2.8070e-05) (hash(x)=153358238)
-500 val loss 7.5393
-500 val perplexity 1880.5006
-500 train 7.423524 (lr=3.5000e-05) (hash(x)=148983354)
-300 val loss 8.7160
-300 val perplexity 6099.9268
-300 train 8.734112 (lr=1.8060e-05) (hash(x)=150071018)
-600 val loss 7.4113
-600 val perplexity 1654.4934
-600 train 7.314175 (lr=3.4989e-05) (hash(x)=150770333)
-700 val loss 7.3267
-700 val perplexity 1520.3972
-400 val loss 7.8636
-400 val perplexity 2600.8687
-700 train 7.248207 (lr=3.4954e-05) (hash(x)=145246201)
-400 train 7.890481 (lr=2.4060e-05) (hash(x)=153358238)
-800 val loss 7.2515
-800 val perplexity 1410.1771
-800 train 7.083712 (lr=3.4897e-05) (hash(x)=142635842)
-500 val loss 7.5315
-500 val perplexity 1865.8441
-500 train 7.414758 (lr=3.0000e-05) (hash(x)=148983354)
-900 val loss 7.1575
-900 val perplexity 1283.6674
-900 train 7.189825 (lr=3.4818e-05) (hash(x)=148299832)
-1000 val loss 7.0754
-1000 val perplexity 1182.5190
-1000 train 7.035053 (lr=3.4715e-05) (hash(x)=152887721)
-600 val loss 7.4045
-600 val perplexity 1643.3911
-600 train 7.309084 (lr=2.9990e-05) (hash(x)=150770333)
-1100 val loss 7.0022
-1100 val perplexity 1099.0203
-1100 train 6.891657 (lr=3.4591e-05) (hash(x)=153721194)
-1200 val loss 6.9192
-1200 val perplexity 1011.5258
-1200 train 6.912433 (lr=3.4444e-05) (hash(x)=153047184)
-700 val loss 7.3257
-700 val perplexity 1518.8248
-700 train 7.243683 (lr=2.9961e-05) (hash(x)=145246201)
-1300 val loss 6.8385
-1300 val perplexity 933.1031
-1300 train 6.749180 (lr=3.4275e-05) (hash(x)=152466045)
-800 val loss 7.2468
-800 val perplexity 1403.6697
-1400 val loss 6.7694
-1400 val perplexity 870.7479
-800 train 7.078628 (lr=2.9912e-05) (hash(x)=142635842)
-1400 train 6.569815 (lr=3.4084e-05) (hash(x)=151991552)
-1500 val loss 6.6936
-1500 val perplexity 807.2139
-1500 train 6.642127 (lr=3.3872e-05) (hash(x)=147892594)
-900 val loss 7.1541
-900 val perplexity 1279.3289
-900 train 7.189307 (lr=2.9844e-05) (hash(x)=148299832)
-1600 val loss 6.6326
-1600 val perplexity 759.4801
-1600 train 6.438039 (lr=3.3638e-05) (hash(x)=135678663)
-1700 val loss 6.5863
-1700 val perplexity 725.0812
-1700 train 6.610061 (lr=3.3384e-05) (hash(x)=143909276)
-1000 val loss 7.0835
-1000 val perplexity 1192.1432
-1000 train 7.044007 (lr=2.9756e-05) (hash(x)=152887721)
-1800 val loss 6.5362
-1800 val perplexity 689.6570
-1800 train 6.684325 (lr=3.3109e-05) (hash(x)=155393468)
-1900 val loss 6.4801
-1900 val perplexity 652.0583
-1900 train 6.526745 (lr=3.2814e-05) (hash(x)=150957388)
-1100 val loss 7.0117
-1100 val perplexity 1109.5219
-1100 train 6.898890 (lr=2.9649e-05) (hash(x)=153721194)
-2000 val loss 6.4394
-2000 val perplexity 626.0261
-2000 train 6.688434 (lr=3.2500e-05) (hash(x)=144030938)
-1200 val loss 6.9380
-1200 val perplexity 1030.7054
-1200 train 6.929457 (lr=2.9523e-05) (hash(x)=153047184)
-2100 val loss 6.4148
-2100 val perplexity 610.8321
-2100 train 6.284521 (lr=3.2166e-05) (hash(x)=146254754)
-2200 val loss 6.3695
-2200 val perplexity 583.7899
-2200 train 6.399682 (lr=3.1813e-05) (hash(x)=150630461)
-1300 val loss 6.8630
-1300 val perplexity 956.2305
-1300 train 6.769284 (lr=2.9378e-05) (hash(x)=152466045)
-2300 val loss 6.3410
-2300 val perplexity 567.3669
-2300 train 6.493802 (lr=3.1443e-05) (hash(x)=165208942)
-2400 val loss 6.3212
-2400 val perplexity 556.2291
-2400 train 6.247215 (lr=3.1054e-05) (hash(x)=153448706)
-1400 val loss 6.8011
-1400 val perplexity 898.8173
-1400 train 6.607848 (lr=2.9215e-05) (hash(x)=151991552)
-2500 val loss 6.2890
-2500 val perplexity 538.6242
-2500 train 6.301908 (lr=3.0649e-05) (hash(x)=148505056)
-1500 val loss 6.7320
-1500 val perplexity 838.8151
-1500 train 6.682234 (lr=2.9033e-05) (hash(x)=147892594)
-2600 val loss 6.2645
-2600 val perplexity 525.5840
-2600 train 6.209645 (lr=3.0227e-05) (hash(x)=142911960)
-2700 val loss 6.2464
-2700 val perplexity 516.1746
-2700 train 6.249057 (lr=2.9789e-05) (hash(x)=149951660)
-1600 val loss 6.6699
-1600 val perplexity 788.3246
-1600 train 6.471530 (lr=2.8833e-05) (hash(x)=135678663)
-2800 val loss 6.2150
-2800 val perplexity 500.2123
-2800 train 6.209450 (lr=2.9336e-05) (hash(x)=152956713)
-2900 val loss 6.1961
-2900 val perplexity 490.8508
-2900 train 6.112799 (lr=2.8868e-05) (hash(x)=147247056)
-1700 val loss 6.6200
-1700 val perplexity 749.9150
-1700 train 6.636775 (lr=2.8615e-05) (hash(x)=143909276)
-3000 val loss 6.1776
-3000 val perplexity 481.8291
-3000 train 5.997092 (lr=2.8386e-05) (hash(x)=146911716)
-3100 val loss 6.1579
-3100 val perplexity 472.4378
-3100 train 6.180588 (lr=2.7891e-05) (hash(x)=153282809)
-1800 val loss 6.5746
-1800 val perplexity 716.6734
-1800 train 6.721464 (lr=2.8379e-05) (hash(x)=155393468)
-3200 val loss 6.1410
-3200 val perplexity 464.5390
-3200 train 6.036775 (lr=2.7383e-05) (hash(x)=152009984)
-1900 val loss 6.5198
-1900 val perplexity 678.4648
-1900 train 6.568262 (lr=2.8127e-05) (hash(x)=150957388)
-3300 val loss 6.1275
-3300 val perplexity 458.2952
-3300 train 6.036661 (lr=2.6864e-05) (hash(x)=150012952)
-3400 val loss 6.1178
-3400 val perplexity 453.8775
-3400 train 5.951714 (lr=2.6333e-05) (hash(x)=146217477)
-2000 val loss 6.4772
-2000 val perplexity 650.1710
-2000 train 6.721745 (lr=2.7857e-05) (hash(x)=144030938)
-3500 val loss 6.0906
-3500 val perplexity 441.6902
-3500 train 6.259603 (lr=2.5793e-05) (hash(x)=180156144)
-3600 val loss 6.0826
-3600 val perplexity 438.1469
-3600 train 6.120096 (lr=2.5243e-05) (hash(x)=154243319)
-2100 val loss 6.4546
-2100 val perplexity 635.6494
-2100 train 6.322638 (lr=2.7571e-05) (hash(x)=146254754)
-3700 val loss 6.0649
-3700 val perplexity 430.4960
-3700 train 5.963401 (lr=2.4684e-05) (hash(x)=154330476)
-2200 val loss 6.4164
-2200 val perplexity 611.7704
-2200 train 6.443453 (lr=2.7269e-05) (hash(x)=150630461)
-3800 val loss 6.0467
-3800 val perplexity 422.7079
-3800 train 5.975914 (lr=2.4117e-05) (hash(x)=145988858)
-3900 val loss 6.0322
-3900 val perplexity 416.6099
-3900 train 6.045941 (lr=2.3543e-05) (hash(x)=149648609)
-2300 val loss 6.3844
-2300 val perplexity 592.5361
-2300 train 6.536951 (lr=2.6951e-05) (hash(x)=165208942)
-4000 val loss 6.0218
-4000 val perplexity 412.3248
-4000 train 5.994764 (lr=2.2963e-05) (hash(x)=154149272)
-4100 val loss 6.0072
-4100 val perplexity 406.3483
-4100 train 5.903815 (lr=2.2378e-05) (hash(x)=132608538)
-2400 val loss 6.3635
-2400 val perplexity 580.2852
-2400 train 6.286649 (lr=2.6618e-05) (hash(x)=153448706)
-4200 val loss 5.9916
-4200 val perplexity 400.0361
-4200 train 5.939827 (lr=2.1788e-05) (hash(x)=140443636)
-2500 val loss 6.3339
-2500 val perplexity 563.3612
-2500 train 6.346560 (lr=2.6270e-05) (hash(x)=148505056)
-4300 val loss 5.9913
-4300 val perplexity 399.9310
-4300 train 5.913877 (lr=2.1194e-05) (hash(x)=138919540)
-4400 val loss 5.9748
-4400 val perplexity 393.4041
-4400 train 5.947241 (lr=2.0598e-05) (hash(x)=153594684)
-2600 val loss 6.3080
-2600 val perplexity 548.9651
-2600 train 6.246406 (lr=2.5909e-05) (hash(x)=142911960)
-4500 val loss 5.9610
-4500 val perplexity 388.0159
-4500 train 5.911044 (lr=1.9999e-05) (hash(x)=144084750)
-4600 val loss 5.9548
-4600 val perplexity 385.5923
-4600 train 5.910222 (lr=1.9400e-05) (hash(x)=147423675)
-2700 val loss 6.2882
-2700 val perplexity 538.1934
-2700 train 6.289870 (lr=2.5533e-05) (hash(x)=149951660)
-4700 val loss 5.9404
-4700 val perplexity 380.0801
-4700 train 6.012012 (lr=1.8800e-05) (hash(x)=162157039)
-4800 val loss 5.9313
-4800 val perplexity 376.6331
-4800 train 5.835851 (lr=1.8201e-05) (hash(x)=134970942)
-2800 val loss 6.2582
-2800 val perplexity 522.2596
-2800 train 6.255882 (lr=2.5145e-05) (hash(x)=152956713)
-4900 val loss 5.9238
-4900 val perplexity 373.8367
-4900 train 5.844675 (lr=1.7604e-05) (hash(x)=160093370)
-2900 val loss 6.2437
-2900 val perplexity 514.7744
-2900 train 6.157321 (lr=2.4744e-05) (hash(x)=147247056)
-5000 val loss 5.9154
-5000 val perplexity 370.6849
-5000 train 5.809307 (lr=1.7009e-05) (hash(x)=154986299)
-5100 val loss 5.9005
-5100 val perplexity 365.2234
-5100 train 5.757463 (lr=1.6417e-05) (hash(x)=145406582)
-3000 val loss 6.2251
-3000 val perplexity 505.2902
-3000 train 6.048134 (lr=2.4331e-05) (hash(x)=146911716)
-5200 val loss 5.8944
-5200 val perplexity 362.9891
-5200 train 5.941761 (lr=1.5829e-05) (hash(x)=148029261)
-5300 val loss 5.8838
-5300 val perplexity 359.1739
-5300 train 5.881489 (lr=1.5246e-05) (hash(x)=157909233)
-3100 val loss 6.2032
-3100 val perplexity 494.3125
-3100 train 6.222645 (lr=2.3906e-05) (hash(x)=153282809)
-5400 val loss 5.8742
-5400 val perplexity 355.7338
-5400 train 5.757402 (lr=1.4669e-05) (hash(x)=142250860)
-3200 val loss 6.1904
-3200 val perplexity 488.0625
-3200 train 6.085604 (lr=2.3471e-05) (hash(x)=152009984)
-5500 val loss 5.8667
-5500 val perplexity 353.0680
-5500 train 5.740597 (lr=1.4099e-05) (hash(x)=145694703)
-5600 val loss 5.8590
-5600 val perplexity 350.3583
-5600 train 5.788935 (lr=1.3536e-05) (hash(x)=141788252)
-3300 val loss 6.1772
-3300 val perplexity 481.6196
-3300 train 6.087312 (lr=2.3026e-05) (hash(x)=150012952)
-5700 val loss 5.8508
-5700 val perplexity 347.4989
-5700 train 5.752746 (lr=1.2981e-05) (hash(x)=150886692)
-5800 val loss 5.8472
-5800 val perplexity 346.2778
-5800 train 5.754390 (lr=1.2436e-05) (hash(x)=151814419)
-3400 val loss 6.1625
-3400 val perplexity 474.5927
-3400 train 5.998357 (lr=2.2572e-05) (hash(x)=146217477)
-5900 val loss 5.8375
-5900 val perplexity 342.9117
-5900 train 5.847322 (lr=1.1900e-05) (hash(x)=146972539)
-6000 val loss 5.8301
-6000 val perplexity 340.3773
-6000 train 5.711991 (lr=1.1375e-05) (hash(x)=150219954)
-3500 val loss 6.1421
-3500 val perplexity 465.0062
-3500 train 6.311218 (lr=2.2108e-05) (hash(x)=180156144)
-6100 val loss 5.8264
-6100 val perplexity 339.1409
-6100 train 5.822284 (lr=1.0861e-05) (hash(x)=161131435)
-3600 val loss 6.1281
-3600 val perplexity 458.5415
-3600 train 6.163641 (lr=2.1637e-05) (hash(x)=154243319)
-6200 val loss 5.8175
-6200 val perplexity 336.1375
-6200 train 5.829421 (lr=1.0360e-05) (hash(x)=150226893)
-6300 val loss 5.8113
-6300 val perplexity 334.0599
-6300 train 5.643476 (lr=9.8715e-06) (hash(x)=144389552)
-3700 val loss 6.1135
-3700 val perplexity 451.9110
-3700 train 6.011994 (lr=2.1158e-05) (hash(x)=154330476)
-6400 val loss 5.8087
-6400 val perplexity 333.1811
-6400 train 5.592588 (lr=9.3966e-06) (hash(x)=147757943)
-6500 val loss 5.7999
-6500 val perplexity 330.2697
-6500 train 5.782908 (lr=8.9359e-06) (hash(x)=144616373)
-3800 val loss 6.0951
-3800 val perplexity 443.6720
-3800 train 6.029699 (lr=2.0672e-05) (hash(x)=145988858)
-6600 val loss 5.7994
-6600 val perplexity 330.1003
-6600 train 5.625536 (lr=8.4903e-06) (hash(x)=145266452)
-3900 val loss 6.0815
-3900 val perplexity 437.6682
-3900 train 6.089600 (lr=2.0180e-05) (hash(x)=149648609)
-6700 val loss 5.7939
-6700 val perplexity 328.2831
-6700 train 5.785510 (lr=8.0602e-06) (hash(x)=149952261)
-6800 val loss 5.7885
-6800 val perplexity 326.5093
-6800 train 5.665899 (lr=7.6463e-06) (hash(x)=143351199)
-4000 val loss 6.0739
-4000 val perplexity 434.3786
-4000 train 6.048274 (lr=1.9683e-05) (hash(x)=154149272)
-6900 val loss 5.7881
-6900 val perplexity 326.3762
-6900 train 5.749453 (lr=7.2493e-06) (hash(x)=156055618)
-7000 val loss 5.7804
-7000 val perplexity 323.9043
-7000 train 5.854444 (lr=6.8697e-06) (hash(x)=164290908)
-4100 val loss 6.0584
-4100 val perplexity 427.6766
-4100 train 5.955194 (lr=1.9181e-05) (hash(x)=132608538)
-7100 val loss 5.7781
-7100 val perplexity 323.1483
-7100 train 5.874842 (lr=6.5080e-06) (hash(x)=150263832)
-4200 val loss 6.0428
-4200 val perplexity 421.0503
-4200 train 5.982690 (lr=1.8675e-05) (hash(x)=140443636)
-7200 val loss 5.7763
-7200 val perplexity 322.5544
-7200 train 5.465231 (lr=6.1648e-06) (hash(x)=139219680)
-7300 val loss 5.7695
-7300 val perplexity 320.3765
-7300 train 5.715026 (lr=5.8405e-06) (hash(x)=150662994)
-4300 val loss 6.0377
-4300 val perplexity 418.9107
-4300 train 5.956387 (lr=1.8166e-05) (hash(x)=138919540)
-7400 val loss 5.7668
-7400 val perplexity 319.5232
-7400 train 5.611770 (lr=5.5357e-06) (hash(x)=139007967)
-7500 val loss 5.7658
-7500 val perplexity 319.1935
-7500 train 5.595723 (lr=5.2508e-06) (hash(x)=138142461)
-4400 val loss 6.0255
-4400 val perplexity 413.8688
-4400 train 5.995091 (lr=1.7655e-05) (hash(x)=153594684)
-7600 val loss 5.7609
-7600 val perplexity 317.6342
-7600 train 5.675484 (lr=4.9862e-06) (hash(x)=150023998)
-7700 val loss 5.7574
-7700 val perplexity 316.5240
-7700 train 5.652620 (lr=4.7423e-06) (hash(x)=143393355)
-4500 val loss 6.0108
-4500 val perplexity 407.7993
-4500 train 5.956801 (lr=1.7142e-05) (hash(x)=144084750)
-7800 val loss 5.7580
-7800 val perplexity 316.7283
-7800 train 5.633855 (lr=4.5194e-06) (hash(x)=144916472)
-4600 val loss 6.0024
-4600 val perplexity 404.3789
-4600 train 5.959134 (lr=1.6629e-05) (hash(x)=147423675)
-7900 val loss 5.7531
-7900 val perplexity 315.1684
-7900 train 5.706028 (lr=4.3179e-06) (hash(x)=150236934)
-8000 val loss 5.7501
-8000 val perplexity 314.2092
-8000 train 5.744445 (lr=4.1380e-06) (hash(x)=146536422)
-4700 val loss 5.9911
-4700 val perplexity 399.8620
-4700 train 6.067215 (lr=1.6114e-05) (hash(x)=162157039)
-8100 val loss 5.7506
-8100 val perplexity 314.3856
-8100 train 5.577597 (lr=3.9800e-06) (hash(x)=151300857)
-8200 val loss 5.7486
-8200 val perplexity 313.7418
-8200 train 5.792922 (lr=3.8442e-06) (hash(x)=160351956)
-4800 val loss 5.9808
-4800 val perplexity 395.7435
-4800 train 5.874648 (lr=1.5601e-05) (hash(x)=134970942)
-8300 val loss 5.7449
-8300 val perplexity 312.5950
-8300 train 5.781292 (lr=3.7307e-06) (hash(x)=150514540)
-4900 val loss 5.9759
-4900 val perplexity 393.8117
-4900 train 5.892208 (lr=1.5089e-05) (hash(x)=160093370)
-8400 val loss 5.7426
-8400 val perplexity 311.8808
-8400 train 5.695298 (lr=3.6397e-06) (hash(x)=155904762)
-8500 val loss 5.7438
-8500 val perplexity 312.2379
-8500 train 5.549510 (lr=3.5713e-06) (hash(x)=146923196)
-5000 val loss 5.9667
-5000 val perplexity 390.2094
-5000 train 5.861556 (lr=1.4579e-05) (hash(x)=154986299)
-8600 val loss 5.7400
-8600 val perplexity 311.0759
-8600 train 5.715517 (lr=3.5257e-06) (hash(x)=138977080)
-8700 val loss 5.7382
-8700 val perplexity 310.5197
-8700 train 5.543046 (lr=3.5029e-06) (hash(x)=146442792)
-5100 val loss 5.9525
-5100 val perplexity 384.7192
-5100 train 5.802466 (lr=1.4071e-05) (hash(x)=145406582)
-8749 val loss 5.7376
-8749 val perplexity 310.3325
+0 train 11.305982 (lr=9.0000e-08) (hash(x)=153418542)
+100 val loss 9.9641
+100 val perplexity 21248.8672
+100 train 9.970656 (lr=9.0900e-06) (hash(x)=143722001)
+200 val loss 9.2898
+200 val perplexity 10826.7256
+200 train 9.287369 (lr=1.8090e-05) (hash(x)=152989689)
+300 val loss 8.1539
+300 val perplexity 3477.0691
+300 train 8.164041 (lr=2.7090e-05) (hash(x)=150071018)
+400 val loss 7.5785
+400 val perplexity 1955.6984
+400 train 7.610251 (lr=3.6090e-05) (hash(x)=153358238)
+500 val loss 7.4310
+500 val perplexity 1687.4906
+500 train 7.303682 (lr=4.5000e-05) (hash(x)=148983354)
+600 val loss 7.3044
+600 val perplexity 1486.8474
+600 train 7.186595 (lr=4.4985e-05) (hash(x)=150770333)
+700 val loss 7.1867
+700 val perplexity 1321.7366
+700 train 7.097453 (lr=4.4941e-05) (hash(x)=145246201)
+800 val loss 7.0878
+800 val perplexity 1197.2869
+800 train 6.912718 (lr=4.4868e-05) (hash(x)=142635842)
+900 val loss 6.9741
+900 val perplexity 1068.5809
+900 train 6.998743 (lr=4.4766e-05) (hash(x)=148299832)
+1000 val loss 6.8867
+1000 val perplexity 979.1664
+1000 train 6.844893 (lr=4.4634e-05) (hash(x)=152887721)
+1100 val loss 6.8123
+1100 val perplexity 908.9700
+1100 train 6.700805 (lr=4.4474e-05) (hash(x)=153721194)
+1200 val loss 6.7234
+1200 val perplexity 831.6400
+1200 train 6.715183 (lr=4.4285e-05) (hash(x)=153047184)
+1300 val loss 6.6363
+1300 val perplexity 762.2647
+1300 train 6.541929 (lr=4.4068e-05) (hash(x)=152466045)
+1400 val loss 6.5742
+1400 val perplexity 716.3878
+1400 train 6.369258 (lr=4.3822e-05) (hash(x)=151991552)
+1500 val loss 6.5181
+1500 val perplexity 677.2753
+1500 train 6.460218 (lr=4.3549e-05) (hash(x)=147892594)
+1600 val loss 6.4664
+1600 val perplexity 643.1600
+1600 train 6.269328 (lr=4.3249e-05) (hash(x)=135678663)
+1700 val loss 6.4287
+1700 val perplexity 619.3771
+1700 train 6.447353 (lr=4.2922e-05) (hash(x)=143909276)
+1800 val loss 6.3942
+1800 val perplexity 598.3675
+1800 train 6.546668 (lr=4.2569e-05) (hash(x)=155393468)
+1900 val loss 6.3449
+1900 val perplexity 569.5897
+1900 train 6.392309 (lr=4.2190e-05) (hash(x)=150957388)
+2000 val loss 6.3159
+2000 val perplexity 553.2760
+2000 train 6.583006 (lr=4.1785e-05) (hash(x)=144030938)
+2100 val loss 6.2922
+2100 val perplexity 540.3519
+2100 train 6.168851 (lr=4.1356e-05) (hash(x)=146254754)
+2200 val loss 6.2523
+2200 val perplexity 519.2087
+2200 train 6.281209 (lr=4.0903e-05) (hash(x)=150630461)
+2300 val loss 6.2278
+2300 val perplexity 506.6485
+2300 train 6.372598 (lr=4.0426e-05) (hash(x)=165208942)
+2400 val loss 6.2082
+2400 val perplexity 496.8154
+2400 train 6.129076 (lr=3.9927e-05) (hash(x)=153448706)
+2500 val loss 6.1753
+2500 val perplexity 480.7440
+2500 train 6.189457 (lr=3.9406e-05) (hash(x)=148505056)
+2600 val loss 6.1536
+2600 val perplexity 470.3942
+2600 train 6.104032 (lr=3.8863e-05) (hash(x)=142911960)
+2700 val loss 6.1333
+2700 val perplexity 460.9367
+2700 train 6.136764 (lr=3.8300e-05) (hash(x)=149951660)
+2800 val loss 6.1089
+2800 val perplexity 449.8466
+2800 train 6.094249 (lr=3.7717e-05) (hash(x)=152956713)
+2900 val loss 6.0913
+2900 val perplexity 442.0087
+2900 train 6.002796 (lr=3.7116e-05) (hash(x)=147247056)
+3000 val loss 6.0699
+3000 val perplexity 432.6271
+3000 train 5.864369 (lr=3.6496e-05) (hash(x)=146911716)
+3100 val loss 6.0500
+3100 val perplexity 424.1339
+3100 train 6.068802 (lr=3.5860e-05) (hash(x)=153282809)
+3200 val loss 6.0386
+3200 val perplexity 419.3048
+3200 train 5.938767 (lr=3.5207e-05) (hash(x)=152009984)
+3300 val loss 6.0237
+3300 val perplexity 413.1028
+3300 train 5.933391 (lr=3.4539e-05) (hash(x)=150012952)
+3400 val loss 6.0107
+3400 val perplexity 407.7699
+3400 train 5.838824 (lr=3.3857e-05) (hash(x)=146217477)
+3500 val loss 5.9852
+3500 val perplexity 397.5202
+3500 train 6.152969 (lr=3.3162e-05) (hash(x)=180156144)
+3600 val loss 5.9755
+3600 val perplexity 393.6608
+3600 train 6.006383 (lr=3.2455e-05) (hash(x)=154243319)
+3700 val loss 5.9587
+3700 val perplexity 387.0885
+3700 train 5.861712 (lr=3.1736e-05) (hash(x)=154330476)
+3800 val loss 5.9375
+3800 val perplexity 379.0009
+3800 train 5.868928 (lr=3.1008e-05) (hash(x)=145988858)
+3900 val loss 5.9250
+3900 val perplexity 374.2960
+3900 train 5.935997 (lr=3.0270e-05) (hash(x)=149648609)
+4000 val loss 5.9114
+4000 val perplexity 369.2115
+4000 train 5.889551 (lr=2.9524e-05) (hash(x)=154149272)
+4100 val loss 5.8974
+4100 val perplexity 364.0854
+4100 train 5.804778 (lr=2.8771e-05) (hash(x)=132608538)
+4200 val loss 5.8807
+4200 val perplexity 358.0486
+4200 train 5.826519 (lr=2.8013e-05) (hash(x)=140443636)
+4300 val loss 5.8788
+4300 val perplexity 357.3949
+4300 train 5.804891 (lr=2.7250e-05) (hash(x)=138919540)
+4400 val loss 5.8603
+4400 val perplexity 350.8134
+4400 train 5.846482 (lr=2.6483e-05) (hash(x)=153594684)
+4500 val loss 5.8472
+4500 val perplexity 346.2659
+4500 train 5.798268 (lr=2.5714e-05) (hash(x)=144084750)
+4600 val loss 5.8362
+4600 val perplexity 342.4863
+4600 train 5.791237 (lr=2.4943e-05) (hash(x)=147423675)
+4700 val loss 5.8220
+4700 val perplexity 337.6465
+4700 train 5.890419 (lr=2.4172e-05) (hash(x)=162157039)
+4800 val loss 5.8095
+4800 val perplexity 333.4402
+4800 train 5.711671 (lr=2.3402e-05) (hash(x)=134970942)
+4900 val loss 5.8021
+4900 val perplexity 330.9785
+4900 train 5.715578 (lr=2.2633e-05) (hash(x)=160093370)
+5000 val loss 5.7940
+5000 val perplexity 328.3248
+5000 train 5.686359 (lr=2.1868e-05) (hash(x)=154986299)
+5100 val loss 5.7770
+5100 val perplexity 322.7944
+5100 train 5.622216 (lr=2.1107e-05) (hash(x)=145406582)
+5200 val loss 5.7733
+5200 val perplexity 321.5952
+5200 train 5.835043 (lr=2.0351e-05) (hash(x)=148029261)
+5300 val loss 5.7598
+5300 val perplexity 317.2803
+5300 train 5.757574 (lr=1.9602e-05) (hash(x)=157909233)
+5400 val loss 5.7508
+5400 val perplexity 314.4500
+5400 train 5.614801 (lr=1.8860e-05) (hash(x)=142250860)
+5500 val loss 5.7426
+5500 val perplexity 311.8788
+5500 train 5.618705 (lr=1.8127e-05) (hash(x)=145694703)
+5600 val loss 5.7324
+5600 val perplexity 308.7081
+5600 train 5.669287 (lr=1.7403e-05) (hash(x)=141788252)
+5700 val loss 5.7260
+5700 val perplexity 306.7326
+5700 train 5.629610 (lr=1.6690e-05) (hash(x)=150886692)
+5800 val loss 5.7198
+5800 val perplexity 304.8435
+5800 train 5.630589 (lr=1.5989e-05) (hash(x)=151814419)
+5900 val loss 5.7119
+5900 val perplexity 302.4598
+5900 train 5.727466 (lr=1.5300e-05) (hash(x)=146972539)
+6000 val loss 5.7033
+6000 val perplexity 299.8523
+6000 train 5.589607 (lr=1.4625e-05) (hash(x)=150219954)
+6100 val loss 5.6995
+6100 val perplexity 298.7126
+6100 train 5.695937 (lr=1.3965e-05) (hash(x)=161131435)
+6200 val loss 5.6894
+6200 val perplexity 295.7209
+6200 train 5.703752 (lr=1.3320e-05) (hash(x)=150226893)
+6300 val loss 5.6839
+6300 val perplexity 294.1045
+6300 train 5.519572 (lr=1.2692e-05) (hash(x)=144389552)
+6400 val loss 5.6800
+6400 val perplexity 292.9378
+6400 train 5.458444 (lr=1.2081e-05) (hash(x)=147757943)
+6500 val loss 5.6721
+6500 val perplexity 290.6318
+6500 train 5.659186 (lr=1.1489e-05) (hash(x)=144616373)
+6600 val loss 5.6706
+6600 val perplexity 290.1995
+6600 train 5.494869 (lr=1.0916e-05) (hash(x)=145266452)
+6700 val loss 5.6648
+6700 val perplexity 288.5276
+6700 train 5.657665 (lr=1.0363e-05) (hash(x)=149952261)
+6800 val loss 5.6608
+6800 val perplexity 287.3798
+6800 train 5.549417 (lr=9.8310e-06) (hash(x)=143351199)
+6900 val loss 5.6579
+6900 val perplexity 286.5487
+6900 train 5.622020 (lr=9.3205e-06) (hash(x)=156055618)
+7000 val loss 5.6511
+7000 val perplexity 284.6044
+7000 train 5.724583 (lr=8.8324e-06) (hash(x)=164290908)
+7100 val loss 5.6484
+7100 val perplexity 283.8316
+7100 train 5.750381 (lr=8.3674e-06) (hash(x)=150263832)
+7200 val loss 5.6447
+7200 val perplexity 282.8003
+7200 train 5.325688 (lr=7.9261e-06) (hash(x)=139219680)
+7300 val loss 5.6390
+7300 val perplexity 281.1682
+7300 train 5.581698 (lr=7.5093e-06) (hash(x)=150662994)
+7400 val loss 5.6358
+7400 val perplexity 280.2739
+7400 train 5.475738 (lr=7.1174e-06) (hash(x)=139007967)
+7500 val loss 5.6339
+7500 val perplexity 279.7443
+7500 train 5.472013 (lr=6.7511e-06) (hash(x)=138142461)
+7600 val loss 5.6296
+7600 val perplexity 278.5492
+7600 train 5.547984 (lr=6.4109e-06) (hash(x)=150023998)
+7700 val loss 5.6253
+7700 val perplexity 277.3461
+7700 train 5.518490 (lr=6.0972e-06) (hash(x)=143393355)
+7800 val loss 5.6247
+7800 val perplexity 277.1978
+7800 train 5.491231 (lr=5.8107e-06) (hash(x)=144916472)
+7900 val loss 5.6204
+7900 val perplexity 275.9962
+7900 train 5.574353 (lr=5.5515e-06) (hash(x)=150236934)
+8000 val loss 5.6175
+8000 val perplexity 275.1998
+8000 train 5.620214 (lr=5.3203e-06) (hash(x)=146536422)
+8100 val loss 5.6170
+8100 val perplexity 275.0581
+8100 train 5.449247 (lr=5.1172e-06) (hash(x)=151300857)
+8200 val loss 5.6160
+8200 val perplexity 274.7760
+8200 train 5.651255 (lr=4.9425e-06) (hash(x)=160351956)
+8300 val loss 5.6122
+8300 val perplexity 273.7557
+8300 train 5.668663 (lr=4.7966e-06) (hash(x)=150514540)
+8400 val loss 5.6096
+8400 val perplexity 273.0242
+8400 train 5.559687 (lr=4.6796e-06) (hash(x)=155904762)
+8500 val loss 5.6098
+8500 val perplexity 273.0965
+8500 train 5.412411 (lr=4.5917e-06) (hash(x)=146923196)
+8600 val loss 5.6073
+8600 val perplexity 272.4052
+8600 train 5.586360 (lr=4.5330e-06) (hash(x)=138977080)
+8700 val loss 5.6043
+8700 val perplexity 271.5973
+8700 train 5.410792 (lr=4.5037e-06) (hash(x)=146442792)
+8749 val loss 5.6043
+8749 val perplexity 271.6046

attention_kindselective_n_heads4_seed1343/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:165fb75f76ce3bebd99414ffb57b2a8b9c8d2cb74bc6b1efd62ef1362ad7c162
+oid sha256:89cbbfe08051cc268518f4ac57eb06686e0ad031912b376d60d4b2ece7f550c6
 size 92843394

attention_kindselective_n_heads4_seed1343/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5fdb83ba5561770f331d41f2e12db5dd14567e223eba95905bc5b55fdec00cbd
+oid sha256:1d40e0b7c47029541a659d00dbbf99d8b7310e2b074f124e697a2d1d9d53a229
 size 179406214