Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 4e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "4e-5_61440_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5e-5_61440_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,105 +1,408 @@
 max_steps: 8750
-7100 val loss 5.9287
-7100 val perplexity 375.6843
-7100 train 5.880068 (lr=6.5080e-06) (hash(x)=151066339)
+2000 val loss 6.4370
+2000 val perplexity 624.5455
+2000 train 6.371128 (lr=3.7143e-05) (hash(x)=158245023)
 0 val loss 11.2647
 0 val perplexity 78014.9766
-7200 val loss 5.9281
-7200 val perplexity 375.4359
-7200 train 5.731686 (lr=6.1648e-06) (hash(x)=155231264)
-0 train 11.267745 (lr=8.0000e-08) (hash(x)=150327452)
-7300 val loss 5.9206
-7300 val perplexity 372.6359
-7300 train 5.929327 (lr=5.8405e-06) (hash(x)=150281149)
-100 val loss 10.0255
-100 val perplexity 22595.3477
-100 train 10.075964 (lr=8.0800e-06) (hash(x)=166780046)
-7400 val loss 5.9179
-7400 val perplexity 371.6358
-7400 train 5.795485 (lr=5.5357e-06) (hash(x)=148421717)
-200 val loss 9.5219
-200 val perplexity 13655.3779
-200 train 9.486035 (lr=1.6080e-05) (hash(x)=155040610)
-7500 val loss 5.9176
-7500 val perplexity 371.5118
-7500 train 5.672206 (lr=5.2508e-06) (hash(x)=146921118)
-300 val loss 8.5181
-300 val perplexity 5004.6064
-300 train 8.538123 (lr=2.4080e-05) (hash(x)=155504036)
-7600 val loss 5.9100
-7600 val perplexity 368.6994
-7600 train 5.960232 (lr=4.9862e-06) (hash(x)=150660048)
-400 val loss 7.7858
-400 val perplexity 2406.2373
-400 train 7.670135 (lr=3.2080e-05) (hash(x)=143823248)
-7700 val loss 5.9087
-7700 val perplexity 368.2145
-7700 train 5.777826 (lr=4.7423e-06) (hash(x)=148059852)
-500 val loss 7.5398
-500 val perplexity 1881.4100
-500 train 7.509302 (lr=4.0000e-05) (hash(x)=143734685)
-7800 val loss 5.9071
-7800 val perplexity 367.6471
-7800 train 5.606631 (lr=4.5194e-06) (hash(x)=148331002)
-600 val loss 7.4574
-600 val perplexity 1732.6132
-600 train 7.243900 (lr=3.9987e-05) (hash(x)=150678249)
-7900 val loss 5.9006
-7900 val perplexity 365.2534
-7900 train 5.859887 (lr=4.3179e-06) (hash(x)=164923883)
-700 val loss 7.3848
-700 val perplexity 1611.2378
-700 train 7.528392 (lr=3.9948e-05) (hash(x)=175802021)
-8000 val loss 5.8995
-8000 val perplexity 364.8397
-8000 train 5.780609 (lr=4.1380e-06) (hash(x)=143545384)
-800 val loss 7.2813
-800 val perplexity 1452.8223
-800 train 7.292555 (lr=3.9883e-05) (hash(x)=158681215)
-8100 val loss 5.8987
-8100 val perplexity 364.5598
-8100 train 5.720141 (lr=3.9800e-06) (hash(x)=160686959)
-900 val loss 7.1971
-900 val perplexity 1335.4854
-900 train 7.089118 (lr=3.9792e-05) (hash(x)=146108145)
-8200 val loss 5.8961
-8200 val perplexity 363.6072
-8200 train 5.852425 (lr=3.8442e-06) (hash(x)=156501889)
-1000 val loss 7.1234
-1000 val perplexity 1240.7209
-1000 train 6.964436 (lr=3.9675e-05) (hash(x)=154996086)
-8300 val loss 5.8938
-8300 val perplexity 362.7742
-8300 train 5.731362 (lr=3.7307e-06) (hash(x)=142716875)
-1100 val loss 7.0250
-1100 val perplexity 1124.4392
-1100 train 7.009098 (lr=3.9532e-05) (hash(x)=153885445)
-8400 val loss 5.8888
-8400 val perplexity 360.9724
-8400 train 5.790694 (lr=3.6397e-06) (hash(x)=154436684)
-1200 val loss 6.9352
-1200 val perplexity 1027.8256
-1200 train 6.891309 (lr=3.9364e-05) (hash(x)=142353087)
-8500 val loss 5.8872
-8500 val perplexity 360.4104
-8500 train 5.953193 (lr=3.5713e-06) (hash(x)=147965839)
-1300 val loss 6.8669
-1300 val perplexity 959.9401
-1300 train 6.776668 (lr=3.9171e-05) (hash(x)=150750353)
-8600 val loss 5.8862
-8600 val perplexity 360.0428
-8600 train 5.690831 (lr=3.5257e-06) (hash(x)=145228097)
-1400 val loss 6.8045
-1400 val perplexity 901.8710
-1400 train 6.626483 (lr=3.8953e-05) (hash(x)=152767913)
-8700 val loss 5.8822
-8700 val perplexity 358.5815
-8700 train 6.058657 (lr=3.5029e-06) (hash(x)=152910357)
-1500 val loss 6.7235
-1500 val perplexity 831.7086
-1500 train 6.764162 (lr=3.8711e-05) (hash(x)=151562048)
-8749 val loss 5.8818
-8749 val perplexity 358.4548
-1600 val loss 6.6411
-1600 val perplexity 765.9436
-1600 train 6.638767 (lr=3.8444e-05) (hash(x)=166486165)
+2100 val loss 6.3948
+2100 val perplexity 598.6997
+2100 train 6.215977 (lr=3.6761e-05) (hash(x)=157204896)
+0 train 11.267745 (lr=1.0000e-07) (hash(x)=150327452)
+2200 val loss 6.3725
+2200 val perplexity 585.5281
+2200 train 6.217300 (lr=3.6358e-05) (hash(x)=137541932)
+100 val loss 9.9617
+100 val perplexity 21199.5801
+100 train 10.014555 (lr=1.0100e-05) (hash(x)=166780046)
+2300 val loss 6.3442
+2300 val perplexity 569.1939
+2300 train 6.298882 (lr=3.5935e-05) (hash(x)=150149692)
+200 val loss 9.2487
+200 val perplexity 10391.1982
+200 train 9.198502 (lr=2.0100e-05) (hash(x)=155040610)
+2400 val loss 6.3008
+2400 val perplexity 545.0193
+2400 train 6.289979 (lr=3.5491e-05) (hash(x)=151730720)
+300 val loss 8.1647
+300 val perplexity 3514.8396
+300 train 8.169972 (lr=3.0100e-05) (hash(x)=155504036)
+2500 val loss 6.2774
+2500 val perplexity 532.3767
+2500 train 6.057900 (lr=3.5027e-05) (hash(x)=143406752)
+400 val loss 7.5999
+400 val perplexity 1998.0118
+400 train 7.465411 (lr=4.0100e-05) (hash(x)=143823248)
+2600 val loss 6.2626
+2600 val perplexity 524.5986
+2600 train 6.088369 (lr=3.4545e-05) (hash(x)=157272496)
+500 val loss 7.4464
+500 val perplexity 1713.7411
+500 train 7.412394 (lr=5.0000e-05) (hash(x)=143734685)
+2700 val loss 6.2352
+2700 val perplexity 510.3958
+2700 train 6.233461 (lr=3.4044e-05) (hash(x)=155342327)
+600 val loss 7.3773
+600 val perplexity 1599.2358
+600 train 7.154145 (lr=4.9984e-05) (hash(x)=150678249)
+2800 val loss 6.2037
+2800 val perplexity 494.5931
+2800 train 6.083380 (lr=3.3526e-05) (hash(x)=140626679)
+700 val loss 7.2967
+700 val perplexity 1475.4744
+700 train 7.442945 (lr=4.9935e-05) (hash(x)=175802021)
+2900 val loss 6.1842
+2900 val perplexity 485.0013
+2900 train 6.041373 (lr=3.2992e-05) (hash(x)=144953350)
+800 val loss 7.1922
+800 val perplexity 1329.0032
+800 train 7.206111 (lr=4.9853e-05) (hash(x)=158681215)
+3000 val loss 6.1690
+3000 val perplexity 477.6920
+3000 train 6.072640 (lr=3.2441e-05) (hash(x)=172449837)
+900 val loss 7.1072
+900 val perplexity 1220.7802
+900 train 6.997451 (lr=4.9739e-05) (hash(x)=146108145)
+3100 val loss 6.1653
+3100 val perplexity 475.9668
+3100 train 5.957810 (lr=3.1875e-05) (hash(x)=141710086)
+1000 val loss 7.0298
+1000 val perplexity 1129.8363
+1000 train 6.867131 (lr=4.9593e-05) (hash(x)=154996086)
+3200 val loss 6.1250
+3200 val perplexity 457.1336
+3200 train 6.054896 (lr=3.1295e-05) (hash(x)=151299772)
+1100 val loss 6.9379
+1100 val perplexity 1030.5953
+1100 train 6.917721 (lr=4.9415e-05) (hash(x)=153885445)
+3300 val loss 6.1114
+3300 val perplexity 450.9624
+3300 train 5.977787 (lr=3.0702e-05) (hash(x)=146473110)
+1200 val loss 6.8587
+1200 val perplexity 952.1497
+1200 train 6.820927 (lr=4.9205e-05) (hash(x)=142353087)
+3400 val loss 6.0891
+3400 val perplexity 441.0146
+3400 train 6.166475 (lr=3.0095e-05) (hash(x)=153954157)
+1300 val loss 6.7949
+1300 val perplexity 893.2373
+1300 train 6.699144 (lr=4.8964e-05) (hash(x)=150750353)
+3500 val loss 6.0733
+3500 val perplexity 434.1111
+3500 train 6.005799 (lr=2.9477e-05) (hash(x)=153717336)
+1400 val loss 6.7528
+1400 val perplexity 856.4910
+1400 train 6.573970 (lr=4.8691e-05) (hash(x)=152767913)
+3600 val loss 6.0607
+3600 val perplexity 428.6727
+3600 train 5.809951 (lr=2.8849e-05) (hash(x)=144965161)
+1500 val loss 6.6831
+1500 val perplexity 798.7867
+1500 train 6.727711 (lr=4.8388e-05) (hash(x)=151562048)
+3700 val loss 6.0523
+3700 val perplexity 425.0963
+3700 train 5.825770 (lr=2.8210e-05) (hash(x)=125969741)
+1600 val loss 6.6191
+1600 val perplexity 749.2616
+1600 train 6.620916 (lr=4.8055e-05) (hash(x)=166486165)
+3800 val loss 6.0290
+3800 val perplexity 415.2866
+3800 train 5.888848 (lr=2.7562e-05) (hash(x)=155070487)
+1700 val loss 6.5918
+1700 val perplexity 729.0835
+1700 train 6.235582 (lr=4.7691e-05) (hash(x)=130835396)
+3900 val loss 6.0102
+3900 val perplexity 407.5825
+3900 train 5.883321 (lr=2.6907e-05) (hash(x)=149444644)
+1800 val loss 6.5444
+1800 val perplexity 695.3355
+1800 train 6.591407 (lr=4.7299e-05) (hash(x)=158851816)
+4000 val loss 6.0051
+4000 val perplexity 405.4889
+4000 train 5.854551 (lr=2.6244e-05) (hash(x)=151663033)
+1900 val loss 6.5057
+1900 val perplexity 668.9247
+1900 train 6.534745 (lr=4.6878e-05) (hash(x)=153313879)
+4100 val loss 6.0012
+4100 val perplexity 403.9044
+4100 train 5.955617 (lr=2.5575e-05) (hash(x)=143688282)
+2000 val loss 6.4488
+2000 val perplexity 631.9535
+2000 train 6.381792 (lr=4.6428e-05) (hash(x)=158245023)
+4200 val loss 5.9756
+4200 val perplexity 393.6931
+4200 train 5.975471 (lr=2.4900e-05) (hash(x)=163361651)
+2100 val loss 6.4032
+2100 val perplexity 603.7603
+2100 train 6.227605 (lr=4.5951e-05) (hash(x)=157204896)
+4300 val loss 5.9607
+4300 val perplexity 387.8712
+4300 train 6.028554 (lr=2.4222e-05) (hash(x)=153619361)
+2200 val loss 6.3820
+2200 val perplexity 591.1212
+4400 val loss 5.9584
+4400 val perplexity 386.9796
+2200 train 6.223777 (lr=4.5448e-05) (hash(x)=137541932)
+4400 train 6.271734 (lr=2.3540e-05) (hash(x)=168527064)
+4500 val loss 5.9460
+4500 val perplexity 382.2105
+4500 train 5.752192 (lr=2.2856e-05) (hash(x)=125588037)
+2300 val loss 6.3599
+2300 val perplexity 578.1849
+2300 train 6.312685 (lr=4.4918e-05) (hash(x)=150149692)
+4600 val loss 5.9269
+4600 val perplexity 374.9906
+4600 train 5.890114 (lr=2.2171e-05) (hash(x)=143710941)
+2400 val loss 6.3222
+2400 val perplexity 556.8002
+2400 train 6.310997 (lr=4.4363e-05) (hash(x)=151730720)
+4700 val loss 5.9159
+4700 val perplexity 370.8951
+4700 train 5.746759 (lr=2.1486e-05) (hash(x)=150952742)
+2500 val loss 6.2898
+2500 val perplexity 539.0579
+2500 train 6.072419 (lr=4.3784e-05) (hash(x)=143406752)
+4800 val loss 5.9073
+4800 val perplexity 367.7109
+4800 train 5.773638 (lr=2.0801e-05) (hash(x)=145323659)
+2600 val loss 6.2791
+2600 val perplexity 533.3013
+2600 train 6.115969 (lr=4.3181e-05) (hash(x)=157272496)
+4900 val loss 5.8995
+4900 val perplexity 364.8397
+4900 train 5.918860 (lr=2.0118e-05) (hash(x)=153151397)
+2700 val loss 6.2556
+2700 val perplexity 520.9327
+2700 train 6.248377 (lr=4.2555e-05) (hash(x)=155342327)
+5000 val loss 5.8912
+5000 val perplexity 361.8272
+5000 train 5.792883 (lr=1.9438e-05) (hash(x)=143182059)
+2800 val loss 6.2198
+2800 val perplexity 502.6056
+2800 train 6.090466 (lr=4.1908e-05) (hash(x)=140626679)
+5100 val loss 5.8823
+5100 val perplexity 358.6324
+5100 train 5.928619 (lr=1.8762e-05) (hash(x)=170083586)
+2900 val loss 6.1933
+2900 val perplexity 489.4789
+2900 train 6.050964 (lr=4.1240e-05) (hash(x)=144953350)
+5200 val loss 5.8692
+5200 val perplexity 353.9545
+5200 train 5.759492 (lr=1.8090e-05) (hash(x)=149363919)
+3000 val loss 6.1754
+3000 val perplexity 480.7940
+3000 train 6.073356 (lr=4.0551e-05) (hash(x)=172449837)
+5300 val loss 5.8625
+5300 val perplexity 351.5884
+5300 train 5.756340 (lr=1.7424e-05) (hash(x)=152033784)
+3100 val loss 6.1695
+3100 val perplexity 477.9614
+3100 train 5.960622 (lr=3.9844e-05) (hash(x)=141710086)
+5400 val loss 5.8494
+5400 val perplexity 347.0114
+5400 train 5.957279 (lr=1.6765e-05) (hash(x)=154614289)
+3200 val loss 6.1283
+3200 val perplexity 458.6766
+3200 train 6.060437 (lr=3.9119e-05) (hash(x)=151299772)
+5500 val loss 5.8406
+5500 val perplexity 343.9975
+5500 train 5.933627 (lr=1.6113e-05) (hash(x)=157745174)
+3300 val loss 6.1232
+3300 val perplexity 456.3084
+3300 train 5.992179 (lr=3.8377e-05) (hash(x)=146473110)
+5600 val loss 5.8365
+5600 val perplexity 342.5832
+5600 train 5.729990 (lr=1.5469e-05) (hash(x)=147693222)
+3400 val loss 6.0933
+3400 val perplexity 442.8724
+3400 train 6.177316 (lr=3.7619e-05) (hash(x)=153954157)
+5700 val loss 5.8250
+5700 val perplexity 338.6544
+5700 train 5.762623 (lr=1.4836e-05) (hash(x)=149784627)
+3500 val loss 6.0865
+3500 val perplexity 439.8985
+3500 train 6.025422 (lr=3.6847e-05) (hash(x)=153717336)
+5800 val loss 5.8244
+5800 val perplexity 338.4496
+5800 train 5.736485 (lr=1.4212e-05) (hash(x)=158620729)
+3600 val loss 6.0678
+3600 val perplexity 431.7099
+3600 train 5.823427 (lr=3.6061e-05) (hash(x)=144965161)
+5900 val loss 5.8146
+5900 val perplexity 335.1426
+5900 train 5.760200 (lr=1.3600e-05) (hash(x)=159763910)
+3700 val loss 6.0644
+3700 val perplexity 430.2674
+3700 train 5.841285 (lr=3.5263e-05) (hash(x)=125969741)
+6000 val loss 5.8055
+6000 val perplexity 332.1231
+6000 train 5.739980 (lr=1.3000e-05) (hash(x)=147640561)
+3800 val loss 6.0351
+3800 val perplexity 417.8310
+3800 train 5.895473 (lr=3.4453e-05) (hash(x)=155070487)
+6100 val loss 5.7986
+6100 val perplexity 329.8218
+6100 train 5.780902 (lr=1.2413e-05) (hash(x)=156613394)
+3900 val loss 6.0204
+3900 val perplexity 411.7543
+3900 train 5.894702 (lr=3.3633e-05) (hash(x)=149444644)
+6200 val loss 5.7929
+6200 val perplexity 327.9590
+6200 train 5.741335 (lr=1.1840e-05) (hash(x)=186221290)
+4000 val loss 6.0050
+4000 val perplexity 405.4474
+4000 train 5.864678 (lr=3.2805e-05) (hash(x)=151663033)
+6300 val loss 5.7831
+6300 val perplexity 324.7564
+6300 train 5.659614 (lr=1.1282e-05) (hash(x)=152081419)
+4100 val loss 5.9989
+4100 val perplexity 403.0023
+4100 train 5.955342 (lr=3.1968e-05) (hash(x)=143688282)
+6400 val loss 5.7833
+6400 val perplexity 324.8156
+6400 train 5.735166 (lr=1.0739e-05) (hash(x)=154808349)
+4200 val loss 5.9733
+4200 val perplexity 392.7840
+4200 train 5.975454 (lr=3.1126e-05) (hash(x)=163361651)
+6500 val loss 5.7760
+6500 val perplexity 322.4663
+6500 train 5.745718 (lr=1.0213e-05) (hash(x)=159437208)
+4300 val loss 5.9635
+4300 val perplexity 388.9864
+4300 train 6.007787 (lr=3.0277e-05) (hash(x)=153619361)
+6600 val loss 5.7731
+6600 val perplexity 321.5303
+6600 train 5.587155 (lr=9.7032e-06) (hash(x)=157933074)
+4400 val loss 5.9564
+4400 val perplexity 386.2238
+4400 train 6.267161 (lr=2.9425e-05) (hash(x)=168527064)
+6700 val loss 5.7666
+6700 val perplexity 319.4617
+6700 train 5.745788 (lr=9.2116e-06) (hash(x)=161560240)
+4500 val loss 5.9458
+4500 val perplexity 382.1341
+4500 train 5.755313 (lr=2.8571e-05) (hash(x)=125588037)
+6800 val loss 5.7636
+6800 val perplexity 318.4973
+6800 train 5.744992 (lr=8.7387e-06) (hash(x)=155424292)
+4600 val loss 5.9250
+4600 val perplexity 374.2863
+4600 train 5.879909 (lr=2.7714e-05) (hash(x)=143710941)
+6900 val loss 5.7626
+6900 val perplexity 318.1587
+6900 train 5.756405 (lr=8.2849e-06) (hash(x)=148561470)
+4700 val loss 5.9121
+4700 val perplexity 369.4801
+4700 train 5.743450 (lr=2.6857e-05) (hash(x)=150952742)
+7000 val loss 5.7545
+7000 val perplexity 315.6169
+7000 train 5.646148 (lr=7.8510e-06) (hash(x)=141527450)
+4800 val loss 5.9022
+4800 val perplexity 365.8246
+4800 train 5.772085 (lr=2.6002e-05) (hash(x)=145323659)
+7100 val loss 5.7503
+7100 val perplexity 314.2946
+7100 train 5.706478 (lr=7.4377e-06) (hash(x)=151066339)
+4900 val loss 5.8952
+4900 val perplexity 363.2955
+4900 train 5.914246 (lr=2.5148e-05) (hash(x)=153151397)
+7200 val loss 5.7511
+7200 val perplexity 314.5356
+7200 train 5.548562 (lr=7.0455e-06) (hash(x)=155231264)
+5000 val loss 5.8868
+5000 val perplexity 360.2437
+5000 train 5.788630 (lr=2.4298e-05) (hash(x)=143182059)
+7300 val loss 5.7421
+7300 val perplexity 311.7135
+7300 train 5.759835 (lr=6.6749e-06) (hash(x)=150281149)
+5100 val loss 5.8739
+5100 val perplexity 355.6327
+5100 train 5.913543 (lr=2.3452e-05) (hash(x)=170083586)
+7400 val loss 5.7399
+7400 val perplexity 311.0442
+7400 train 5.622359 (lr=6.3266e-06) (hash(x)=148421717)
+5200 val loss 5.8581
+5200 val perplexity 350.0584
+5200 train 5.746659 (lr=2.2613e-05) (hash(x)=149363919)
+7500 val loss 5.7394
+7500 val perplexity 310.8757
+7500 train 5.503174 (lr=6.0010e-06) (hash(x)=146921118)
+5300 val loss 5.8530
+5300 val perplexity 348.2945
+5300 train 5.744291 (lr=2.1780e-05) (hash(x)=152033784)
+7600 val loss 5.7331
+7600 val perplexity 308.9312
+7600 train 5.784346 (lr=5.6986e-06) (hash(x)=150660048)
+5400 val loss 5.8368
+5400 val perplexity 342.6903
+5400 train 5.944860 (lr=2.0956e-05) (hash(x)=154614289)
+7700 val loss 5.7314
+7700 val perplexity 308.3879
+7700 train 5.606787 (lr=5.4198e-06) (hash(x)=148059852)
+5500 val loss 5.8313
+5500 val perplexity 340.8127
+5500 train 5.921842 (lr=2.0141e-05) (hash(x)=157745174)
+7800 val loss 5.7305
+7800 val perplexity 308.1266
+7800 train 5.426733 (lr=5.1650e-06) (hash(x)=148331002)
+5600 val loss 5.8229
+5600 val perplexity 337.9419
+5600 train 5.719602 (lr=1.9337e-05) (hash(x)=147693222)
+7900 val loss 5.7251
+7900 val perplexity 306.4575
+7900 train 5.675220 (lr=4.9347e-06) (hash(x)=164923883)
+5700 val loss 5.8136
+5700 val perplexity 334.8164
+5700 train 5.748807 (lr=1.8545e-05) (hash(x)=149784627)
+8000 val loss 5.7236
+8000 val perplexity 306.0140
+8000 train 5.605746 (lr=4.7291e-06) (hash(x)=143545384)
+5800 val loss 5.8120
+5800 val perplexity 334.2779
+5800 train 5.732249 (lr=1.7765e-05) (hash(x)=158620729)
+8100 val loss 5.7227
+8100 val perplexity 305.7214
+8100 train 5.539697 (lr=4.5486e-06) (hash(x)=160686959)
+5900 val loss 5.8007
+5900 val perplexity 330.5159
+5900 train 5.749244 (lr=1.7000e-05) (hash(x)=159763910)
+8200 val loss 5.7206
+8200 val perplexity 305.1013
+8200 train 5.676816 (lr=4.3933e-06) (hash(x)=156501889)
+6000 val loss 5.7906
+6000 val perplexity 327.2256
+6000 train 5.725507 (lr=1.6250e-05) (hash(x)=147640561)
+8300 val loss 5.7185
+8300 val perplexity 304.4521
+8300 train 5.560667 (lr=4.2636e-06) (hash(x)=142716875)
+6100 val loss 5.7819
+6100 val perplexity 324.3689
+6100 train 5.764189 (lr=1.5516e-05) (hash(x)=156613394)
+8400 val loss 5.7141
+8400 val perplexity 303.1260
+8400 train 5.627058 (lr=4.1596e-06) (hash(x)=154436684)
+6200 val loss 5.7755
+6200 val perplexity 322.3004
+6200 train 5.712209 (lr=1.4800e-05) (hash(x)=186221290)
+8500 val loss 5.7127
+8500 val perplexity 302.6869
+8500 train 5.795162 (lr=4.0815e-06) (hash(x)=147965839)
+6300 val loss 5.7650
+6300 val perplexity 318.9550
+6300 train 5.640378 (lr=1.4102e-05) (hash(x)=152081419)
+8600 val loss 5.7114
+8600 val perplexity 302.3018
+8600 train 5.512054 (lr=4.0294e-06) (hash(x)=145228097)
+6400 val loss 5.7646
+6400 val perplexity 318.8096
+6400 train 5.714537 (lr=1.3424e-05) (hash(x)=154808349)
+8700 val loss 5.7076
+8700 val perplexity 301.1422
+8700 train 5.898399 (lr=4.0033e-06) (hash(x)=152910357)
+8749 val loss 5.7076
+8749 val perplexity 301.1334
+6500 val loss 5.7571
+6500 val perplexity 316.4409
+6500 train 5.723630 (lr=1.2766e-05) (hash(x)=159437208)
+6600 val loss 5.7538
+6600 val perplexity 315.3894
+6600 train 5.563268 (lr=1.2129e-05) (hash(x)=157933074)

attention_kindselective_n_heads4_seed1338/model_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d1bac2e88faa57534b7341e4dc64586426fcadfd3c4f0cca0c9fa558bf960b18
+oid sha256:5d9aea93b1a4a003ae1bb2610aa8e99a57ab1cbb4737ed231f282363c6467892
 size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_08749.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ffaba6e69d30b87c050e2bc7acc71bb66c4439938865e3f0d226c5e8348474a9
+oid sha256:b8ad9c642a2e4de42b4ccaecdd362a1673e76da42d6280c1a1635b4fe2515f05
 size 179406214