Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1338", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_2_1338", "n_embd": 128}

attention_kindselective_n_heads2_seed1338/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.7630
 0 val perplexity 128408.9922
-0 train 11.769258 (lr=7.5000e-07) (hash(x)=150327452)
-100 val loss 9.0521
-100 val perplexity 8536.5449
-100 train 9.044237 (lr=7.5750e-05) (hash(x)=166441190)
-200 val loss 7.6605
-200 val perplexity 2122.7275
-200 train 7.890631 (lr=1.5000e-04) (hash(x)=166780046)
-300 val loss 7.6222
-300 val perplexity 2043.0656
-300 train 7.604558 (lr=1.4997e-04) (hash(x)=159835303)
-400 val loss 7.6228
-400 val perplexity 2044.2993
-400 train 7.408037 (lr=1.4986e-04) (hash(x)=155040610)
-500 val loss 7.6274
-500 val perplexity 2053.7896
-500 train 7.338044 (lr=1.4969e-04) (hash(x)=130190460)
-600 val loss 7.5507
-600 val perplexity 1902.0396
-600 train 7.583073 (lr=1.4945e-04) (hash(x)=155504036)
-700 val loss 7.5274
-700 val perplexity 1858.2739
-700 train 7.362857 (lr=1.4913e-04) (hash(x)=137347213)
-800 val loss 7.4907
-800 val perplexity 1791.3402
-800 train 7.345782 (lr=1.4876e-04) (hash(x)=143823248)
-900 val loss 7.4892
-900 val perplexity 1788.5781
-900 train 7.601803 (lr=1.4831e-04) (hash(x)=156260416)
-1000 val loss 7.5226
-1000 val perplexity 1849.3644
-1000 train 7.496523 (lr=1.4779e-04) (hash(x)=143734685)
-1100 val loss 7.5623
-1100 val perplexity 1924.2355
-1100 train 7.504193 (lr=1.4721e-04) (hash(x)=160013925)
-1200 val loss 7.5167
-1200 val perplexity 1838.4960
-1200 train 7.274280 (lr=1.4656e-04) (hash(x)=150678249)
-1300 val loss 7.5381
-1300 val perplexity 1878.2494
-1300 train 7.334253 (lr=1.4585e-04) (hash(x)=149073315)
-1400 val loss 7.4529
-1400 val perplexity 1724.9286
-1400 train 7.732779 (lr=1.4507e-04) (hash(x)=175802021)
-1500 val loss 7.4782
-1500 val perplexity 1768.9734
-1500 train 7.721244 (lr=1.4422e-04) (hash(x)=171034639)
-1600 val loss 7.3972
-1600 val perplexity 1631.4352
-1600 train 7.618242 (lr=1.4332e-04) (hash(x)=158681215)
-1700 val loss 7.3786
-1700 val perplexity 1601.3541
-1700 train 7.325286 (lr=1.4235e-04) (hash(x)=152116061)
-1800 val loss 7.3584
-1800 val perplexity 1569.2958
-1800 train 7.297063 (lr=1.4131e-04) (hash(x)=146108145)
-1900 val loss 7.3448
-1900 val perplexity 1548.2006
-1900 train 7.178658 (lr=1.4022e-04) (hash(x)=147598108)
-2000 val loss 7.3579
-2000 val perplexity 1568.5356
-2000 train 7.214629 (lr=1.3907e-04) (hash(x)=154996086)
-2100 val loss 7.3745
-2100 val perplexity 1594.8114
-2100 train 7.154959 (lr=1.3786e-04) (hash(x)=153396183)
-2200 val loss 7.3039
-2200 val perplexity 1486.0160
-2200 train 7.297114 (lr=1.3660e-04) (hash(x)=153885445)
-2300 val loss 7.2644
-2300 val perplexity 1428.4915
-2300 train 7.247360 (lr=1.3527e-04) (hash(x)=159666385)
-2400 val loss 7.2269
-2400 val perplexity 1375.9486
-2400 train 7.192606 (lr=1.3390e-04) (hash(x)=142353087)
-2500 val loss 7.2318
-2500 val perplexity 1382.7686
-2500 train 7.238693 (lr=1.3247e-04) (hash(x)=146491718)
-2600 val loss 7.2232
-2600 val perplexity 1370.8739
-2600 train 7.170867 (lr=1.3099e-04) (hash(x)=150750353)
-2700 val loss 7.2291
-2700 val perplexity 1378.9740
-2700 train 6.991965 (lr=1.2946e-04) (hash(x)=129849193)
-2800 val loss 7.2272
-2800 val perplexity 1376.4000
-2800 train 7.102032 (lr=1.2788e-04) (hash(x)=152767913)
-2900 val loss 7.2269
-2900 val perplexity 1375.8988
-2900 train 7.011504 (lr=1.2626e-04) (hash(x)=146531140)
-3000 val loss 7.2099
-3000 val perplexity 1352.7839
-3000 train 7.265077 (lr=1.2459e-04) (hash(x)=151562048)
-3100 val loss 7.1710
-3100 val perplexity 1301.0862
-3100 train 7.163158 (lr=1.2287e-04) (hash(x)=146001424)
-3200 val loss 7.1383
-3200 val perplexity 1259.2461
-3200 train 7.142620 (lr=1.2112e-04) (hash(x)=166486165)
-3300 val loss 7.1243
-3300 val perplexity 1241.7349
-3300 train 7.075697 (lr=1.1932e-04) (hash(x)=150866680)
-3400 val loss 7.0989
-3400 val perplexity 1210.6708
-3400 train 7.022961 (lr=1.1749e-04) (hash(x)=143900419)
-3500 val loss 7.0771
-3500 val perplexity 1184.4706
-3500 train 6.928240 (lr=1.1562e-04) (hash(x)=148845794)
-3600 val loss 7.0529
-3600 val perplexity 1156.2266
-3600 train 6.923109 (lr=1.1372e-04) (hash(x)=145667796)
-3700 val loss 7.0336
-3700 val perplexity 1134.1205
-3700 train 7.037069 (lr=1.1179e-04) (hash(x)=163563851)
-3800 val loss 7.0171
-3800 val perplexity 1115.5350
-3800 train 6.973225 (lr=1.0982e-04) (hash(x)=147488689)
-3900 val loss 6.9559
-3900 val perplexity 1049.3538
-3900 train 6.944118 (lr=1.0783e-04) (hash(x)=148186608)
-4000 val loss 6.9276
-4000 val perplexity 1020.0370
-4000 train 6.799710 (lr=1.0581e-04) (hash(x)=142970187)
-4100 val loss 6.8930
-4100 val perplexity 985.3903
-4100 train 6.878218 (lr=1.0377e-04) (hash(x)=141584883)
-4200 val loss 6.8809
-4200 val perplexity 973.5345
-4200 train 6.712707 (lr=1.0171e-04) (hash(x)=145664585)
-4300 val loss 6.8585
-4300 val perplexity 951.9722
-4300 train 6.715063 (lr=9.9622e-05) (hash(x)=143736499)
-4400 val loss 6.8377
-4400 val perplexity 932.3591
-4400 train 6.658221 (lr=9.7520e-05) (hash(x)=151883322)
-4500 val loss 6.8341
-4500 val perplexity 928.9695
-4500 train 6.731677 (lr=9.5403e-05) (hash(x)=153904871)
-4600 val loss 6.8100
-4600 val perplexity 906.8578
-4600 train 6.856602 (lr=9.3273e-05) (hash(x)=154893521)
-4700 val loss 6.7685
-4700 val perplexity 869.9999
-4700 train 6.820551 (lr=9.1132e-05) (hash(x)=152323949)
-4800 val loss 6.7547
-4800 val perplexity 858.0624
-4800 train 6.703928 (lr=8.8982e-05) (hash(x)=154104619)
-4900 val loss 6.7486
-4900 val perplexity 852.8326
-4900 train 6.794868 (lr=8.6825e-05) (hash(x)=146311426)
-5000 val loss 6.7308
-5000 val perplexity 837.7810
-5000 train 6.760775 (lr=8.4663e-05) (hash(x)=156741847)
-5100 val loss 6.7207
-5100 val perplexity 829.4147
-5100 train 6.523645 (lr=8.2500e-05) (hash(x)=142086346)
-5200 val loss 6.7151
-5200 val perplexity 824.7618
-5200 train 6.511878 (lr=8.0337e-05) (hash(x)=150265428)
-5300 val loss 6.7084
-5300 val perplexity 819.2867
-5300 train 6.611176 (lr=7.8175e-05) (hash(x)=151339108)
-5400 val loss 6.6909
-5400 val perplexity 805.0494
-5400 train 6.707263 (lr=7.6018e-05) (hash(x)=154654372)
-5500 val loss 6.6492
-5500 val perplexity 772.1855
-5500 train 6.639955 (lr=7.3868e-05) (hash(x)=150575051)
-5600 val loss 6.6394
-5600 val perplexity 764.6109
-5600 train 6.526065 (lr=7.1727e-05) (hash(x)=140396423)
-5700 val loss 6.6357
-5700 val perplexity 761.8185
-5700 train 6.524915 (lr=6.9597e-05) (hash(x)=144678758)
-5800 val loss 6.6252
-5800 val perplexity 753.8713
-5800 train 6.742186 (lr=6.7480e-05) (hash(x)=151992743)
-5900 val loss 6.6310
-5900 val perplexity 758.2639
-5900 train 6.428650 (lr=6.5378e-05) (hash(x)=144396927)
-6000 val loss 6.6086
-6000 val perplexity 741.4727
-6000 train 6.562921 (lr=6.3294e-05) (hash(x)=165478625)
-6100 val loss 6.6103
-6100 val perplexity 742.7289
-6100 train 6.391157 (lr=6.1230e-05) (hash(x)=147088621)
-6200 val loss 6.6126
-6200 val perplexity 744.4166
-6200 train 6.403824 (lr=5.9188e-05) (hash(x)=140794994)
-6300 val loss 6.5798
-6300 val perplexity 720.4310
-6300 train 6.508721 (lr=5.7169e-05) (hash(x)=134780906)
-6400 val loss 6.5572
-6400 val perplexity 704.3224
-6400 train 6.560293 (lr=5.5177e-05) (hash(x)=149023655)
-6500 val loss 6.5482
-6500 val perplexity 698.0131
-6500 train 6.456416 (lr=5.3213e-05) (hash(x)=147497796)
-6600 val loss 6.5457
-6600 val perplexity 696.2134
-6600 train 6.418583 (lr=5.1279e-05) (hash(x)=152902689)
-6700 val loss 6.5409
-6700 val perplexity 692.8889
-6700 train 6.581048 (lr=4.9377e-05) (hash(x)=153846046)
-6800 val loss 6.5169
-6800 val perplexity 676.4545
-6800 train 6.625207 (lr=4.7509e-05) (hash(x)=158512738)
-6900 val loss 6.5168
-6900 val perplexity 676.3990
-6900 train 6.928324 (lr=4.5676e-05) (hash(x)=156849968)
-7000 val loss 6.5009
-7000 val perplexity 665.7363
-7000 train 6.337559 (lr=4.3882e-05) (hash(x)=142395855)
-7100 val loss 6.4916
-7100 val perplexity 659.5842
-7100 train 6.373418 (lr=4.2128e-05) (hash(x)=147114884)
-7200 val loss 6.4952
-7200 val perplexity 661.9579
-7200 train 6.517949 (lr=4.0414e-05) (hash(x)=156979839)
-7300 val loss 6.4884
-7300 val perplexity 657.4925
-7300 train 6.252299 (lr=3.8745e-05) (hash(x)=145584373)
-7400 val loss 6.4907
-7400 val perplexity 658.9988
-7400 train 6.238008 (lr=3.7120e-05) (hash(x)=141508204)
-7500 val loss 6.4769
-7500 val perplexity 649.9462
-7500 train 6.531075 (lr=3.5541e-05) (hash(x)=148803965)
-7600 val loss 6.4656
-7600 val perplexity 642.6309
-7600 train 6.525251 (lr=3.4011e-05) (hash(x)=151019676)
-7700 val loss 6.4557
-7700 val perplexity 636.3220
-7700 train 6.477695 (lr=3.2531e-05) (hash(x)=143155750)
-7800 val loss 6.4516
-7800 val perplexity 633.7315
-7800 train 6.449757 (lr=3.1102e-05) (hash(x)=152569653)
-7900 val loss 6.4499
-7900 val perplexity 632.6575
-7900 train 6.330617 (lr=2.9726e-05) (hash(x)=143519455)
-8000 val loss 6.4452
-8000 val perplexity 629.6876
-8000 train 6.569186 (lr=2.8405e-05) (hash(x)=161180944)
-8100 val loss 6.4395
-8100 val perplexity 626.1073
-8100 train 6.452418 (lr=2.7138e-05) (hash(x)=154107345)
-8200 val loss 6.4417
-8200 val perplexity 627.4514
-8200 train 6.449410 (lr=2.5929e-05) (hash(x)=152486517)
-8300 val loss 6.4259
-8300 val perplexity 617.6229
-8300 train 6.451607 (lr=2.4778e-05) (hash(x)=156167749)
-8400 val loss 6.4192
-8400 val perplexity 613.5302
-8400 train 6.499967 (lr=2.3686e-05) (hash(x)=149155006)
-8500 val loss 6.4125
-8500 val perplexity 609.4304
-8500 train 6.375862 (lr=2.2655e-05) (hash(x)=147844390)
-8600 val loss 6.4072
-8600 val perplexity 606.1641
-8600 train 6.633799 (lr=2.1685e-05) (hash(x)=165753320)
-8700 val loss 6.4076
-8700 val perplexity 606.4477
-8700 train 6.207141 (lr=2.0777e-05) (hash(x)=146079979)
-8800 val loss 6.4047
-8800 val perplexity 604.6834
-8800 train 6.595518 (lr=1.9933e-05) (hash(x)=172259509)
-8900 val loss 6.4076
-8900 val perplexity 606.4255
-8900 train 6.130263 (lr=1.9153e-05) (hash(x)=145148314)
-9000 val loss 6.4070
-9000 val perplexity 606.0482
-9000 train 6.334520 (lr=1.8439e-05) (hash(x)=144250633)
-9100 val loss 6.3994
-9100 val perplexity 601.4957
-9100 train 6.510191 (lr=1.7790e-05) (hash(x)=157219797)
-9200 val loss 6.3903
-9200 val perplexity 596.0293
-9200 train 6.373724 (lr=1.7208e-05) (hash(x)=142743778)
-9300 val loss 6.3861
-9300 val perplexity 593.5485
-9300 train 6.297025 (lr=1.6692e-05) (hash(x)=139669771)
-9400 val loss 6.3821
-9400 val perplexity 591.1829
-9400 train 6.361324 (lr=1.6245e-05) (hash(x)=145916843)
-9500 val loss 6.3828
-9500 val perplexity 591.5715
-9500 train 6.313448 (lr=1.5865e-05) (hash(x)=150196125)
-9600 val loss 6.3831
-9600 val perplexity 591.7433
-9600 train 6.660431 (lr=1.5554e-05) (hash(x)=160041419)
-9700 val loss 6.3839
-9700 val perplexity 592.2127
-9700 train 6.210125 (lr=1.5312e-05) (hash(x)=139931627)
-9800 val loss 6.3800
-9800 val perplexity 589.9517
-9800 train 6.333031 (lr=1.5139e-05) (hash(x)=150370792)
-9900 val loss 6.3764
-9900 val perplexity 587.8044
-9900 train 6.237167 (lr=1.5035e-05) (hash(x)=153014886)
-9999 val loss 6.3805
-9999 val perplexity 590.2499
+0 train 11.769258 (lr=2.5000e-07) (hash(x)=150327452)
+100 val loss 10.2404
+100 val perplexity 28012.3633
+100 train 10.227105 (lr=2.5250e-05) (hash(x)=166441190)
+200 val loss 8.6924
+200 val perplexity 5957.4800
+200 train 8.835118 (lr=5.0000e-05) (hash(x)=166780046)
+300 val loss 7.7964
+300 val perplexity 2431.7542
+300 train 7.786602 (lr=4.9988e-05) (hash(x)=159835303)
+400 val loss 7.6744
+400 val perplexity 2152.5623
+400 train 7.489794 (lr=4.9954e-05) (hash(x)=155040610)
+500 val loss 7.6406
+500 val perplexity 2080.9221
+500 train 7.391425 (lr=4.9896e-05) (hash(x)=130190460)
+600 val loss 7.6032
+600 val perplexity 2004.5076
+600 train 7.638520 (lr=4.9815e-05) (hash(x)=155504036)
+700 val loss 7.5646
+700 val perplexity 1928.7394
+700 train 7.406923 (lr=4.9712e-05) (hash(x)=137347213)
+800 val loss 7.5406
+800 val perplexity 1882.9725
+800 train 7.397386 (lr=4.9585e-05) (hash(x)=143823248)
+900 val loss 7.5339
+900 val perplexity 1870.4193
+900 train 7.654196 (lr=4.9436e-05) (hash(x)=156260416)
+1000 val loss 7.5107
+1000 val perplexity 1827.5564
+1000 train 7.477935 (lr=4.9264e-05) (hash(x)=143734685)
+1100 val loss 7.4936
+1100 val perplexity 1796.4606
+1100 train 7.441096 (lr=4.9070e-05) (hash(x)=160013925)
+1200 val loss 7.4976
+1200 val perplexity 1803.6320
+1200 train 7.258204 (lr=4.8854e-05) (hash(x)=150678249)
+1300 val loss 7.4832
+1300 val perplexity 1777.9482
+1300 train 7.282100 (lr=4.8616e-05) (hash(x)=149073315)
+1400 val loss 7.4629
+1400 val perplexity 1742.1178
+1400 train 7.883915 (lr=4.8356e-05) (hash(x)=175802021)
+1500 val loss 7.4301
+1500 val perplexity 1685.9038
+1500 train 7.686608 (lr=4.8074e-05) (hash(x)=171034639)
+1600 val loss 7.4090
+1600 val perplexity 1650.8046
+1600 train 7.636811 (lr=4.7772e-05) (hash(x)=158681215)
+1700 val loss 7.3883
+1700 val perplexity 1616.9302
+1700 train 7.329289 (lr=4.7448e-05) (hash(x)=152116061)
+1800 val loss 7.3710
+1800 val perplexity 1589.1628
+1800 train 7.307854 (lr=4.7105e-05) (hash(x)=146108145)
+1900 val loss 7.3425
+1900 val perplexity 1544.5565
+1900 train 7.182374 (lr=4.6741e-05) (hash(x)=147598108)
+2000 val loss 7.3217
+2000 val perplexity 1512.8422
+2000 train 7.187046 (lr=4.6357e-05) (hash(x)=154996086)
+2100 val loss 7.3005
+2100 val perplexity 1481.1086
+2100 train 7.081825 (lr=4.5954e-05) (hash(x)=153396183)
+2200 val loss 7.2452
+2200 val perplexity 1401.3156
+2200 train 7.233769 (lr=4.5532e-05) (hash(x)=153885445)
+2300 val loss 7.2313
+2300 val perplexity 1382.0316
+2300 train 7.206238 (lr=4.5091e-05) (hash(x)=159666385)
+2400 val loss 7.1577
+2400 val perplexity 1283.9122
+2400 train 7.111083 (lr=4.4633e-05) (hash(x)=142353087)
+2500 val loss 7.1407
+2500 val perplexity 1262.2935
+2500 train 7.140073 (lr=4.4156e-05) (hash(x)=146491718)
+2600 val loss 7.1162
+2600 val perplexity 1231.7444
+2600 train 7.048321 (lr=4.3663e-05) (hash(x)=150750353)
+2700 val loss 7.0994
+2700 val perplexity 1211.1899
+2700 train 6.828227 (lr=4.3153e-05) (hash(x)=129849193)
+2800 val loss 7.0820
+2800 val perplexity 1190.2938
+2800 train 6.952722 (lr=4.2627e-05) (hash(x)=152767913)
+2900 val loss 7.0613
+2900 val perplexity 1165.9741
+2900 train 6.857185 (lr=4.2085e-05) (hash(x)=146531140)
+3000 val loss 7.0340
+3000 val perplexity 1134.5959
+3000 train 7.094491 (lr=4.1529e-05) (hash(x)=151562048)
+3100 val loss 6.9862
+3100 val perplexity 1081.6365
+3100 train 6.972751 (lr=4.0957e-05) (hash(x)=146001424)
+3200 val loss 6.9622
+3200 val perplexity 1055.9473
+3200 train 6.967948 (lr=4.0373e-05) (hash(x)=166486165)
+3300 val loss 6.9229
+3300 val perplexity 1015.2311
+3300 train 6.851268 (lr=3.9775e-05) (hash(x)=150866680)
+3400 val loss 6.8889
+3400 val perplexity 981.2885
+3400 train 6.802827 (lr=3.9164e-05) (hash(x)=143900419)
+3500 val loss 6.8543
+3500 val perplexity 947.9136
+3500 train 6.685315 (lr=3.8541e-05) (hash(x)=148845794)
+3600 val loss 6.8242
+3600 val perplexity 919.8097
+3600 train 6.676590 (lr=3.7907e-05) (hash(x)=145667796)
+3700 val loss 6.7948
+3700 val perplexity 893.1960
+3700 train 6.810184 (lr=3.7262e-05) (hash(x)=163563851)
+3800 val loss 6.7596
+3800 val perplexity 862.3146
+3800 train 6.712791 (lr=3.6608e-05) (hash(x)=147488689)
+3900 val loss 6.7088
+3900 val perplexity 819.6110
+3900 train 6.699515 (lr=3.5944e-05) (hash(x)=148186608)
+4000 val loss 6.6668
+4000 val perplexity 785.9120
+4000 train 6.524370 (lr=3.5271e-05) (hash(x)=142970187)
+4100 val loss 6.6322
+4100 val perplexity 759.1307
+4100 train 6.648249 (lr=3.4590e-05) (hash(x)=141584883)
+4200 val loss 6.6028
+4200 val perplexity 737.1482
+4200 train 6.413905 (lr=3.3902e-05) (hash(x)=145664585)
+4300 val loss 6.5812
+4300 val perplexity 721.4269
+4300 train 6.433772 (lr=3.3207e-05) (hash(x)=143736499)
+4400 val loss 6.5652
+4400 val perplexity 709.9598
+4400 train 6.385487 (lr=3.2507e-05) (hash(x)=151883322)
+4500 val loss 6.5521
+4500 val perplexity 700.7370
+4500 train 6.444251 (lr=3.1801e-05) (hash(x)=153904871)
+4600 val loss 6.5329
+4600 val perplexity 687.4182
+4600 train 6.596064 (lr=3.1091e-05) (hash(x)=154893521)
+4700 val loss 6.4969
+4700 val perplexity 663.1146
+4700 train 6.560022 (lr=3.0377e-05) (hash(x)=152323949)
+4800 val loss 6.4743
+4800 val perplexity 648.2936
+4800 train 6.412653 (lr=2.9661e-05) (hash(x)=154104619)
+4900 val loss 6.4596
+4900 val perplexity 638.8070
+4900 train 6.544138 (lr=2.8942e-05) (hash(x)=146311426)
+5000 val loss 6.4390
+5000 val perplexity 625.7906
+5000 train 6.456110 (lr=2.8221e-05) (hash(x)=156741847)
+5100 val loss 6.4319
+5100 val perplexity 621.3528
+5100 train 6.222180 (lr=2.7500e-05) (hash(x)=142086346)
+5200 val loss 6.4287
+5200 val perplexity 619.3978
+5200 train 6.202271 (lr=2.6779e-05) (hash(x)=150265428)
+5300 val loss 6.4182
+5300 val perplexity 612.8869
+5300 train 6.315850 (lr=2.6058e-05) (hash(x)=151339108)
+5400 val loss 6.4048
+5400 val perplexity 604.7270
+5400 train 6.392514 (lr=2.5339e-05) (hash(x)=154654372)
+5500 val loss 6.3853
+5500 val perplexity 593.0718
+5500 train 6.397058 (lr=2.4623e-05) (hash(x)=150575051)
+5600 val loss 6.3629
+5600 val perplexity 579.9388
+5600 train 6.260410 (lr=2.3909e-05) (hash(x)=140396423)
+5700 val loss 6.3533
+5700 val perplexity 574.3911
+5700 train 6.218089 (lr=2.3199e-05) (hash(x)=144678758)
+5800 val loss 6.3443
+5800 val perplexity 569.2311
+5800 train 6.426583 (lr=2.2493e-05) (hash(x)=151992743)
+5900 val loss 6.3338
+5900 val perplexity 563.2771
+5900 train 6.116439 (lr=2.1793e-05) (hash(x)=144396927)
+6000 val loss 6.3250
+6000 val perplexity 558.3319
+6000 train 6.296118 (lr=2.1098e-05) (hash(x)=165478625)
+6100 val loss 6.3288
+6100 val perplexity 560.4827
+6100 train 6.075109 (lr=2.0410e-05) (hash(x)=147088621)
+6200 val loss 6.3158
+6200 val perplexity 553.2230
+6200 train 6.100374 (lr=1.9729e-05) (hash(x)=140794994)
+6300 val loss 6.2978
+6300 val perplexity 543.3687
+6300 train 6.230536 (lr=1.9056e-05) (hash(x)=134780906)
+6400 val loss 6.2830
+6400 val perplexity 535.3809
+6400 train 6.271345 (lr=1.8392e-05) (hash(x)=149023655)
+6500 val loss 6.2739
+6500 val perplexity 530.5635
+6500 train 6.142905 (lr=1.7738e-05) (hash(x)=147497796)
+6600 val loss 6.2718
+6600 val perplexity 529.4250
+6600 train 6.123541 (lr=1.7093e-05) (hash(x)=152902689)
+6700 val loss 6.2679
+6700 val perplexity 527.3727
+6700 train 6.298330 (lr=1.6459e-05) (hash(x)=153846046)
+6800 val loss 6.2524
+6800 val perplexity 519.2451
+6800 train 6.383592 (lr=1.5836e-05) (hash(x)=158512738)
+6900 val loss 6.2441
+6900 val perplexity 514.9430
+6900 train 6.754239 (lr=1.5225e-05) (hash(x)=156849968)
+7000 val loss 6.2357
+7000 val perplexity 510.6797
+7000 train 6.079944 (lr=1.4627e-05) (hash(x)=142395855)
+7100 val loss 6.2332
+7100 val perplexity 509.3864
+7100 train 6.096469 (lr=1.4043e-05) (hash(x)=147114884)
+7200 val loss 6.2246
+7200 val perplexity 505.0182
+7200 train 6.257214 (lr=1.3471e-05) (hash(x)=156979839)
+7300 val loss 6.2214
+7300 val perplexity 503.4091
+7300 train 5.995510 (lr=1.2915e-05) (hash(x)=145584373)
+7400 val loss 6.2212
+7400 val perplexity 503.3155
+7400 train 5.960420 (lr=1.2373e-05) (hash(x)=141508204)
+7500 val loss 6.2089
+7500 val perplexity 497.1344
+7500 train 6.253492 (lr=1.1847e-05) (hash(x)=148803965)
+7600 val loss 6.1995
+7600 val perplexity 492.4985
+7600 train 6.262319 (lr=1.1337e-05) (hash(x)=151019676)
+7700 val loss 6.1925
+7700 val perplexity 489.0664
+7700 train 6.230704 (lr=1.0844e-05) (hash(x)=143155750)
+7800 val loss 6.1897
+7800 val perplexity 487.7184
+7800 train 6.212282 (lr=1.0367e-05) (hash(x)=152569653)
+7900 val loss 6.1841
+7900 val perplexity 484.9865
+7900 train 6.054117 (lr=9.9088e-06) (hash(x)=143519455)
+8000 val loss 6.1830
+8000 val perplexity 484.4512
+8000 train 6.301287 (lr=9.4682e-06) (hash(x)=161180944)
+8100 val loss 6.1832
+8100 val perplexity 484.5494
+8100 train 6.211595 (lr=9.0461e-06) (hash(x)=154107345)
+8200 val loss 6.1815
+8200 val perplexity 483.7320
+8200 train 6.209146 (lr=8.6430e-06) (hash(x)=152486517)
+8300 val loss 6.1705
+8300 val perplexity 478.4457
+8300 train 6.188221 (lr=8.2593e-06) (hash(x)=156167749)
+8400 val loss 6.1650
+8400 val perplexity 475.7902
+8400 train 6.239934 (lr=7.8953e-06) (hash(x)=149155006)
+8500 val loss 6.1602
+8500 val perplexity 473.5081
+8500 train 6.126613 (lr=7.5515e-06) (hash(x)=147844390)
+8600 val loss 6.1572
+8600 val perplexity 472.1252
+8600 train 6.423381 (lr=7.2282e-06) (hash(x)=165753320)
+8700 val loss 6.1558
+8700 val perplexity 471.4586
+8700 train 5.938447 (lr=6.9257e-06) (hash(x)=146079979)
+8800 val loss 6.1564
+8800 val perplexity 471.7098
+8800 train 6.352519 (lr=6.6444e-06) (hash(x)=172259509)
+8900 val loss 6.1557
+8900 val perplexity 471.4083
+8900 train 5.836442 (lr=6.3845e-06) (hash(x)=145148314)
+9000 val loss 6.1548
+9000 val perplexity 470.9663
+9000 train 6.078220 (lr=6.1462e-06) (hash(x)=144250633)
+9100 val loss 6.1462
+9100 val perplexity 466.9475
+9100 train 6.256772 (lr=5.9300e-06) (hash(x)=157219797)
+9200 val loss 6.1410
+9200 val perplexity 464.5114
+9200 train 6.155202 (lr=5.7359e-06) (hash(x)=142743778)
+9300 val loss 6.1387
+9300 val perplexity 463.4443
+9300 train 6.043135 (lr=5.5641e-06) (hash(x)=139669771)
+9400 val loss 6.1351
+9400 val perplexity 461.7762
+9400 train 6.132237 (lr=5.4149e-06) (hash(x)=145916843)
+9500 val loss 6.1342
+9500 val perplexity 461.3840
+9500 train 6.058453 (lr=5.2884e-06) (hash(x)=150196125)
+9600 val loss 6.1338
+9600 val perplexity 461.1818
+9600 train 6.431315 (lr=5.1847e-06) (hash(x)=160041419)
+9700 val loss 6.1329
+9700 val perplexity 460.7543
+9700 train 5.923302 (lr=5.1040e-06) (hash(x)=139931627)
+9800 val loss 6.1306
+9800 val perplexity 459.7270
+9800 train 6.095959 (lr=5.0462e-06) (hash(x)=150370792)
+9900 val loss 6.1289
+9900 val perplexity 458.9510
+9900 train 5.972460 (lr=5.0116e-06) (hash(x)=153014886)
+9999 val loss 6.1289
+9999 val perplexity 458.9519

attention_kindselective_n_heads2_seed1338/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17b69f47997f43c837a595496e6d70cca6c006bf194d37b7282b2e6bba0f3434
+oid sha256:ec6bd7e87f52a5c93d8861916e3d82c1ba6393fe528c2119a53e9b72aa371446
 size 38587970

attention_kindselective_n_heads2_seed1338/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:627ddc0f75f88ce7a8abf1351ce99479e8fd0956e09337cd06eb25d2215f3777
+oid sha256:9cadf2af730975458af4444f28d26a5cc30c93a8ecab502f773cbe1fb1dc91e8
 size 38587970

attention_kindselective_n_heads2_seed1338/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53046326141d737465741cc3baaf4b8d005b56404efac0e1c9d0b69af27a65ef
+oid sha256:da964b640be4bb6a0d647549527e5cb4389be18c0ad3ea8604a371383a3a15b8
 size 38587970

attention_kindselective_n_heads2_seed1338/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b666d62c268d8acc9824fd57ac056016f9ee37069c8864d0489fc364c48841f
+oid sha256:e0c5cb0d1631c98e783b7f9c6dc0f347e2490fc6240b77dc62fe3a2869067972
 size 38587970

attention_kindselective_n_heads2_seed1338/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fed2eb2cef71e7046f8c0ae38cf87aa7c67165ce01ba2633af6fb10675749b52
+oid sha256:7914d38ea51a0b603e2bd0d7b49b0845b68b726646e48c15d0132680add44d7c
 size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31b1902932930152572eef398aae2372a0eb42b414ecee1ebc50373d0d0114ec
+oid sha256:f4074661b3b029bf8d3f6f3d765e43fa8d2c27f0ce0850708b58af6f36cbf9dc
 size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d8e46734116785cdf40b2fa62ced5d13bd1f2d517e897572824bc266d7f74383
+oid sha256:c222bd337a0e135a7bd57ddee8e40f76f8c7f26e8b95251a5f508a0910654717
 size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d721d2e577c0bd12c5293e00af8818a4f9d5b9ba9e7775acefd217b5f02bb09f
+oid sha256:76c9b27422cd0a25887e6e5a39f1d7a35087a1e53dd1953799b22114616b0758
 size 70895430