Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_3/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_3", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1339", "n_embd": 256}

attention_kindselective_n_heads4_seed1339/log2.txt

@@ -2,302 +2,302 @@ max_steps: 10000
 0 val loss 11.2810
 0 val perplexity 79298.4922
 0 train 11.295413 (lr=7.5000e-07) (hash(x)=150724848)
-100 val loss 8.6250
-100 val perplexity 5569.2051
-100 train 8.675070 (lr=7.5750e-05) (hash(x)=149217061)
-200 val loss 7.5646
-200 val perplexity 1928.7081
-200 train 7.551589 (lr=1.5000e-04) (hash(x)=149910534)
-300 val loss 7.4554
-300 val perplexity 1729.2274
-300 train 7.312988 (lr=1.4997e-04) (hash(x)=142185643)
-400 val loss 7.3929
-400 val perplexity 1624.3813
-400 train 7.405525 (lr=1.4986e-04) (hash(x)=148123706)
-500 val loss 7.3476
-500 val perplexity 1552.4868
-500 train 7.195226 (lr=1.4969e-04) (hash(x)=149952383)
-600 val loss 7.2760
-600 val perplexity 1445.1543
-600 train 7.072387 (lr=1.4945e-04) (hash(x)=146678221)
-700 val loss 7.2279
-700 val perplexity 1377.3153
-700 train 7.264610 (lr=1.4913e-04) (hash(x)=156180736)
-800 val loss 7.2208
-800 val perplexity 1367.6276
-800 train 7.212437 (lr=1.4876e-04) (hash(x)=151700982)
-900 val loss 7.1983
-900 val perplexity 1337.2192
-900 train 6.981043 (lr=1.4831e-04) (hash(x)=147288467)
-1000 val loss 7.2724
-1000 val perplexity 1440.0166
-1000 train 7.326461 (lr=1.4779e-04) (hash(x)=156182087)
-1100 val loss 7.2104
-1100 val perplexity 1353.4854
-1100 train 7.078388 (lr=1.4721e-04) (hash(x)=147861550)
-1200 val loss 7.2043
-1200 val perplexity 1345.2571
-1200 train 7.095018 (lr=1.4656e-04) (hash(x)=149318660)
-1300 val loss 7.1332
-1300 val perplexity 1252.8716
-1300 train 6.962538 (lr=1.4585e-04) (hash(x)=151631103)
-1400 val loss 7.0923
-1400 val perplexity 1202.7006
-1400 train 7.109515 (lr=1.4507e-04) (hash(x)=150482428)
-1500 val loss 7.0494
-1500 val perplexity 1152.1907
-1500 train 6.899590 (lr=1.4422e-04) (hash(x)=144458617)
-1600 val loss 7.0726
-1600 val perplexity 1179.2318
-1600 train 6.963625 (lr=1.4332e-04) (hash(x)=143268605)
-1700 val loss 7.0272
-1700 val perplexity 1126.8760
-1700 train 6.971726 (lr=1.4235e-04) (hash(x)=150817011)
-1800 val loss 7.0082
-1800 val perplexity 1105.6932
-1800 train 6.986873 (lr=1.4131e-04) (hash(x)=152322423)
-1900 val loss 6.9633
-1900 val perplexity 1057.0878
-1900 train 6.767703 (lr=1.4022e-04) (hash(x)=143574126)
-2000 val loss 6.9497
-2000 val perplexity 1042.8232
-2000 train 6.723602 (lr=1.3907e-04) (hash(x)=147904298)
-2100 val loss 6.9780
-2100 val perplexity 1072.7555
-2100 train 6.723686 (lr=1.3786e-04) (hash(x)=144719400)
-2200 val loss 6.9285
-2200 val perplexity 1020.9377
-2200 train 7.179186 (lr=1.3660e-04) (hash(x)=154343147)
-2300 val loss 6.8777
-2300 val perplexity 970.3969
-2300 train 6.908420 (lr=1.3527e-04) (hash(x)=154372635)
-2400 val loss 6.8547
-2400 val perplexity 948.3024
-2400 train 6.695008 (lr=1.3390e-04) (hash(x)=141843115)
-2500 val loss 6.8764
-2500 val perplexity 969.1234
-2500 train 6.780936 (lr=1.3247e-04) (hash(x)=149417679)
-2600 val loss 6.8460
-2600 val perplexity 940.0710
-2600 train 6.705399 (lr=1.3099e-04) (hash(x)=145279030)
-2700 val loss 6.8267
-2700 val perplexity 922.1456
-2700 train 6.738108 (lr=1.2946e-04) (hash(x)=146363911)
-2800 val loss 6.7896
-2800 val perplexity 888.5368
-2800 train 6.743748 (lr=1.2788e-04) (hash(x)=152507639)
-2900 val loss 6.7851
-2900 val perplexity 884.5744
-2900 train 6.561275 (lr=1.2626e-04) (hash(x)=139296191)
-3000 val loss 6.7609
-3000 val perplexity 863.4239
-3000 train 6.757187 (lr=1.2459e-04) (hash(x)=148473774)
-3100 val loss 6.6914
-3100 val perplexity 805.4599
-3100 train 6.768880 (lr=1.2287e-04) (hash(x)=154130527)
-3200 val loss 6.6348
-3200 val perplexity 761.1155
-3200 train 6.586649 (lr=1.2112e-04) (hash(x)=151117002)
-3300 val loss 6.6200
-3300 val perplexity 749.9819
-3300 train 6.764714 (lr=1.1932e-04) (hash(x)=151035383)
-3400 val loss 6.6029
-3400 val perplexity 737.2410
-3400 train 6.731507 (lr=1.1749e-04) (hash(x)=150614749)
-3500 val loss 6.5882
-3500 val perplexity 726.4773
-3500 train 6.565275 (lr=1.1562e-04) (hash(x)=151431391)
-3600 val loss 6.6085
-3600 val perplexity 741.3655
-3600 train 6.543498 (lr=1.1372e-04) (hash(x)=151140797)
-3700 val loss 6.5567
-3700 val perplexity 703.9423
-3700 train 6.750622 (lr=1.1179e-04) (hash(x)=154608340)
-3800 val loss 6.5366
-3800 val perplexity 689.9665
-3800 train 6.559993 (lr=1.0982e-04) (hash(x)=169641309)
-3900 val loss 6.5218
-3900 val perplexity 679.8000
-3900 train 6.492042 (lr=1.0783e-04) (hash(x)=150592479)
-4000 val loss 6.5130
-4000 val perplexity 673.8427
-4000 train 6.484363 (lr=1.0581e-04) (hash(x)=155495317)
-4100 val loss 6.5028
-4100 val perplexity 667.0012
-4100 train 6.692910 (lr=1.0377e-04) (hash(x)=170323970)
-4200 val loss 6.5015
-4200 val perplexity 666.1364
-4200 train 6.316307 (lr=1.0171e-04) (hash(x)=140854360)
-4300 val loss 6.4855
-4300 val perplexity 655.5953
-4300 train 6.486406 (lr=9.9622e-05) (hash(x)=148926644)
-4400 val loss 6.4532
-4400 val perplexity 634.7213
-4400 train 6.490174 (lr=9.7520e-05) (hash(x)=149997871)
-4500 val loss 6.4522
-4500 val perplexity 634.0930
-4500 train 6.414519 (lr=9.5403e-05) (hash(x)=149219289)
-4600 val loss 6.4631
-4600 val perplexity 641.0709
-4600 train 6.281563 (lr=9.3273e-05) (hash(x)=139836197)
-4700 val loss 6.4744
-4700 val perplexity 648.3419
-4700 train 6.421967 (lr=9.1132e-05) (hash(x)=150722883)
-4800 val loss 6.4518
-4800 val perplexity 633.8252
-4800 train 6.639651 (lr=8.8982e-05) (hash(x)=166864648)
-4900 val loss 6.4374
-4900 val perplexity 624.7933
-4900 train 6.549968 (lr=8.6825e-05) (hash(x)=157746743)
-5000 val loss 6.4179
-5000 val perplexity 612.7011
-5000 train 6.368554 (lr=8.4663e-05) (hash(x)=153032257)
-5100 val loss 6.4126
-5100 val perplexity 609.4542
-5100 train 6.476406 (lr=8.2500e-05) (hash(x)=151862371)
-5200 val loss 6.4037
-5200 val perplexity 604.0690
-5200 train 6.694038 (lr=8.0337e-05) (hash(x)=168862995)
-5300 val loss 6.4165
-5300 val perplexity 611.8710
-5300 train 6.378269 (lr=7.8175e-05) (hash(x)=148243567)
-5400 val loss 6.4033
-5400 val perplexity 603.8110
-5400 train 6.334872 (lr=7.6018e-05) (hash(x)=145381742)
-5500 val loss 6.3863
-5500 val perplexity 593.6646
-5500 train 6.534186 (lr=7.3868e-05) (hash(x)=155602174)
-5600 val loss 6.3683
-5600 val perplexity 583.0696
-5600 train 6.443361 (lr=7.1727e-05) (hash(x)=160637672)
-5700 val loss 6.3710
-5700 val perplexity 584.6398
-5700 train 6.291743 (lr=6.9597e-05) (hash(x)=152101131)
-5800 val loss 6.3493
-5800 val perplexity 572.1193
-5800 train 6.318473 (lr=6.7480e-05) (hash(x)=152327089)
-5900 val loss 6.3517
-5900 val perplexity 573.4702
-5900 train 6.292232 (lr=6.5378e-05) (hash(x)=151094543)
-6000 val loss 6.3507
-6000 val perplexity 572.9124
-6000 train 6.437282 (lr=6.3294e-05) (hash(x)=160233315)
-6100 val loss 6.3443
-6100 val perplexity 569.2604
-6100 train 6.436557 (lr=6.1230e-05) (hash(x)=158295337)
-6200 val loss 6.3255
-6200 val perplexity 558.6296
-6200 train 6.284213 (lr=5.9188e-05) (hash(x)=151269817)
-6300 val loss 6.3158
-6300 val perplexity 553.2232
-6300 train 6.439045 (lr=5.7169e-05) (hash(x)=156912756)
-6400 val loss 6.3036
-6400 val perplexity 546.5519
-6400 train 6.262898 (lr=5.5177e-05) (hash(x)=125779637)
-6500 val loss 6.3109
-6500 val perplexity 550.5519
-6500 train 6.200621 (lr=5.3213e-05) (hash(x)=148157661)
-6600 val loss 6.3018
-6600 val perplexity 545.5410
-6600 train 6.336145 (lr=5.1279e-05) (hash(x)=152820049)
-6700 val loss 6.2946
-6700 val perplexity 541.6356
-6700 train 6.257331 (lr=4.9377e-05) (hash(x)=155818717)
-6800 val loss 6.2828
-6800 val perplexity 535.2729
-6800 train 6.318388 (lr=4.7509e-05) (hash(x)=158833881)
-6900 val loss 6.2741
-6900 val perplexity 530.6553
-6900 train 6.197567 (lr=4.5676e-05) (hash(x)=164984909)
-7000 val loss 6.2719
-7000 val perplexity 529.5022
-7000 train 6.165509 (lr=4.3882e-05) (hash(x)=151172493)
-7100 val loss 6.2759
-7100 val perplexity 531.5972
-7100 train 6.307955 (lr=4.2128e-05) (hash(x)=154023633)
-7200 val loss 6.2661
-7200 val perplexity 526.4122
-7200 train 6.149921 (lr=4.0414e-05) (hash(x)=139351250)
-7300 val loss 6.2632
-7300 val perplexity 524.9023
-7300 train 6.218886 (lr=3.8745e-05) (hash(x)=158719458)
-7400 val loss 6.2493
-7400 val perplexity 517.6301
-7400 train 6.330258 (lr=3.7120e-05) (hash(x)=162250810)
-7500 val loss 6.2381
-7500 val perplexity 511.8631
-7500 train 6.112332 (lr=3.5541e-05) (hash(x)=142827557)
-7600 val loss 6.2380
-7600 val perplexity 511.8196
-7600 train 6.153459 (lr=3.4011e-05) (hash(x)=147830801)
-7700 val loss 6.2311
-7700 val perplexity 508.3083
-7700 train 6.335352 (lr=3.2531e-05) (hash(x)=165233314)
-7800 val loss 6.2284
-7800 val perplexity 506.9622
-7800 train 6.220976 (lr=3.1102e-05) (hash(x)=160654717)
-7900 val loss 6.2270
-7900 val perplexity 506.2233
-7900 train 6.102557 (lr=2.9726e-05) (hash(x)=140712586)
-8000 val loss 6.2113
-8000 val perplexity 498.3417
-8000 train 6.367906 (lr=2.8405e-05) (hash(x)=150309061)
-8100 val loss 6.2063
-8100 val perplexity 495.8723
-8100 train 6.201856 (lr=2.7138e-05) (hash(x)=158805466)
-8200 val loss 6.2057
-8200 val perplexity 495.5759
-8200 train 6.074079 (lr=2.5929e-05) (hash(x)=142814078)
-8300 val loss 6.1993
-8300 val perplexity 492.4088
-8300 train 6.161883 (lr=2.4778e-05) (hash(x)=148340417)
-8400 val loss 6.1987
-8400 val perplexity 492.1197
-8400 train 6.151392 (lr=2.3686e-05) (hash(x)=157132639)
-8500 val loss 6.1978
-8500 val perplexity 491.6466
-8500 train 6.153172 (lr=2.2655e-05) (hash(x)=152994240)
-8600 val loss 6.1860
-8600 val perplexity 485.8925
-8600 train 6.145603 (lr=2.1685e-05) (hash(x)=152210154)
-8700 val loss 6.1859
-8700 val perplexity 485.8365
-8700 train 6.241817 (lr=2.0777e-05) (hash(x)=149631247)
-8800 val loss 6.1828
-8800 val perplexity 484.3528
-8800 train 6.051007 (lr=1.9933e-05) (hash(x)=145617784)
-8900 val loss 6.1769
-8900 val perplexity 481.4896
-8900 train 6.254554 (lr=1.9153e-05) (hash(x)=155984970)
-9000 val loss 6.1798
-9000 val perplexity 482.8825
-9000 train 5.940945 (lr=1.8439e-05) (hash(x)=143433013)
-9100 val loss 6.1810
-9100 val perplexity 483.4917
-9100 train 6.057093 (lr=1.7790e-05) (hash(x)=152507533)
-9200 val loss 6.1740
-9200 val perplexity 480.1161
-9200 train 6.288232 (lr=1.7208e-05) (hash(x)=162400650)
-9300 val loss 6.1685
-9300 val perplexity 477.4570
-9300 train 6.225449 (lr=1.6692e-05) (hash(x)=148419068)
-9400 val loss 6.1678
-9400 val perplexity 477.1325
-9400 train 6.139578 (lr=1.6245e-05) (hash(x)=145025169)
-9500 val loss 6.1637
-9500 val perplexity 475.2030
-9500 train 6.133398 (lr=1.5865e-05) (hash(x)=145104235)
-9600 val loss 6.1667
-9600 val perplexity 476.6320
-9600 train 6.100382 (lr=1.5554e-05) (hash(x)=150499797)
-9700 val loss 6.1654
-9700 val perplexity 476.0076
-9700 train 5.987143 (lr=1.5312e-05) (hash(x)=139690022)
-9800 val loss 6.1580
-9800 val perplexity 472.4623
-9800 train 6.160158 (lr=1.5139e-05) (hash(x)=145645279)
-9900 val loss 6.1531
-9900 val perplexity 470.1617
-9900 train 6.052269 (lr=1.5035e-05) (hash(x)=150103432)
-9999 val loss 6.1522
-9999 val perplexity 469.7554
+100 val loss 8.5817
+100 val perplexity 5332.9116
+100 train 8.631360 (lr=7.5750e-05) (hash(x)=149217061)
+200 val loss 7.5932
+200 val perplexity 1984.5913
+200 train 7.581491 (lr=1.5000e-04) (hash(x)=149910534)
+300 val loss 7.4699
+300 val perplexity 1754.4465
+300 train 7.326682 (lr=1.4997e-04) (hash(x)=142185643)
+400 val loss 7.3927
+400 val perplexity 1624.1583
+400 train 7.405558 (lr=1.4986e-04) (hash(x)=148123706)
+500 val loss 7.3282
+500 val perplexity 1522.6230
+500 train 7.174585 (lr=1.4969e-04) (hash(x)=149952383)
+600 val loss 7.2701
+600 val perplexity 1436.7457
+600 train 7.065297 (lr=1.4945e-04) (hash(x)=146678221)
+700 val loss 7.1607
+700 val perplexity 1287.7505
+700 train 7.205720 (lr=1.4913e-04) (hash(x)=156180736)
+800 val loss 7.0975
+800 val perplexity 1209.0013
+800 train 7.072469 (lr=1.4876e-04) (hash(x)=151700982)
+900 val loss 7.0589
+900 val perplexity 1163.1676
+900 train 6.836958 (lr=1.4831e-04) (hash(x)=147288467)
+1000 val loss 7.0223
+1000 val perplexity 1121.3513
+1000 train 7.066115 (lr=1.4779e-04) (hash(x)=156182087)
+1100 val loss 7.0102
+1100 val perplexity 1107.9083
+1100 train 6.860521 (lr=1.4721e-04) (hash(x)=147861550)
+1200 val loss 6.9932
+1200 val perplexity 1089.1798
+1200 train 6.866424 (lr=1.4656e-04) (hash(x)=149318660)
+1300 val loss 7.0027
+1300 val perplexity 1099.6331
+1300 train 6.823798 (lr=1.4585e-04) (hash(x)=151631103)
+1400 val loss 7.0197
+1400 val perplexity 1118.4031
+1400 train 7.047281 (lr=1.4507e-04) (hash(x)=150482428)
+1500 val loss 6.9541
+1500 val perplexity 1047.3906
+1500 train 6.805685 (lr=1.4422e-04) (hash(x)=144458617)
+1600 val loss 6.8616
+1600 val perplexity 954.9236
+1600 train 6.756378 (lr=1.4332e-04) (hash(x)=143268605)
+1700 val loss 6.8326
+1700 val perplexity 927.6204
+1700 train 6.767980 (lr=1.4235e-04) (hash(x)=150817011)
+1800 val loss 6.8296
+1800 val perplexity 924.8635
+1800 train 6.823513 (lr=1.4131e-04) (hash(x)=152322423)
+1900 val loss 6.8104
+1900 val perplexity 907.2120
+1900 train 6.606685 (lr=1.4022e-04) (hash(x)=143574126)
+2000 val loss 6.8044
+2000 val perplexity 901.8112
+2000 train 6.589591 (lr=1.3907e-04) (hash(x)=147904298)
+2100 val loss 6.7997
+2100 val perplexity 897.5615
+2100 train 6.542164 (lr=1.3786e-04) (hash(x)=144719400)
+2200 val loss 6.7954
+2200 val perplexity 893.6961
+2200 train 7.016779 (lr=1.3660e-04) (hash(x)=154343147)
+2300 val loss 6.7536
+2300 val perplexity 857.1774
+2300 train 6.790805 (lr=1.3527e-04) (hash(x)=154372635)
+2400 val loss 6.7246
+2400 val perplexity 832.6526
+2400 train 6.568871 (lr=1.3390e-04) (hash(x)=141843115)
+2500 val loss 6.6865
+2500 val perplexity 801.4880
+2500 train 6.583528 (lr=1.3247e-04) (hash(x)=149417679)
+2600 val loss 6.6823
+2600 val perplexity 798.1768
+2600 train 6.545222 (lr=1.3099e-04) (hash(x)=145279030)
+2700 val loss 6.6431
+2700 val perplexity 767.4857
+2700 train 6.573290 (lr=1.2946e-04) (hash(x)=146363911)
+2800 val loss 6.6954
+2800 val perplexity 808.7022
+2800 train 6.648605 (lr=1.2788e-04) (hash(x)=152507639)
+2900 val loss 6.6802
+2900 val perplexity 796.4598
+2900 train 6.488822 (lr=1.2626e-04) (hash(x)=139296191)
+3000 val loss 6.6775
+3000 val perplexity 794.3075
+3000 train 6.686038 (lr=1.2459e-04) (hash(x)=148473774)
+3100 val loss 6.6638
+3100 val perplexity 783.4847
+3100 train 6.742999 (lr=1.2287e-04) (hash(x)=154130527)
+3200 val loss 6.6479
+3200 val perplexity 771.1449
+3200 train 6.599888 (lr=1.2112e-04) (hash(x)=151117002)
+3300 val loss 6.6349
+3300 val perplexity 761.2346
+3300 train 6.778743 (lr=1.1932e-04) (hash(x)=151035383)
+3400 val loss 6.6253
+3400 val perplexity 753.8972
+3400 train 6.759344 (lr=1.1749e-04) (hash(x)=150614749)
+3500 val loss 6.6271
+3500 val perplexity 755.2767
+3500 train 6.640333 (lr=1.1562e-04) (hash(x)=151431391)
+3600 val loss 6.5977
+3600 val perplexity 733.3723
+3600 train 6.536980 (lr=1.1372e-04) (hash(x)=151140797)
+3700 val loss 6.6127
+3700 val perplexity 744.4752
+3700 train 6.792097 (lr=1.1179e-04) (hash(x)=154608340)
+3800 val loss 6.6042
+3800 val perplexity 738.1901
+3800 train 6.620290 (lr=1.0982e-04) (hash(x)=169641309)
+3900 val loss 6.5557
+3900 val perplexity 703.2455
+3900 train 6.523308 (lr=1.0783e-04) (hash(x)=150592479)
+4000 val loss 6.5413
+4000 val perplexity 693.1867
+4000 train 6.511438 (lr=1.0581e-04) (hash(x)=155495317)
+4100 val loss 6.5415
+4100 val perplexity 693.3563
+4100 train 6.727324 (lr=1.0377e-04) (hash(x)=170323970)
+4200 val loss 6.5481
+4200 val perplexity 697.9368
+4200 train 6.369248 (lr=1.0171e-04) (hash(x)=140854360)
+4300 val loss 6.5413
+4300 val perplexity 693.1804
+4300 train 6.532832 (lr=9.9622e-05) (hash(x)=148926644)
+4400 val loss 6.5165
+4400 val perplexity 676.1769
+4400 train 6.555954 (lr=9.7520e-05) (hash(x)=149997871)
+4500 val loss 6.4623
+4500 val perplexity 640.5529
+4500 train 6.415502 (lr=9.5403e-05) (hash(x)=149219289)
+4600 val loss 6.4509
+4600 val perplexity 633.2473
+4600 train 6.264147 (lr=9.3273e-05) (hash(x)=139836197)
+4700 val loss 6.4489
+4700 val perplexity 631.9807
+4700 train 6.410443 (lr=9.1132e-05) (hash(x)=150722883)
+4800 val loss 6.4401
+4800 val perplexity 626.4460
+4800 train 6.639719 (lr=8.8982e-05) (hash(x)=166864648)
+4900 val loss 6.4142
+4900 val perplexity 610.4495
+4900 train 6.536889 (lr=8.6825e-05) (hash(x)=157746743)
+5000 val loss 6.3877
+5000 val perplexity 594.4929
+5000 train 6.344072 (lr=8.4663e-05) (hash(x)=153032257)
+5100 val loss 6.3855
+5100 val perplexity 593.1589
+5100 train 6.445783 (lr=8.2500e-05) (hash(x)=151862371)
+5200 val loss 6.3667
+5200 val perplexity 582.1065
+5200 train 6.667581 (lr=8.0337e-05) (hash(x)=168862995)
+5300 val loss 6.3696
+5300 val perplexity 583.8088
+5300 train 6.328732 (lr=7.8175e-05) (hash(x)=148243567)
+5400 val loss 6.3508
+5400 val perplexity 572.9389
+5400 train 6.276412 (lr=7.6018e-05) (hash(x)=145381742)
+5500 val loss 6.3349
+5500 val perplexity 563.9183
+5500 train 6.471667 (lr=7.3868e-05) (hash(x)=155602174)
+5600 val loss 6.3206
+5600 val perplexity 555.9157
+5600 train 6.392186 (lr=7.1727e-05) (hash(x)=160637672)
+5700 val loss 6.3222
+5700 val perplexity 556.7697
+5700 train 6.250708 (lr=6.9597e-05) (hash(x)=152101131)
+5800 val loss 6.3087
+5800 val perplexity 549.3273
+5800 train 6.284104 (lr=6.7480e-05) (hash(x)=152327089)
+5900 val loss 6.3093
+5900 val perplexity 549.6396
+5900 train 6.256981 (lr=6.5378e-05) (hash(x)=151094543)
+6000 val loss 6.3130
+6000 val perplexity 551.7166
+6000 train 6.390558 (lr=6.3294e-05) (hash(x)=160233315)
+6100 val loss 6.2926
+6100 val perplexity 540.5599
+6100 train 6.394215 (lr=6.1230e-05) (hash(x)=158295337)
+6200 val loss 6.2789
+6200 val perplexity 533.2273
+6200 train 6.239454 (lr=5.9188e-05) (hash(x)=151269817)
+6300 val loss 6.2694
+6300 val perplexity 528.1562
+6300 train 6.401673 (lr=5.7169e-05) (hash(x)=156912756)
+6400 val loss 6.2619
+6400 val perplexity 524.2102
+6400 train 6.249959 (lr=5.5177e-05) (hash(x)=125779637)
+6500 val loss 6.2738
+6500 val perplexity 530.4722
+6500 train 6.169670 (lr=5.3213e-05) (hash(x)=148157661)
+6600 val loss 6.2593
+6600 val perplexity 522.8664
+6600 train 6.271819 (lr=5.1279e-05) (hash(x)=152820049)
+6700 val loss 6.2570
+6700 val perplexity 521.6345
+6700 train 6.216044 (lr=4.9377e-05) (hash(x)=155818717)
+6800 val loss 6.2368
+6800 val perplexity 511.1964
+6800 train 6.283014 (lr=4.7509e-05) (hash(x)=158833881)
+6900 val loss 6.2285
+6900 val perplexity 506.9743
+6900 train 6.179168 (lr=4.5676e-05) (hash(x)=164984909)
+7000 val loss 6.2311
+7000 val perplexity 508.3391
+7000 train 6.126690 (lr=4.3882e-05) (hash(x)=151172493)
+7100 val loss 6.2327
+7100 val perplexity 509.1287
+7100 train 6.266987 (lr=4.2128e-05) (hash(x)=154023633)
+7200 val loss 6.2152
+7200 val perplexity 500.3194
+7200 train 6.093438 (lr=4.0414e-05) (hash(x)=139351250)
+7300 val loss 6.2126
+7300 val perplexity 498.9968
+7300 train 6.167672 (lr=3.8745e-05) (hash(x)=158719458)
+7400 val loss 6.2025
+7400 val perplexity 494.0034
+7400 train 6.270429 (lr=3.7120e-05) (hash(x)=162250810)
+7500 val loss 6.1907
+7500 val perplexity 488.2101
+7500 train 6.070210 (lr=3.5541e-05) (hash(x)=142827557)
+7600 val loss 6.1884
+7600 val perplexity 487.0707
+7600 train 6.085032 (lr=3.4011e-05) (hash(x)=147830801)
+7700 val loss 6.1889
+7700 val perplexity 487.3230
+7700 train 6.269209 (lr=3.2531e-05) (hash(x)=165233314)
+7800 val loss 6.1830
+7800 val perplexity 484.4642
+7800 train 6.176233 (lr=3.1102e-05) (hash(x)=160654717)
+7900 val loss 6.1814
+7900 val perplexity 483.6577
+7900 train 6.050529 (lr=2.9726e-05) (hash(x)=140712586)
+8000 val loss 6.1706
+8000 val perplexity 478.4890
+8000 train 6.319698 (lr=2.8405e-05) (hash(x)=150309061)
+8100 val loss 6.1642
+8100 val perplexity 475.4011
+8100 train 6.150677 (lr=2.7138e-05) (hash(x)=158805466)
+8200 val loss 6.1579
+8200 val perplexity 472.4475
+8200 train 6.029180 (lr=2.5929e-05) (hash(x)=142814078)
+8300 val loss 6.1567
+8300 val perplexity 471.8569
+8300 train 6.110010 (lr=2.4778e-05) (hash(x)=148340417)
+8400 val loss 6.1595
+8400 val perplexity 473.1881
+8400 train 6.101810 (lr=2.3686e-05) (hash(x)=157132639)
+8500 val loss 6.1568
+8500 val perplexity 471.9323
+8500 train 6.116642 (lr=2.2655e-05) (hash(x)=152994240)
+8600 val loss 6.1439
+8600 val perplexity 465.8618
+8600 train 6.103668 (lr=2.1685e-05) (hash(x)=152210154)
+8700 val loss 6.1421
+8700 val perplexity 465.0104
+8700 train 6.190649 (lr=2.0777e-05) (hash(x)=149631247)
+8800 val loss 6.1399
+8800 val perplexity 464.0101
+8800 train 5.996617 (lr=1.9933e-05) (hash(x)=145617784)
+8900 val loss 6.1353
+8900 val perplexity 461.8795
+8900 train 6.206507 (lr=1.9153e-05) (hash(x)=155984970)
+9000 val loss 6.1374
+9000 val perplexity 462.8537
+9000 train 5.890743 (lr=1.8439e-05) (hash(x)=143433013)
+9100 val loss 6.1376
+9100 val perplexity 462.9261
+9100 train 6.009195 (lr=1.7790e-05) (hash(x)=152507533)
+9200 val loss 6.1288
+9200 val perplexity 458.8862
+9200 train 6.234875 (lr=1.7208e-05) (hash(x)=162400650)
+9300 val loss 6.1247
+9300 val perplexity 457.0203
+9300 train 6.179266 (lr=1.6692e-05) (hash(x)=148419068)
+9400 val loss 6.1257
+9400 val perplexity 457.4461
+9400 train 6.100061 (lr=1.6245e-05) (hash(x)=145025169)
+9500 val loss 6.1228
+9500 val perplexity 456.1235
+9500 train 6.096772 (lr=1.5865e-05) (hash(x)=145104235)
+9600 val loss 6.1236
+9600 val perplexity 456.4965
+9600 train 6.053626 (lr=1.5554e-05) (hash(x)=150499797)
+9700 val loss 6.1195
+9700 val perplexity 454.6506
+9700 train 5.945662 (lr=1.5312e-05) (hash(x)=139690022)
+9800 val loss 6.1149
+9800 val perplexity 452.5396
+9800 train 6.116775 (lr=1.5139e-05) (hash(x)=145645279)
+9900 val loss 6.1106
+9900 val perplexity 450.5890
+9900 train 6.013401 (lr=1.5035e-05) (hash(x)=150103432)
+9999 val loss 6.1084
+9999 val perplexity 449.6328

attention_kindselective_n_heads4_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e716d019135dfe3a2162e5557ea36fbb3de4f3a202bd9edbebb7262813fb5f5f
+oid sha256:f9294532d9d2d4c7db014f7c1f5323cf5a4a2a16513b50cbc1b1039191702c59
 size 92843394

attention_kindselective_n_heads4_seed1339/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a29171b6ea1a71ea214bfb3b23186077674ba2b36ea6818a1ecb25d083c5a24
+oid sha256:71bdb80b8d183c8b995a678d3b770c11ac172bc535692db54ed7801762c64c1b
 size 92843394

attention_kindselective_n_heads4_seed1339/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9cc9582bdfd4241e8a7e5a7ac2d18b080ce3fc67ed7721340f67a55d44f36b1c
+oid sha256:ce765557cd271eef6368a61da11f4937b2fa47747c4d797eb63cb9c6318ee3dc
 size 92843394

attention_kindselective_n_heads4_seed1339/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c20142fb9b1c16799d1e6996ace274d6582244a691ec9343774e0f59016276ab
+oid sha256:9bf858df261be317cfb63794b256e4a8d2da111417ac052e97654337dcdab941
 size 92843394

attention_kindselective_n_heads4_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bb13a645e3fb0d0a7ad40ae6b27063e7881bc03ee558ced863ad4d5e2b4e0c43
+oid sha256:202df9b4ec19476f10069e10e4da133d7a1786cb35ccd20c1a5c3bf5c23dc384
 size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56b9061bc444d656c038eb3f035450cd31039dd78d555ec8510c390924b9e5ac
+oid sha256:d94d2d71ffe79e132aabc8895a5207065010391979dca509b8e3d88bad546633
 size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:537491d8da9df92ce0fe0f70ddbf61245ac4a12374ab5b94c15fd36c90057ff5
+oid sha256:bba53bf30935669f2bae08d31556a325db6d8c9f1cb754cb0ff4ef1cbe276934
 size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:514e3f12687baea816c1ef563b9a785864000f1cb4641b8ced72d92cdab0c005
+oid sha256:6fa363931f59651b3dfe70400beca8db629ec599879e5ac22dc68db271c6bb19
 size 179406214