Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1340", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_2_1340", "n_embd": 128}

attention_kindselective_n_heads2_seed1340/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.8210
 0 val perplexity 136085.8281
-0 train 11.833096 (lr=7.5000e-07) (hash(x)=164406924)
-100 val loss 8.9632
-100 val perplexity 7809.9502
-100 train 9.364941 (lr=7.5750e-05) (hash(x)=177407419)
-200 val loss 7.6800
-200 val perplexity 2164.7278
-200 train 7.591533 (lr=1.5000e-04) (hash(x)=144903932)
-300 val loss 7.6212
-300 val perplexity 2041.0771
-300 train 7.965369 (lr=1.4997e-04) (hash(x)=173839165)
-400 val loss 7.6203
-400 val perplexity 2039.1880
-400 train 7.712651 (lr=1.4986e-04) (hash(x)=167734596)
-500 val loss 7.6629
-500 val perplexity 2127.8159
-500 train 7.718594 (lr=1.4969e-04) (hash(x)=153224076)
-600 val loss 7.6136
-600 val perplexity 2025.6000
-600 train 7.535222 (lr=1.4945e-04) (hash(x)=149619098)
-700 val loss 7.5827
-700 val perplexity 1963.9933
-700 train 7.542725 (lr=1.4913e-04) (hash(x)=146539909)
-800 val loss 7.5429
-800 val perplexity 1887.2891
-800 train 7.505376 (lr=1.4876e-04) (hash(x)=153710890)
-900 val loss 7.5786
-900 val perplexity 1955.8300
-900 train 7.526095 (lr=1.4831e-04) (hash(x)=155873620)
-1000 val loss 7.4960
-1000 val perplexity 1800.8063
-1000 train 7.421547 (lr=1.4779e-04) (hash(x)=145450636)
-1100 val loss 7.4722
-1100 val perplexity 1758.4231
-1100 train 7.541608 (lr=1.4721e-04) (hash(x)=154123388)
-1200 val loss 7.4480
-1200 val perplexity 1716.4382
-1200 train 7.308753 (lr=1.4656e-04) (hash(x)=145249251)
-1300 val loss 7.4397
-1300 val perplexity 1702.2803
-1300 train 7.368898 (lr=1.4585e-04) (hash(x)=148937127)
-1400 val loss 7.4124
-1400 val perplexity 1656.4629
-1400 train 7.518748 (lr=1.4507e-04) (hash(x)=150475545)
-1500 val loss 7.3749
-1500 val perplexity 1595.4633
-1500 train 7.291466 (lr=1.4422e-04) (hash(x)=154653428)
-1600 val loss 7.3791
-1600 val perplexity 1602.1072
-1600 train 7.349000 (lr=1.4332e-04) (hash(x)=144483776)
-1700 val loss 7.3710
-1700 val perplexity 1589.2416
-1700 train 7.533147 (lr=1.4235e-04) (hash(x)=157395496)
-1800 val loss 7.3049
-1800 val perplexity 1487.5573
-1800 train 7.328281 (lr=1.4131e-04) (hash(x)=157916369)
-1900 val loss 7.2586
-1900 val perplexity 1420.2678
-1900 train 7.444204 (lr=1.4022e-04) (hash(x)=166073923)
-2000 val loss 7.2832
-2000 val perplexity 1455.6466
-2000 train 7.365619 (lr=1.3907e-04) (hash(x)=154856891)
-2100 val loss 7.2291
-2100 val perplexity 1378.9766
-2100 train 7.213472 (lr=1.3786e-04) (hash(x)=151925203)
-2200 val loss 7.1903
-2200 val perplexity 1326.5510
-2200 train 6.975736 (lr=1.3660e-04) (hash(x)=136191502)
-2300 val loss 7.1517
-2300 val perplexity 1276.3109
-2300 train 7.243627 (lr=1.3527e-04) (hash(x)=153273362)
-2400 val loss 7.1472
-2400 val perplexity 1270.4921
-2400 train 7.079123 (lr=1.3390e-04) (hash(x)=148021541)
-2500 val loss 7.1173
-2500 val perplexity 1233.0848
-2500 train 7.058460 (lr=1.3247e-04) (hash(x)=141356608)
-2600 val loss 7.0719
-2600 val perplexity 1178.3792
-2600 train 7.030075 (lr=1.3099e-04) (hash(x)=146005217)
-2700 val loss 7.0647
-2700 val perplexity 1169.9490
-2700 train 6.929686 (lr=1.2946e-04) (hash(x)=144511718)
-2800 val loss 7.0291
-2800 val perplexity 1128.9994
-2800 train 6.924961 (lr=1.2788e-04) (hash(x)=146019502)
-2900 val loss 7.0272
-2900 val perplexity 1126.9019
-2900 train 6.943810 (lr=1.2626e-04) (hash(x)=146496200)
-3000 val loss 6.9735
-3000 val perplexity 1067.9238
-3000 train 6.912158 (lr=1.2459e-04) (hash(x)=150127281)
-3100 val loss 6.9307
-3100 val perplexity 1023.2532
-3100 train 6.903213 (lr=1.2287e-04) (hash(x)=142022255)
-3200 val loss 6.8967
-3200 val perplexity 988.9994
-3200 train 6.954908 (lr=1.2112e-04) (hash(x)=154120875)
-3300 val loss 6.8727
-3300 val perplexity 965.5895
-3300 train 6.896562 (lr=1.1932e-04) (hash(x)=153999717)
-3400 val loss 6.8499
-3400 val perplexity 943.7463
-3400 train 6.647655 (lr=1.1749e-04) (hash(x)=139694097)
-3500 val loss 6.8276
-3500 val perplexity 922.9308
-3500 train 7.016836 (lr=1.1562e-04) (hash(x)=162992732)
-3600 val loss 6.7817
-3600 val perplexity 881.5263
-3600 train 6.761508 (lr=1.1372e-04) (hash(x)=147574101)
-3700 val loss 6.7609
-3700 val perplexity 863.3860
-3700 train 6.856951 (lr=1.1179e-04) (hash(x)=157763099)
-3800 val loss 6.7505
-3800 val perplexity 854.4914
-3800 train 6.888644 (lr=1.0982e-04) (hash(x)=170800034)
-3900 val loss 6.7345
-3900 val perplexity 840.8956
-3900 train 6.812279 (lr=1.0783e-04) (hash(x)=164984528)
-4000 val loss 6.7104
-4000 val perplexity 820.8669
-4000 train 6.596583 (lr=1.0581e-04) (hash(x)=141743323)
-4100 val loss 6.7152
-4100 val perplexity 824.8502
-4100 train 6.724424 (lr=1.0377e-04) (hash(x)=153392872)
-4200 val loss 6.6934
-4200 val perplexity 807.0927
-4200 train 6.549108 (lr=1.0171e-04) (hash(x)=149074933)
-4300 val loss 6.6564
-4300 val perplexity 777.7588
-4300 train 7.006297 (lr=9.9622e-05) (hash(x)=167823423)
-4400 val loss 6.6221
-4400 val perplexity 751.5302
-4400 train 6.387350 (lr=9.7520e-05) (hash(x)=141203114)
-4500 val loss 6.5965
-4500 val perplexity 732.5110
-4500 train 6.602117 (lr=9.5403e-05) (hash(x)=146284780)
-4600 val loss 6.5730
-4600 val perplexity 715.4841
-4600 train 6.407486 (lr=9.3273e-05) (hash(x)=141126464)
-4700 val loss 6.5855
-4700 val perplexity 724.4916
-4700 train 6.621264 (lr=9.1132e-05) (hash(x)=154751926)
-4800 val loss 6.5644
-4800 val perplexity 709.3859
-4800 train 6.642868 (lr=8.8982e-05) (hash(x)=154793198)
-4900 val loss 6.5719
-4900 val perplexity 714.7520
-4900 train 6.330755 (lr=8.6825e-05) (hash(x)=139406392)
-5000 val loss 6.5733
-5000 val perplexity 715.7223
-5000 train 6.305210 (lr=8.4663e-05) (hash(x)=153548741)
-5100 val loss 6.5487
-5100 val perplexity 698.3330
-5100 train 6.591603 (lr=8.2500e-05) (hash(x)=160488568)
-5200 val loss 6.5172
-5200 val perplexity 676.7101
-5200 train 6.530630 (lr=8.0337e-05) (hash(x)=149645053)
-5300 val loss 6.4955
-5300 val perplexity 662.1673
-5300 train 6.574218 (lr=7.8175e-05) (hash(x)=155820556)
-5400 val loss 6.4987
-5400 val perplexity 664.2599
-5400 train 6.438256 (lr=7.6018e-05) (hash(x)=147538134)
-5500 val loss 6.4771
-5500 val perplexity 650.0532
-5500 train 6.676089 (lr=7.3868e-05) (hash(x)=166889307)
-5600 val loss 6.4716
-5600 val perplexity 646.5319
-5600 train 6.210559 (lr=7.1727e-05) (hash(x)=139516699)
-5700 val loss 6.4662
-5700 val perplexity 643.0082
-5700 train 6.176149 (lr=6.9597e-05) (hash(x)=140453511)
-5800 val loss 6.4738
-5800 val perplexity 647.9169
-5800 train 6.412430 (lr=6.7480e-05) (hash(x)=162964847)
-5900 val loss 6.4570
-5900 val perplexity 637.1229
-5900 train 6.458410 (lr=6.5378e-05) (hash(x)=150606634)
-6000 val loss 6.4347
-6000 val perplexity 623.0748
-6000 train 6.529973 (lr=6.3294e-05) (hash(x)=149890857)
-6100 val loss 6.4225
-6100 val perplexity 615.5125
-6100 train 6.529283 (lr=6.1230e-05) (hash(x)=173884145)
-6200 val loss 6.4154
-6200 val perplexity 611.2152
-6200 train 6.487962 (lr=5.9188e-05) (hash(x)=151987098)
-6300 val loss 6.4154
-6300 val perplexity 611.1697
-6300 train 6.371587 (lr=5.7169e-05) (hash(x)=148853562)
-6400 val loss 6.4049
-6400 val perplexity 604.7910
-6400 train 6.175096 (lr=5.5177e-05) (hash(x)=141530101)
-6500 val loss 6.4083
-6500 val perplexity 606.8895
-6500 train 6.273750 (lr=5.3213e-05) (hash(x)=142297809)
-6600 val loss 6.3913
-6600 val perplexity 596.6281
-6600 train 6.285662 (lr=5.1279e-05) (hash(x)=142447782)
-6700 val loss 6.3852
-6700 val perplexity 592.9855
-6700 train 6.334586 (lr=4.9377e-05) (hash(x)=147004686)
-6800 val loss 6.3813
-6800 val perplexity 590.7168
-6800 train 6.122798 (lr=4.7509e-05) (hash(x)=133438702)
-6900 val loss 6.3817
-6900 val perplexity 590.9413
-6900 train 6.400922 (lr=4.5676e-05) (hash(x)=157085143)
-7000 val loss 6.3686
-7000 val perplexity 583.2387
-7000 train 6.254630 (lr=4.3882e-05) (hash(x)=139437666)
-7100 val loss 6.3700
-7100 val perplexity 584.0773
-7100 train 6.429934 (lr=4.2128e-05) (hash(x)=159792986)
-7200 val loss 6.3505
-7200 val perplexity 572.7610
-7200 train 6.315082 (lr=4.0414e-05) (hash(x)=144930687)
-7300 val loss 6.3462
-7300 val perplexity 570.3040
-7300 train 6.420286 (lr=3.8745e-05) (hash(x)=156242690)
-7400 val loss 6.3418
-7400 val perplexity 567.8328
-7400 train 6.196812 (lr=3.7120e-05) (hash(x)=148183719)
-7500 val loss 6.3468
-7500 val perplexity 570.6543
-7500 train 6.363414 (lr=3.5541e-05) (hash(x)=152494758)
-7600 val loss 6.3398
-7600 val perplexity 566.7075
-7600 train 6.055812 (lr=3.4011e-05) (hash(x)=142485027)
-7700 val loss 6.3250
-7700 val perplexity 558.3545
-7700 train 6.183957 (lr=3.2531e-05) (hash(x)=147512165)
-7800 val loss 6.3210
-7800 val perplexity 556.1392
-7800 train 6.250357 (lr=3.1102e-05) (hash(x)=160346994)
-7900 val loss 6.3207
-7900 val perplexity 555.9703
-7900 train 6.175825 (lr=2.9726e-05) (hash(x)=144488254)
-8000 val loss 6.3116
-8000 val perplexity 550.9532
-8000 train 6.130961 (lr=2.8405e-05) (hash(x)=147637019)
-8100 val loss 6.3146
-8100 val perplexity 552.5546
-8100 train 6.177629 (lr=2.7138e-05) (hash(x)=147340534)
-8200 val loss 6.3078
-8200 val perplexity 548.8185
-8200 train 6.342015 (lr=2.5929e-05) (hash(x)=151630665)
-8300 val loss 6.3006
-8300 val perplexity 544.8951
-8300 train 6.376939 (lr=2.4778e-05) (hash(x)=149747064)
-8400 val loss 6.2979
-8400 val perplexity 543.4130
-8400 train 6.496303 (lr=2.3686e-05) (hash(x)=154245770)
-8500 val loss 6.2967
-8500 val perplexity 542.7586
-8500 train 6.180349 (lr=2.2655e-05) (hash(x)=152559100)
-8600 val loss 6.2924
-8600 val perplexity 540.4501
-8600 train 6.865870 (lr=2.1685e-05) (hash(x)=181365926)
-8700 val loss 6.2913
-8700 val perplexity 539.8574
-8700 train 6.130272 (lr=2.0777e-05) (hash(x)=154405991)
-8800 val loss 6.2885
-8800 val perplexity 538.3657
-8800 train 6.236886 (lr=1.9933e-05) (hash(x)=153755904)
-8900 val loss 6.2817
-8900 val perplexity 534.7005
-8900 train 6.219995 (lr=1.9153e-05) (hash(x)=152120568)
-9000 val loss 6.2762
-9000 val perplexity 531.7777
-9000 train 6.081510 (lr=1.8439e-05) (hash(x)=142797279)
-9100 val loss 6.2735
-9100 val perplexity 530.3247
-9100 train 6.146645 (lr=1.7790e-05) (hash(x)=143037503)
-9200 val loss 6.2750
-9200 val perplexity 531.1204
-9200 train 6.164870 (lr=1.7208e-05) (hash(x)=113690273)
-9300 val loss 6.2728
-9300 val perplexity 529.9344
-9300 train 6.205205 (lr=1.6692e-05) (hash(x)=158025077)
-9400 val loss 6.2699
-9400 val perplexity 528.4240
-9400 train 6.318002 (lr=1.6245e-05) (hash(x)=158251718)
-9500 val loss 6.2629
-9500 val perplexity 524.7199
-9500 train 6.295635 (lr=1.5865e-05) (hash(x)=154752610)
-9600 val loss 6.2630
-9600 val perplexity 524.7664
-9600 train 6.138709 (lr=1.5554e-05) (hash(x)=146889093)
-9700 val loss 6.2586
-9700 val perplexity 522.4791
-9700 train 6.264400 (lr=1.5312e-05) (hash(x)=156906516)
-9800 val loss 6.2603
-9800 val perplexity 523.3580
-9800 train 6.122754 (lr=1.5139e-05) (hash(x)=153841927)
-9900 val loss 6.2590
-9900 val perplexity 522.6864
-9900 train 6.450033 (lr=1.5035e-05) (hash(x)=163514334)
-9999 val loss 6.2538
-9999 val perplexity 519.9728
+0 train 11.833096 (lr=2.5000e-07) (hash(x)=164406924)
+100 val loss 10.1314
+100 val perplexity 25120.3730
+100 train 10.435864 (lr=2.5250e-05) (hash(x)=177407419)
+200 val loss 8.5877
+200 val perplexity 5365.3306
+200 train 8.530438 (lr=5.0000e-05) (hash(x)=144903932)
+300 val loss 7.9051
+300 val perplexity 2711.0681
+300 train 8.196055 (lr=4.9988e-05) (hash(x)=173839165)
+400 val loss 7.6891
+400 val perplexity 2184.3062
+400 train 7.772710 (lr=4.9954e-05) (hash(x)=167734596)
+500 val loss 7.5944
+500 val perplexity 1987.0854
+500 train 7.608729 (lr=4.9896e-05) (hash(x)=153224076)
+600 val loss 7.5572
+600 val perplexity 1914.4473
+600 train 7.472777 (lr=4.9815e-05) (hash(x)=149619098)
+700 val loss 7.5444
+700 val perplexity 1890.2152
+700 train 7.501118 (lr=4.9712e-05) (hash(x)=146539909)
+800 val loss 7.5187
+800 val perplexity 1842.2115
+800 train 7.481338 (lr=4.9585e-05) (hash(x)=153710890)
+900 val loss 7.5052
+900 val perplexity 1817.4755
+900 train 7.457563 (lr=4.9436e-05) (hash(x)=155873620)
+1000 val loss 7.4734
+1000 val perplexity 1760.6692
+1000 train 7.401876 (lr=4.9264e-05) (hash(x)=145450636)
+1100 val loss 7.4466
+1100 val perplexity 1714.0623
+1100 train 7.503413 (lr=4.9070e-05) (hash(x)=154123388)
+1200 val loss 7.4085
+1200 val perplexity 1650.0238
+1200 train 7.273043 (lr=4.8854e-05) (hash(x)=145249251)
+1300 val loss 7.3725
+1300 val perplexity 1591.5873
+1300 train 7.281743 (lr=4.8616e-05) (hash(x)=148937127)
+1400 val loss 7.3240
+1400 val perplexity 1516.2676
+1400 train 7.429045 (lr=4.8356e-05) (hash(x)=150475545)
+1500 val loss 7.2841
+1500 val perplexity 1456.8834
+1500 train 7.223948 (lr=4.8074e-05) (hash(x)=154653428)
+1600 val loss 7.2210
+1600 val perplexity 1367.9034
+1600 train 7.197338 (lr=4.7772e-05) (hash(x)=144483776)
+1700 val loss 7.1577
+1700 val perplexity 1283.9692
+1700 train 7.327583 (lr=4.7448e-05) (hash(x)=157395496)
+1800 val loss 7.0988
+1800 val perplexity 1210.5161
+1800 train 7.118510 (lr=4.7105e-05) (hash(x)=157916369)
+1900 val loss 7.0482
+1900 val perplexity 1150.7883
+1900 train 7.250220 (lr=4.6741e-05) (hash(x)=166073923)
+2000 val loss 7.0099
+2000 val perplexity 1107.5649
+2000 train 7.078530 (lr=4.6357e-05) (hash(x)=154856891)
+2100 val loss 6.9708
+2100 val perplexity 1065.0917
+2100 train 6.957214 (lr=4.5954e-05) (hash(x)=151925203)
+2200 val loss 6.9150
+2200 val perplexity 1007.2465
+2200 train 6.623221 (lr=4.5532e-05) (hash(x)=136191502)
+2300 val loss 6.8516
+2300 val perplexity 945.3943
+2300 train 6.992955 (lr=4.5091e-05) (hash(x)=153273362)
+2400 val loss 6.8002
+2400 val perplexity 898.0457
+2400 train 6.748407 (lr=4.4633e-05) (hash(x)=148021541)
+2500 val loss 6.7577
+2500 val perplexity 860.6304
+2500 train 6.699186 (lr=4.4156e-05) (hash(x)=141356608)
+2600 val loss 6.7101
+2600 val perplexity 820.6888
+2600 train 6.662726 (lr=4.3663e-05) (hash(x)=146005217)
+2700 val loss 6.6873
+2700 val perplexity 802.1870
+2700 train 6.533102 (lr=4.3153e-05) (hash(x)=144511718)
+2800 val loss 6.6459
+2800 val perplexity 769.6336
+2800 train 6.522305 (lr=4.2627e-05) (hash(x)=146019502)
+2900 val loss 6.6170
+2900 val perplexity 747.7319
+2900 train 6.526833 (lr=4.2085e-05) (hash(x)=146496200)
+3000 val loss 6.5904
+3000 val perplexity 728.0774
+3000 train 6.532321 (lr=4.1529e-05) (hash(x)=150127281)
+3100 val loss 6.5528
+3100 val perplexity 701.1899
+3100 train 6.537275 (lr=4.0957e-05) (hash(x)=142022255)
+3200 val loss 6.5277
+3200 val perplexity 683.8523
+3200 train 6.587036 (lr=4.0373e-05) (hash(x)=154120875)
+3300 val loss 6.5102
+3300 val perplexity 671.9406
+3300 train 6.563637 (lr=3.9775e-05) (hash(x)=153999717)
+3400 val loss 6.4894
+3400 val perplexity 658.1267
+3400 train 6.270551 (lr=3.9164e-05) (hash(x)=139694097)
+3500 val loss 6.4582
+3500 val perplexity 637.9100
+3500 train 6.644126 (lr=3.8541e-05) (hash(x)=162992732)
+3600 val loss 6.4246
+3600 val perplexity 616.8521
+3600 train 6.398591 (lr=3.7907e-05) (hash(x)=147574101)
+3700 val loss 6.4115
+3700 val perplexity 608.8231
+3700 train 6.528100 (lr=3.7262e-05) (hash(x)=157763099)
+3800 val loss 6.3859
+3800 val perplexity 593.4160
+3800 train 6.540219 (lr=3.6608e-05) (hash(x)=170800034)
+3900 val loss 6.3712
+3900 val perplexity 584.7700
+3900 train 6.450828 (lr=3.5944e-05) (hash(x)=164984528)
+4000 val loss 6.3565
+4000 val perplexity 576.2042
+4000 train 6.230247 (lr=3.5271e-05) (hash(x)=141743323)
+4100 val loss 6.3548
+4100 val perplexity 575.2441
+4100 train 6.368493 (lr=3.4590e-05) (hash(x)=153392872)
+4200 val loss 6.3445
+4200 val perplexity 569.3611
+4200 train 6.203356 (lr=3.3902e-05) (hash(x)=149074933)
+4300 val loss 6.3186
+4300 val perplexity 554.7869
+4300 train 6.702516 (lr=3.3207e-05) (hash(x)=167823423)
+4400 val loss 6.2979
+4400 val perplexity 543.4063
+4400 train 6.061867 (lr=3.2507e-05) (hash(x)=141203114)
+4500 val loss 6.2891
+4500 val perplexity 538.6491
+4500 train 6.289015 (lr=3.1801e-05) (hash(x)=146284780)
+4600 val loss 6.2654
+4600 val perplexity 526.0757
+4600 train 6.073493 (lr=3.1091e-05) (hash(x)=141126464)
+4700 val loss 6.2576
+4700 val perplexity 521.9659
+4700 train 6.287447 (lr=3.0377e-05) (hash(x)=154751926)
+4800 val loss 6.2461
+4800 val perplexity 516.0110
+4800 train 6.331287 (lr=2.9661e-05) (hash(x)=154793198)
+4900 val loss 6.2420
+4900 val perplexity 513.8834
+4900 train 5.956399 (lr=2.8942e-05) (hash(x)=139406392)
+5000 val loss 6.2666
+5000 val perplexity 526.6661
+5000 train 5.975211 (lr=2.8221e-05) (hash(x)=153548741)
+5100 val loss 6.2267
+5100 val perplexity 506.0816
+5100 train 6.303495 (lr=2.7500e-05) (hash(x)=160488568)
+5200 val loss 6.2044
+5200 val perplexity 494.9333
+5200 train 6.213053 (lr=2.6779e-05) (hash(x)=149645053)
+5300 val loss 6.1867
+5300 val perplexity 486.2615
+5300 train 6.276705 (lr=2.6058e-05) (hash(x)=155820556)
+5400 val loss 6.1806
+5400 val perplexity 483.2999
+5400 train 6.128071 (lr=2.5339e-05) (hash(x)=147538134)
+5500 val loss 6.1716
+5500 val perplexity 478.9390
+5500 train 6.377961 (lr=2.4623e-05) (hash(x)=166889307)
+5600 val loss 6.1622
+5600 val perplexity 474.4739
+5600 train 5.880041 (lr=2.3909e-05) (hash(x)=139516699)
+5700 val loss 6.1641
+5700 val perplexity 475.3632
+5700 train 5.843537 (lr=2.3199e-05) (hash(x)=140453511)
+5800 val loss 6.1614
+5800 val perplexity 474.1037
+5800 train 6.102824 (lr=2.2493e-05) (hash(x)=162964847)
+5900 val loss 6.1491
+5900 val perplexity 468.2912
+5900 train 6.153123 (lr=2.1793e-05) (hash(x)=150606634)
+6000 val loss 6.1313
+6000 val perplexity 460.0215
+6000 train 6.210247 (lr=2.1098e-05) (hash(x)=149890857)
+6100 val loss 6.1292
+6100 val perplexity 459.0655
+6100 train 6.234848 (lr=2.0410e-05) (hash(x)=173884145)
+6200 val loss 6.1213
+6200 val perplexity 455.4413
+6200 train 6.167700 (lr=1.9729e-05) (hash(x)=151987098)
+6300 val loss 6.1115
+6300 val perplexity 451.0340
+6300 train 6.057473 (lr=1.9056e-05) (hash(x)=148853562)
+6400 val loss 6.1064
+6400 val perplexity 448.7010
+6400 train 5.853988 (lr=1.8392e-05) (hash(x)=141530101)
+6500 val loss 6.1060
+6500 val perplexity 448.5262
+6500 train 5.964704 (lr=1.7738e-05) (hash(x)=142297809)
+6600 val loss 6.0976
+6600 val perplexity 444.7705
+6600 train 5.990374 (lr=1.7093e-05) (hash(x)=142447782)
+6700 val loss 6.0943
+6700 val perplexity 443.3159
+6700 train 6.063104 (lr=1.6459e-05) (hash(x)=147004686)
+6800 val loss 6.0880
+6800 val perplexity 440.5413
+6800 train 5.831447 (lr=1.5836e-05) (hash(x)=133438702)
+6900 val loss 6.0855
+6900 val perplexity 439.4597
+6900 train 6.103887 (lr=1.5225e-05) (hash(x)=157085143)
+7000 val loss 6.0821
+7000 val perplexity 437.9416
+7000 train 5.957407 (lr=1.4627e-05) (hash(x)=139437666)
+7100 val loss 6.0744
+7100 val perplexity 434.5800
+7100 train 6.096306 (lr=1.4043e-05) (hash(x)=159792986)
+7200 val loss 6.0653
+7200 val perplexity 430.6621
+7200 train 6.041170 (lr=1.3471e-05) (hash(x)=144930687)
+7300 val loss 6.0612
+7300 val perplexity 428.8893
+7300 train 6.113462 (lr=1.2915e-05) (hash(x)=156242690)
+7400 val loss 6.0626
+7400 val perplexity 429.4924
+7400 train 5.896077 (lr=1.2373e-05) (hash(x)=148183719)
+7500 val loss 6.0577
+7500 val perplexity 427.3765
+7500 train 6.091302 (lr=1.1847e-05) (hash(x)=152494758)
+7600 val loss 6.0547
+7600 val perplexity 426.1260
+7600 train 5.754051 (lr=1.1337e-05) (hash(x)=142485027)
+7700 val loss 6.0560
+7700 val perplexity 426.6740
+7700 train 5.928751 (lr=1.0844e-05) (hash(x)=147512165)
+7800 val loss 6.0404
+7800 val perplexity 420.0570
+7800 train 5.927812 (lr=1.0367e-05) (hash(x)=160346994)
+7900 val loss 6.0410
+7900 val perplexity 420.3325
+7900 train 5.907988 (lr=9.9088e-06) (hash(x)=144488254)
+8000 val loss 6.0358
+8000 val perplexity 418.1136
+8000 train 5.854425 (lr=9.4682e-06) (hash(x)=147637019)
+8100 val loss 6.0361
+8100 val perplexity 418.2736
+8100 train 5.901888 (lr=9.0461e-06) (hash(x)=147340534)
+8200 val loss 6.0326
+8200 val perplexity 416.7949
+8200 train 6.087666 (lr=8.6430e-06) (hash(x)=151630665)
+8300 val loss 6.0267
+8300 val perplexity 414.3367
+8300 train 6.126692 (lr=8.2593e-06) (hash(x)=149747064)
+8400 val loss 6.0232
+8400 val perplexity 412.8936
+8400 train 6.212832 (lr=7.8953e-06) (hash(x)=154245770)
+8500 val loss 6.0230
+8500 val perplexity 412.8336
+8500 train 5.904833 (lr=7.5515e-06) (hash(x)=152559100)
+8600 val loss 6.0187
+8600 val perplexity 411.0398
+8600 train 6.535484 (lr=7.2282e-06) (hash(x)=181365926)
+8700 val loss 6.0188
+8700 val perplexity 411.0829
+8700 train 5.863358 (lr=6.9257e-06) (hash(x)=154405991)
+8800 val loss 6.0153
+8800 val perplexity 409.6678
+8800 train 5.970211 (lr=6.6444e-06) (hash(x)=153755904)
+8900 val loss 6.0107
+8900 val perplexity 407.7833
+8900 train 5.926088 (lr=6.3845e-06) (hash(x)=152120568)
+9000 val loss 6.0077
+9000 val perplexity 406.5505
+9000 train 5.799891 (lr=6.1462e-06) (hash(x)=142797279)
+9100 val loss 6.0051
+9100 val perplexity 405.5031
+9100 train 5.871176 (lr=5.9300e-06) (hash(x)=143037503)
+9200 val loss 6.0051
+9200 val perplexity 405.4734
+9200 train 5.892231 (lr=5.7359e-06) (hash(x)=113690273)
+9300 val loss 6.0039
+9300 val perplexity 405.0020
+9300 train 5.939885 (lr=5.5641e-06) (hash(x)=158025077)
+9400 val loss 6.0019
+9400 val perplexity 404.1820
+9400 train 6.068281 (lr=5.4149e-06) (hash(x)=158251718)
+9500 val loss 5.9988
+9500 val perplexity 402.9359
+9500 train 6.039095 (lr=5.2884e-06) (hash(x)=154752610)
+9600 val loss 5.9971
+9600 val perplexity 402.2651
+9600 train 5.874845 (lr=5.1847e-06) (hash(x)=146889093)
+9700 val loss 5.9940
+9700 val perplexity 401.0011
+9700 train 6.001687 (lr=5.1040e-06) (hash(x)=156906516)
+9800 val loss 5.9932
+9800 val perplexity 400.7096
+9800 train 5.831883 (lr=5.0462e-06) (hash(x)=153841927)
+9900 val loss 5.9949
+9900 val perplexity 401.3906
+9900 train 6.192818 (lr=5.0116e-06) (hash(x)=163514334)
+9999 val loss 5.9891
+9999 val perplexity 399.0517

attention_kindselective_n_heads2_seed1340/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e5fcb57a54c44092399cfd81ff1ac237270f67858a3cbfcd984c918e5f518b3
+oid sha256:e5aeb7b7022a7ef095259330e13212b1ff8ec10d9fe91f9cfe5321726439400e
 size 38587970

attention_kindselective_n_heads2_seed1340/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:85ad5d27f8738f65092f79c3eeee065788669448b947dec7932419f34cd6f1a0
+oid sha256:ad1dcff0aef7e71a44d4bafe3d6fa2064b4970bceaa1db35e56cacedd4bce39b
 size 38587970

attention_kindselective_n_heads2_seed1340/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:74f8572300b137d51dd45501fa67ab3f549533819957f432f7f5d55c9f76c581
+oid sha256:df381c5d857ee4e2185137efa7971ff6ecca01a7071f0e3523bebefff488b94d
 size 38587970

attention_kindselective_n_heads2_seed1340/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:deb71d2e49ed3051060f7ebba35294fc1e13af32124e503566ccc123a84c22dc
+oid sha256:3260dd2ac1f7c16a62ed6d3c8b157a3f0a33efe14c610d63fec80ece19e6295c
 size 38587970

attention_kindselective_n_heads2_seed1340/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f5bf95d69aa7f485da54020766d70183985312d3ac8860e1443297ebb1e31b8
+oid sha256:2f494e8b5205e169278e98dc46247c57b4eecddb1d447131b03a3e8d70073d9e
 size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63a4b101db6ae2a081903120ed89323d2034db639836f58e6be1ba93a3bcd119
+oid sha256:c7f3fc56d8050fdf69a5ced1176d19cd3d7141742bed03ead99504850e90b4be
 size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:431eee67e4652c32b9c022648a5dc62e869954c5f106c4d37472fb51133c83c2
+oid sha256:4c05973832a45530c4593bc265a28c7ae61596a93a88b25f70012a7d34e37954
 size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9c8dbdc83a59eafb4e6d02f7622375e007acbd97945d3ec79e4cc349aaf7c51
+oid sha256:2084254e2cd8f9f5a63850ea73b259adc84ee7fc54797ca79e71efee5351e261
 size 70895430