Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1341/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_2_1341", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1341", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1341, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_2_1341", "n_embd": 128}

attention_kindselective_n_heads2_seed1341/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.7722
 0 val perplexity 129595.2031
-0 train 11.774795 (lr=2.5000e-07) (hash(x)=145079536)
-100 val loss 10.1967
-100 val perplexity 26815.2441
-100 train 10.143024 (lr=2.5250e-05) (hash(x)=154745873)
-200 val loss 8.4253
-200 val perplexity 4561.1460
-200 train 8.489046 (lr=5.0000e-05) (hash(x)=155800595)
-300 val loss 7.7144
-300 val perplexity 2240.3523
-300 train 7.665938 (lr=4.9988e-05) (hash(x)=148595389)
-400 val loss 7.5699
-400 val perplexity 1938.9187
-400 train 7.476192 (lr=4.9954e-05) (hash(x)=145606733)
-500 val loss 7.5245
-500 val perplexity 1852.8209
-500 train 7.344064 (lr=4.9896e-05) (hash(x)=138221231)
-600 val loss 7.4760
-600 val perplexity 1765.1185
-600 train 7.476169 (lr=4.9815e-05) (hash(x)=150367139)
-700 val loss 7.4224
-700 val perplexity 1672.9674
-700 train 7.559739 (lr=4.9712e-05) (hash(x)=155579314)
-800 val loss 7.3439
-800 val perplexity 1546.7035
-800 train 7.421778 (lr=4.9585e-05) (hash(x)=155747374)
-900 val loss 7.2824
-900 val perplexity 1454.4935
-900 train 7.492918 (lr=4.9436e-05) (hash(x)=159334575)
-1000 val loss 7.2183
-1000 val perplexity 1364.1555
-1000 train 7.090861 (lr=4.9264e-05) (hash(x)=140604760)
-1100 val loss 7.1651
-1100 val perplexity 1293.5281
-1100 train 6.987471 (lr=4.9070e-05) (hash(x)=146208052)
-1200 val loss 7.0891
-1200 val perplexity 1198.8540
-1200 train 6.984112 (lr=4.8854e-05) (hash(x)=148404734)
-1300 val loss 7.0370
-1300 val perplexity 1137.9492
-1300 train 6.965640 (lr=4.8616e-05) (hash(x)=155681970)
-1400 val loss 6.9815
-1400 val perplexity 1076.5845
-1400 train 6.933797 (lr=4.8356e-05) (hash(x)=148115934)
-1500 val loss 6.9014
-1500 val perplexity 993.6962
-1500 train 7.060465 (lr=4.8074e-05) (hash(x)=157074034)
-1600 val loss 6.8488
-1600 val perplexity 942.7806
-1600 train 6.613491 (lr=4.7772e-05) (hash(x)=137464699)
-1700 val loss 6.7931
-1700 val perplexity 891.7070
-1700 train 6.929006 (lr=4.7448e-05) (hash(x)=166955614)
-1800 val loss 6.7475
-1800 val perplexity 851.9409
-1800 train 6.565822 (lr=4.7105e-05) (hash(x)=143886042)
-1900 val loss 6.7145
-1900 val perplexity 824.3010
-1900 train 7.249403 (lr=4.6741e-05) (hash(x)=193610391)
-2000 val loss 6.6721
-2000 val perplexity 790.0560
-2000 train 6.862562 (lr=4.6357e-05) (hash(x)=163799796)
-2100 val loss 6.6205
-2100 val perplexity 750.3120
-2100 train 6.607678 (lr=4.5954e-05) (hash(x)=154107339)
-2200 val loss 6.5971
-2200 val perplexity 732.9810
-2200 train 6.600620 (lr=4.5532e-05) (hash(x)=144592844)
-2300 val loss 6.5494
-2300 val perplexity 698.8473
-2300 train 6.613638 (lr=4.5091e-05) (hash(x)=154535861)
-2400 val loss 6.5211
-2400 val perplexity 679.3499
-2400 train 7.256778 (lr=4.4633e-05) (hash(x)=204706354)
-2500 val loss 6.5063
-2500 val perplexity 669.3500
-2500 train 6.299736 (lr=4.4156e-05) (hash(x)=143783202)
-2600 val loss 6.4746
-2600 val perplexity 648.4813
-2600 train 6.395596 (lr=4.3663e-05) (hash(x)=150862210)
-2700 val loss 6.4500
-2700 val perplexity 632.7221
-2700 train 6.488371 (lr=4.3153e-05) (hash(x)=150753426)
-2800 val loss 6.4271
-2800 val perplexity 618.3502
-2800 train 6.448690 (lr=4.2627e-05) (hash(x)=147766811)
-2900 val loss 6.4014
-2900 val perplexity 602.6843
-2900 train 6.334109 (lr=4.2085e-05) (hash(x)=151177814)
-3000 val loss 6.3806
-3000 val perplexity 590.3071
-3000 train 6.256360 (lr=4.1529e-05) (hash(x)=135925327)
-3100 val loss 6.3712
-3100 val perplexity 584.7452
-3100 train 6.015045 (lr=4.0957e-05) (hash(x)=124842353)
-3200 val loss 6.3610
-3200 val perplexity 578.8417
-3200 train 6.456865 (lr=4.0373e-05) (hash(x)=160440642)
-3300 val loss 6.3370
-3300 val perplexity 565.1111
-3300 train 6.252707 (lr=3.9775e-05) (hash(x)=143017131)
-3400 val loss 6.3147
-3400 val perplexity 552.6184
-3400 train 6.692260 (lr=3.9164e-05) (hash(x)=148143474)
-3500 val loss 6.3072
-3500 val perplexity 548.5033
-3500 train 6.128360 (lr=3.8541e-05) (hash(x)=148732639)
-3600 val loss 6.2925
-3600 val perplexity 540.4887
-3600 train 6.286038 (lr=3.7907e-05) (hash(x)=146333803)
-3700 val loss 6.2816
-3700 val perplexity 534.6521
-3700 train 6.090304 (lr=3.7262e-05) (hash(x)=143611445)
-3800 val loss 6.2777
-3800 val perplexity 532.5809
-3800 train 6.061575 (lr=3.6608e-05) (hash(x)=141272115)
-3900 val loss 6.2569
-3900 val perplexity 521.6227
-3900 train 6.195599 (lr=3.5944e-05) (hash(x)=149161848)
-4000 val loss 6.2324
-4000 val perplexity 508.9707
-4000 train 6.309767 (lr=3.5271e-05) (hash(x)=152036955)
-4100 val loss 6.2213
-4100 val perplexity 503.3548
-4100 train 6.268515 (lr=3.4590e-05) (hash(x)=153260968)
-4200 val loss 6.2097
-4200 val perplexity 497.5497
-4200 train 6.101007 (lr=3.3902e-05) (hash(x)=147629478)
-4300 val loss 6.1937
-4300 val perplexity 489.6672
-4300 train 6.167351 (lr=3.3207e-05) (hash(x)=161701080)
-4400 val loss 6.1911
-4400 val perplexity 488.3745
-4400 train 6.231541 (lr=3.2507e-05) (hash(x)=155290524)
-4500 val loss 6.1868
-4500 val perplexity 486.3091
-4500 train 5.971059 (lr=3.1801e-05) (hash(x)=144726593)
-4600 val loss 6.1760
-4600 val perplexity 481.0795
-4600 train 6.129531 (lr=3.1091e-05) (hash(x)=149517202)
-4700 val loss 6.1587
-4700 val perplexity 472.8268
-4700 train 6.162160 (lr=3.0377e-05) (hash(x)=139403008)
-4800 val loss 6.1452
-4800 val perplexity 466.4960
-4800 train 6.247012 (lr=2.9661e-05) (hash(x)=153387238)
-4900 val loss 6.1427
-4900 val perplexity 465.2972
-4900 train 5.897235 (lr=2.8942e-05) (hash(x)=139881569)
-5000 val loss 6.1239
-5000 val perplexity 456.6504
-5000 train 6.141126 (lr=2.8221e-05) (hash(x)=152602580)
-5100 val loss 6.1204
-5100 val perplexity 455.0395
-5100 train 6.160268 (lr=2.7500e-05) (hash(x)=150598460)
-5200 val loss 6.1112
-5200 val perplexity 450.8787
-5200 train 5.984895 (lr=2.6779e-05) (hash(x)=147660496)
-5300 val loss 6.1046
-5300 val perplexity 447.9008
-5300 train 5.845158 (lr=2.6058e-05) (hash(x)=147693848)
-5400 val loss 6.1049
-5400 val perplexity 448.0625
-5400 train 5.862473 (lr=2.5339e-05) (hash(x)=136048145)
-5500 val loss 6.0878
-5500 val perplexity 440.4329
-5500 train 6.121768 (lr=2.4623e-05) (hash(x)=140313123)
-5600 val loss 6.0770
-5600 val perplexity 435.7415
-5600 train 6.093797 (lr=2.3909e-05) (hash(x)=156997993)
-5700 val loss 6.0683
-5700 val perplexity 431.9543
-5700 train 6.087464 (lr=2.3199e-05) (hash(x)=143513880)
-5800 val loss 6.0614
-5800 val perplexity 428.9725
-5800 train 6.172104 (lr=2.2493e-05) (hash(x)=153736628)
-5900 val loss 6.0547
-5900 val perplexity 426.1208
-5900 train 6.190841 (lr=2.1793e-05) (hash(x)=166385417)
-6000 val loss 6.0458
-6000 val perplexity 422.3309
-6000 train 6.277092 (lr=2.1098e-05) (hash(x)=153216517)
-6100 val loss 6.0446
-6100 val perplexity 421.8097
-6100 train 5.932088 (lr=2.0410e-05) (hash(x)=145387545)
-6200 val loss 6.0460
-6200 val perplexity 422.4171
-6200 train 6.175024 (lr=1.9729e-05) (hash(x)=156306460)
-6300 val loss 6.0292
-6300 val perplexity 415.3701
-6300 train 5.960823 (lr=1.9056e-05) (hash(x)=148610447)
-6400 val loss 6.0213
-6400 val perplexity 412.0996
-6400 train 6.243478 (lr=1.8392e-05) (hash(x)=158343535)
-6500 val loss 6.0134
-6500 val perplexity 408.8782
-6500 train 6.185011 (lr=1.7738e-05) (hash(x)=157742557)
-6600 val loss 6.0124
-6600 val perplexity 408.4781
-6600 train 5.903574 (lr=1.7093e-05) (hash(x)=150457308)
-6700 val loss 6.0066
-6700 val perplexity 406.1186
-6700 train 5.866361 (lr=1.6459e-05) (hash(x)=142426205)
-6800 val loss 6.0069
-6800 val perplexity 406.2187
-6800 train 5.806967 (lr=1.5836e-05) (hash(x)=151522525)
-6900 val loss 5.9981
-6900 val perplexity 402.6668
-6900 train 5.947786 (lr=1.5225e-05) (hash(x)=152029630)
-7000 val loss 5.9889
-7000 val perplexity 398.9860
-7000 train 5.835184 (lr=1.4627e-05) (hash(x)=142978954)
-7100 val loss 5.9839
-7100 val perplexity 396.9924
-7100 train 5.866302 (lr=1.4043e-05) (hash(x)=148064359)
-7200 val loss 5.9817
-7200 val perplexity 396.0956
-7200 train 5.878535 (lr=1.3471e-05) (hash(x)=155771845)
-7300 val loss 5.9780
-7300 val perplexity 394.6485
-7300 train 5.788899 (lr=1.2915e-05) (hash(x)=141639844)
-7400 val loss 5.9782
-7400 val perplexity 394.7098
-7400 train 6.030018 (lr=1.2373e-05) (hash(x)=153466021)
-7500 val loss 5.9703
-7500 val perplexity 391.6344
-7500 train 5.951066 (lr=1.1847e-05) (hash(x)=148843587)
-7600 val loss 5.9636
-7600 val perplexity 388.9968
-7600 train 6.093093 (lr=1.1337e-05) (hash(x)=152053933)
-7700 val loss 5.9594
-7700 val perplexity 387.3708
-7700 train 5.819657 (lr=1.0844e-05) (hash(x)=141564003)
-7800 val loss 5.9565
-7800 val perplexity 386.2691
-7800 train 5.836637 (lr=1.0367e-05) (hash(x)=148617843)
-7900 val loss 5.9547
-7900 val perplexity 385.5467
-7900 train 5.925571 (lr=9.9088e-06) (hash(x)=146417668)
-8000 val loss 5.9517
-8000 val perplexity 384.4099
-8000 train 5.756671 (lr=9.4682e-06) (hash(x)=150165488)
-8100 val loss 5.9492
-8100 val perplexity 383.4286
-8100 train 5.928113 (lr=9.0461e-06) (hash(x)=144818855)
-8200 val loss 5.9425
-8200 val perplexity 380.8962
-8200 train 6.034739 (lr=8.6430e-06) (hash(x)=143074954)
-8300 val loss 5.9406
-8300 val perplexity 380.1790
-8300 train 5.819485 (lr=8.2593e-06) (hash(x)=145172494)
-8400 val loss 5.9388
-8400 val perplexity 379.4785
-8400 train 5.967767 (lr=7.8953e-06) (hash(x)=152284393)
-8500 val loss 5.9353
-8500 val perplexity 378.1628
-8500 train 5.779359 (lr=7.5515e-06) (hash(x)=147044705)
-8600 val loss 5.9345
-8600 val perplexity 377.8589
-8600 train 5.811585 (lr=7.2282e-06) (hash(x)=147762608)
-8700 val loss 5.9323
-8700 val perplexity 377.0360
-8700 train 5.942141 (lr=6.9257e-06) (hash(x)=141994567)
-8800 val loss 5.9272
-8800 val perplexity 375.1202
-8800 train 6.021777 (lr=6.6444e-06) (hash(x)=152552896)
-8900 val loss 5.9233
-8900 val perplexity 373.6460
-8900 train 5.762463 (lr=6.3845e-06) (hash(x)=139714454)
-9000 val loss 5.9218
-9000 val perplexity 373.0761
-9000 train 5.954716 (lr=6.1462e-06) (hash(x)=152274419)
-9100 val loss 5.9213
-9100 val perplexity 372.9054
-9100 train 6.007237 (lr=5.9300e-06) (hash(x)=160536885)
-9200 val loss 5.9212
-9200 val perplexity 372.8654
-9200 train 5.734900 (lr=5.7359e-06) (hash(x)=138386899)
-9300 val loss 5.9195
-9300 val perplexity 372.2228
-9300 train 5.970142 (lr=5.5641e-06) (hash(x)=149957457)
-9400 val loss 5.9159
-9400 val perplexity 370.8707
-9400 train 5.931352 (lr=5.4149e-06) (hash(x)=161671548)
-9500 val loss 5.9132
-9500 val perplexity 369.8911
-9500 train 6.040483 (lr=5.2884e-06) (hash(x)=171857400)
-9600 val loss 5.9124
-9600 val perplexity 369.6096
-9600 train 5.686675 (lr=5.1847e-06) (hash(x)=135378471)
-9700 val loss 5.9109
-9700 val perplexity 369.0223
-9700 train 5.833959 (lr=5.1040e-06) (hash(x)=149514591)
-9800 val loss 5.9099
-9800 val perplexity 368.6873
-9800 train 5.888211 (lr=5.0462e-06) (hash(x)=152679072)
-9900 val loss 5.9073
-9900 val perplexity 367.6994
-9900 train 5.844470 (lr=5.0116e-06) (hash(x)=150415193)
-9999 val loss 5.9055
-9999 val perplexity 367.0529
+0 train 11.774795 (lr=1.5000e-07) (hash(x)=145079536)
+100 val loss 10.3638
+100 val perplexity 31690.6855
+100 train 10.310390 (lr=1.5150e-05) (hash(x)=154745873)
+200 val loss 9.4344
+200 val perplexity 12511.1768
+200 train 9.417608 (lr=3.0000e-05) (hash(x)=155800595)
+300 val loss 8.1776
+300 val perplexity 3560.4011
+300 train 8.142024 (lr=2.9993e-05) (hash(x)=148595389)
+400 val loss 7.7316
+400 val perplexity 2279.2727
+400 train 7.647736 (lr=2.9972e-05) (hash(x)=145606733)
+500 val loss 7.5773
+500 val perplexity 1953.3274
+500 train 7.406966 (lr=2.9938e-05) (hash(x)=138221231)
+600 val loss 7.5203
+600 val perplexity 1845.0748
+600 train 7.525246 (lr=2.9889e-05) (hash(x)=150367139)
+700 val loss 7.4772
+700 val perplexity 1767.2173
+700 train 7.618254 (lr=2.9827e-05) (hash(x)=155579314)
+800 val loss 7.4219
+800 val perplexity 1672.1436
+800 train 7.494927 (lr=2.9751e-05) (hash(x)=155747374)
+900 val loss 7.3563
+900 val perplexity 1566.0425
+900 train 7.568688 (lr=2.9662e-05) (hash(x)=159334575)
+1000 val loss 7.2908
+1000 val perplexity 1466.7249
+1000 train 7.166207 (lr=2.9558e-05) (hash(x)=140604760)
+1100 val loss 7.2271
+1100 val perplexity 1376.2406
+1100 train 7.053119 (lr=2.9442e-05) (hash(x)=146208052)
+1200 val loss 7.1591
+1200 val perplexity 1285.7166
+1200 train 7.065541 (lr=2.9312e-05) (hash(x)=148404734)
+1300 val loss 7.0993
+1300 val perplexity 1211.0946
+1300 train 7.030000 (lr=2.9169e-05) (hash(x)=155681970)
+1400 val loss 7.0369
+1400 val perplexity 1137.8608
+1400 train 6.990728 (lr=2.9013e-05) (hash(x)=148115934)
+1500 val loss 6.9659
+1500 val perplexity 1059.9108
+1500 train 7.128124 (lr=2.8845e-05) (hash(x)=157074034)
+1600 val loss 6.9095
+1600 val perplexity 1001.7750
+1600 train 6.662536 (lr=2.8663e-05) (hash(x)=137464699)
+1700 val loss 6.8588
+1700 val perplexity 952.2664
+1700 train 7.000303 (lr=2.8469e-05) (hash(x)=166955614)
+1800 val loss 6.8125
+1800 val perplexity 909.1738
+1800 train 6.636445 (lr=2.8263e-05) (hash(x)=143886042)
+1900 val loss 6.7785
+1900 val perplexity 878.7321
+1900 train 7.344856 (lr=2.8044e-05) (hash(x)=193610391)
+2000 val loss 6.7289
+2000 val perplexity 836.2564
+2000 train 6.923176 (lr=2.7814e-05) (hash(x)=163799796)
+2100 val loss 6.6765
+2100 val perplexity 793.5648
+2100 train 6.666200 (lr=2.7572e-05) (hash(x)=154107339)
+2200 val loss 6.6449
+2200 val perplexity 768.8541
+2200 train 6.642457 (lr=2.7319e-05) (hash(x)=144592844)
+2300 val loss 6.6128
+2300 val perplexity 744.5838
+2300 train 6.680394 (lr=2.7055e-05) (hash(x)=154535861)
+2400 val loss 6.5805
+2400 val perplexity 720.9152
+2400 train 7.345472 (lr=2.6780e-05) (hash(x)=204706354)
+2500 val loss 6.5576
+2500 val perplexity 704.6126
+2500 train 6.345363 (lr=2.6494e-05) (hash(x)=143783202)
+2600 val loss 6.5345
+2600 val perplexity 688.5145
+2600 train 6.449786 (lr=2.6198e-05) (hash(x)=150862210)
+2700 val loss 6.5102
+2700 val perplexity 671.9762
+2700 train 6.551790 (lr=2.5892e-05) (hash(x)=150753426)
+2800 val loss 6.4929
+2800 val perplexity 660.4471
+2800 train 6.511770 (lr=2.5576e-05) (hash(x)=147766811)
+2900 val loss 6.4571
+2900 val perplexity 637.2186
+2900 train 6.394937 (lr=2.5251e-05) (hash(x)=151177814)
+3000 val loss 6.4394
+3000 val perplexity 626.0252
+3000 train 6.307624 (lr=2.4917e-05) (hash(x)=135925327)
+3100 val loss 6.4255
+3100 val perplexity 617.3714
+3100 train 6.066052 (lr=2.4574e-05) (hash(x)=124842353)
+3200 val loss 6.4126
+3200 val perplexity 609.4990
+3200 train 6.509407 (lr=2.4224e-05) (hash(x)=160440642)
+3300 val loss 6.3849
+3300 val perplexity 592.8026
+3300 train 6.300091 (lr=2.3865e-05) (hash(x)=143017131)
+3400 val loss 6.3705
+3400 val perplexity 584.3425
+3400 train 6.720880 (lr=2.3498e-05) (hash(x)=148143474)
+3500 val loss 6.3546
+3500 val perplexity 575.1470
+3500 train 6.168210 (lr=2.3125e-05) (hash(x)=148732639)
+3600 val loss 6.3427
+3600 val perplexity 568.3472
+3600 train 6.334900 (lr=2.2744e-05) (hash(x)=146333803)
+3700 val loss 6.3290
+3700 val perplexity 560.5775
+3700 train 6.129919 (lr=2.2357e-05) (hash(x)=143611445)
+3800 val loss 6.3270
+3800 val perplexity 559.4955
+3800 train 6.108684 (lr=2.1965e-05) (hash(x)=141272115)
+3900 val loss 6.3116
+3900 val perplexity 550.9077
+3900 train 6.246768 (lr=2.1566e-05) (hash(x)=149161848)
+4000 val loss 6.2831
+4000 val perplexity 535.4652
+4000 train 6.355687 (lr=2.1162e-05) (hash(x)=152036955)
+4100 val loss 6.2690
+4100 val perplexity 527.9527
+4100 train 6.317615 (lr=2.0754e-05) (hash(x)=153260968)
+4200 val loss 6.2611
+4200 val perplexity 523.7897
+4200 train 6.157646 (lr=2.0341e-05) (hash(x)=147629478)
+4300 val loss 6.2524
+4300 val perplexity 519.2458
+4300 train 6.222133 (lr=1.9924e-05) (hash(x)=161701080)
+4400 val loss 6.2445
+4400 val perplexity 515.1727
+4400 train 6.297623 (lr=1.9504e-05) (hash(x)=155290524)
+4500 val loss 6.2363
+4500 val perplexity 510.9671
+4500 train 6.020789 (lr=1.9081e-05) (hash(x)=144726593)
+4600 val loss 6.2316
+4600 val perplexity 508.5655
+4600 train 6.176429 (lr=1.8655e-05) (hash(x)=149517202)
+4700 val loss 6.2135
+4700 val perplexity 499.4560
+4700 train 6.223347 (lr=1.8226e-05) (hash(x)=139403008)
+4800 val loss 6.1999
+4800 val perplexity 492.6792
+4800 train 6.289749 (lr=1.7796e-05) (hash(x)=153387238)
+4900 val loss 6.1991
+4900 val perplexity 492.2868
+4900 train 5.946225 (lr=1.7365e-05) (hash(x)=139881569)
+5000 val loss 6.1858
+5000 val perplexity 485.8128
+5000 train 6.200039 (lr=1.6933e-05) (hash(x)=152602580)
+5100 val loss 6.1775
+5100 val perplexity 481.7774
+5100 train 6.222368 (lr=1.6500e-05) (hash(x)=150598460)
+5200 val loss 6.1737
+5200 val perplexity 479.9440
+5200 train 6.049888 (lr=1.6067e-05) (hash(x)=147660496)
+5300 val loss 6.1657
+5300 val perplexity 476.1298
+5300 train 5.907603 (lr=1.5635e-05) (hash(x)=147693848)
+5400 val loss 6.1717
+5400 val perplexity 478.9965
+5400 train 5.929591 (lr=1.5204e-05) (hash(x)=136048145)
+5500 val loss 6.1501
+5500 val perplexity 468.7782
+5500 train 6.171415 (lr=1.4774e-05) (hash(x)=140313123)
+5600 val loss 6.1436
+5600 val perplexity 465.7047
+5600 train 6.169950 (lr=1.4345e-05) (hash(x)=156997993)
+5700 val loss 6.1348
+5700 val perplexity 461.6267
+5700 train 6.146784 (lr=1.3919e-05) (hash(x)=143513880)
+5800 val loss 6.1298
+5800 val perplexity 459.3479
+5800 train 6.255326 (lr=1.3496e-05) (hash(x)=153736628)
+5900 val loss 6.1278
+5900 val perplexity 458.4455
+5900 train 6.270812 (lr=1.3076e-05) (hash(x)=166385417)
+6000 val loss 6.1195
+6000 val perplexity 454.6567
+6000 train 6.350556 (lr=1.2659e-05) (hash(x)=153216517)
+6100 val loss 6.1183
+6100 val perplexity 454.0780
+6100 train 6.004011 (lr=1.2246e-05) (hash(x)=145387545)
+6200 val loss 6.1155
+6200 val perplexity 452.8155
+6200 train 6.235756 (lr=1.1838e-05) (hash(x)=156306460)
+6300 val loss 6.1022
+6300 val perplexity 446.8536
+6300 train 6.034924 (lr=1.1434e-05) (hash(x)=148610447)
+6400 val loss 6.0974
+6400 val perplexity 444.7107
+6400 train 6.314558 (lr=1.1035e-05) (hash(x)=158343535)
+6500 val loss 6.0914
+6500 val perplexity 442.0321
+6500 train 6.262345 (lr=1.0643e-05) (hash(x)=157742557)
+6600 val loss 6.0901
+6600 val perplexity 441.4777
+6600 train 5.989834 (lr=1.0256e-05) (hash(x)=150457308)
+6700 val loss 6.0879
+6700 val perplexity 440.4988
+6700 train 5.953698 (lr=9.8753e-06) (hash(x)=142426205)
+6800 val loss 6.0852
+6800 val perplexity 439.3283
+6800 train 5.884865 (lr=9.5017e-06) (hash(x)=151522525)
+6900 val loss 6.0779
+6900 val perplexity 436.1267
+6900 train 6.016484 (lr=9.1353e-06) (hash(x)=152029630)
+7000 val loss 6.0745
+7000 val perplexity 434.6117
+7000 train 5.922952 (lr=8.7764e-06) (hash(x)=142978954)
+7100 val loss 6.0687
+7100 val perplexity 432.1179
+7100 train 5.947993 (lr=8.4255e-06) (hash(x)=148064359)
+7200 val loss 6.0660
+7200 val perplexity 430.9639
+7200 train 5.966048 (lr=8.0829e-06) (hash(x)=155771845)
+7300 val loss 6.0634
+7300 val perplexity 429.8407
+7300 train 5.861534 (lr=7.7489e-06) (hash(x)=141639844)
+7400 val loss 6.0639
+7400 val perplexity 430.0664
+7400 train 6.113844 (lr=7.4239e-06) (hash(x)=153466021)
+7500 val loss 6.0577
+7500 val perplexity 427.3977
+7500 train 6.058067 (lr=7.1083e-06) (hash(x)=148843587)
+7600 val loss 6.0520
+7600 val perplexity 424.9706
+7600 train 6.189840 (lr=6.8023e-06) (hash(x)=152053933)
+7700 val loss 6.0507
+7700 val perplexity 424.4133
+7700 train 5.898217 (lr=6.5062e-06) (hash(x)=141564003)
+7800 val loss 6.0465
+7800 val perplexity 422.6144
+7800 train 5.920815 (lr=6.2205e-06) (hash(x)=148617843)
+7900 val loss 6.0456
+7900 val perplexity 422.2512
+7900 train 6.015367 (lr=5.9453e-06) (hash(x)=146417668)
+8000 val loss 6.0450
+8000 val perplexity 421.9865
+8000 train 5.850304 (lr=5.6809e-06) (hash(x)=150165488)
+8100 val loss 6.0408
+8100 val perplexity 420.2381
+8100 train 6.013474 (lr=5.4277e-06) (hash(x)=144818855)
+8200 val loss 6.0372
+8200 val perplexity 418.7222
+8200 train 6.119427 (lr=5.1858e-06) (hash(x)=143074954)
+8300 val loss 6.0340
+8300 val perplexity 417.3623
+8300 train 5.923309 (lr=4.9556e-06) (hash(x)=145172494)
+8400 val loss 6.0337
+8400 val perplexity 417.2652
+8400 train 6.065529 (lr=4.7372e-06) (hash(x)=152284393)
+8500 val loss 6.0314
+8500 val perplexity 416.2838
+8500 train 5.882653 (lr=4.5309e-06) (hash(x)=147044705)
+8600 val loss 6.0320
+8600 val perplexity 416.5469
+8600 train 5.929805 (lr=4.3369e-06) (hash(x)=147762608)
+8700 val loss 6.0279
+8700 val perplexity 414.8306
+8700 train 6.042439 (lr=4.1554e-06) (hash(x)=141994567)
+8800 val loss 6.0255
+8800 val perplexity 413.8477
+8800 train 6.112911 (lr=3.9866e-06) (hash(x)=152552896)
+8900 val loss 6.0231
+8900 val perplexity 412.8680
+8900 train 5.859866 (lr=3.8307e-06) (hash(x)=139714454)
+9000 val loss 6.0218
+9000 val perplexity 412.3357
+9000 train 6.052674 (lr=3.6877e-06) (hash(x)=152274419)
+9100 val loss 6.0214
+9100 val perplexity 412.1412
+9100 train 6.103989 (lr=3.5580e-06) (hash(x)=160536885)
+9200 val loss 6.0210
+9200 val perplexity 411.9943
+9200 train 5.841023 (lr=3.4415e-06) (hash(x)=138386899)
+9300 val loss 6.0192
+9300 val perplexity 411.2347
+9300 train 6.074156 (lr=3.3385e-06) (hash(x)=149957457)
+9400 val loss 6.0161
+9400 val perplexity 409.9883
+9400 train 6.065225 (lr=3.2490e-06) (hash(x)=161671548)
+9500 val loss 6.0157
+9500 val perplexity 409.8131
+9500 train 6.159393 (lr=3.1730e-06) (hash(x)=171857400)
+9600 val loss 6.0129
+9600 val perplexity 408.6819
+9600 train 5.782595 (lr=3.1108e-06) (hash(x)=135378471)
+9700 val loss 6.0124
+9700 val perplexity 408.4582
+9700 train 5.940501 (lr=3.0624e-06) (hash(x)=149514591)
+9800 val loss 6.0114
+9800 val perplexity 408.0481
+9800 train 5.993114 (lr=3.0277e-06) (hash(x)=152679072)
+9900 val loss 6.0094
+9900 val perplexity 407.2200
+9900 train 5.938741 (lr=3.0069e-06) (hash(x)=150415193)
+9999 val loss 6.0089
+9999 val perplexity 407.0517

attention_kindselective_n_heads2_seed1341/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:74ba334bd0101aaadc01e26184cae068161abfa0ae4d46c807c3442a5ced88c6
+oid sha256:61917b2061d550c01c6a13d915f7eddbd256ab6e650148187f75cdce6cacbc7b
 size 38587970

attention_kindselective_n_heads2_seed1341/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b16d132361577f9fe65063258b176bda59304031f30e5218be876604b731f064
+oid sha256:4cf822d46768dc68a2e287a6477f6a972e531f4ac2b6bee3659c053eddf6833b
 size 38587970

attention_kindselective_n_heads2_seed1341/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad2fa889095cb76098814eb3719349a2a88f7883160df398bb236eae2e0dd419
+oid sha256:0d74f40d28c5bcaeacec475043051481eb3c1cac5909bf8266749547ca031e5c
 size 38587970

attention_kindselective_n_heads2_seed1341/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a9877f2d1585c73ca2a2d1529be8d7c23b4402dfb2488a72c5b2e4f1e7f289f
+oid sha256:92515d22afc83810717ffa953962bfbf259bbbb4453961f59c3ddb677c9b5e0e
 size 38587970

attention_kindselective_n_heads2_seed1341/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c592f311f13824dc70dda18105f0323da847beb468ad3662628d16c3f1f38bdf
+oid sha256:0691eaba89da72b0b9e028c1b0f83917e0f6c2436c82b31b0d7e187f615e83e6
 size 70895430

attention_kindselective_n_heads2_seed1341/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf007fb4171da723c27811bcb2b34a91e5f5b47f68dfe98b97dd74e61c5ec415
+oid sha256:302fd87b96897189d938f9764920b673f8b54755f7112d2a04192fea75f999e4
 size 70895430

attention_kindselective_n_heads2_seed1341/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da0337e3b4da4c98a4a70a4614120ce958de075a8f71d2460b983614a4d736a7
+oid sha256:3bf07f379520069eceb45d10e1bc257b623074ee5e9d9c913d6d34a09b07f476
 size 70895430

attention_kindselective_n_heads2_seed1341/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:849ec8ad2b98bcb82237741dbdd657d3a1463e7ec0077e7ae9bca8af3a98c953
+oid sha256:03daeb14bf6ca8dcc9eded806955bb32da689c8c248d32e57fe4b5e2d430be28
 size 70895430