Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "0.5e-4_30720_2_1339", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindselective_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_2_1339", "n_embd": 128}

attention_kindselective_n_heads2_seed1339/log2.txt

@@ -1,303 +1,303 @@
 max_steps: 10000
 0 val loss 11.7467
 0 val perplexity 126331.5391
-0 train 11.740535 (lr=2.5000e-07) (hash(x)=150724848)
-100 val loss 10.4025
-100 val perplexity 32942.0703
-100 train 10.435096 (lr=2.5250e-05) (hash(x)=149217061)
-200 val loss 8.6411
-200 val perplexity 5659.7891
-200 train 8.616951 (lr=5.0000e-05) (hash(x)=149910534)
-300 val loss 7.7488
-300 val perplexity 2318.7954
-300 train 7.632362 (lr=4.9988e-05) (hash(x)=142185643)
-400 val loss 7.6018
-400 val perplexity 2001.8092
-400 train 7.636534 (lr=4.9954e-05) (hash(x)=148123706)
-500 val loss 7.5696
-500 val perplexity 1938.4028
-500 train 7.433709 (lr=4.9896e-05) (hash(x)=149952383)
-600 val loss 7.5472
-600 val perplexity 1895.3499
-600 train 7.349388 (lr=4.9815e-05) (hash(x)=146678221)
-700 val loss 7.5005
-700 val perplexity 1808.9504
-700 train 7.523438 (lr=4.9712e-05) (hash(x)=156180736)
-800 val loss 7.4568
-800 val perplexity 1731.6691
-800 train 7.442268 (lr=4.9585e-05) (hash(x)=151700982)
-900 val loss 7.4146
-900 val perplexity 1660.1090
-900 train 7.219617 (lr=4.9436e-05) (hash(x)=147288467)
-1000 val loss 7.3795
-1000 val perplexity 1602.7788
-1000 train 7.424051 (lr=4.9264e-05) (hash(x)=156182087)
-1100 val loss 7.3479
-1100 val perplexity 1552.8918
-1100 train 7.208760 (lr=4.9070e-05) (hash(x)=147861550)
-1200 val loss 7.3147
-1200 val perplexity 1502.2614
-1200 train 7.178025 (lr=4.8854e-05) (hash(x)=149318660)
-1300 val loss 7.3030
-1300 val perplexity 1484.6936
-1300 train 7.139358 (lr=4.8616e-05) (hash(x)=151631103)
-1400 val loss 7.2869
-1400 val perplexity 1461.0060
-1400 train 7.325649 (lr=4.8356e-05) (hash(x)=150482428)
-1500 val loss 7.2215
-1500 val perplexity 1368.5865
-1500 train 7.062772 (lr=4.8074e-05) (hash(x)=144458617)
-1600 val loss 7.1778
-1600 val perplexity 1310.0883
-1600 train 7.060380 (lr=4.7772e-05) (hash(x)=143268605)
-1700 val loss 7.1428
-1700 val perplexity 1264.9622
-1700 train 7.076792 (lr=4.7448e-05) (hash(x)=150817011)
-1800 val loss 7.1078
-1800 val perplexity 1221.4225
-1800 train 7.102525 (lr=4.7105e-05) (hash(x)=152322423)
-1900 val loss 7.0664
-1900 val perplexity 1171.8892
-1900 train 6.872022 (lr=4.6741e-05) (hash(x)=143574126)
-2000 val loss 7.0288
-2000 val perplexity 1128.6533
-2000 train 6.792527 (lr=4.6357e-05) (hash(x)=147904298)
-2100 val loss 6.9943
-2100 val perplexity 1090.4375
-2100 train 6.740319 (lr=4.5954e-05) (hash(x)=144719400)
-2200 val loss 6.9503
-2200 val perplexity 1043.4559
-2200 train 7.198068 (lr=4.5532e-05) (hash(x)=154343147)
-2300 val loss 6.8955
-2300 val perplexity 987.8442
-2300 train 6.939746 (lr=4.5091e-05) (hash(x)=154372635)
-2400 val loss 6.8531
-2400 val perplexity 946.8014
-2400 train 6.698762 (lr=4.4633e-05) (hash(x)=141843115)
-2500 val loss 6.8142
-2500 val perplexity 910.7258
-2500 train 6.709142 (lr=4.4156e-05) (hash(x)=149417679)
-2600 val loss 6.7830
-2600 val perplexity 882.6696
-2600 train 6.663291 (lr=4.3663e-05) (hash(x)=145279030)
-2700 val loss 6.7474
-2700 val perplexity 851.8775
-2700 train 6.656663 (lr=4.3153e-05) (hash(x)=146363911)
-2800 val loss 6.7253
-2800 val perplexity 833.2512
-2800 train 6.681330 (lr=4.2627e-05) (hash(x)=152507639)
-2900 val loss 6.6957
-2900 val perplexity 808.9321
-2900 train 6.486669 (lr=4.2085e-05) (hash(x)=139296191)
-3000 val loss 6.6649
-3000 val perplexity 784.4163
-3000 train 6.682261 (lr=4.1529e-05) (hash(x)=148473774)
-3100 val loss 6.6050
-3100 val perplexity 738.7633
-3100 train 6.694517 (lr=4.0957e-05) (hash(x)=154130527)
-3200 val loss 6.5747
-3200 val perplexity 716.7247
-3200 train 6.522137 (lr=4.0373e-05) (hash(x)=151117002)
-3300 val loss 6.5659
-3300 val perplexity 710.4384
-3300 train 6.724684 (lr=3.9775e-05) (hash(x)=151035383)
-3400 val loss 6.5302
-3400 val perplexity 685.5111
-3400 train 6.655223 (lr=3.9164e-05) (hash(x)=150614749)
-3500 val loss 6.5049
-3500 val perplexity 668.4155
-3500 train 6.523757 (lr=3.8541e-05) (hash(x)=151431391)
-3600 val loss 6.4827
-3600 val perplexity 653.7342
-3600 train 6.420113 (lr=3.7907e-05) (hash(x)=151140797)
-3700 val loss 6.4595
-3700 val perplexity 638.7455
-3700 train 6.653959 (lr=3.7262e-05) (hash(x)=154608340)
-3800 val loss 6.4378
-3800 val perplexity 625.0529
-3800 train 6.450654 (lr=3.6608e-05) (hash(x)=169641309)
-3900 val loss 6.4161
-3900 val perplexity 611.6103
-3900 train 6.381898 (lr=3.5944e-05) (hash(x)=150592479)
-4000 val loss 6.3932
-4000 val perplexity 597.7495
-4000 train 6.379524 (lr=3.5271e-05) (hash(x)=155495317)
-4100 val loss 6.3845
-4100 val perplexity 592.6045
-4100 train 6.582943 (lr=3.4590e-05) (hash(x)=170323970)
-4200 val loss 6.3710
-4200 val perplexity 584.6144
-4200 train 6.205431 (lr=3.3902e-05) (hash(x)=140854360)
-4300 val loss 6.3471
-4300 val perplexity 570.8184
-4300 train 6.348463 (lr=3.3207e-05) (hash(x)=148926644)
-4400 val loss 6.3300
-4400 val perplexity 561.1761
-4400 train 6.365700 (lr=3.2507e-05) (hash(x)=149997871)
-4500 val loss 6.3119
-4500 val perplexity 551.0798
-4500 train 6.272457 (lr=3.1801e-05) (hash(x)=149219289)
-4600 val loss 6.3005
-4600 val perplexity 544.8549
-4600 train 6.106970 (lr=3.1091e-05) (hash(x)=139836197)
-4700 val loss 6.2902
-4700 val perplexity 539.2803
-4700 train 6.239379 (lr=3.0377e-05) (hash(x)=150722883)
-4800 val loss 6.2791
-4800 val perplexity 533.3046
-4800 train 6.481926 (lr=2.9661e-05) (hash(x)=166864648)
-4900 val loss 6.2589
-4900 val perplexity 522.6470
-4900 train 6.376144 (lr=2.8942e-05) (hash(x)=157746743)
-5000 val loss 6.2425
-5000 val perplexity 514.1351
-5000 train 6.196634 (lr=2.8221e-05) (hash(x)=153032257)
-5100 val loss 6.2367
-5100 val perplexity 511.1816
-5100 train 6.311671 (lr=2.7500e-05) (hash(x)=151862371)
-5200 val loss 6.2239
-5200 val perplexity 504.6796
-5200 train 6.545998 (lr=2.6779e-05) (hash(x)=168862995)
-5300 val loss 6.2230
-5300 val perplexity 504.2054
-5300 train 6.190140 (lr=2.6058e-05) (hash(x)=148243567)
-5400 val loss 6.2093
-5400 val perplexity 497.3521
-5400 train 6.134599 (lr=2.5339e-05) (hash(x)=145381742)
-5500 val loss 6.2002
-5500 val perplexity 492.8568
-5500 train 6.354118 (lr=2.4623e-05) (hash(x)=155602174)
-5600 val loss 6.1799
-5600 val perplexity 482.9541
-5600 train 6.256996 (lr=2.3909e-05) (hash(x)=160637672)
-5700 val loss 6.1744
-5700 val perplexity 480.3039
-5700 train 6.102977 (lr=2.3199e-05) (hash(x)=152101131)
-5800 val loss 6.1643
-5800 val perplexity 475.4475
-5800 train 6.129929 (lr=2.2493e-05) (hash(x)=152327089)
-5900 val loss 6.1617
-5900 val perplexity 474.2258
-5900 train 6.115116 (lr=2.1793e-05) (hash(x)=151094543)
-6000 val loss 6.1547
-6000 val perplexity 470.9344
-6000 train 6.266313 (lr=2.1098e-05) (hash(x)=160233315)
-6100 val loss 6.1427
-6100 val perplexity 465.3227
-6100 train 6.246418 (lr=2.0410e-05) (hash(x)=158295337)
-6200 val loss 6.1317
-6200 val perplexity 460.2089
-6200 train 6.082817 (lr=1.9729e-05) (hash(x)=151269817)
-6300 val loss 6.1250
-6300 val perplexity 457.1342
-6300 train 6.272613 (lr=1.9056e-05) (hash(x)=156912756)
-6400 val loss 6.1183
-6400 val perplexity 454.1139
-6400 train 6.069808 (lr=1.8392e-05) (hash(x)=125779637)
-6500 val loss 6.1209
-6500 val perplexity 455.2741
-6500 train 6.025056 (lr=1.7738e-05) (hash(x)=148157661)
-6600 val loss 6.1137
-6600 val perplexity 451.9875
-6600 train 6.132632 (lr=1.7093e-05) (hash(x)=152820049)
-6700 val loss 6.1073
-6700 val perplexity 449.1447
-6700 train 6.042549 (lr=1.6459e-05) (hash(x)=155818717)
-6800 val loss 6.0981
-6800 val perplexity 445.0075
-6800 train 6.135156 (lr=1.5836e-05) (hash(x)=158833881)
-6900 val loss 6.0922
-6900 val perplexity 442.4055
-6900 train 6.018761 (lr=1.5225e-05) (hash(x)=164984909)
-7000 val loss 6.0899
-7000 val perplexity 441.3766
-7000 train 5.979160 (lr=1.4627e-05) (hash(x)=151172493)
-7100 val loss 6.0860
-7100 val perplexity 439.6446
-7100 train 6.136322 (lr=1.4043e-05) (hash(x)=154023633)
-7200 val loss 6.0815
-7200 val perplexity 437.6871
-7200 train 5.934602 (lr=1.3471e-05) (hash(x)=139351250)
-7300 val loss 6.0764
-7300 val perplexity 435.4692
-7300 train 6.026505 (lr=1.2915e-05) (hash(x)=158719458)
-7400 val loss 6.0684
-7400 val perplexity 431.9809
-7400 train 6.137913 (lr=1.2373e-05) (hash(x)=162250810)
-7500 val loss 6.0626
-7500 val perplexity 429.5063
-7500 train 5.942214 (lr=1.1847e-05) (hash(x)=142827557)
-7600 val loss 6.0585
-7600 val perplexity 427.7453
-7600 train 5.955554 (lr=1.1337e-05) (hash(x)=147830801)
-7700 val loss 6.0555
-7700 val perplexity 426.4401
-7700 train 6.161001 (lr=1.0844e-05) (hash(x)=165233314)
-7800 val loss 6.0556
-7800 val perplexity 426.4869
-7800 train 6.048510 (lr=1.0367e-05) (hash(x)=160654717)
-7900 val loss 6.0524
-7900 val perplexity 425.1184
-7900 train 5.907644 (lr=9.9088e-06) (hash(x)=140712586)
-8000 val loss 6.0453
-8000 val perplexity 422.1395
-8000 train 6.212321 (lr=9.4682e-06) (hash(x)=150309061)
-8100 val loss 6.0427
-8100 val perplexity 421.0160
-8100 train 6.045451 (lr=9.0461e-06) (hash(x)=158805466)
-8200 val loss 6.0367
-8200 val perplexity 418.5128
-8200 train 5.912158 (lr=8.6430e-06) (hash(x)=142814078)
-8300 val loss 6.0354
-8300 val perplexity 417.9462
-8300 train 5.992675 (lr=8.2593e-06) (hash(x)=148340417)
-8400 val loss 6.0360
-8400 val perplexity 418.2068
-8400 train 5.979285 (lr=7.8953e-06) (hash(x)=157132639)
-8500 val loss 6.0357
-8500 val perplexity 418.0764
-8500 train 5.972235 (lr=7.5515e-06) (hash(x)=152994240)
-8600 val loss 6.0275
-8600 val perplexity 414.6911
-8600 train 5.985933 (lr=7.2282e-06) (hash(x)=152210154)
-8700 val loss 6.0254
-8700 val perplexity 413.8256
-8700 train 6.090410 (lr=6.9257e-06) (hash(x)=149631247)
-8800 val loss 6.0220
-8800 val perplexity 412.4035
-8800 train 5.890935 (lr=6.6444e-06) (hash(x)=145617784)
-8900 val loss 6.0217
-8900 val perplexity 412.2670
-8900 train 6.111032 (lr=6.3845e-06) (hash(x)=155984970)
-9000 val loss 6.0222
-9000 val perplexity 412.4674
-9000 train 5.757221 (lr=6.1462e-06) (hash(x)=143433013)
-9100 val loss 6.0210
-9100 val perplexity 411.9792
-9100 train 5.914127 (lr=5.9300e-06) (hash(x)=152507533)
-9200 val loss 6.0152
-9200 val perplexity 409.5916
-9200 train 6.130023 (lr=5.7359e-06) (hash(x)=162400650)
-9300 val loss 6.0124
-9300 val perplexity 408.4705
-9300 train 6.066665 (lr=5.5641e-06) (hash(x)=148419068)
-9400 val loss 6.0114
-9400 val perplexity 408.0426
-9400 train 5.982956 (lr=5.4149e-06) (hash(x)=145025169)
-9500 val loss 6.0094
-9500 val perplexity 407.2299
-9500 train 5.996061 (lr=5.2884e-06) (hash(x)=145104235)
-9600 val loss 6.0103
-9600 val perplexity 407.6214
-9600 train 5.956517 (lr=5.1847e-06) (hash(x)=150499797)
-9700 val loss 6.0096
-9700 val perplexity 407.3307
-9700 train 5.820603 (lr=5.1040e-06) (hash(x)=139690022)
-9800 val loss 6.0035
-9800 val perplexity 404.8292
-9800 train 5.991593 (lr=5.0462e-06) (hash(x)=145645279)
-9900 val loss 6.0030
-9900 val perplexity 404.6236
-9900 train 5.909470 (lr=5.0116e-06) (hash(x)=150103432)
-9999 val loss 6.0007
-9999 val perplexity 403.7148
+0 train 11.740535 (lr=1.5000e-07) (hash(x)=150724848)
+100 val loss 10.5531
+100 val perplexity 38296.7969
+100 train 10.582304 (lr=1.5150e-05) (hash(x)=149217061)
+200 val loss 9.7262
+200 val perplexity 16750.0762
+200 train 9.707881 (lr=3.0000e-05) (hash(x)=149910534)
+300 val loss 8.5429
+300 val perplexity 5130.0444
+300 train 8.481003 (lr=2.9993e-05) (hash(x)=142185643)
+400 val loss 7.9918
+400 val perplexity 2956.5107
+400 train 8.018432 (lr=2.9972e-05) (hash(x)=148123706)
+500 val loss 7.7440
+500 val perplexity 2307.6658
+500 train 7.625622 (lr=2.9938e-05) (hash(x)=149952383)
+600 val loss 7.6583
+600 val perplexity 2118.1472
+600 train 7.492473 (lr=2.9889e-05) (hash(x)=146678221)
+700 val loss 7.6015
+700 val perplexity 2001.2423
+700 train 7.618642 (lr=2.9827e-05) (hash(x)=156180736)
+800 val loss 7.5754
+800 val perplexity 1949.5923
+800 train 7.563412 (lr=2.9751e-05) (hash(x)=151700982)
+900 val loss 7.5546
+900 val perplexity 1909.5804
+900 train 7.378692 (lr=2.9662e-05) (hash(x)=147288467)
+1000 val loss 7.5409
+1000 val perplexity 1883.4944
+1000 train 7.593865 (lr=2.9558e-05) (hash(x)=156182087)
+1100 val loss 7.5328
+1100 val perplexity 1868.2631
+1100 train 7.420054 (lr=2.9442e-05) (hash(x)=147861550)
+1200 val loss 7.5242
+1200 val perplexity 1852.2574
+1200 train 7.404797 (lr=2.9312e-05) (hash(x)=149318660)
+1300 val loss 7.5252
+1300 val perplexity 1854.2634
+1300 train 7.397731 (lr=2.9169e-05) (hash(x)=151631103)
+1400 val loss 7.5227
+1400 val perplexity 1849.5629
+1400 train 7.571448 (lr=2.9013e-05) (hash(x)=150482428)
+1500 val loss 7.5023
+1500 val perplexity 1812.2666
+1500 train 7.358995 (lr=2.8845e-05) (hash(x)=144458617)
+1600 val loss 7.4812
+1600 val perplexity 1774.3268
+1600 train 7.361270 (lr=2.8663e-05) (hash(x)=143268605)
+1700 val loss 7.4645
+1700 val perplexity 1744.9453
+1700 train 7.396727 (lr=2.8469e-05) (hash(x)=150817011)
+1800 val loss 7.4566
+1800 val perplexity 1731.2513
+1800 train 7.454478 (lr=2.8263e-05) (hash(x)=152322423)
+1900 val loss 7.4449
+1900 val perplexity 1711.1436
+1900 train 7.286794 (lr=2.8044e-05) (hash(x)=143574126)
+2000 val loss 7.4327
+2000 val perplexity 1690.3496
+2000 train 7.217881 (lr=2.7814e-05) (hash(x)=147904298)
+2100 val loss 7.4200
+2100 val perplexity 1669.0591
+2100 train 7.191989 (lr=2.7572e-05) (hash(x)=144719400)
+2200 val loss 7.4000
+2200 val perplexity 1635.9027
+2200 train 7.653648 (lr=2.7319e-05) (hash(x)=154343147)
+2300 val loss 7.3701
+2300 val perplexity 1587.8517
+2300 train 7.430760 (lr=2.7055e-05) (hash(x)=154372635)
+2400 val loss 7.3509
+2400 val perplexity 1557.6462
+2400 train 7.204432 (lr=2.6780e-05) (hash(x)=141843115)
+2500 val loss 7.3293
+2500 val perplexity 1524.2460
+2500 train 7.223310 (lr=2.6494e-05) (hash(x)=149417679)
+2600 val loss 7.3065
+2600 val perplexity 1489.9070
+2600 train 7.167060 (lr=2.6198e-05) (hash(x)=145279030)
+2700 val loss 7.2811
+2700 val perplexity 1452.5431
+2700 train 7.161134 (lr=2.5892e-05) (hash(x)=146363911)
+2800 val loss 7.2621
+2800 val perplexity 1425.2209
+2800 train 7.244130 (lr=2.5576e-05) (hash(x)=152507639)
+2900 val loss 7.2392
+2900 val perplexity 1392.9362
+2900 train 7.028590 (lr=2.5251e-05) (hash(x)=139296191)
+3000 val loss 7.2215
+3000 val perplexity 1368.5748
+3000 train 7.244356 (lr=2.4917e-05) (hash(x)=148473774)
+3100 val loss 7.1779
+3100 val perplexity 1310.1744
+3100 train 7.289028 (lr=2.4574e-05) (hash(x)=154130527)
+3200 val loss 7.1512
+3200 val perplexity 1275.6593
+3200 train 7.077964 (lr=2.4224e-05) (hash(x)=151117002)
+3300 val loss 7.1260
+3300 val perplexity 1243.9489
+3300 train 7.227322 (lr=2.3865e-05) (hash(x)=151035383)
+3400 val loss 7.0967
+3400 val perplexity 1207.9542
+3400 train 7.183464 (lr=2.3498e-05) (hash(x)=150614749)
+3500 val loss 7.0740
+3500 val perplexity 1180.8242
+3500 train 7.129180 (lr=2.3125e-05) (hash(x)=151431391)
+3600 val loss 7.0466
+3600 val perplexity 1148.9330
+3600 train 6.987071 (lr=2.2744e-05) (hash(x)=151140797)
+3700 val loss 7.0165
+3700 val perplexity 1114.8676
+3700 train 7.249604 (lr=2.2357e-05) (hash(x)=154608340)
+3800 val loss 6.9886
+3800 val perplexity 1084.2018
+3800 train 7.016414 (lr=2.1965e-05) (hash(x)=169641309)
+3900 val loss 6.9617
+3900 val perplexity 1055.3744
+3900 train 6.910209 (lr=2.1566e-05) (hash(x)=150592479)
+4000 val loss 6.9326
+4000 val perplexity 1025.2000
+4000 train 6.902418 (lr=2.1162e-05) (hash(x)=155495317)
+4100 val loss 6.9115
+4100 val perplexity 1003.7407
+4100 train 7.114289 (lr=2.0754e-05) (hash(x)=170323970)
+4200 val loss 6.8869
+4200 val perplexity 979.3690
+4200 train 6.689180 (lr=2.0341e-05) (hash(x)=140854360)
+4300 val loss 6.8560
+4300 val perplexity 949.5716
+4300 train 6.847187 (lr=1.9924e-05) (hash(x)=148926644)
+4400 val loss 6.8237
+4400 val perplexity 919.4199
+4400 train 6.867269 (lr=1.9504e-05) (hash(x)=149997871)
+4500 val loss 6.7989
+4500 val perplexity 896.8582
+4500 train 6.745916 (lr=1.9081e-05) (hash(x)=149219289)
+4600 val loss 6.7731
+4600 val perplexity 874.0312
+4600 train 6.579295 (lr=1.8655e-05) (hash(x)=139836197)
+4700 val loss 6.7550
+4700 val perplexity 858.2997
+4700 train 6.710022 (lr=1.8226e-05) (hash(x)=150722883)
+4800 val loss 6.7303
+4800 val perplexity 837.4395
+4800 train 6.927665 (lr=1.7796e-05) (hash(x)=166864648)
+4900 val loss 6.7064
+4900 val perplexity 817.6269
+4900 train 6.820303 (lr=1.7365e-05) (hash(x)=157746743)
+5000 val loss 6.6780
+5000 val perplexity 794.7618
+5000 train 6.646445 (lr=1.6933e-05) (hash(x)=153032257)
+5100 val loss 6.6568
+5100 val perplexity 778.0281
+5100 train 6.741875 (lr=1.6500e-05) (hash(x)=151862371)
+5200 val loss 6.6357
+5200 val perplexity 761.8054
+5200 train 6.965239 (lr=1.6067e-05) (hash(x)=168862995)
+5300 val loss 6.6194
+5300 val perplexity 749.4685
+5300 train 6.594924 (lr=1.5635e-05) (hash(x)=148243567)
+5400 val loss 6.6047
+5400 val perplexity 738.5341
+5400 train 6.526024 (lr=1.5204e-05) (hash(x)=145381742)
+5500 val loss 6.5829
+5500 val perplexity 722.6185
+5500 train 6.733404 (lr=1.4774e-05) (hash(x)=155602174)
+5600 val loss 6.5620
+5600 val perplexity 707.6592
+5600 train 6.655446 (lr=1.4345e-05) (hash(x)=160637672)
+5700 val loss 6.5480
+5700 val perplexity 697.8254
+5700 train 6.487575 (lr=1.3919e-05) (hash(x)=152101131)
+5800 val loss 6.5325
+5800 val perplexity 687.1360
+5800 train 6.491645 (lr=1.3496e-05) (hash(x)=152327089)
+5900 val loss 6.5261
+5900 val perplexity 682.7640
+5900 train 6.454029 (lr=1.3076e-05) (hash(x)=151094543)
+6000 val loss 6.5148
+6000 val perplexity 675.0368
+6000 train 6.621383 (lr=1.2659e-05) (hash(x)=160233315)
+6100 val loss 6.4954
+6100 val perplexity 662.1202
+6100 train 6.595299 (lr=1.2246e-05) (hash(x)=158295337)
+6200 val loss 6.4802
+6200 val perplexity 652.1130
+6200 train 6.442608 (lr=1.1838e-05) (hash(x)=151269817)
+6300 val loss 6.4713
+6300 val perplexity 646.3232
+6300 train 6.575415 (lr=1.1434e-05) (hash(x)=156912756)
+6400 val loss 6.4603
+6400 val perplexity 639.2235
+6400 train 6.395647 (lr=1.1035e-05) (hash(x)=125779637)
+6500 val loss 6.4595
+6500 val perplexity 638.7330
+6500 train 6.357163 (lr=1.0643e-05) (hash(x)=148157661)
+6600 val loss 6.4493
+6600 val perplexity 632.2552
+6600 train 6.514656 (lr=1.0256e-05) (hash(x)=152820049)
+6700 val loss 6.4382
+6700 val perplexity 625.2979
+6700 train 6.391938 (lr=9.8753e-06) (hash(x)=155818717)
+6800 val loss 6.4283
+6800 val perplexity 619.1412
+6800 train 6.463234 (lr=9.5017e-06) (hash(x)=158833881)
+6900 val loss 6.4202
+6900 val perplexity 614.1420
+6900 train 6.413707 (lr=9.1353e-06) (hash(x)=164984909)
+7000 val loss 6.4139
+7000 val perplexity 610.2620
+7000 train 6.291857 (lr=8.7764e-06) (hash(x)=151172493)
+7100 val loss 6.4110
+7100 val perplexity 608.5116
+7100 train 6.417180 (lr=8.4255e-06) (hash(x)=154023633)
+7200 val loss 6.4018
+7200 val perplexity 602.9565
+7200 train 6.265404 (lr=8.0829e-06) (hash(x)=139351250)
+7300 val loss 6.3948
+7300 val perplexity 598.6977
+7300 train 6.346671 (lr=7.7489e-06) (hash(x)=158719458)
+7400 val loss 6.3858
+7400 val perplexity 593.3685
+7400 train 6.486239 (lr=7.4239e-06) (hash(x)=162250810)
+7500 val loss 6.3777
+7500 val perplexity 588.5457
+7500 train 6.237173 (lr=7.1083e-06) (hash(x)=142827557)
+7600 val loss 6.3751
+7600 val perplexity 587.0210
+7600 train 6.279722 (lr=6.8023e-06) (hash(x)=147830801)
+7700 val loss 6.3701
+7700 val perplexity 584.0976
+7700 train 6.498811 (lr=6.5062e-06) (hash(x)=165233314)
+7800 val loss 6.3656
+7800 val perplexity 581.5134
+7800 train 6.361984 (lr=6.2205e-06) (hash(x)=160654717)
+7900 val loss 6.3624
+7900 val perplexity 579.6442
+7900 train 6.218457 (lr=5.9453e-06) (hash(x)=140712586)
+8000 val loss 6.3538
+8000 val perplexity 574.6667
+8000 train 6.490506 (lr=5.6809e-06) (hash(x)=150309061)
+8100 val loss 6.3507
+8100 val perplexity 572.8998
+8100 train 6.403825 (lr=5.4277e-06) (hash(x)=158805466)
+8200 val loss 6.3458
+8200 val perplexity 570.1182
+8200 train 6.204818 (lr=5.1858e-06) (hash(x)=142814078)
+8300 val loss 6.3424
+8300 val perplexity 568.1640
+8300 train 6.298479 (lr=4.9556e-06) (hash(x)=148340417)
+8400 val loss 6.3400
+8400 val perplexity 566.7823
+8400 train 6.272801 (lr=4.7372e-06) (hash(x)=157132639)
+8500 val loss 6.3395
+8500 val perplexity 566.5011
+8500 train 6.289837 (lr=4.5309e-06) (hash(x)=152994240)
+8600 val loss 6.3312
+8600 val perplexity 561.8377
+8600 train 6.268798 (lr=4.3369e-06) (hash(x)=152210154)
+8700 val loss 6.3291
+8700 val perplexity 560.6460
+8700 train 6.383264 (lr=4.1554e-06) (hash(x)=149631247)
+8800 val loss 6.3247
+8800 val perplexity 558.1961
+8800 train 6.177348 (lr=3.9866e-06) (hash(x)=145617784)
+8900 val loss 6.3224
+8900 val perplexity 556.9269
+8900 train 6.424631 (lr=3.8307e-06) (hash(x)=155984970)
+9000 val loss 6.3215
+9000 val perplexity 556.4344
+9000 train 6.062396 (lr=3.6877e-06) (hash(x)=143433013)
+9100 val loss 6.3194
+9100 val perplexity 555.2563
+9100 train 6.223702 (lr=3.5580e-06) (hash(x)=152507533)
+9200 val loss 6.3143
+9200 val perplexity 552.4195
+9200 train 6.450141 (lr=3.4415e-06) (hash(x)=162400650)
+9300 val loss 6.3106
+9300 val perplexity 550.4012
+9300 train 6.372156 (lr=3.3385e-06) (hash(x)=148419068)
+9400 val loss 6.3096
+9400 val perplexity 549.8129
+9400 train 6.264310 (lr=3.2490e-06) (hash(x)=145025169)
+9500 val loss 6.3079
+9500 val perplexity 548.9128
+9500 train 6.279797 (lr=3.1730e-06) (hash(x)=145104235)
+9600 val loss 6.3075
+9600 val perplexity 548.6482
+9600 train 6.231697 (lr=3.1108e-06) (hash(x)=150499797)
+9700 val loss 6.3053
+9700 val perplexity 547.4417
+9700 train 6.115508 (lr=3.0624e-06) (hash(x)=139690022)
+9800 val loss 6.3002
+9800 val perplexity 544.6938
+9800 train 6.280835 (lr=3.0277e-06) (hash(x)=145645279)
+9900 val loss 6.2976
+9900 val perplexity 543.2441
+9900 train 6.197763 (lr=3.0069e-06) (hash(x)=150103432)
+9999 val loss 6.2956
+9999 val perplexity 542.2076

attention_kindselective_n_heads2_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:26725685fb50827dc83ff9f82278f8ba42e9d22b96701aec4a728d4a8dbd0f9c
+oid sha256:992fce12960917e67fe333978c01ab8a1168c2d58c5ba978e00dcd7988513450
 size 38587970

attention_kindselective_n_heads2_seed1339/model_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21ec684a166123440416fba30806e36722f229e82332d1d22abc1a74df7a99a9
+oid sha256:5d43af45ee6b4af132415e0df0d43a6b60b0f73a569bd2830225f0bc786cc385
 size 38587970

attention_kindselective_n_heads2_seed1339/model_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5986d3501491e985e8e00c8458eb498d763d7f9a78ece6147050485e100bcdb5
+oid sha256:bc7a6bea127db219249fd3c5b87fd5cef8b65412dd927d2d06137bf15ecd6810
 size 38587970

attention_kindselective_n_heads2_seed1339/model_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8ee4d4d9b92e53fb7fed22798ddd855f33919f39d84d10bc1b01b00ea2240bf
+oid sha256:ae43b53e4572a8ffdc6daa9c7dfa2cf2422a024f985558fc1223d73da608af4c
 size 38587970

attention_kindselective_n_heads2_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:034ef43e482023b6bd9f0964b417fbdd8203f1186153ac6987211b9695b65496
+oid sha256:4638b256edfc13bb0531ad0b4c5a3e9dc58e85390cc0e5f22b814d704239c6bf
 size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_05000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d8857e209bbc4d1f53aa2895a05bd2b6a1c84e646a62b371d514bc1acb38a4cb
+oid sha256:581649abe991b6fa8953ce33ce20d06ca6d4b8019177d33625611a2777a77bc5
 size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_07500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b56dc8c39a61d3f18c12d3b72d372b1ab49fdbe6bcc2e8bfbf9bc6140bb0953
+oid sha256:125f33f6a9ef4d45f8d4d711ef5cf84fa22e33cd6388b65ac4876c47b07070f9
 size 70895430

attention_kindselective_n_heads2_seed1339/optimizer_09999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab36f6184851e8f0b703753114d767fbac91c51fd4f93a77d98854f772b45afa
+oid sha256:cf6bff0001314e87e1c262e977deff42b8efa0a930960c8b56a22c26f720e4bb
 size 70895430