Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1342/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1342", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1342, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1342", "n_embd": 256}

attention_kindselective_n_heads4_seed1342/dataloader_17499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:584ddc45f1addd41e01adcc6cf70f582d8774c147c912cd238caff8ebf7fb069
+size 964

attention_kindselective_n_heads4_seed1342/log2.txt

@@ -0,0 +1,528 @@
+max_steps: 17500
+0 val loss 11.2907
+0 val perplexity 80077.4453
+0 train 11.286213 (lr=1.6250e-07) (hash(x)=150332693)
+100 val loss 9.9210
+100 val perplexity 20352.7031
+100 train 9.943149 (lr=1.6412e-05) (hash(x)=148948662)
+200 val loss 8.8456
+200 val perplexity 6943.5645
+200 train 8.764943 (lr=3.2500e-05) (hash(x)=149277319)
+300 val loss 7.8158
+300 val perplexity 2479.5896
+300 train 7.699074 (lr=3.2498e-05) (hash(x)=146429920)
+400 val loss 7.5422
+400 val perplexity 1886.0404
+400 train 7.658668 (lr=3.2490e-05) (hash(x)=155934075)
+500 val loss 7.4469
+500 val perplexity 1714.5961
+500 train 7.449210 (lr=3.2478e-05) (hash(x)=157510366)
+600 val loss 7.3775
+600 val perplexity 1599.5120
+600 train 7.494113 (lr=3.2461e-05) (hash(x)=165402628)
+700 val loss 7.3098
+700 val perplexity 1494.8848
+700 train 7.372377 (lr=3.2440e-05) (hash(x)=166427433)
+800 val loss 7.2453
+800 val perplexity 1401.5107
+800 train 7.043656 (lr=3.2413e-05) (hash(x)=139168939)
+900 val loss 7.1972
+900 val perplexity 1335.7433
+900 train 7.097889 (lr=3.2382e-05) (hash(x)=144698667)
+1000 val loss 7.1481
+1000 val perplexity 1271.6613
+1000 train 7.133333 (lr=3.2346e-05) (hash(x)=147134338)
+1100 val loss 7.0959
+1100 val perplexity 1207.0054
+1100 train 7.058327 (lr=3.2305e-05) (hash(x)=147172640)
+1200 val loss 7.0486
+1200 val perplexity 1151.2061
+1200 train 7.053260 (lr=3.2260e-05) (hash(x)=159086459)
+1300 val loss 7.0029
+1300 val perplexity 1099.8055
+1300 train 6.757029 (lr=3.2209e-05) (hash(x)=141578940)
+1400 val loss 6.9632
+1400 val perplexity 1057.0172
+1400 train 7.025721 (lr=3.2154e-05) (hash(x)=152399099)
+1500 val loss 6.9023
+1500 val perplexity 994.5381
+1500 train 6.985551 (lr=3.2094e-05) (hash(x)=158569712)
+1600 val loss 6.8529
+1600 val perplexity 946.5969
+1600 train 6.953071 (lr=3.2030e-05) (hash(x)=156548843)
+1700 val loss 6.7977
+1700 val perplexity 895.8251
+1700 train 6.806039 (lr=3.1961e-05) (hash(x)=152608173)
+1800 val loss 6.7500
+1800 val perplexity 854.0690
+1800 train 6.826288 (lr=3.1887e-05) (hash(x)=157889911)
+1900 val loss 6.7062
+1900 val perplexity 817.4597
+1900 train 6.672256 (lr=3.1809e-05) (hash(x)=140839139)
+2000 val loss 6.6697
+2000 val perplexity 788.1506
+2000 train 6.555364 (lr=3.1726e-05) (hash(x)=142943707)
+2100 val loss 6.6246
+2100 val perplexity 753.4261
+2100 train 6.591334 (lr=3.1638e-05) (hash(x)=151025344)
+2200 val loss 6.5868
+2200 val perplexity 725.4727
+2200 train 6.594067 (lr=3.1546e-05) (hash(x)=148938689)
+2300 val loss 6.5577
+2300 val perplexity 704.6324
+2300 train 6.518873 (lr=3.1449e-05) (hash(x)=150321226)
+2400 val loss 6.5091
+2400 val perplexity 671.1996
+2400 train 6.753350 (lr=3.1348e-05) (hash(x)=161962903)
+2500 val loss 6.4833
+2500 val perplexity 654.1196
+2500 train 6.418224 (lr=3.1243e-05) (hash(x)=153085222)
+2600 val loss 6.4487
+2600 val perplexity 631.8550
+2600 train 6.508657 (lr=3.1133e-05) (hash(x)=145979235)
+2700 val loss 6.4258
+2700 val perplexity 617.5670
+2700 train 6.490528 (lr=3.1019e-05) (hash(x)=149091945)
+2800 val loss 6.4006
+2800 val perplexity 602.2289
+2800 train 6.484369 (lr=3.0900e-05) (hash(x)=159157507)
+2900 val loss 6.3882
+2900 val perplexity 594.7667
+2900 train 6.396844 (lr=3.0777e-05) (hash(x)=155377499)
+3000 val loss 6.3600
+3000 val perplexity 578.2189
+3000 train 6.319705 (lr=3.0650e-05) (hash(x)=145943130)
+3100 val loss 6.3322
+3100 val perplexity 562.4135
+3100 train 6.391010 (lr=3.0518e-05) (hash(x)=153264973)
+3200 val loss 6.3080
+3200 val perplexity 548.9350
+3200 train 6.146701 (lr=3.0383e-05) (hash(x)=143249173)
+3300 val loss 6.2994
+3300 val perplexity 544.2535
+3300 train 6.307387 (lr=3.0243e-05) (hash(x)=155558767)
+3400 val loss 6.2723
+3400 val perplexity 529.6979
+3400 train 6.214176 (lr=3.0099e-05) (hash(x)=144137265)
+3500 val loss 6.2528
+3500 val perplexity 519.4819
+3500 train 6.451112 (lr=2.9952e-05) (hash(x)=154379201)
+3600 val loss 6.2379
+3600 val perplexity 511.8052
+3600 train 6.370066 (lr=2.9800e-05) (hash(x)=158728918)
+3700 val loss 6.2239
+3700 val perplexity 504.6459
+3700 train 5.971830 (lr=2.9644e-05) (hash(x)=139902844)
+3800 val loss 6.2162
+3800 val perplexity 500.8030
+3800 train 6.299497 (lr=2.9485e-05) (hash(x)=160738928)
+3900 val loss 6.1909
+3900 val perplexity 488.2827
+3900 train 6.231839 (lr=2.9321e-05) (hash(x)=156122233)
+4000 val loss 6.1714
+4000 val perplexity 478.8499
+4000 train 6.007371 (lr=2.9154e-05) (hash(x)=136691174)
+4100 val loss 6.1697
+4100 val perplexity 478.0639
+4100 train 6.138481 (lr=2.8983e-05) (hash(x)=150135523)
+4200 val loss 6.1566
+4200 val perplexity 471.8432
+4200 train 6.149134 (lr=2.8808e-05) (hash(x)=154053307)
+4300 val loss 6.1458
+4300 val perplexity 466.7565
+4300 train 5.931820 (lr=2.8630e-05) (hash(x)=136076439)
+4400 val loss 6.1281
+4400 val perplexity 458.5824
+4400 train 6.025293 (lr=2.8448e-05) (hash(x)=147368042)
+4500 val loss 6.1132
+4500 val perplexity 451.7802
+4500 train 6.206073 (lr=2.8263e-05) (hash(x)=163726147)
+4600 val loss 6.0976
+4600 val perplexity 444.7792
+4600 train 6.113523 (lr=2.8075e-05) (hash(x)=151996388)
+4700 val loss 6.0855
+4700 val perplexity 439.4369
+4700 train 6.137928 (lr=2.7883e-05) (hash(x)=148952763)
+4800 val loss 6.0812
+4800 val perplexity 437.5425
+4800 train 6.004941 (lr=2.7687e-05) (hash(x)=139747769)
+4900 val loss 6.0717
+4900 val perplexity 433.4262
+4900 train 5.939377 (lr=2.7489e-05) (hash(x)=145011383)
+5000 val loss 6.0599
+5000 val perplexity 428.3344
+5000 train 6.082546 (lr=2.7287e-05) (hash(x)=153449428)
+5100 val loss 6.0476
+5100 val perplexity 423.1035
+5100 train 6.091887 (lr=2.7082e-05) (hash(x)=161397672)
+5200 val loss 6.0315
+5200 val perplexity 416.3513
+5200 train 6.013671 (lr=2.6874e-05) (hash(x)=164813156)
+5300 val loss 6.0209
+5300 val perplexity 411.9349
+5300 train 6.068652 (lr=2.6664e-05) (hash(x)=146155759)
+5400 val loss 6.0193
+5400 val perplexity 411.3075
+5400 train 6.090827 (lr=2.6450e-05) (hash(x)=152337652)
+5500 val loss 6.0072
+5500 val perplexity 406.3346
+5500 train 5.868421 (lr=2.6233e-05) (hash(x)=148346631)
+5600 val loss 5.9981
+5600 val perplexity 402.6518
+5600 train 5.910365 (lr=2.6014e-05) (hash(x)=146652463)
+5700 val loss 5.9894
+5700 val perplexity 399.1822
+5700 train 5.936297 (lr=2.5792e-05) (hash(x)=147553632)
+5800 val loss 5.9710
+5800 val perplexity 391.8972
+5800 train 6.067301 (lr=2.5567e-05) (hash(x)=153512314)
+5900 val loss 5.9650
+5900 val perplexity 389.5436
+5900 train 5.942406 (lr=2.5340e-05) (hash(x)=137351707)
+6000 val loss 5.9541
+6000 val perplexity 385.3337
+6000 train 5.976089 (lr=2.5111e-05) (hash(x)=143375055)
+6100 val loss 5.9517
+6100 val perplexity 384.4194
+6100 train 5.977683 (lr=2.4879e-05) (hash(x)=144781857)
+6200 val loss 5.9378
+6200 val perplexity 379.0844
+6200 train 6.171885 (lr=2.4644e-05) (hash(x)=167556812)
+6300 val loss 5.9337
+6300 val perplexity 377.5300
+6300 train 6.092905 (lr=2.4408e-05) (hash(x)=162290597)
+6400 val loss 5.9223
+6400 val perplexity 373.2822
+6400 train 5.821190 (lr=2.4169e-05) (hash(x)=144836361)
+6500 val loss 5.9054
+6500 val perplexity 367.0234
+6500 train 5.778796 (lr=2.3928e-05) (hash(x)=150949131)
+6600 val loss 5.9223
+6600 val perplexity 373.2838
+6600 train 5.985580 (lr=2.3686e-05) (hash(x)=151386312)
+6700 val loss 5.8876
+6700 val perplexity 360.5525
+6700 train 5.970166 (lr=2.3441e-05) (hash(x)=159840718)
+6800 val loss 5.8800
+6800 val perplexity 357.8123
+6800 train 5.971055 (lr=2.3195e-05) (hash(x)=158843242)
+6900 val loss 5.8727
+6900 val perplexity 355.2012
+6900 train 6.100095 (lr=2.2946e-05) (hash(x)=160544793)
+7000 val loss 5.8631
+7000 val perplexity 351.8195
+7000 train 5.713056 (lr=2.2696e-05) (hash(x)=139031423)
+7100 val loss 5.8606
+7100 val perplexity 350.9410
+7100 train 5.813060 (lr=2.2445e-05) (hash(x)=148444150)
+7200 val loss 5.8529
+7200 val perplexity 348.2327
+7200 train 6.190878 (lr=2.2192e-05) (hash(x)=162348298)
+7300 val loss 5.8435
+7300 val perplexity 345.0013
+7300 train 5.890631 (lr=2.1937e-05) (hash(x)=160070640)
+7400 val loss 5.8353
+7400 val perplexity 342.1762
+7400 train 5.759228 (lr=2.1682e-05) (hash(x)=142907242)
+7500 val loss 5.8300
+7500 val perplexity 340.3468
+7500 train 5.815858 (lr=2.1425e-05) (hash(x)=155567735)
+7600 val loss 5.8206
+7600 val perplexity 337.1619
+7600 train 5.675367 (lr=2.1166e-05) (hash(x)=149026190)
+7700 val loss 5.8122
+7700 val perplexity 334.3678
+7700 train 6.354319 (lr=2.0907e-05) (hash(x)=172973386)
+7800 val loss 5.8100
+7800 val perplexity 333.6221
+7800 train 5.794294 (lr=2.0647e-05) (hash(x)=148166384)
+7900 val loss 5.8004
+7900 val perplexity 330.4375
+7900 train 6.223359 (lr=2.0386e-05) (hash(x)=174580801)
+8000 val loss 5.7914
+8000 val perplexity 327.4816
+8000 train 5.803993 (lr=2.0123e-05) (hash(x)=153576391)
+8100 val loss 5.7841
+8100 val perplexity 325.0847
+8100 train 5.681134 (lr=1.9861e-05) (hash(x)=143483378)
+8200 val loss 5.7820
+8200 val perplexity 324.3921
+8200 train 6.249942 (lr=1.9597e-05) (hash(x)=125210829)
+8300 val loss 5.7756
+8300 val perplexity 322.3530
+8300 train 5.698894 (lr=1.9333e-05) (hash(x)=152432601)
+8400 val loss 5.7777
+8400 val perplexity 323.0283
+8400 train 5.651137 (lr=1.9069e-05) (hash(x)=148759990)
+8500 val loss 5.7614
+8500 val perplexity 317.8048
+8500 train 5.744183 (lr=1.8804e-05) (hash(x)=156974533)
+8600 val loss 5.7548
+8600 val perplexity 315.6933
+8600 train 5.871877 (lr=1.8539e-05) (hash(x)=155441439)
+8700 val loss 5.7501
+8700 val perplexity 314.2266
+8700 train 5.732445 (lr=1.8273e-05) (hash(x)=149001504)
+8800 val loss 5.7452
+8800 val perplexity 312.6789
+8800 train 5.619228 (lr=1.8008e-05) (hash(x)=144721722)
+8900 val loss 5.7395
+8900 val perplexity 310.9202
+8900 train 5.648099 (lr=1.7742e-05) (hash(x)=151789511)
+9000 val loss 5.7362
+9000 val perplexity 309.8998
+9000 train 5.668345 (lr=1.7477e-05) (hash(x)=151307797)
+9100 val loss 5.7284
+9100 val perplexity 307.4894
+9100 train 5.858921 (lr=1.7211e-05) (hash(x)=155534554)
+9200 val loss 5.7233
+9200 val perplexity 305.9167
+9200 train 5.622472 (lr=1.6946e-05) (hash(x)=138467356)
+9300 val loss 5.7209
+9300 val perplexity 305.1840
+9300 train 5.654151 (lr=1.6681e-05) (hash(x)=148622510)
+9400 val loss 5.7137
+9400 val perplexity 303.0007
+9400 train 5.844534 (lr=1.6417e-05) (hash(x)=154343672)
+9500 val loss 5.7113
+9500 val perplexity 302.2711
+9500 train 5.699077 (lr=1.6153e-05) (hash(x)=156831628)
+9600 val loss 5.7072
+9600 val perplexity 301.0198
+9600 train 5.483487 (lr=1.5889e-05) (hash(x)=142190812)
+9700 val loss 5.6971
+9700 val perplexity 298.0132
+9700 train 5.597317 (lr=1.5627e-05) (hash(x)=146487376)
+9800 val loss 5.6923
+9800 val perplexity 296.5642
+9800 train 5.762100 (lr=1.5364e-05) (hash(x)=164178945)
+9900 val loss 5.6894
+9900 val perplexity 295.7070
+9900 train 5.551723 (lr=1.5103e-05) (hash(x)=142224918)
+10000 val loss 5.6884
+10000 val perplexity 295.4115
+10000 train 5.614565 (lr=1.4843e-05) (hash(x)=148509740)
+10100 val loss 5.6823
+10100 val perplexity 293.6255
+10100 train 5.746253 (lr=1.4584e-05) (hash(x)=153536520)
+10200 val loss 5.6733
+10200 val perplexity 290.9957
+10200 train 5.608801 (lr=1.4325e-05) (hash(x)=153740168)
+10300 val loss 5.6889
+10300 val perplexity 295.5677
+10300 train 5.645096 (lr=1.4068e-05) (hash(x)=160410260)
+10400 val loss 5.6682
+10400 val perplexity 289.5168
+10400 train 5.824097 (lr=1.3813e-05) (hash(x)=136788294)
+10500 val loss 5.6622
+10500 val perplexity 287.7668
+10500 train 5.517078 (lr=1.3558e-05) (hash(x)=153988760)
+10600 val loss 5.6585
+10600 val perplexity 286.7173
+10600 train 5.546037 (lr=1.3305e-05) (hash(x)=141896522)
+10700 val loss 5.6583
+10700 val perplexity 286.6493
+10700 train 6.007439 (lr=1.3054e-05) (hash(x)=147057702)
+10800 val loss 5.6483
+10800 val perplexity 283.8185
+10800 train 5.720152 (lr=1.2804e-05) (hash(x)=157542287)
+10900 val loss 5.6443
+10900 val perplexity 282.6749
+10900 train 5.721572 (lr=1.2555e-05) (hash(x)=153860040)
+11000 val loss 5.6408
+11000 val perplexity 281.6780
+11000 train 5.501269 (lr=1.2309e-05) (hash(x)=142932555)
+11100 val loss 5.6365
+11100 val perplexity 280.4904
+11100 train 5.674755 (lr=1.2064e-05) (hash(x)=146260760)
+11200 val loss 5.6382
+11200 val perplexity 280.9466
+11200 train 5.741637 (lr=1.1822e-05) (hash(x)=148970533)
+11300 val loss 5.6323
+11300 val perplexity 279.3171
+11300 train 5.654080 (lr=1.1581e-05) (hash(x)=149421645)
+11400 val loss 5.6261
+11400 val perplexity 277.5911
+11400 train 5.560232 (lr=1.1342e-05) (hash(x)=158728429)
+11500 val loss 5.6225
+11500 val perplexity 276.5673
+11500 train 5.730070 (lr=1.1106e-05) (hash(x)=139524335)
+11600 val loss 5.6225
+11600 val perplexity 276.5689
+11600 train 5.659407 (lr=1.0871e-05) (hash(x)=152760174)
+11700 val loss 5.6180
+11700 val perplexity 275.3245
+11700 train 5.519762 (lr=1.0639e-05) (hash(x)=140740682)
+11800 val loss 5.6172
+11800 val perplexity 275.1151
+11800 train 5.465315 (lr=1.0410e-05) (hash(x)=153665407)
+11900 val loss 5.6156
+11900 val perplexity 274.6813
+11900 train 5.564779 (lr=1.0183e-05) (hash(x)=163068424)
+12000 val loss 5.6077
+12000 val perplexity 272.5207
+12000 train 5.665866 (lr=9.9580e-06) (hash(x)=145166541)
+12100 val loss 5.6057
+12100 val perplexity 271.9738
+12100 train 5.569032 (lr=9.7360e-06) (hash(x)=143228042)
+12200 val loss 5.6028
+12200 val perplexity 271.1841
+12200 train 5.703817 (lr=9.5167e-06) (hash(x)=159222680)
+12300 val loss 5.6012
+12300 val perplexity 270.7473
+12300 train 5.551047 (lr=9.3001e-06) (hash(x)=145196157)
+12400 val loss 5.5979
+12400 val perplexity 269.8643
+12400 train 5.523250 (lr=9.0864e-06) (hash(x)=143033742)
+12500 val loss 5.5977
+12500 val perplexity 269.8084
+12500 train 5.637140 (lr=8.8756e-06) (hash(x)=151133346)
+12600 val loss 5.5936
+12600 val perplexity 268.7136
+12600 train 5.616040 (lr=8.6677e-06) (hash(x)=153607184)
+12700 val loss 5.5890
+12700 val perplexity 267.4702
+12700 train 5.499484 (lr=8.4629e-06) (hash(x)=136931911)
+12800 val loss 5.5876
+12800 val perplexity 267.0939
+12800 train 5.697507 (lr=8.2612e-06) (hash(x)=155105402)
+12900 val loss 5.5868
+12900 val perplexity 266.8801
+12900 train 5.419711 (lr=8.0627e-06) (hash(x)=144141636)
+13000 val loss 5.5876
+13000 val perplexity 267.0950
+13000 train 5.509489 (lr=7.8674e-06) (hash(x)=139380218)
+13100 val loss 5.5797
+13100 val perplexity 264.9858
+13100 train 5.683111 (lr=7.6754e-06) (hash(x)=157695174)
+13200 val loss 5.5790
+13200 val perplexity 264.8096
+13200 train 5.524367 (lr=7.4867e-06) (hash(x)=143697560)
+13300 val loss 5.5759
+13300 val perplexity 263.9969
+13300 train 5.406800 (lr=7.3015e-06) (hash(x)=156314952)
+13400 val loss 5.5758
+13400 val perplexity 263.9568
+13400 train 5.547408 (lr=7.1198e-06) (hash(x)=136720279)
+13500 val loss 5.5748
+13500 val perplexity 263.7006
+13500 train 5.532155 (lr=6.9416e-06) (hash(x)=142579957)
+13600 val loss 5.5756
+13600 val perplexity 263.9139
+13600 train 5.289562 (lr=6.7670e-06) (hash(x)=140719440)
+13700 val loss 5.5755
+13700 val perplexity 263.8818
+13700 train 5.670376 (lr=6.5961e-06) (hash(x)=164396003)
+13800 val loss 5.5686
+13800 val perplexity 262.0556
+13800 train 5.625392 (lr=6.4289e-06) (hash(x)=152712862)
+13900 val loss 5.5644
+13900 val perplexity 260.9780
+13900 train 5.587784 (lr=6.2655e-06) (hash(x)=160985139)
+14000 val loss 5.5626
+14000 val perplexity 260.4993
+14000 train 5.565255 (lr=6.1059e-06) (hash(x)=147429379)
+14100 val loss 5.5613
+14100 val perplexity 260.1722
+14100 train 5.627103 (lr=5.9502e-06) (hash(x)=131830470)
+14200 val loss 5.5633
+14200 val perplexity 260.6945
+14200 train 5.305113 (lr=5.7984e-06) (hash(x)=146383028)
+14300 val loss 5.5595
+14300 val perplexity 259.6993
+14300 train 5.609846 (lr=5.6506e-06) (hash(x)=157158747)
+14400 val loss 5.5622
+14400 val perplexity 260.3876
+14400 train 5.420488 (lr=5.5068e-06) (hash(x)=143911520)
+14500 val loss 5.5625
+14500 val perplexity 260.4818
+14500 train 5.335642 (lr=5.3671e-06) (hash(x)=159555347)
+14600 val loss 5.5564
+14600 val perplexity 258.8873
+14600 train 5.560647 (lr=5.2316e-06) (hash(x)=152957062)
+14700 val loss 5.5522
+14700 val perplexity 257.7971
+14700 train 5.625054 (lr=5.1002e-06) (hash(x)=156001594)
+14800 val loss 5.5512
+14800 val perplexity 257.5373
+14800 train 5.657235 (lr=4.9730e-06) (hash(x)=145885458)
+14900 val loss 5.5487
+14900 val perplexity 256.9017
+14900 train 5.622381 (lr=4.8501e-06) (hash(x)=144544291)
+15000 val loss 5.5491
+15000 val perplexity 257.0116
+15000 train 5.556798 (lr=4.7314e-06) (hash(x)=154606709)
+15100 val loss 5.5484
+15100 val perplexity 256.8292
+15100 train 5.593152 (lr=4.6171e-06) (hash(x)=156005324)
+15200 val loss 5.5506
+15200 val perplexity 257.3899
+15200 train 5.648378 (lr=4.5072e-06) (hash(x)=158847373)
+15300 val loss 5.5508
+15300 val perplexity 257.4484
+15300 train 5.334248 (lr=4.4017e-06) (hash(x)=159609594)
+15400 val loss 5.5455
+15400 val perplexity 256.0708
+15400 train 5.643424 (lr=4.3006e-06) (hash(x)=152428492)
+15500 val loss 5.5420
+15500 val perplexity 255.1837
+15500 train 5.612495 (lr=4.2040e-06) (hash(x)=162251192)
+15600 val loss 5.5408
+15600 val perplexity 254.8778
+15600 train 5.792667 (lr=4.1119e-06) (hash(x)=155072002)
+15700 val loss 5.5408
+15700 val perplexity 254.8784
+15700 train 5.562519 (lr=4.0244e-06) (hash(x)=151203094)
+15800 val loss 5.5395
+15800 val perplexity 254.5390
+15800 train 5.467110 (lr=3.9414e-06) (hash(x)=146125932)
+15900 val loss 5.5390
+15900 val perplexity 254.4307
+15900 train 5.412131 (lr=3.8630e-06) (hash(x)=148060195)
+16000 val loss 5.5403
+16000 val perplexity 254.7613
+16000 train 5.244052 (lr=3.7892e-06) (hash(x)=137406416)
+16100 val loss 5.5406
+16100 val perplexity 254.8330
+16100 train 5.603872 (lr=3.7201e-06) (hash(x)=160149755)
+16200 val loss 5.5347
+16200 val perplexity 253.3356
+16200 train 5.715085 (lr=3.6556e-06) (hash(x)=151371894)
+16300 val loss 5.5332
+16300 val perplexity 252.9408
+16300 train 5.589443 (lr=3.5959e-06) (hash(x)=151810252)
+16400 val loss 5.5319
+16400 val perplexity 252.6117
+16400 train 5.517071 (lr=3.5408e-06) (hash(x)=153088303)
+16500 val loss 5.5314
+16500 val perplexity 252.5005
+16500 train 5.410724 (lr=3.4905e-06) (hash(x)=151706615)
+16600 val loss 5.5295
+16600 val perplexity 252.0252
+16600 train 5.427650 (lr=3.4449e-06) (hash(x)=142110626)
+16700 val loss 5.5305
+16700 val perplexity 252.2643
+16700 train 5.427150 (lr=3.4041e-06) (hash(x)=153222439)
+16800 val loss 5.5297
+16800 val perplexity 252.0710
+16800 train 5.448761 (lr=3.3680e-06) (hash(x)=154964902)
+16900 val loss 5.5264
+16900 val perplexity 251.2368
+16900 train 5.686378 (lr=3.3367e-06) (hash(x)=158893204)
+17000 val loss 5.5250
+17000 val perplexity 250.8958
+17000 train 5.454472 (lr=3.3102e-06) (hash(x)=148929955)
+17100 val loss 5.5259
+17100 val perplexity 251.1136
+17100 train 5.539378 (lr=3.2886e-06) (hash(x)=147818677)
+17200 val loss 5.5243
+17200 val perplexity 250.7198
+17200 train 5.456254 (lr=3.2717e-06) (hash(x)=147119446)
+17300 val loss 5.5240
+17300 val perplexity 250.6337
+17300 train 5.575593 (lr=3.2596e-06) (hash(x)=150494142)
+17400 val loss 5.5245
+17400 val perplexity 250.7691
+17400 train 5.557147 (lr=3.2524e-06) (hash(x)=155918058)
+17499 val loss 5.5225
+17499 val perplexity 250.2664

attention_kindselective_n_heads4_seed1342/model_17499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bfcb58747e9c1a758d5d5cc9c3751f077e21cf4336de84d9b33c35dcc8c3ccb
+size 92843394

attention_kindselective_n_heads4_seed1342/optimizer_17499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d724ae9cc4ad82d4bca42989b48a19daf300f0696210809e9b80107bbccec269
+size 179406214