Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1344/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1344", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1344, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3e-5_30720_4_1344", "n_embd": 256}

attention_kindselective_n_heads4_seed1344/dataloader_17499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:584ddc45f1addd41e01adcc6cf70f582d8774c147c912cd238caff8ebf7fb069
+size 964

attention_kindselective_n_heads4_seed1344/log2.txt

@@ -0,0 +1,528 @@
+max_steps: 17500
+0 val loss 11.2848
+0 val perplexity 79602.4766
+0 train 11.283560 (lr=1.5000e-07) (hash(x)=145830960)
+100 val loss 9.9692
+100 val perplexity 21358.1699
+100 train 9.986609 (lr=1.5150e-05) (hash(x)=156335112)
+200 val loss 9.1055
+200 val perplexity 9005.0107
+200 train 9.164769 (lr=3.0000e-05) (hash(x)=144157453)
+300 val loss 8.1325
+300 val perplexity 3403.3740
+300 train 8.135631 (lr=2.9998e-05) (hash(x)=149245414)
+400 val loss 7.6795
+400 val perplexity 2163.4895
+400 train 7.722869 (lr=2.9991e-05) (hash(x)=146764602)
+500 val loss 7.5390
+500 val perplexity 1879.9814
+500 train 7.497005 (lr=2.9980e-05) (hash(x)=152540261)
+600 val loss 7.4717
+600 val perplexity 1757.6008
+600 train 7.247765 (lr=2.9964e-05) (hash(x)=140559124)
+700 val loss 7.4065
+700 val perplexity 1646.7147
+700 train 7.458422 (lr=2.9944e-05) (hash(x)=156003556)
+800 val loss 7.3485
+800 val perplexity 1553.9318
+800 train 7.892264 (lr=2.9920e-05) (hash(x)=166024176)
+900 val loss 7.3074
+900 val perplexity 1491.2965
+900 train 7.358513 (lr=2.9891e-05) (hash(x)=156196976)
+1000 val loss 7.2559
+1000 val perplexity 1416.4850
+1000 train 7.332314 (lr=2.9858e-05) (hash(x)=161040668)
+1100 val loss 7.2075
+1100 val perplexity 1349.4982
+1100 train 7.274787 (lr=2.9820e-05) (hash(x)=175899671)
+1200 val loss 7.1561
+1200 val perplexity 1281.8519
+1200 train 7.172896 (lr=2.9778e-05) (hash(x)=148878990)
+1300 val loss 7.0955
+1300 val perplexity 1206.5381
+1300 train 7.090349 (lr=2.9732e-05) (hash(x)=147253797)
+1400 val loss 7.0572
+1400 val perplexity 1161.1394
+1400 train 7.078465 (lr=2.9681e-05) (hash(x)=150420695)
+1500 val loss 6.9883
+1500 val perplexity 1083.8916
+1500 train 6.935720 (lr=2.9626e-05) (hash(x)=146933283)
+1600 val loss 6.9395
+1600 val perplexity 1032.2332
+1600 train 6.901734 (lr=2.9566e-05) (hash(x)=151995229)
+1700 val loss 6.9046
+1700 val perplexity 996.8332
+1700 train 7.139846 (lr=2.9502e-05) (hash(x)=154468626)
+1800 val loss 6.8615
+1800 val perplexity 954.8381
+1800 train 6.561424 (lr=2.9434e-05) (hash(x)=139853932)
+1900 val loss 6.8130
+1900 val perplexity 909.5905
+1900 train 6.602960 (lr=2.9362e-05) (hash(x)=142492588)
+2000 val loss 6.7646
+2000 val perplexity 866.5991
+2000 train 7.098177 (lr=2.9285e-05) (hash(x)=174104207)
+2100 val loss 6.7188
+2100 val perplexity 827.8433
+2100 train 6.714899 (lr=2.9204e-05) (hash(x)=151046997)
+2200 val loss 6.6761
+2200 val perplexity 793.2269
+2200 train 6.536441 (lr=2.9119e-05) (hash(x)=146275038)
+2300 val loss 6.6381
+2300 val perplexity 763.6530
+2300 train 6.574053 (lr=2.9030e-05) (hash(x)=147779512)
+2400 val loss 6.6098
+2400 val perplexity 742.3422
+2400 train 6.482061 (lr=2.8937e-05) (hash(x)=141403655)
+2500 val loss 6.5681
+2500 val perplexity 712.0099
+2500 train 6.546371 (lr=2.8840e-05) (hash(x)=160143436)
+2600 val loss 6.5341
+2600 val perplexity 688.2089
+2600 train 6.783262 (lr=2.8738e-05) (hash(x)=173675040)
+2700 val loss 6.5074
+2700 val perplexity 670.1088
+2700 train 6.542911 (lr=2.8633e-05) (hash(x)=150124772)
+2800 val loss 6.4810
+2800 val perplexity 652.6323
+2800 train 6.611236 (lr=2.8523e-05) (hash(x)=162475906)
+2900 val loss 6.4627
+2900 val perplexity 640.7689
+2900 train 6.393878 (lr=2.8410e-05) (hash(x)=152789231)
+3000 val loss 6.4435
+3000 val perplexity 628.6244
+3000 train 6.368320 (lr=2.8292e-05) (hash(x)=152785612)
+3100 val loss 6.4100
+3100 val perplexity 607.8701
+3100 train 6.428837 (lr=2.8171e-05) (hash(x)=156140235)
+3200 val loss 6.3834
+3200 val perplexity 591.9541
+3200 train 6.398758 (lr=2.8046e-05) (hash(x)=151987383)
+3300 val loss 6.3576
+3300 val perplexity 576.8705
+3300 train 6.542221 (lr=2.7917e-05) (hash(x)=176259299)
+3400 val loss 6.3385
+3400 val perplexity 565.9324
+3400 train 6.146057 (lr=2.7784e-05) (hash(x)=144540187)
+3500 val loss 6.3221
+3500 val perplexity 556.7604
+3500 train 6.537164 (lr=2.7648e-05) (hash(x)=156773596)
+3600 val loss 6.3115
+3600 val perplexity 550.8785
+3600 train 6.537225 (lr=2.7508e-05) (hash(x)=160934110)
+3700 val loss 6.2907
+3700 val perplexity 539.5314
+3700 train 6.405750 (lr=2.7364e-05) (hash(x)=156100357)
+3800 val loss 6.2763
+3800 val perplexity 531.8310
+3800 train 6.380016 (lr=2.7216e-05) (hash(x)=158048867)
+3900 val loss 6.2723
+3900 val perplexity 529.6952
+3900 train 6.104307 (lr=2.7066e-05) (hash(x)=148195960)
+4000 val loss 6.2419
+4000 val perplexity 513.8405
+4000 train 6.238558 (lr=2.6911e-05) (hash(x)=151976963)
+4100 val loss 6.2256
+4100 val perplexity 505.5300
+4100 train 6.186838 (lr=2.6754e-05) (hash(x)=157598041)
+4200 val loss 6.2065
+4200 val perplexity 495.9598
+4200 train 6.071241 (lr=2.6592e-05) (hash(x)=139121993)
+4300 val loss 6.1931
+4300 val perplexity 489.3787
+4300 train 6.278595 (lr=2.6428e-05) (hash(x)=154546719)
+4400 val loss 6.1838
+4400 val perplexity 484.8242
+4400 train 6.113948 (lr=2.6260e-05) (hash(x)=148310119)
+4500 val loss 6.1640
+4500 val perplexity 475.3333
+4500 train 6.048776 (lr=2.6089e-05) (hash(x)=147070971)
+4600 val loss 6.1615
+4600 val perplexity 474.1256
+4600 train 7.687497 (lr=2.5915e-05) (hash(x)=232298831)
+4700 val loss 6.1528
+4700 val perplexity 470.0258
+4700 train 6.114225 (lr=2.5738e-05) (hash(x)=149549680)
+4800 val loss 6.1339
+4800 val perplexity 461.2135
+4800 train 6.275864 (lr=2.5558e-05) (hash(x)=155703970)
+4900 val loss 6.1186
+4900 val perplexity 454.2499
+4900 train 6.110617 (lr=2.5374e-05) (hash(x)=158725785)
+5000 val loss 6.1065
+5000 val perplexity 448.7472
+5000 train 6.052335 (lr=2.5188e-05) (hash(x)=146983088)
+5100 val loss 6.0916
+5100 val perplexity 442.1179
+5100 train 6.025856 (lr=2.4999e-05) (hash(x)=144882522)
+5200 val loss 6.0840
+5200 val perplexity 438.7811
+5200 train 5.971626 (lr=2.4807e-05) (hash(x)=145880132)
+5300 val loss 6.0749
+5300 val perplexity 434.8229
+5300 train 5.903040 (lr=2.4613e-05) (hash(x)=140482543)
+5400 val loss 6.0653
+5400 val perplexity 430.6625
+5400 train 5.909417 (lr=2.4415e-05) (hash(x)=148902645)
+5500 val loss 6.0568
+5500 val perplexity 427.0222
+5500 train 5.798812 (lr=2.4215e-05) (hash(x)=149411471)
+5600 val loss 6.0457
+5600 val perplexity 422.3114
+5600 train 5.997655 (lr=2.4013e-05) (hash(x)=148673383)
+5700 val loss 6.0285
+5700 val perplexity 415.1040
+5700 train 6.085971 (lr=2.3808e-05) (hash(x)=157502794)
+5800 val loss 6.0220
+5800 val perplexity 412.3956
+5800 train 5.868658 (lr=2.3601e-05) (hash(x)=148057218)
+5900 val loss 6.0099
+5900 val perplexity 407.4564
+5900 train 5.854727 (lr=2.3391e-05) (hash(x)=141239696)
+6000 val loss 6.0013
+6000 val perplexity 403.9641
+6000 train 6.088947 (lr=2.3179e-05) (hash(x)=152338728)
+6100 val loss 5.9970
+6100 val perplexity 402.2112
+6100 train 5.684302 (lr=2.2965e-05) (hash(x)=143749713)
+6200 val loss 5.9905
+6200 val perplexity 399.6147
+6200 train 6.051906 (lr=2.2749e-05) (hash(x)=152128518)
+6300 val loss 5.9858
+6300 val perplexity 397.7259
+6300 train 5.860623 (lr=2.2530e-05) (hash(x)=152992195)
+6400 val loss 5.9688
+6400 val perplexity 391.0414
+6400 train 5.868921 (lr=2.2310e-05) (hash(x)=143799591)
+6500 val loss 5.9531
+6500 val perplexity 384.9623
+6500 train 6.056067 (lr=2.2088e-05) (hash(x)=144266888)
+6600 val loss 5.9429
+6600 val perplexity 381.0557
+6600 train 6.216046 (lr=2.1864e-05) (hash(x)=167374028)
+6700 val loss 5.9368
+6700 val perplexity 378.7207
+6700 train 5.912586 (lr=2.1638e-05) (hash(x)=151301787)
+6800 val loss 5.9301
+6800 val perplexity 376.1754
+6800 train 5.752751 (lr=2.1410e-05) (hash(x)=143204898)
+6900 val loss 5.9215
+6900 val perplexity 372.9644
+6900 train 6.009975 (lr=2.1181e-05) (hash(x)=148058660)
+7000 val loss 5.9250
+7000 val perplexity 374.2801
+7000 train 5.896644 (lr=2.0951e-05) (hash(x)=157509461)
+7100 val loss 5.9075
+7100 val perplexity 367.7890
+7100 train 6.227095 (lr=2.0718e-05) (hash(x)=171579342)
+7200 val loss 5.8980
+7200 val perplexity 364.3020
+7200 train 5.892464 (lr=2.0485e-05) (hash(x)=158180787)
+7300 val loss 5.8878
+7300 val perplexity 360.5948
+7300 train 5.758845 (lr=2.0250e-05) (hash(x)=150122338)
+7400 val loss 5.8823
+7400 val perplexity 358.6264
+7400 train 5.820298 (lr=2.0014e-05) (hash(x)=147561852)
+7500 val loss 5.8777
+7500 val perplexity 356.9875
+7500 train 5.868729 (lr=1.9777e-05) (hash(x)=149353924)
+7600 val loss 5.8703
+7600 val perplexity 354.3709
+7600 train 5.894732 (lr=1.9538e-05) (hash(x)=144545768)
+7700 val loss 5.8606
+7700 val perplexity 350.9321
+7700 train 5.883697 (lr=1.9299e-05) (hash(x)=142691770)
+7800 val loss 5.8543
+7800 val perplexity 348.7289
+7800 train 5.985229 (lr=1.9059e-05) (hash(x)=158378295)
+7900 val loss 5.8550
+7900 val perplexity 348.9877
+7900 train 5.711920 (lr=1.8817e-05) (hash(x)=150691918)
+8000 val loss 5.8351
+8000 val perplexity 342.1070
+8000 train 5.726991 (lr=1.8576e-05) (hash(x)=147837896)
+8100 val loss 5.8347
+8100 val perplexity 341.9752
+8100 train 5.806161 (lr=1.8333e-05) (hash(x)=144751226)
+8200 val loss 5.8267
+8200 val perplexity 339.2332
+8200 train 5.600180 (lr=1.8090e-05) (hash(x)=139168726)
+8300 val loss 5.8215
+8300 val perplexity 337.4666
+8300 train 5.742377 (lr=1.7846e-05) (hash(x)=154279434)
+8400 val loss 5.8104
+8400 val perplexity 333.7566
+8400 train 5.684167 (lr=1.7602e-05) (hash(x)=140111678)
+8500 val loss 5.8033
+8500 val perplexity 331.4041
+8500 train 5.792879 (lr=1.7357e-05) (hash(x)=152124088)
+8600 val loss 5.8023
+8600 val perplexity 331.0701
+8600 train 5.577775 (lr=1.7113e-05) (hash(x)=142869347)
+8700 val loss 5.7923
+8700 val perplexity 327.7723
+8700 train 5.696801 (lr=1.6868e-05) (hash(x)=145697606)
+8800 val loss 5.7945
+8800 val perplexity 328.4869
+8800 train 5.763039 (lr=1.6623e-05) (hash(x)=157314231)
+8900 val loss 5.7809
+8900 val perplexity 324.0471
+8900 train 5.758891 (lr=1.6377e-05) (hash(x)=149357509)
+9000 val loss 5.7770
+9000 val perplexity 322.7733
+9000 train 5.710581 (lr=1.6132e-05) (hash(x)=140434750)
+9100 val loss 5.7688
+9100 val perplexity 320.1650
+9100 train 5.773588 (lr=1.5887e-05) (hash(x)=144954543)
+9200 val loss 5.7651
+9200 val perplexity 318.9758
+9200 train 5.689574 (lr=1.5643e-05) (hash(x)=156329199)
+9300 val loss 5.7582
+9300 val perplexity 316.7898
+9300 train 5.556169 (lr=1.5398e-05) (hash(x)=137925654)
+9400 val loss 5.7588
+9400 val perplexity 316.9700
+9400 train 5.634879 (lr=1.5154e-05) (hash(x)=146385919)
+9500 val loss 5.7472
+9500 val perplexity 313.3140
+9500 train 5.686854 (lr=1.4910e-05) (hash(x)=144400019)
+9600 val loss 5.7486
+9600 val perplexity 313.7517
+9600 train 5.701641 (lr=1.4667e-05) (hash(x)=130562204)
+9700 val loss 5.7357
+9700 val perplexity 309.7330
+9700 train 5.676045 (lr=1.4424e-05) (hash(x)=152732805)
+9800 val loss 5.7323
+9800 val perplexity 308.6738
+9800 train 5.795771 (lr=1.4183e-05) (hash(x)=151860383)
+9900 val loss 5.7268
+9900 val perplexity 306.9808
+9900 train 5.828072 (lr=1.3941e-05) (hash(x)=156732259)
+10000 val loss 5.7220
+10000 val perplexity 305.5055
+10000 train 5.650548 (lr=1.3701e-05) (hash(x)=139943350)
+10100 val loss 5.7186
+10100 val perplexity 304.4856
+10100 train 5.526080 (lr=1.3462e-05) (hash(x)=145593924)
+10200 val loss 5.7150
+10200 val perplexity 303.3745
+10200 train 5.586910 (lr=1.3223e-05) (hash(x)=153996667)
+10300 val loss 5.7164
+10300 val perplexity 303.8144
+10300 train 5.620425 (lr=1.2986e-05) (hash(x)=154200987)
+10400 val loss 5.7155
+10400 val perplexity 303.5305
+10400 train 5.528822 (lr=1.2750e-05) (hash(x)=156872686)
+10500 val loss 5.7024
+10500 val perplexity 299.5930
+10500 train 5.794130 (lr=1.2515e-05) (hash(x)=155008723)
+10600 val loss 5.6952
+10600 val perplexity 297.4219
+10600 train 5.627241 (lr=1.2282e-05) (hash(x)=152715441)
+10700 val loss 5.6918
+10700 val perplexity 296.4389
+10700 train 5.847895 (lr=1.2049e-05) (hash(x)=166668700)
+10800 val loss 5.6873
+10800 val perplexity 295.1098
+10800 train 5.695483 (lr=1.1819e-05) (hash(x)=150698847)
+10900 val loss 5.6859
+10900 val perplexity 294.6873
+10900 train 5.507511 (lr=1.1590e-05) (hash(x)=137094574)
+11000 val loss 5.6800
+11000 val perplexity 292.9599
+11000 train 5.522943 (lr=1.1362e-05) (hash(x)=144225378)
+11100 val loss 5.6841
+11100 val perplexity 294.1578
+11100 train 5.603326 (lr=1.1136e-05) (hash(x)=151666772)
+11200 val loss 5.6811
+11200 val perplexity 293.2774
+11200 train 5.681242 (lr=1.0912e-05) (hash(x)=157843445)
+11300 val loss 5.6706
+11300 val perplexity 290.1950
+11300 train 5.572571 (lr=1.0690e-05) (hash(x)=140305735)
+11400 val loss 5.6664
+11400 val perplexity 288.9919
+11400 train 5.594697 (lr=1.0470e-05) (hash(x)=143435929)
+11500 val loss 5.6605
+11500 val perplexity 287.3021
+11500 train 5.694790 (lr=1.0251e-05) (hash(x)=150858606)
+11600 val loss 5.6594
+11600 val perplexity 286.9900
+11600 train 5.477522 (lr=1.0035e-05) (hash(x)=141643887)
+11700 val loss 5.6571
+11700 val perplexity 286.3209
+11700 train 5.519454 (lr=9.8209e-06) (hash(x)=151097313)
+11800 val loss 5.6553
+11800 val perplexity 285.7881
+11800 train 5.460770 (lr=9.6090e-06) (hash(x)=150639616)
+11900 val loss 5.6575
+11900 val perplexity 286.4227
+11900 train 5.337720 (lr=9.3993e-06) (hash(x)=154557688)
+12000 val loss 5.6531
+12000 val perplexity 285.1815
+12000 train 5.787493 (lr=9.1920e-06) (hash(x)=159324159)
+12100 val loss 5.6444
+12100 val perplexity 282.7010
+12100 train 5.700291 (lr=8.9871e-06) (hash(x)=151185357)
+12200 val loss 5.6409
+12200 val perplexity 281.7193
+12200 train 5.533353 (lr=8.7846e-06) (hash(x)=137457899)
+12300 val loss 5.6375
+12300 val perplexity 280.7644
+12300 train 5.693140 (lr=8.5847e-06) (hash(x)=146370056)
+12400 val loss 5.6356
+12400 val perplexity 280.2367
+12400 train 5.485180 (lr=8.3875e-06) (hash(x)=148352898)
+12500 val loss 5.6327
+12500 val perplexity 279.4252
+12500 train 5.597754 (lr=8.1929e-06) (hash(x)=161514681)
+12600 val loss 5.6327
+12600 val perplexity 279.4115
+12600 train 5.494642 (lr=8.0010e-06) (hash(x)=146207280)
+12700 val loss 5.6343
+12700 val perplexity 279.8726
+12700 train 5.465546 (lr=7.8119e-06) (hash(x)=142615619)
+12800 val loss 5.6293
+12800 val perplexity 278.4552
+12800 train 5.659070 (lr=7.6257e-06) (hash(x)=144604802)
+12900 val loss 5.6242
+12900 val perplexity 277.0586
+12900 train 5.622487 (lr=7.4425e-06) (hash(x)=143212941)
+13000 val loss 5.6172
+13000 val perplexity 275.1291
+13000 train 5.696371 (lr=7.2622e-06) (hash(x)=155987866)
+13100 val loss 5.6161
+13100 val perplexity 274.8131
+13100 train 5.900623 (lr=7.0850e-06) (hash(x)=155920713)
+13200 val loss 5.6141
+13200 val perplexity 274.2628
+13200 train 5.793137 (lr=6.9108e-06) (hash(x)=152145878)
+13300 val loss 5.6115
+13300 val perplexity 273.5495
+13300 train 5.438443 (lr=6.7398e-06) (hash(x)=134141709)
+13400 val loss 5.6135
+13400 val perplexity 274.1128
+13400 train 5.531283 (lr=6.5721e-06) (hash(x)=146096299)
+13500 val loss 5.6123
+13500 val perplexity 273.7640
+13500 train 5.370831 (lr=6.4076e-06) (hash(x)=144011431)
+13600 val loss 5.6047
+13600 val perplexity 271.6991
+13600 train 5.862495 (lr=6.2465e-06) (hash(x)=163950932)
+13700 val loss 5.6026
+13700 val perplexity 271.1429
+13700 train 5.737296 (lr=6.0887e-06) (hash(x)=156652406)
+13800 val loss 5.5982
+13800 val perplexity 269.9393
+13800 train 5.509688 (lr=5.9344e-06) (hash(x)=145753166)
+13900 val loss 5.5959
+13900 val perplexity 269.3103
+13900 train 5.459958 (lr=5.7835e-06) (hash(x)=147065551)
+14000 val loss 5.5955
+14000 val perplexity 269.1991
+14000 train 6.079162 (lr=5.6362e-06) (hash(x)=179077542)
+14100 val loss 5.5942
+14100 val perplexity 268.8653
+14100 train 5.684788 (lr=5.4925e-06) (hash(x)=171962802)
+14200 val loss 5.5939
+14200 val perplexity 268.7884
+14200 train 5.663789 (lr=5.3524e-06) (hash(x)=153299913)
+14300 val loss 5.5935
+14300 val perplexity 268.6757
+14300 train 5.508217 (lr=5.2159e-06) (hash(x)=154750565)
+14400 val loss 5.5911
+14400 val perplexity 268.0197
+14400 train 5.859001 (lr=5.0832e-06) (hash(x)=154536694)
+14500 val loss 5.5876
+14500 val perplexity 267.0922
+14500 train 5.642450 (lr=4.9543e-06) (hash(x)=151236383)
+14600 val loss 5.5843
+14600 val perplexity 266.2125
+14600 train 5.612250 (lr=4.8291e-06) (hash(x)=152584272)
+14700 val loss 5.5824
+14700 val perplexity 265.7056
+14700 train 5.555599 (lr=4.7079e-06) (hash(x)=175301826)
+14800 val loss 5.5823
+14800 val perplexity 265.6867
+14800 train 5.505548 (lr=4.5905e-06) (hash(x)=148729408)
+14900 val loss 5.5806
+14900 val perplexity 265.2391
+14900 train 5.777581 (lr=4.4770e-06) (hash(x)=150978283)
+15000 val loss 5.5811
+15000 val perplexity 265.3698
+15000 train 5.508034 (lr=4.3675e-06) (hash(x)=153621389)
+15100 val loss 5.5811
+15100 val perplexity 265.3546
+15100 train 5.494666 (lr=4.2620e-06) (hash(x)=150653793)
+15200 val loss 5.5787
+15200 val perplexity 264.7404
+15200 train 5.763510 (lr=4.1605e-06) (hash(x)=140802934)
+15300 val loss 5.5737
+15300 val perplexity 263.4139
+15300 train 5.551564 (lr=4.0631e-06) (hash(x)=152007556)
+15400 val loss 5.5736
+15400 val perplexity 263.3871
+15400 train 5.518318 (lr=3.9698e-06) (hash(x)=149057446)
+15500 val loss 5.5717
+15500 val perplexity 262.8695
+15500 train 5.481091 (lr=3.8806e-06) (hash(x)=147521774)
+15600 val loss 5.5706
+15600 val perplexity 262.5835
+15600 train 5.547856 (lr=3.7956e-06) (hash(x)=148944405)
+15700 val loss 5.5698
+15700 val perplexity 262.3817
+15700 train 5.518208 (lr=3.7148e-06) (hash(x)=148472532)
+15800 val loss 5.5692
+15800 val perplexity 262.2169
+15800 train 5.329184 (lr=3.6382e-06) (hash(x)=149048780)
+15900 val loss 5.5697
+15900 val perplexity 262.3451
+15900 train 5.534845 (lr=3.5658e-06) (hash(x)=150175778)
+16000 val loss 5.5708
+16000 val perplexity 262.6354
+16000 train 5.352664 (lr=3.4977e-06) (hash(x)=145943831)
+16100 val loss 5.5642
+16100 val perplexity 260.9034
+16100 train 5.749448 (lr=3.4339e-06) (hash(x)=134314842)
+16200 val loss 5.5623
+16200 val perplexity 260.4116
+16200 train 5.599839 (lr=3.3744e-06) (hash(x)=153755135)
+16300 val loss 5.5628
+16300 val perplexity 260.5445
+16300 train 5.281954 (lr=3.3193e-06) (hash(x)=140070802)
+16400 val loss 5.5612
+16400 val perplexity 260.1248
+16400 train 5.618970 (lr=3.2684e-06) (hash(x)=151132329)
+16500 val loss 5.5586
+16500 val perplexity 259.4483
+16500 train 5.575374 (lr=3.2220e-06) (hash(x)=151585550)
+16600 val loss 5.5569
+16600 val perplexity 259.0252
+16600 train 5.718242 (lr=3.1799e-06) (hash(x)=164389325)
+16700 val loss 5.5559
+16700 val perplexity 258.7569
+16700 train 5.469742 (lr=3.1422e-06) (hash(x)=146016298)
+16800 val loss 5.5566
+16800 val perplexity 258.9420
+16800 train 5.464295 (lr=3.1089e-06) (hash(x)=147668192)
+16900 val loss 5.5556
+16900 val perplexity 258.6944
+16900 train 5.519043 (lr=3.0801e-06) (hash(x)=155692530)
+17000 val loss 5.5567
+17000 val perplexity 258.9579
+17000 train 5.459873 (lr=3.0556e-06) (hash(x)=156598351)
+17100 val loss 5.5573
+17100 val perplexity 259.1112
+17100 train 5.451344 (lr=3.0356e-06) (hash(x)=157344236)
+17200 val loss 5.5540
+17200 val perplexity 258.2800
+17200 train 5.380807 (lr=3.0200e-06) (hash(x)=126288656)
+17300 val loss 5.5513
+17300 val perplexity 257.5620
+17300 train 5.597789 (lr=3.0089e-06) (hash(x)=146365959)
+17400 val loss 5.5494
+17400 val perplexity 257.0894
+17400 train 5.534318 (lr=3.0022e-06) (hash(x)=152729388)
+17499 val loss 5.5481
+17499 val perplexity 256.7398

attention_kindselective_n_heads4_seed1344/model_17499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87806558a4cf546cf3180d16eda42f953540aad9c91c4a39b6d87c196a91d101
+size 92843394

attention_kindselective_n_heads4_seed1344/optimizer_17499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c7ed780c8b3f89184cf2d18cf5f0126733793e07acad9e400c342d53dbf8fdc
+size 179406214