Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1345/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_9/attention_kindselective_n_heads4_seed1345", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 17500, "warmup_steps": 200, "group": "wider_is_better_9", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1345, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 3.25e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "3.25e-5_30720_4_1345", "n_embd": 256}

attention_kindselective_n_heads4_seed1345/dataloader_17499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:584ddc45f1addd41e01adcc6cf70f582d8774c147c912cd238caff8ebf7fb069
+size 964

attention_kindselective_n_heads4_seed1345/log2.txt

@@ -0,0 +1,528 @@
+max_steps: 17500
+0 val loss 11.2682
+0 val perplexity 78293.8047
+0 train 11.260221 (lr=1.6250e-07) (hash(x)=134751525)
+100 val loss 9.9509
+100 val perplexity 20971.9102
+100 train 10.005062 (lr=1.6412e-05) (hash(x)=144692155)
+200 val loss 9.1865
+200 val perplexity 9764.0762
+200 train 9.186141 (lr=3.2500e-05) (hash(x)=150358957)
+300 val loss 8.2416
+300 val perplexity 3795.5759
+300 train 8.247768 (lr=3.2498e-05) (hash(x)=150268299)
+400 val loss 7.8259
+400 val perplexity 2504.7554
+400 train 7.739417 (lr=3.2490e-05) (hash(x)=126319983)
+500 val loss 7.6182
+500 val perplexity 2034.9335
+500 train 7.662487 (lr=3.2478e-05) (hash(x)=153934799)
+600 val loss 7.5104
+600 val perplexity 1827.0092
+600 train 7.547043 (lr=3.2461e-05) (hash(x)=159305874)
+700 val loss 7.4383
+700 val perplexity 1699.8833
+700 train 7.282788 (lr=3.2440e-05) (hash(x)=141986303)
+800 val loss 7.3900
+800 val perplexity 1619.6603
+800 train 7.376721 (lr=3.2413e-05) (hash(x)=149855631)
+900 val loss 7.3498
+900 val perplexity 1555.8484
+900 train 7.228592 (lr=3.2382e-05) (hash(x)=149870240)
+1000 val loss 7.2929
+1000 val perplexity 1469.7900
+1000 train 7.360933 (lr=3.2346e-05) (hash(x)=150706760)
+1100 val loss 7.2379
+1100 val perplexity 1391.1984
+1100 train 7.074641 (lr=3.2305e-05) (hash(x)=139020621)
+1200 val loss 7.1946
+1200 val perplexity 1332.1755
+1200 train 7.057891 (lr=3.2260e-05) (hash(x)=146858163)
+1300 val loss 7.1576
+1300 val perplexity 1283.8492
+1300 train 7.224605 (lr=3.2209e-05) (hash(x)=156383408)
+1400 val loss 7.1333
+1400 val perplexity 1252.9684
+1400 train 7.060022 (lr=3.2154e-05) (hash(x)=144262072)
+1500 val loss 7.0878
+1500 val perplexity 1197.3103
+1500 train 6.939501 (lr=3.2094e-05) (hash(x)=144528048)
+1600 val loss 7.0391
+1600 val perplexity 1140.3126
+1600 train 6.955491 (lr=3.2030e-05) (hash(x)=135443880)
+1700 val loss 6.9995
+1700 val perplexity 1096.1010
+1700 train 6.916674 (lr=3.1961e-05) (hash(x)=145067496)
+1800 val loss 6.9578
+1800 val perplexity 1051.3185
+1800 train 6.918793 (lr=3.1887e-05) (hash(x)=153147388)
+1900 val loss 6.9111
+1900 val perplexity 1003.3770
+1900 train 6.801150 (lr=3.1809e-05) (hash(x)=151331979)
+2000 val loss 6.8813
+2000 val perplexity 973.9022
+2000 train 6.678115 (lr=3.1726e-05) (hash(x)=151448445)
+2100 val loss 6.8426
+2100 val perplexity 936.9084
+2100 train 6.747893 (lr=3.1638e-05) (hash(x)=157223072)
+2200 val loss 6.8096
+2200 val perplexity 906.5486
+2200 train 6.822385 (lr=3.1546e-05) (hash(x)=153161010)
+2300 val loss 6.7690
+2300 val perplexity 870.4435
+2300 train 6.856603 (lr=3.1449e-05) (hash(x)=156906742)
+2400 val loss 6.7498
+2400 val perplexity 853.9199
+2400 train 6.636610 (lr=3.1348e-05) (hash(x)=143227423)
+2500 val loss 6.7043
+2500 val perplexity 815.8697
+2500 train 6.638939 (lr=3.1243e-05) (hash(x)=151827079)
+2600 val loss 6.6780
+2600 val perplexity 794.7333
+2600 train 7.153996 (lr=3.1133e-05) (hash(x)=176373796)
+2700 val loss 6.6564
+2700 val perplexity 777.7272
+2700 train 6.593842 (lr=3.1019e-05) (hash(x)=148437519)
+2800 val loss 6.6258
+2800 val perplexity 754.2744
+2800 train 6.623206 (lr=3.0900e-05) (hash(x)=155989503)
+2900 val loss 6.5986
+2900 val perplexity 734.0786
+2900 train 6.736583 (lr=3.0777e-05) (hash(x)=159227845)
+3000 val loss 6.5678
+3000 val perplexity 711.8181
+3000 train 6.593345 (lr=3.0650e-05) (hash(x)=156507542)
+3100 val loss 6.5396
+3100 val perplexity 691.9895
+3100 train 6.525067 (lr=3.0518e-05) (hash(x)=155771797)
+3200 val loss 6.5217
+3200 val perplexity 679.7400
+3200 train 6.444234 (lr=3.0383e-05) (hash(x)=156078901)
+3300 val loss 6.4924
+3300 val perplexity 660.1191
+3300 train 6.362537 (lr=3.0243e-05) (hash(x)=143722886)
+3400 val loss 6.4715
+3400 val perplexity 646.4329
+3400 train 6.399912 (lr=3.0099e-05) (hash(x)=143793373)
+3500 val loss 6.4543
+3500 val perplexity 635.4442
+3500 train 6.709872 (lr=2.9952e-05) (hash(x)=147278080)
+3600 val loss 6.4274
+3600 val perplexity 618.5666
+3600 train 7.150148 (lr=2.9800e-05) (hash(x)=175299205)
+3700 val loss 6.4063
+3700 val perplexity 605.6692
+3700 train 6.446000 (lr=2.9644e-05) (hash(x)=150047891)
+3800 val loss 6.3847
+3800 val perplexity 592.7319
+3800 train 6.414546 (lr=2.9485e-05) (hash(x)=153419668)
+3900 val loss 6.3675
+3900 val perplexity 582.6219
+3900 train 6.473610 (lr=2.9321e-05) (hash(x)=153147630)
+4000 val loss 6.3503
+4000 val perplexity 572.6838
+4000 train 6.184350 (lr=2.9154e-05) (hash(x)=142428269)
+4100 val loss 6.3389
+4100 val perplexity 566.1708
+4100 train 6.225300 (lr=2.8983e-05) (hash(x)=148836993)
+4200 val loss 6.3193
+4200 val perplexity 555.1793
+4200 train 6.366936 (lr=2.8808e-05) (hash(x)=159557219)
+4300 val loss 6.3010
+4300 val perplexity 545.0898
+4300 train 6.417048 (lr=2.8630e-05) (hash(x)=154137450)
+4400 val loss 6.2802
+4400 val perplexity 533.8776
+4400 train 6.427896 (lr=2.8448e-05) (hash(x)=153336405)
+4500 val loss 6.2660
+4500 val perplexity 526.3508
+4500 train 6.238311 (lr=2.8263e-05) (hash(x)=145729145)
+4600 val loss 6.2631
+4600 val perplexity 524.8270
+4600 train 6.266792 (lr=2.8075e-05) (hash(x)=157789272)
+4700 val loss 6.2471
+4700 val perplexity 516.4915
+4700 train 6.222703 (lr=2.7883e-05) (hash(x)=146146781)
+4800 val loss 6.2314
+4800 val perplexity 508.4698
+4800 train 6.240023 (lr=2.7687e-05) (hash(x)=149454469)
+4900 val loss 6.2125
+4900 val perplexity 498.9409
+4900 train 6.299968 (lr=2.7489e-05) (hash(x)=155966467)
+5000 val loss 6.2044
+5000 val perplexity 494.9347
+5000 train 6.158088 (lr=2.7287e-05) (hash(x)=147184546)
+5100 val loss 6.1919
+5100 val perplexity 488.7724
+5100 train 6.167572 (lr=2.7082e-05) (hash(x)=149019501)
+5200 val loss 6.1775
+5200 val perplexity 481.8079
+5200 train 6.079477 (lr=2.6874e-05) (hash(x)=143521586)
+5300 val loss 6.1643
+5300 val perplexity 475.4693
+5300 train 6.025494 (lr=2.6664e-05) (hash(x)=143870567)
+5400 val loss 6.1496
+5400 val perplexity 468.5291
+5400 train 6.164584 (lr=2.6450e-05) (hash(x)=150658725)
+5500 val loss 6.1392
+5500 val perplexity 463.6912
+5500 train 6.165554 (lr=2.6233e-05) (hash(x)=147645530)
+5600 val loss 6.1279
+5600 val perplexity 458.4820
+5600 train 6.166198 (lr=2.6014e-05) (hash(x)=146088410)
+5700 val loss 6.1144
+5700 val perplexity 452.3112
+5700 train 6.213290 (lr=2.5792e-05) (hash(x)=151193083)
+5800 val loss 6.1091
+5800 val perplexity 449.9164
+5800 train 5.871378 (lr=2.5567e-05) (hash(x)=137474865)
+5900 val loss 6.1057
+5900 val perplexity 448.4180
+5900 train 6.060199 (lr=2.5340e-05) (hash(x)=142426763)
+6000 val loss 6.0816
+6000 val perplexity 437.7228
+6000 train 5.857378 (lr=2.5111e-05) (hash(x)=140492670)
+6100 val loss 6.0692
+6100 val perplexity 432.3176
+6100 train 6.123445 (lr=2.4879e-05) (hash(x)=175687796)
+6200 val loss 6.0592
+6200 val perplexity 428.0246
+6200 train 6.116088 (lr=2.4644e-05) (hash(x)=165841139)
+6300 val loss 6.0548
+6300 val perplexity 426.1531
+6300 train 6.104833 (lr=2.4408e-05) (hash(x)=145328367)
+6400 val loss 6.0470
+6400 val perplexity 422.8510
+6400 train 6.033661 (lr=2.4169e-05) (hash(x)=148213323)
+6500 val loss 6.0358
+6500 val perplexity 418.1525
+6500 train 6.119610 (lr=2.3928e-05) (hash(x)=153344426)
+6600 val loss 6.0248
+6600 val perplexity 413.5683
+6600 train 5.989787 (lr=2.3686e-05) (hash(x)=145840902)
+6700 val loss 6.0156
+6700 val perplexity 409.7875
+6700 train 5.850917 (lr=2.3441e-05) (hash(x)=142812723)
+6800 val loss 6.0129
+6800 val perplexity 408.6497
+6800 train 5.857504 (lr=2.3195e-05) (hash(x)=147420964)
+6900 val loss 6.0023
+6900 val perplexity 404.3388
+6900 train 6.047520 (lr=2.2946e-05) (hash(x)=152637292)
+7000 val loss 5.9893
+7000 val perplexity 399.1236
+7000 train 5.988328 (lr=2.2696e-05) (hash(x)=144728159)
+7100 val loss 5.9800
+7100 val perplexity 395.4302
+7100 train 5.956685 (lr=2.2445e-05) (hash(x)=143450899)
+7200 val loss 5.9678
+7200 val perplexity 390.6588
+7200 train 6.042522 (lr=2.2192e-05) (hash(x)=156229897)
+7300 val loss 5.9604
+7300 val perplexity 387.7628
+7300 train 5.833129 (lr=2.1937e-05) (hash(x)=147447250)
+7400 val loss 5.9574
+7400 val perplexity 386.6089
+7400 train 5.701279 (lr=2.1682e-05) (hash(x)=134836112)
+7500 val loss 5.9543
+7500 val perplexity 385.4186
+7500 train 5.960179 (lr=2.1425e-05) (hash(x)=155027499)
+7600 val loss 5.9378
+7600 val perplexity 379.0817
+7600 train 5.817950 (lr=2.1166e-05) (hash(x)=145744849)
+7700 val loss 5.9286
+7700 val perplexity 375.6128
+7700 train 6.050512 (lr=2.0907e-05) (hash(x)=158082973)
+7800 val loss 5.9247
+7800 val perplexity 374.1752
+7800 train 6.063409 (lr=2.0647e-05) (hash(x)=161538768)
+7900 val loss 5.9136
+7900 val perplexity 370.0218
+7900 train 6.012225 (lr=2.0386e-05) (hash(x)=152103064)
+8000 val loss 5.9084
+8000 val perplexity 368.1076
+8000 train 5.911327 (lr=2.0123e-05) (hash(x)=152409525)
+8100 val loss 5.9014
+8100 val perplexity 365.5402
+8100 train 5.876762 (lr=1.9861e-05) (hash(x)=150986182)
+8200 val loss 5.8890
+8200 val perplexity 361.0322
+8200 train 5.892981 (lr=1.9597e-05) (hash(x)=159555650)
+8300 val loss 5.8882
+8300 val perplexity 360.7704
+8300 train 6.041511 (lr=1.9333e-05) (hash(x)=139602404)
+8400 val loss 5.8763
+8400 val perplexity 356.4845
+8400 train 5.777814 (lr=1.9069e-05) (hash(x)=154432574)
+8500 val loss 5.8692
+8500 val perplexity 353.9774
+8500 train 5.809097 (lr=1.8804e-05) (hash(x)=142554189)
+8600 val loss 5.8649
+8600 val perplexity 352.4355
+8600 train 5.816896 (lr=1.8539e-05) (hash(x)=151300791)
+8700 val loss 5.8605
+8700 val perplexity 350.9152
+8700 train 5.849548 (lr=1.8273e-05) (hash(x)=148488790)
+8800 val loss 5.8597
+8800 val perplexity 350.6023
+8800 train 5.884410 (lr=1.8008e-05) (hash(x)=156797847)
+8900 val loss 5.8430
+8900 val perplexity 344.7959
+8900 train 5.666561 (lr=1.7742e-05) (hash(x)=143919631)
+9000 val loss 5.8368
+9000 val perplexity 342.6675
+9000 train 5.780665 (lr=1.7477e-05) (hash(x)=148706835)
+9100 val loss 5.8310
+9100 val perplexity 340.7010
+9100 train 5.844406 (lr=1.7211e-05) (hash(x)=153074259)
+9200 val loss 5.8274
+9200 val perplexity 339.4596
+9200 train 5.747160 (lr=1.6946e-05) (hash(x)=151127077)
+9300 val loss 5.8239
+9300 val perplexity 338.2773
+9300 train 5.715285 (lr=1.6681e-05) (hash(x)=152170602)
+9400 val loss 5.8309
+9400 val perplexity 340.6801
+9400 train 5.866196 (lr=1.6417e-05) (hash(x)=153874803)
+9500 val loss 5.8052
+9500 val perplexity 332.0251
+9500 train 5.803330 (lr=1.6153e-05) (hash(x)=157021298)
+9600 val loss 5.7999
+9600 val perplexity 330.2598
+9600 train 5.633519 (lr=1.5889e-05) (hash(x)=135620494)
+9700 val loss 5.7949
+9700 val perplexity 328.6039
+9700 train 5.519158 (lr=1.5627e-05) (hash(x)=142434463)
+9800 val loss 5.7910
+9800 val perplexity 327.3397
+9800 train 5.659460 (lr=1.5364e-05) (hash(x)=149075541)
+9900 val loss 5.7879
+9900 val perplexity 326.3245
+9900 train 5.656864 (lr=1.5103e-05) (hash(x)=141901425)
+10000 val loss 5.7953
+10000 val perplexity 328.7514
+10000 train 5.779858 (lr=1.4843e-05) (hash(x)=133062699)
+10100 val loss 5.7730
+10100 val perplexity 321.5122
+10100 train 5.806496 (lr=1.4584e-05) (hash(x)=157339196)
+10200 val loss 5.7692
+10200 val perplexity 320.2876
+10200 train 5.539021 (lr=1.4325e-05) (hash(x)=131082121)
+10300 val loss 5.7667
+10300 val perplexity 319.4867
+10300 train 5.867615 (lr=1.4068e-05) (hash(x)=144553757)
+10400 val loss 5.7597
+10400 val perplexity 317.2390
+10400 train 5.552857 (lr=1.3813e-05) (hash(x)=143398861)
+10500 val loss 5.7583
+10500 val perplexity 316.8132
+10500 train 5.756111 (lr=1.3558e-05) (hash(x)=157633809)
+10600 val loss 5.7498
+10600 val perplexity 314.1412
+10600 train 5.741689 (lr=1.3305e-05) (hash(x)=147294315)
+10700 val loss 5.7437
+10700 val perplexity 312.2167
+10700 train 5.868924 (lr=1.3054e-05) (hash(x)=157735028)
+10800 val loss 5.7377
+10800 val perplexity 310.3455
+10800 train 5.709660 (lr=1.2804e-05) (hash(x)=149846113)
+10900 val loss 5.7336
+10900 val perplexity 309.0825
+10900 train 6.117226 (lr=1.2555e-05) (hash(x)=157353556)
+11000 val loss 5.7338
+11000 val perplexity 309.1367
+11000 train 6.042399 (lr=1.2309e-05) (hash(x)=160672627)
+11100 val loss 5.7295
+11100 val perplexity 307.8035
+11100 train 5.947324 (lr=1.2064e-05) (hash(x)=166692335)
+11200 val loss 5.7213
+11200 val perplexity 305.3054
+11200 train 5.786076 (lr=1.1822e-05) (hash(x)=158899869)
+11300 val loss 5.7176
+11300 val perplexity 304.1765
+11300 train 5.999375 (lr=1.1581e-05) (hash(x)=166306828)
+11400 val loss 5.7132
+11400 val perplexity 302.8427
+11400 train 5.638960 (lr=1.1342e-05) (hash(x)=149545840)
+11500 val loss 5.7106
+11500 val perplexity 302.0578
+11500 train 5.682230 (lr=1.1106e-05) (hash(x)=139076910)
+11600 val loss 5.7091
+11600 val perplexity 301.5869
+11600 train 5.547813 (lr=1.0871e-05) (hash(x)=151871279)
+11700 val loss 5.7054
+11700 val perplexity 300.4809
+11700 train 5.650321 (lr=1.0639e-05) (hash(x)=152940196)
+11800 val loss 5.7005
+11800 val perplexity 299.0240
+11800 train 5.795137 (lr=1.0410e-05) (hash(x)=150754129)
+11900 val loss 5.6965
+11900 val perplexity 297.8329
+11900 train 5.651140 (lr=1.0183e-05) (hash(x)=148982597)
+12000 val loss 5.6934
+12000 val perplexity 296.9042
+12000 train 5.739422 (lr=9.9580e-06) (hash(x)=167798390)
+12100 val loss 5.6929
+12100 val perplexity 296.7506
+12100 train 5.770573 (lr=9.7360e-06) (hash(x)=146590810)
+12200 val loss 5.6882
+12200 val perplexity 295.3570
+12200 train 5.914321 (lr=9.5167e-06) (hash(x)=159875780)
+12300 val loss 5.6856
+12300 val perplexity 294.5880
+12300 train 5.556276 (lr=9.3001e-06) (hash(x)=135887861)
+12400 val loss 5.6782
+12400 val perplexity 292.4197
+12400 train 5.741988 (lr=9.0864e-06) (hash(x)=150484221)
+12500 val loss 5.6753
+12500 val perplexity 291.5758
+12500 train 5.678569 (lr=8.8756e-06) (hash(x)=157629455)
+12600 val loss 5.6724
+12600 val perplexity 290.7240
+12600 train 5.573870 (lr=8.6677e-06) (hash(x)=147308692)
+12700 val loss 5.6697
+12700 val perplexity 289.9527
+12700 train 5.501825 (lr=8.4629e-06) (hash(x)=144614722)
+12800 val loss 5.6691
+12800 val perplexity 289.7650
+12800 train 5.460669 (lr=8.2612e-06) (hash(x)=137633606)
+12900 val loss 5.6668
+12900 val perplexity 289.1100
+12900 train 5.705952 (lr=8.0627e-06) (hash(x)=155534772)
+13000 val loss 5.6622
+13000 val perplexity 287.7739
+13000 train 5.678839 (lr=7.8674e-06) (hash(x)=148501470)
+13100 val loss 5.6638
+13100 val perplexity 288.2448
+13100 train 5.714143 (lr=7.6754e-06) (hash(x)=154570618)
+13200 val loss 5.6586
+13200 val perplexity 286.7396
+13200 train 5.754503 (lr=7.4867e-06) (hash(x)=154153766)
+13300 val loss 5.6554
+13300 val perplexity 285.8399
+13300 train 5.440732 (lr=7.3015e-06) (hash(x)=162760888)
+13400 val loss 5.6491
+13400 val perplexity 284.0410
+13400 train 5.471959 (lr=7.1198e-06) (hash(x)=143223528)
+13500 val loss 5.6572
+13500 val perplexity 286.3402
+13500 train 5.813179 (lr=6.9416e-06) (hash(x)=156416986)
+13600 val loss 5.6509
+13600 val perplexity 284.5387
+13600 train 5.653998 (lr=6.7670e-06) (hash(x)=147521418)
+13700 val loss 5.6469
+13700 val perplexity 283.4187
+13700 train 5.624448 (lr=6.5961e-06) (hash(x)=151127796)
+13800 val loss 5.6449
+13800 val perplexity 282.8326
+13800 train 5.705256 (lr=6.4289e-06) (hash(x)=156863042)
+13900 val loss 5.6405
+13900 val perplexity 281.6073
+13900 train 5.639185 (lr=6.2655e-06) (hash(x)=170986545)
+14000 val loss 5.6390
+14000 val perplexity 281.1731
+14000 train 6.109547 (lr=6.1059e-06) (hash(x)=170744741)
+14100 val loss 5.6369
+14100 val perplexity 280.5912
+14100 train 6.027566 (lr=5.9502e-06) (hash(x)=142328767)
+14200 val loss 5.6352
+14200 val perplexity 280.1212
+14200 train 5.782619 (lr=5.7984e-06) (hash(x)=156141076)
+14300 val loss 5.6390
+14300 val perplexity 281.1808
+14300 train 5.619442 (lr=5.6506e-06) (hash(x)=155447298)
+14400 val loss 5.6338
+14400 val perplexity 279.7176
+14400 train 5.563756 (lr=5.5068e-06) (hash(x)=151991844)
+14500 val loss 5.6289
+14500 val perplexity 278.3420
+14500 train 5.609061 (lr=5.3671e-06) (hash(x)=149773009)
+14600 val loss 5.6268
+14600 val perplexity 277.7750
+14600 train 5.578854 (lr=5.2316e-06) (hash(x)=153836753)
+14700 val loss 5.6251
+14700 val perplexity 277.3060
+14700 train 5.757289 (lr=5.1002e-06) (hash(x)=155466746)
+14800 val loss 5.6251
+14800 val perplexity 277.3113
+14800 train 5.559618 (lr=4.9730e-06) (hash(x)=150110933)
+14900 val loss 5.6249
+14900 val perplexity 277.2558
+14900 train 5.509522 (lr=4.8501e-06) (hash(x)=144558528)
+15000 val loss 5.6233
+15000 val perplexity 276.8149
+15000 train 5.560644 (lr=4.7314e-06) (hash(x)=148315702)
+15100 val loss 5.6197
+15100 val perplexity 275.8152
+15100 train 5.463153 (lr=4.6171e-06) (hash(x)=148452301)
+15200 val loss 5.6183
+15200 val perplexity 275.4343
+15200 train 5.877840 (lr=4.5072e-06) (hash(x)=162181769)
+15300 val loss 5.6160
+15300 val perplexity 274.7894
+15300 train 5.540632 (lr=4.4017e-06) (hash(x)=147256707)
+15400 val loss 5.6156
+15400 val perplexity 274.6822
+15400 train 5.716757 (lr=4.3006e-06) (hash(x)=154947797)
+15500 val loss 5.6154
+15500 val perplexity 274.6259
+15500 train 5.637792 (lr=4.2040e-06) (hash(x)=164200136)
+15600 val loss 5.6145
+15600 val perplexity 274.3805
+15600 train 5.486223 (lr=4.1119e-06) (hash(x)=145086131)
+15700 val loss 5.6103
+15700 val perplexity 273.2360
+15700 train 5.546430 (lr=4.0244e-06) (hash(x)=152070006)
+15800 val loss 5.6083
+15800 val perplexity 272.6676
+15800 train 5.639931 (lr=3.9414e-06) (hash(x)=150969760)
+15900 val loss 5.6074
+15900 val perplexity 272.4455
+15900 train 5.629735 (lr=3.8630e-06) (hash(x)=149495681)
+16000 val loss 5.6077
+16000 val perplexity 272.5269
+16000 train 5.721963 (lr=3.7892e-06) (hash(x)=159383533)
+16100 val loss 5.6071
+16100 val perplexity 272.3398
+16100 train 5.477306 (lr=3.7201e-06) (hash(x)=154365611)
+16200 val loss 5.6055
+16200 val perplexity 271.9151
+16200 train 5.766246 (lr=3.6556e-06) (hash(x)=154284815)
+16300 val loss 5.6043
+16300 val perplexity 271.5936
+16300 train 5.592812 (lr=3.5959e-06) (hash(x)=148416573)
+16400 val loss 5.6020
+16400 val perplexity 270.9808
+16400 train 5.550043 (lr=3.5408e-06) (hash(x)=150447087)
+16500 val loss 5.6026
+16500 val perplexity 271.1364
+16500 train 5.598300 (lr=3.4905e-06) (hash(x)=156112678)
+16600 val loss 5.6009
+16600 val perplexity 270.6564
+16600 train 5.541498 (lr=3.4449e-06) (hash(x)=150449728)
+16700 val loss 5.5990
+16700 val perplexity 270.1480
+16700 train 5.659224 (lr=3.4041e-06) (hash(x)=153905935)
+16800 val loss 5.5963
+16800 val perplexity 269.4264
+16800 train 5.567451 (lr=3.3680e-06) (hash(x)=144086573)
+16900 val loss 5.5958
+16900 val perplexity 269.2966
+16900 train 5.576754 (lr=3.3367e-06) (hash(x)=145863379)
+17000 val loss 5.5965
+17000 val perplexity 269.4939
+17000 train 5.673267 (lr=3.3102e-06) (hash(x)=149054226)
+17100 val loss 5.5967
+17100 val perplexity 269.5238
+17100 train 5.496261 (lr=3.2886e-06) (hash(x)=147109203)
+17200 val loss 5.5944
+17200 val perplexity 268.9105
+17200 train 5.528384 (lr=3.2717e-06) (hash(x)=141837353)
+17300 val loss 5.5926
+17300 val perplexity 268.4294
+17300 train 5.605858 (lr=3.2596e-06) (hash(x)=148542064)
+17400 val loss 5.5909
+17400 val perplexity 267.9839
+17400 train 5.420087 (lr=3.2524e-06) (hash(x)=140939390)
+17499 val loss 5.5898
+17499 val perplexity 267.6847

attention_kindselective_n_heads4_seed1345/model_17499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7465d118f10208aa44d9be65c8c05e7770abd7b6f628f9af7961a0958fb2d5bb
+size 92843394

attention_kindselective_n_heads4_seed1345/optimizer_17499.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28b671bf6a03e491a6abea866c6d5275e23c192b9e2bca279386cb1e389e045b
+size 179406214