Upload folder using huggingface_hub

sdpa_16_spooky/sdpa_64_16_v2_4gpu_seed_1339/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "self", "log_dir": "sdpa_16_spooky/sdpa_64_16_v2_4gpu_seed_1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 32, "n_embd": 256, "head_dim": 32, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 4375, "warmup_steps": 250, "group": "sdpa_16_spooky", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "disable_selection_head_linear_combo_bias": false, "assert_latent_matches_no_head": false, "n_latent_masks": null, "init_latent_masks_to_identity": false, "init_latent_masks_to_inverse": false, "latent_mask_scale": null, "latent_mask_runtime_multiplier": null, "latent_mask_sigmoid": false, "S_layernorm": false, "one_head_per_latent_mask": false, "att_conv": false, "att_conv_init": "none", "att_conv_scale": 1.0, "att_conv_weight_decay": false, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 64, "total_batch_size": 131072, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.003, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "mup_zero_init": true, "l1_loss": false, "debugpy": false, "key": "sdpa_64_16_v2_4gpu", "latent_mask_precision": "bfloat16", "att_conv_precision": "bfloat16", "attn_precision": "bfloat16", "profile_kind": "none", "dense_attention_kind": "mha", "head_dim_value": 32, "sdpa_iter_size": null, "stabilize_attn_scores": false, "override_use_sdpa": false, "autocast_precision": "bfloat16", "simulate_micro_bs": 16, "simulate_micro_bs_2": null}

sdpa_16_spooky/sdpa_64_16_v2_4gpu_seed_1339/dataloader_04374.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:994809a5ef065b3b39ff8d94ca69f209f3eedb103954655f09fa348a0918b20e
+size 964

sdpa_16_spooky/sdpa_64_16_v2_4gpu_seed_1339/log2.txt

@@ -0,0 +1,529 @@
+max_steps: 4375
+0 val loss 11.1896
+0 val perplexity 72370.2344
+0 train 89.594803 (lr=4.1958e-06) (hash(x)=76691771)
+10 train 78.734932 (lr=4.6154e-05) (hash(x)=75933188)
+20 train 73.359154 (lr=8.8112e-05) (hash(x)=68847503)
+30 train 67.552238 (lr=1.3007e-04) (hash(x)=87478177)
+40 train 61.505035 (lr=1.7203e-04) (hash(x)=77619364)
+50 train 59.242107 (lr=2.1399e-04) (hash(x)=87055734)
+60 train 59.086449 (lr=2.5594e-04) (hash(x)=89066335)
+70 train 55.846954 (lr=2.9790e-04) (hash(x)=82863987)
+80 train 56.036930 (lr=3.3986e-04) (hash(x)=77000796)
+90 train 54.680637 (lr=3.8182e-04) (hash(x)=77641148)
+100 val loss 6.7877
+100 val perplexity 886.9126
+100 train 52.900608 (lr=4.2378e-04) (hash(x)=78737324)
+110 train 52.998318 (lr=4.6573e-04) (hash(x)=72644243)
+120 train 52.089241 (lr=5.0769e-04) (hash(x)=80676401)
+130 train 51.169708 (lr=5.4965e-04) (hash(x)=77832835)
+140 train 51.196060 (lr=5.9161e-04) (hash(x)=77534009)
+150 train 51.877510 (lr=6.3357e-04) (hash(x)=78944480)
+160 train 51.633690 (lr=6.7552e-04) (hash(x)=86921535)
+170 train 50.916847 (lr=7.1748e-04) (hash(x)=87834135)
+180 train 50.600456 (lr=7.5944e-04) (hash(x)=83084625)
+190 train 50.074940 (lr=8.0140e-04) (hash(x)=79541240)
+200 val loss 6.1889
+200 val perplexity 487.2940
+200 train 49.615055 (lr=8.4336e-04) (hash(x)=78525299)
+210 train 48.870117 (lr=8.8531e-04) (hash(x)=71150038)
+220 train 48.464638 (lr=9.2727e-04) (hash(x)=74625826)
+230 train 49.584770 (lr=9.6923e-04) (hash(x)=85058035)
+240 train 47.241882 (lr=1.0112e-03) (hash(x)=73373171)
+250 train 47.840767 (lr=1.0531e-03) (hash(x)=83146187)
+260 train 46.880207 (lr=1.0951e-03) (hash(x)=82519170)
+270 train 46.470028 (lr=1.1371e-03) (hash(x)=73708471)
+280 train 45.968445 (lr=1.1790e-03) (hash(x)=80885544)
+290 train 45.320183 (lr=1.2210e-03) (hash(x)=72024214)
+300 val loss 5.7917
+300 val perplexity 327.5742
+300 train 44.353485 (lr=1.2629e-03) (hash(x)=76713609)
+310 train 43.914761 (lr=1.3049e-03) (hash(x)=77123891)
+320 train 44.473331 (lr=1.3469e-03) (hash(x)=84559347)
+330 train 45.370079 (lr=1.3888e-03) (hash(x)=84336973)
+340 train 45.479454 (lr=1.4308e-03) (hash(x)=82229953)
+350 train 45.315609 (lr=1.4727e-03) (hash(x)=87428214)
+360 train 44.825932 (lr=1.5147e-03) (hash(x)=74189124)
+370 train 44.881054 (lr=1.5566e-03) (hash(x)=85402705)
+380 train 43.330696 (lr=1.5986e-03) (hash(x)=77928239)
+390 train 43.164780 (lr=1.6406e-03) (hash(x)=79810709)
+400 val loss 5.4183
+400 val perplexity 225.5007
+400 train 43.113373 (lr=1.6825e-03) (hash(x)=75725642)
+410 train 42.655056 (lr=1.7245e-03) (hash(x)=80368890)
+420 train 42.700764 (lr=1.7664e-03) (hash(x)=74315414)
+430 train 42.903759 (lr=1.8084e-03) (hash(x)=81777043)
+440 train 41.972836 (lr=1.8503e-03) (hash(x)=81753282)
+450 train 41.940903 (lr=1.8923e-03) (hash(x)=65988100)
+460 train 41.374969 (lr=1.9343e-03) (hash(x)=85722536)
+470 train 39.709789 (lr=1.9762e-03) (hash(x)=67049135)
+480 train 40.765583 (lr=2.0182e-03) (hash(x)=87459125)
+490 train 39.299068 (lr=2.0601e-03) (hash(x)=76021500)
+500 val loss 5.2164
+500 val perplexity 184.2785
+500 train 40.000660 (lr=2.1021e-03) (hash(x)=76616342)
+510 train 41.191933 (lr=2.1441e-03) (hash(x)=89072022)
+520 train 41.371655 (lr=2.1860e-03) (hash(x)=80991374)
+530 train 41.982674 (lr=2.2280e-03) (hash(x)=81200828)
+540 train 41.441757 (lr=2.2699e-03) (hash(x)=75839384)
+550 train 41.146313 (lr=2.3119e-03) (hash(x)=87671011)
+560 train 41.531075 (lr=2.3538e-03) (hash(x)=73409992)
+570 train 41.140175 (lr=2.3958e-03) (hash(x)=81856439)
+580 train 40.565685 (lr=2.4378e-03) (hash(x)=83303820)
+590 train 40.645008 (lr=2.4797e-03) (hash(x)=85439233)
+600 val loss 5.0292
+600 val perplexity 152.8040
+600 train 40.375832 (lr=2.5217e-03) (hash(x)=76051873)
+610 train 39.853619 (lr=2.5636e-03) (hash(x)=80428074)
+620 train 39.765263 (lr=2.6056e-03) (hash(x)=79200864)
+630 train 38.903976 (lr=2.6476e-03) (hash(x)=82398005)
+640 train 38.904942 (lr=2.6895e-03) (hash(x)=85232109)
+650 train 38.604221 (lr=2.7315e-03) (hash(x)=73769233)
+660 train 37.731190 (lr=2.7734e-03) (hash(x)=80620545)
+670 train 37.776546 (lr=2.8154e-03) (hash(x)=90656221)
+680 train 36.883106 (lr=2.8573e-03) (hash(x)=76185157)
+690 train 37.324097 (lr=2.8993e-03) (hash(x)=75225592)
+700 val loss 4.9078
+700 val perplexity 135.3403
+700 train 37.032436 (lr=2.9413e-03) (hash(x)=80940753)
+710 train 39.156757 (lr=2.9832e-03) (hash(x)=72866907)
+720 train 38.916431 (lr=3.0000e-03) (hash(x)=85236086)
+730 train 39.220139 (lr=2.9999e-03) (hash(x)=82308154)
+740 train 38.862595 (lr=2.9997e-03) (hash(x)=91052832)
+750 train 38.467579 (lr=2.9994e-03) (hash(x)=68972346)
+760 train 38.647392 (lr=2.9990e-03) (hash(x)=74064342)
+770 train 38.242683 (lr=2.9985e-03) (hash(x)=89648204)
+780 train 38.052948 (lr=2.9979e-03) (hash(x)=83708293)
+790 train 38.008602 (lr=2.9972e-03) (hash(x)=84336512)
+800 val loss 4.7452
+800 val perplexity 115.0333
+800 train 37.761009 (lr=2.9964e-03) (hash(x)=75025285)
+810 train 36.864212 (lr=2.9955e-03) (hash(x)=79436536)
+820 train 37.281097 (lr=2.9945e-03) (hash(x)=90467390)
+830 train 37.892864 (lr=2.9934e-03) (hash(x)=77292020)
+840 train 37.004944 (lr=2.9922e-03) (hash(x)=75568927)
+850 train 36.892899 (lr=2.9909e-03) (hash(x)=79671781)
+860 train 38.068909 (lr=2.9896e-03) (hash(x)=84280943)
+870 train 37.797688 (lr=2.9881e-03) (hash(x)=80901294)
+880 train 37.407326 (lr=2.9865e-03) (hash(x)=81437584)
+890 train 37.193985 (lr=2.9848e-03) (hash(x)=73893778)
+900 val loss 4.6255
+900 val perplexity 102.0532
+900 train 36.855549 (lr=2.9830e-03) (hash(x)=74987794)
+910 train 36.748474 (lr=2.9811e-03) (hash(x)=93721374)
+920 train 37.171329 (lr=2.9792e-03) (hash(x)=79149678)
+930 train 36.768913 (lr=2.9771e-03) (hash(x)=83179387)
+940 train 36.285965 (lr=2.9749e-03) (hash(x)=89009978)
+950 train 36.638844 (lr=2.9726e-03) (hash(x)=76483735)
+960 train 36.709927 (lr=2.9703e-03) (hash(x)=78638850)
+970 train 36.265255 (lr=2.9678e-03) (hash(x)=90415206)
+980 train 36.013649 (lr=2.9652e-03) (hash(x)=90819195)
+990 train 36.458954 (lr=2.9626e-03) (hash(x)=85938852)
+1000 val loss 4.5361
+1000 val perplexity 93.3278
+1000 train 36.317154 (lr=2.9598e-03) (hash(x)=82436789)
+1010 train 37.451481 (lr=2.9570e-03) (hash(x)=63022149)
+1020 train 36.758629 (lr=2.9540e-03) (hash(x)=83501199)
+1030 train 36.401451 (lr=2.9510e-03) (hash(x)=77093326)
+1040 train 36.056324 (lr=2.9478e-03) (hash(x)=73437559)
+1050 train 36.206715 (lr=2.9446e-03) (hash(x)=84550388)
+1060 train 36.373260 (lr=2.9412e-03) (hash(x)=86184566)
+1070 train 36.598675 (lr=2.9378e-03) (hash(x)=85644922)
+1080 train 35.388744 (lr=2.9343e-03) (hash(x)=86179801)
+1090 train 35.841125 (lr=2.9307e-03) (hash(x)=82868303)
+1100 val loss 4.4824
+1100 val perplexity 88.4423
+1100 train 36.125500 (lr=2.9270e-03) (hash(x)=96780388)
+1110 train 35.484268 (lr=2.9231e-03) (hash(x)=87223122)
+1120 train 35.175117 (lr=2.9192e-03) (hash(x)=77292786)
+1130 train 35.662804 (lr=2.9152e-03) (hash(x)=88761375)
+1140 train 35.768356 (lr=2.9112e-03) (hash(x)=80956468)
+1150 train 36.864716 (lr=2.9070e-03) (hash(x)=84725462)
+1160 train 36.133995 (lr=2.9027e-03) (hash(x)=76098113)
+1170 train 35.826149 (lr=2.8983e-03) (hash(x)=84387685)
+1180 train 35.404144 (lr=2.8939e-03) (hash(x)=76120321)
+1190 train 35.534580 (lr=2.8893e-03) (hash(x)=85974065)
+1200 val loss 4.4088
+1200 val perplexity 82.1683
+1200 train 34.659821 (lr=2.8847e-03) (hash(x)=79340644)
+1210 train 35.336857 (lr=2.8800e-03) (hash(x)=88345025)
+1220 train 35.030281 (lr=2.8751e-03) (hash(x)=86406230)
+1230 train 34.597775 (lr=2.8702e-03) (hash(x)=80608975)
+1240 train 35.297741 (lr=2.8652e-03) (hash(x)=81635225)
+1250 train 34.136818 (lr=2.8601e-03) (hash(x)=82126376)
+1260 train 34.783695 (lr=2.8550e-03) (hash(x)=83737972)
+1270 train 34.423489 (lr=2.8497e-03) (hash(x)=81132345)
+1280 train 35.113976 (lr=2.8443e-03) (hash(x)=85850404)
+1290 train 35.253349 (lr=2.8389e-03) (hash(x)=83490640)
+1300 val loss 4.3604
+1300 val perplexity 78.2892
+1300 train 35.196747 (lr=2.8333e-03) (hash(x)=77545187)
+1310 train 36.346542 (lr=2.8277e-03) (hash(x)=86412685)
+1320 train 34.209190 (lr=2.8220e-03) (hash(x)=86429640)
+1330 train 34.598312 (lr=2.8162e-03) (hash(x)=79645281)
+1340 train 35.834160 (lr=2.8103e-03) (hash(x)=82668541)
+1350 train 34.205555 (lr=2.8044e-03) (hash(x)=73627248)
+1360 train 35.019566 (lr=2.7983e-03) (hash(x)=74845133)
+1370 train 34.141846 (lr=2.7922e-03) (hash(x)=77850497)
+1380 train 34.366634 (lr=2.7860e-03) (hash(x)=76143954)
+1390 train 34.603092 (lr=2.7797e-03) (hash(x)=80675544)
+1400 val loss 4.3456
+1400 val perplexity 77.1375
+1400 train 34.248360 (lr=2.7733e-03) (hash(x)=76010938)
+1410 train 35.831196 (lr=2.7668e-03) (hash(x)=81260300)
+1420 train 36.239288 (lr=2.7603e-03) (hash(x)=83803629)
+1430 train 34.149944 (lr=2.7536e-03) (hash(x)=83840110)
+1440 train 35.321217 (lr=2.7469e-03) (hash(x)=97711831)
+1450 train 34.623787 (lr=2.7401e-03) (hash(x)=81110139)
+1460 train 34.096939 (lr=2.7333e-03) (hash(x)=90942839)
+1470 train 34.375980 (lr=2.7263e-03) (hash(x)=80721564)
+1480 train 34.701187 (lr=2.7193e-03) (hash(x)=77852059)
+1490 train 34.162258 (lr=2.7121e-03) (hash(x)=80011365)
+1500 val loss 4.3002
+1500 val perplexity 73.7125
+1500 train 35.238487 (lr=2.7050e-03) (hash(x)=84496142)
+1510 train 34.025356 (lr=2.6977e-03) (hash(x)=70186729)
+1520 train 34.334843 (lr=2.6903e-03) (hash(x)=74854227)
+1530 train 33.544765 (lr=2.6829e-03) (hash(x)=83433616)
+1540 train 34.167000 (lr=2.6754e-03) (hash(x)=77708777)
+1550 train 34.490234 (lr=2.6678e-03) (hash(x)=77532793)
+1560 train 33.965343 (lr=2.6602e-03) (hash(x)=71028924)
+1570 train 34.556160 (lr=2.6525e-03) (hash(x)=83230753)
+1580 train 34.124702 (lr=2.6447e-03) (hash(x)=76346638)
+1590 train 35.171978 (lr=2.6368e-03) (hash(x)=80597281)
+1600 val loss 4.2753
+1600 val perplexity 71.9034
+1600 train 34.818813 (lr=2.6289e-03) (hash(x)=83951685)
+1610 train 35.444691 (lr=2.6208e-03) (hash(x)=88368104)
+1620 train 34.229553 (lr=2.6128e-03) (hash(x)=71445651)
+1630 train 34.256180 (lr=2.6046e-03) (hash(x)=78160810)
+1640 train 34.868225 (lr=2.5964e-03) (hash(x)=76271845)
+1650 train 34.303349 (lr=2.5881e-03) (hash(x)=74157814)
+1660 train 34.139549 (lr=2.5797e-03) (hash(x)=83737211)
+1670 train 33.832939 (lr=2.5713e-03) (hash(x)=81712257)
+1680 train 32.577461 (lr=2.5628e-03) (hash(x)=67066692)
+1690 train 32.842590 (lr=2.5542e-03) (hash(x)=71523362)
+1700 val loss 4.2671
+1700 val perplexity 71.3158
+1700 train 34.502762 (lr=2.5455e-03) (hash(x)=103934022)
+1710 train 34.355049 (lr=2.5368e-03) (hash(x)=75660833)
+1720 train 34.322891 (lr=2.5281e-03) (hash(x)=77122585)
+1730 train 34.382030 (lr=2.5192e-03) (hash(x)=86612393)
+1740 train 34.533035 (lr=2.5103e-03) (hash(x)=72362150)
+1750 train 34.327129 (lr=2.5014e-03) (hash(x)=84556331)
+1760 train 34.026077 (lr=2.4924e-03) (hash(x)=72748089)
+1770 train 33.210400 (lr=2.4833e-03) (hash(x)=79347166)
+1780 train 33.999897 (lr=2.4741e-03) (hash(x)=75049407)
+1790 train 33.609283 (lr=2.4649e-03) (hash(x)=81637865)
+1800 val loss 4.2312
+1800 val perplexity 68.8011
+1800 train 34.534370 (lr=2.4556e-03) (hash(x)=74570021)
+1810 train 33.759403 (lr=2.4463e-03) (hash(x)=75340962)
+1820 train 32.564598 (lr=2.4369e-03) (hash(x)=80785751)
+1830 train 33.675694 (lr=2.4275e-03) (hash(x)=81575523)
+1840 train 33.061745 (lr=2.4180e-03) (hash(x)=81012062)
+1850 train 33.384033 (lr=2.4084e-03) (hash(x)=83576703)
+1860 train 34.488945 (lr=2.3988e-03) (hash(x)=78801555)
+1870 train 34.839165 (lr=2.3891e-03) (hash(x)=88128926)
+1880 train 33.326145 (lr=2.3794e-03) (hash(x)=78733953)
+1890 train 33.743610 (lr=2.3696e-03) (hash(x)=79055333)
+1900 val loss 4.2005
+1900 val perplexity 66.7167
+1900 train 33.267452 (lr=2.3598e-03) (hash(x)=78810696)
+1910 train 32.994705 (lr=2.3499e-03) (hash(x)=74571315)
+1920 train 33.733540 (lr=2.3400e-03) (hash(x)=75184481)
+1930 train 32.642403 (lr=2.3300e-03) (hash(x)=74737950)
+1940 train 33.465607 (lr=2.3200e-03) (hash(x)=78798651)
+1950 train 33.113796 (lr=2.3099e-03) (hash(x)=78797414)
+1960 train 32.309814 (lr=2.2998e-03) (hash(x)=83565875)
+1970 train 34.018867 (lr=2.2896e-03) (hash(x)=85304651)
+1980 train 33.364532 (lr=2.2793e-03) (hash(x)=79070376)
+1990 train 33.514977 (lr=2.2691e-03) (hash(x)=80368757)
+2000 val loss 4.1770
+2000 val perplexity 65.1675
+2000 train 33.909748 (lr=2.2588e-03) (hash(x)=84961527)
+2010 train 33.780983 (lr=2.2484e-03) (hash(x)=81100660)
+2020 train 33.812714 (lr=2.2380e-03) (hash(x)=76580449)
+2030 train 34.087017 (lr=2.2275e-03) (hash(x)=76085130)
+2040 train 34.155151 (lr=2.2170e-03) (hash(x)=80455505)
+2050 train 33.360039 (lr=2.2065e-03) (hash(x)=79850444)
+2060 train 34.623158 (lr=2.1959e-03) (hash(x)=73385358)
+2070 train 33.468533 (lr=2.1853e-03) (hash(x)=73991806)
+2080 train 33.510674 (lr=2.1746e-03) (hash(x)=75770410)
+2090 train 32.231144 (lr=2.1639e-03) (hash(x)=76045445)
+2100 val loss 4.1549
+2100 val perplexity 63.7475
+2100 train 32.628990 (lr=2.1532e-03) (hash(x)=74713873)
+2110 train 32.906570 (lr=2.1424e-03) (hash(x)=77134197)
+2120 train 32.595146 (lr=2.1316e-03) (hash(x)=84512261)
+2130 train 33.831520 (lr=2.1208e-03) (hash(x)=78144154)
+2140 train 34.353172 (lr=2.1099e-03) (hash(x)=86642533)
+2150 train 33.644135 (lr=2.0990e-03) (hash(x)=77253201)
+2160 train 33.485092 (lr=2.0881e-03) (hash(x)=75012925)
+2170 train 33.423801 (lr=2.0771e-03) (hash(x)=77605401)
+2180 train 33.552120 (lr=2.0661e-03) (hash(x)=87115530)
+2190 train 33.637871 (lr=2.0550e-03) (hash(x)=92928096)
+2200 val loss 4.1314
+2200 val perplexity 62.2670
+2200 train 33.473251 (lr=2.0440e-03) (hash(x)=60602440)
+2210 train 33.418804 (lr=2.0329e-03) (hash(x)=73881980)
+2220 train 32.688492 (lr=2.0217e-03) (hash(x)=81122001)
+2230 train 32.230110 (lr=2.0106e-03) (hash(x)=69540551)
+2240 train 32.610458 (lr=1.9994e-03) (hash(x)=86014800)
+2250 train 32.475037 (lr=1.9882e-03) (hash(x)=73654615)
+2260 train 32.093620 (lr=1.9770e-03) (hash(x)=74915254)
+2270 train 32.406082 (lr=1.9657e-03) (hash(x)=81427223)
+2280 train 34.196445 (lr=1.9544e-03) (hash(x)=86853418)
+2290 train 33.681976 (lr=1.9431e-03) (hash(x)=77559086)
+2300 val loss 4.1098
+2300 val perplexity 60.9326
+2300 train 32.838013 (lr=1.9318e-03) (hash(x)=75853865)
+2310 train 33.240131 (lr=1.9205e-03) (hash(x)=92795447)
+2320 train 33.496132 (lr=1.9091e-03) (hash(x)=84772445)
+2330 train 32.545212 (lr=1.8977e-03) (hash(x)=69400537)
+2340 train 33.470570 (lr=1.8863e-03) (hash(x)=71568646)
+2350 train 32.819603 (lr=1.8749e-03) (hash(x)=101824801)
+2360 train 33.517059 (lr=1.8635e-03) (hash(x)=72926972)
+2370 train 33.811874 (lr=1.8520e-03) (hash(x)=73404444)
+2380 train 33.011055 (lr=1.8406e-03) (hash(x)=80751524)
+2390 train 33.864826 (lr=1.8291e-03) (hash(x)=102286313)
+2400 val loss 4.1023
+2400 val perplexity 60.4806
+2400 train 32.202892 (lr=1.8176e-03) (hash(x)=77503472)
+2410 train 31.712212 (lr=1.8061e-03) (hash(x)=72581599)
+2420 train 32.195599 (lr=1.7946e-03) (hash(x)=80315981)
+2430 train 31.613575 (lr=1.7830e-03) (hash(x)=81541840)
+2440 train 31.867249 (lr=1.7715e-03) (hash(x)=77869695)
+2450 train 30.831442 (lr=1.7600e-03) (hash(x)=76386856)
+2460 train 30.986675 (lr=1.7484e-03) (hash(x)=83886755)
+2470 train 30.385708 (lr=1.7368e-03) (hash(x)=81675987)
+2480 train 33.769558 (lr=1.7253e-03) (hash(x)=81946772)
+2490 train 33.361172 (lr=1.7137e-03) (hash(x)=78414140)
+2500 val loss 4.0803
+2500 val perplexity 59.1640
+2500 train 32.830513 (lr=1.7021e-03) (hash(x)=74205036)
+2510 train 32.190952 (lr=1.6906e-03) (hash(x)=88890588)
+2520 train 33.119507 (lr=1.6790e-03) (hash(x)=75881329)
+2530 train 32.556881 (lr=1.6674e-03) (hash(x)=79807944)
+2540 train 32.704185 (lr=1.6558e-03) (hash(x)=74194315)
+2550 train 32.595757 (lr=1.6442e-03) (hash(x)=73494342)
+2560 train 32.320976 (lr=1.6326e-03) (hash(x)=79105223)
+2570 train 32.843399 (lr=1.6210e-03) (hash(x)=81813280)
+2580 train 33.798035 (lr=1.6094e-03) (hash(x)=67789519)
+2590 train 32.302319 (lr=1.5979e-03) (hash(x)=87018523)
+2600 val loss 4.0586
+2600 val perplexity 57.8960
+2600 train 31.999454 (lr=1.5863e-03) (hash(x)=64060102)
+2610 train 31.881586 (lr=1.5747e-03) (hash(x)=105831163)
+2620 train 31.192923 (lr=1.5632e-03) (hash(x)=77946087)
+2630 train 30.524557 (lr=1.5516e-03) (hash(x)=75131318)
+2640 train 31.281031 (lr=1.5400e-03) (hash(x)=77703984)
+2650 train 31.429098 (lr=1.5285e-03) (hash(x)=87699577)
+2660 train 31.917547 (lr=1.5170e-03) (hash(x)=76223950)
+2670 train 33.848061 (lr=1.5054e-03) (hash(x)=83089727)
+2680 train 32.771275 (lr=1.4939e-03) (hash(x)=87321261)
+2690 train 32.475224 (lr=1.4824e-03) (hash(x)=73058533)
+2700 val loss 4.0442
+2700 val perplexity 57.0644
+2700 train 33.682911 (lr=1.4709e-03) (hash(x)=90545205)
+2710 train 33.413921 (lr=1.4594e-03) (hash(x)=78161927)
+2720 train 32.718746 (lr=1.4480e-03) (hash(x)=88950268)
+2730 train 31.779388 (lr=1.4365e-03) (hash(x)=91057017)
+2740 train 31.358248 (lr=1.4251e-03) (hash(x)=77976939)
+2750 train 32.559406 (lr=1.4137e-03) (hash(x)=82776730)
+2760 train 31.596546 (lr=1.4023e-03) (hash(x)=65350965)
+2770 train 31.977615 (lr=1.3909e-03) (hash(x)=83069971)
+2780 train 31.809006 (lr=1.3795e-03) (hash(x)=85938141)
+2790 train 31.503971 (lr=1.3682e-03) (hash(x)=89679963)
+2800 val loss 4.0299
+2800 val perplexity 56.2578
+2800 train 30.609516 (lr=1.3569e-03) (hash(x)=81467913)
+2810 train 31.007336 (lr=1.3456e-03) (hash(x)=84834030)
+2820 train 30.195187 (lr=1.3343e-03) (hash(x)=91811998)
+2830 train 31.093758 (lr=1.3230e-03) (hash(x)=80042148)
+2840 train 31.223520 (lr=1.3118e-03) (hash(x)=85923024)
+2850 train 32.689907 (lr=1.3006e-03) (hash(x)=82459205)
+2860 train 31.690012 (lr=1.2894e-03) (hash(x)=98201612)
+2870 train 33.204929 (lr=1.2783e-03) (hash(x)=76138971)
+2880 train 33.223900 (lr=1.2671e-03) (hash(x)=77640789)
+2890 train 31.333981 (lr=1.2560e-03) (hash(x)=78740034)
+2900 val loss 4.0037
+2900 val perplexity 54.8000
+2900 train 31.887011 (lr=1.2450e-03) (hash(x)=81491894)
+2910 train 33.708611 (lr=1.2339e-03) (hash(x)=59234190)
+2920 train 32.430660 (lr=1.2229e-03) (hash(x)=78269949)
+2930 train 32.788620 (lr=1.2119e-03) (hash(x)=77187206)
+2940 train 31.973703 (lr=1.2010e-03) (hash(x)=74932260)
+2950 train 31.820637 (lr=1.1901e-03) (hash(x)=75624095)
+2960 train 31.235863 (lr=1.1792e-03) (hash(x)=82737672)
+2970 train 31.280039 (lr=1.1684e-03) (hash(x)=80433787)
+2980 train 31.676556 (lr=1.1576e-03) (hash(x)=84243181)
+2990 train 30.773487 (lr=1.1468e-03) (hash(x)=80151516)
+3000 val loss 4.0045
+3000 val perplexity 54.8443
+3000 train 31.260944 (lr=1.1361e-03) (hash(x)=79916251)
+3010 train 30.142429 (lr=1.1254e-03) (hash(x)=71552868)
+3020 train 30.898556 (lr=1.1147e-03) (hash(x)=99081923)
+3030 train 32.886570 (lr=1.1041e-03) (hash(x)=80803974)
+3040 train 32.324642 (lr=1.0935e-03) (hash(x)=78265369)
+3050 train 32.274330 (lr=1.0830e-03) (hash(x)=80084875)
+3060 train 33.769733 (lr=1.0725e-03) (hash(x)=79433776)
+3070 train 32.225716 (lr=1.0620e-03) (hash(x)=82745707)
+3080 train 32.383987 (lr=1.0516e-03) (hash(x)=77373872)
+3090 train 32.335289 (lr=1.0412e-03) (hash(x)=80445472)
+3100 val loss 3.9810
+3100 val perplexity 53.5703
+3100 train 31.411505 (lr=1.0309e-03) (hash(x)=85089133)
+3110 train 31.503607 (lr=1.0207e-03) (hash(x)=87805479)
+3120 train 30.837189 (lr=1.0104e-03) (hash(x)=77570798)
+3130 train 30.964579 (lr=1.0002e-03) (hash(x)=72706309)
+3140 train 31.613523 (lr=9.9011e-04) (hash(x)=86236392)
+3150 train 31.806561 (lr=9.8002e-04) (hash(x)=76741376)
+3160 train 31.913166 (lr=9.6999e-04) (hash(x)=85906371)
+3170 train 32.190453 (lr=9.6000e-04) (hash(x)=78263563)
+3180 train 32.417839 (lr=9.5007e-04) (hash(x)=70752762)
+3190 train 32.737190 (lr=9.4019e-04) (hash(x)=69681127)
+3200 val loss 3.9614
+3200 val perplexity 52.5288
+3200 train 32.005856 (lr=9.3036e-04) (hash(x)=81427864)
+3210 train 31.812721 (lr=9.2058e-04) (hash(x)=80173166)
+3220 train 32.257793 (lr=9.1085e-04) (hash(x)=84285649)
+3230 train 33.030239 (lr=9.0118e-04) (hash(x)=80201533)
+3240 train 32.616707 (lr=8.9157e-04) (hash(x)=80539734)
+3250 train 31.824553 (lr=8.8201e-04) (hash(x)=74526818)
+3260 train 31.750336 (lr=8.7251e-04) (hash(x)=84942377)
+3270 train 31.903740 (lr=8.6307e-04) (hash(x)=76979747)
+3280 train 31.381081 (lr=8.5368e-04) (hash(x)=76291360)
+3290 train 32.186043 (lr=8.4435e-04) (hash(x)=91713479)
+3300 val loss 3.9540
+3300 val perplexity 52.1446
+3300 train 31.045033 (lr=8.3508e-04) (hash(x)=69663736)
+3310 train 30.430214 (lr=8.2588e-04) (hash(x)=77057686)
+3320 train 32.309132 (lr=8.1673e-04) (hash(x)=75771339)
+3330 train 29.530054 (lr=8.0764e-04) (hash(x)=80713994)
+3340 train 30.529428 (lr=7.9862e-04) (hash(x)=76392589)
+3350 train 32.231606 (lr=7.8966e-04) (hash(x)=89979162)
+3360 train 32.188267 (lr=7.8076e-04) (hash(x)=83165923)
+3370 train 31.781431 (lr=7.7192e-04) (hash(x)=76814496)
+3380 train 32.170074 (lr=7.6315e-04) (hash(x)=81346232)
+3390 train 33.150108 (lr=7.5445e-04) (hash(x)=79523353)
+3400 val loss 3.9324
+3400 val perplexity 51.0287
+3400 train 32.288147 (lr=7.4581e-04) (hash(x)=76851520)
+3410 train 32.572983 (lr=7.3724e-04) (hash(x)=80538862)
+3420 train 31.943865 (lr=7.2874e-04) (hash(x)=75534281)
+3430 train 33.428356 (lr=7.2030e-04) (hash(x)=80968090)
+3440 train 30.942419 (lr=7.1193e-04) (hash(x)=81988449)
+3450 train 30.951815 (lr=7.0363e-04) (hash(x)=67149756)
+3460 train 32.096252 (lr=6.9541e-04) (hash(x)=75371655)
+3470 train 31.864641 (lr=6.8725e-04) (hash(x)=91252896)
+3480 train 31.264151 (lr=6.7916e-04) (hash(x)=81157003)
+3490 train 30.035397 (lr=6.7114e-04) (hash(x)=70648556)
+3500 val loss 3.9276
+3500 val perplexity 50.7832
+3500 train 30.095478 (lr=6.6320e-04) (hash(x)=82835399)
+3510 train 30.858608 (lr=6.5533e-04) (hash(x)=76337725)
+3520 train 30.231667 (lr=6.4753e-04) (hash(x)=72750380)
+3530 train 32.247643 (lr=6.3981e-04) (hash(x)=92694491)
+3540 train 32.460007 (lr=6.3216e-04) (hash(x)=87796051)
+3550 train 32.368217 (lr=6.2458e-04) (hash(x)=84986356)
+3560 train 31.636837 (lr=6.1708e-04) (hash(x)=84701151)
+3570 train 31.771643 (lr=6.0966e-04) (hash(x)=74380933)
+3580 train 31.878460 (lr=6.0231e-04) (hash(x)=84853287)
+3590 train 31.162991 (lr=5.9504e-04) (hash(x)=77294073)
+3600 val loss 3.9103
+3600 val perplexity 49.9153
+3600 train 31.195284 (lr=5.8785e-04) (hash(x)=76193378)
+3610 train 31.222033 (lr=5.8074e-04) (hash(x)=80328497)
+3620 train 31.890903 (lr=5.7370e-04) (hash(x)=84600168)
+3630 train 31.204929 (lr=5.6675e-04) (hash(x)=77591381)
+3640 train 31.033089 (lr=5.5987e-04) (hash(x)=72730975)
+3650 train 31.676502 (lr=5.5308e-04) (hash(x)=79725116)
+3660 train 30.507154 (lr=5.4636e-04) (hash(x)=85486383)
+3670 train 30.974043 (lr=5.3973e-04) (hash(x)=77645109)
+3680 train 29.806555 (lr=5.3318e-04) (hash(x)=80077172)
+3690 train 29.463531 (lr=5.2671e-04) (hash(x)=76035882)
+3700 val loss 3.9093
+3700 val perplexity 49.8664
+3700 train 30.563705 (lr=5.2033e-04) (hash(x)=81241654)
+3710 train 32.367451 (lr=5.1402e-04) (hash(x)=80618274)
+3720 train 31.466162 (lr=5.0780e-04) (hash(x)=76084071)
+3730 train 31.890373 (lr=5.0167e-04) (hash(x)=83261144)
+3740 train 33.022507 (lr=4.9562e-04) (hash(x)=84240299)
+3750 train 31.777517 (lr=4.8965e-04) (hash(x)=83166802)
+3760 train 32.127674 (lr=4.8377e-04) (hash(x)=81950683)
+3770 train 31.652536 (lr=4.7798e-04) (hash(x)=81159986)
+3780 train 31.610956 (lr=4.7227e-04) (hash(x)=70146825)
+3790 train 32.413826 (lr=4.6665e-04) (hash(x)=85742517)
+3800 val loss 3.8912
+3800 val perplexity 48.9675
+3800 train 31.364937 (lr=4.6112e-04) (hash(x)=80631642)
+3810 train 31.419365 (lr=4.5567e-04) (hash(x)=79221222)
+3820 train 31.703186 (lr=4.5031e-04) (hash(x)=73382566)
+3830 train 31.263662 (lr=4.4504e-04) (hash(x)=73607760)
+3840 train 30.911791 (lr=4.3986e-04) (hash(x)=88462624)
+3850 train 30.690807 (lr=4.3477e-04) (hash(x)=85445704)
+3860 train 30.796759 (lr=4.2977e-04) (hash(x)=84453421)
+3870 train 30.326805 (lr=4.2486e-04) (hash(x)=77703736)
+3880 train 30.510948 (lr=4.2004e-04) (hash(x)=76861628)
+3890 train 31.787260 (lr=4.1530e-04) (hash(x)=91861367)
+3900 val loss 3.8812
+3900 val perplexity 48.4839
+3900 train 31.071960 (lr=4.1066e-04) (hash(x)=79850346)
+3910 train 30.329073 (lr=4.0611e-04) (hash(x)=74649220)
+3920 train 31.937607 (lr=4.0166e-04) (hash(x)=88170186)
+3930 train 31.741817 (lr=3.9729e-04) (hash(x)=91305485)
+3940 train 30.648165 (lr=3.9302e-04) (hash(x)=78171582)
+3950 train 31.029249 (lr=3.8884e-04) (hash(x)=77128117)
+3960 train 31.269829 (lr=3.8475e-04) (hash(x)=81482093)
+3970 train 31.481092 (lr=3.8076e-04) (hash(x)=87519971)
+3980 train 31.553108 (lr=3.7685e-04) (hash(x)=79088888)
+3990 train 30.220152 (lr=3.7305e-04) (hash(x)=79434846)
+4000 val loss 3.8748
+4000 val perplexity 48.1710
+4000 train 31.013145 (lr=3.6933e-04) (hash(x)=72548798)
+4010 train 30.808750 (lr=3.6572e-04) (hash(x)=80728820)
+4020 train 30.586391 (lr=3.6219e-04) (hash(x)=79040400)
+4030 train 31.367241 (lr=3.5876e-04) (hash(x)=83688774)
+4040 train 30.901951 (lr=3.5543e-04) (hash(x)=83462910)
+4050 train 30.649294 (lr=3.5219e-04) (hash(x)=93956780)
+4060 train 30.717396 (lr=3.4905e-04) (hash(x)=80411353)
+4070 train 31.095230 (lr=3.4600e-04) (hash(x)=84846580)
+4080 train 31.873154 (lr=3.4305e-04) (hash(x)=86067575)
+4090 train 30.719452 (lr=3.4019e-04) (hash(x)=80948397)
+4100 val loss 3.8624
+4100 val perplexity 47.5786
+4100 train 31.213221 (lr=3.3744e-04) (hash(x)=73715841)
+4110 train 30.908978 (lr=3.3477e-04) (hash(x)=79184144)
+4120 train 31.116632 (lr=3.3221e-04) (hash(x)=80443210)
+4130 train 31.420317 (lr=3.2974e-04) (hash(x)=73858890)
+4140 train 31.155661 (lr=3.2737e-04) (hash(x)=74971668)
+4150 train 30.922594 (lr=3.2510e-04) (hash(x)=76620081)
+4160 train 30.047548 (lr=3.2292e-04) (hash(x)=82591676)
+4170 train 29.907494 (lr=3.2085e-04) (hash(x)=78014141)
+4180 train 31.484308 (lr=3.1887e-04) (hash(x)=76985629)
+4190 train 31.568949 (lr=3.1699e-04) (hash(x)=83506406)
+4200 val loss 3.8589
+4200 val perplexity 47.4139
+4200 train 32.545586 (lr=3.1520e-04) (hash(x)=85532796)
+4210 train 30.750162 (lr=3.1352e-04) (hash(x)=86569681)
+4220 train 30.926287 (lr=3.1193e-04) (hash(x)=94963039)
+4230 train 31.329437 (lr=3.1044e-04) (hash(x)=82700738)
+4240 train 31.105350 (lr=3.0905e-04) (hash(x)=75417525)
+4250 train 31.264894 (lr=3.0776e-04) (hash(x)=74924678)
+4260 train 31.448677 (lr=3.0657e-04) (hash(x)=71385873)
+4270 train 30.426973 (lr=3.0548e-04) (hash(x)=80042762)
+4280 train 30.783865 (lr=3.0449e-04) (hash(x)=75931192)
+4290 train 29.930367 (lr=3.0359e-04) (hash(x)=83292525)
+4300 val loss 3.8587
+4300 val perplexity 47.4042
+4300 train 30.731203 (lr=3.0280e-04) (hash(x)=93079234)
+4310 train 30.414040 (lr=3.0210e-04) (hash(x)=77127265)
+4320 train 30.719185 (lr=3.0150e-04) (hash(x)=80691627)
+4330 train 31.227200 (lr=3.0101e-04) (hash(x)=74571796)
+4340 train 30.589293 (lr=3.0061e-04) (hash(x)=77693727)
+4350 train 30.768841 (lr=3.0031e-04) (hash(x)=66605379)
+4360 train 30.538492 (lr=3.0011e-04) (hash(x)=81093605)
+4370 train 31.211048 (lr=3.0001e-04) (hash(x)=80765500)
+4374 val loss 3.8511
+4374 val perplexity 47.0428

sdpa_16_spooky/sdpa_64_16_v2_4gpu_seed_1339/model_04374.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:301df75bde5458ff40fca727a65bd5063ad541ee2d8fe738bbf441514d3d8522
+size 127551402

sdpa_16_spooky/sdpa_64_16_v2_4gpu_seed_1339/optimizer_04374.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5708273c5cb50469fc3d175a070ef265c6a511413a2ca48e940add0e6d66ea88
+size 255125126