Upload add_sub_sorl_v1_abs10_K1_25K_2L1H128d

Browse files

Files changed (3) hide show

add_sub_sorl_v1_abs10_K1_25K_2L1H128d/metrics.json +1540 -960
add_sub_sorl_v1_abs10_K1_25K_2L1H128d/model.safetensors +1 -1
add_sub_sorl_v1_abs10_K1_25K_2L1H128d/train_config.json +12 -8

add_sub_sorl_v1_abs10_K1_25K_2L1H128d/metrics.json CHANGED Viewed

@@ -70,444 +70,936 @@
       3719,
       3769,
       3819,
-      3869
     ],
     "loss": [
-      13.454885482788086,
-      11.478792190551758,
-      13.050385475158691,
-      14.577110290527344,
-      14.17066764831543,
-      13.036783218383789,
-      11.886457443237305,
-      10.250679016113281,
-      9.566696166992188,
-      8.962414741516113,
-      8.374322891235352,
-      7.972871780395508,
-      7.557723045349121,
-      7.117671012878418,
-      6.462696075439453,
-      6.050843238830566,
-      5.741342067718506,
-      5.391325950622559,
-      5.264486789703369,
-      4.956804275512695,
-      4.797525405883789,
-      4.500096321105957,
-      4.433718681335449,
-      4.317060470581055,
-      4.279264450073242,
-      4.234098434448242,
-      4.042954921722412,
-      4.035151481628418,
-      4.045926094055176,
-      3.9358861446380615,
-      3.753159761428833,
-      3.7763824462890625,
-      3.8419253826141357,
-      3.6180100440979004,
-      3.3646740913391113,
-      3.16855525970459,
-      3.360339641571045,
-      2.8317813873291016,
-      2.9290781021118164,
-      2.785353660583496,
-      2.54134464263916,
-      2.228949785232544,
-      2.0700716972351074,
-      1.8415426015853882,
-      1.5988508462905884,
-      1.569923758506775,
-      1.2837797403335571,
-      1.1172637939453125,
-      0.7710850834846497,
-      0.6829363107681274,
-      0.49595534801483154,
-      0.17024922370910645,
-      0.21459567546844482,
-      -0.10922026634216309,
-      -0.38342714309692383,
-      -0.10504281520843506,
-      -0.5501455068588257,
-      -0.7160437107086182,
-      -0.707771897315979,
-      -0.9133639335632324,
-      -1.1110361814498901,
-      -0.8491053581237793,
-      -0.9960892200469971,
-      -1.0281953811645508,
-      -1.2926632165908813,
-      -1.2633980512619019,
-      -1.2680187225341797,
-      -1.4050925970077515,
-      -1.2741034030914307,
-      -1.4242706298828125
     ],
     "base_loss": [
-      11.880859375,
-      11.633416175842285,
-      11.221420288085938,
-      10.539042472839355,
-      10.006148338317871,
-      9.441154479980469,
-      8.854585647583008,
-      7.859694480895996,
-      7.329966068267822,
-      6.851498603820801,
-      6.352517127990723,
-      6.018871784210205,
-      5.628158092498779,
-      5.2450385093688965,
-      4.57428503036499,
-      4.198834419250488,
-      3.868682861328125,
-      3.5653653144836426,
-      3.412782907485962,
-      3.165587902069092,
-      2.9751205444335938,
-      2.7248756885528564,
-      2.6281933784484863,
-      2.598451852798462,
-      2.5072898864746094,
-      2.5161983966827393,
-      2.393761157989502,
-      2.3599026203155518,
-      2.3197877407073975,
-      2.3430163860321045,
-      2.2583587169647217,
-      2.2844526767730713,
-      2.254804849624634,
-      2.290438413619995,
-      2.2097489833831787,
-      2.197057008743286,
-      2.2262990474700928,
-      2.1319544315338135,
-      2.107748031616211,
-      2.173170328140259,
-      2.1438701152801514,
-      2.181342363357544,
-      2.0934159755706787,
-      2.126227617263794,
-      2.09466814994812,
-      2.0434868335723877,
-      2.1309261322021484,
-      2.075498342514038,
-      2.1343188285827637,
-      2.0033481121063232,
-      2.0886590480804443,
-      2.092684507369995,
-      2.020324945449829,
-      1.984065055847168,
-      2.0311124324798584,
-      2.026329755783081,
-      1.9996259212493896,
-      2.0779316425323486,
-      1.9736812114715576,
-      2.030266284942627,
-      2.1000616550445557,
-      2.038893461227417,
-      1.9640614986419678,
-      2.036902904510498,
-      2.0785582065582275,
-      2.0229272842407227,
-      2.0268771648406982,
-      2.0155155658721924,
-      1.9266163110733032,
-      2.076498508453369
     ],
     "info_loss": [
-      -0.7863264083862305,
-      -0.9126081466674805,
-      -0.61785888671875,
-      -0.2595643997192383,
-      -0.1060333251953125,
-      -0.05567169189453125,
-      -0.03524017333984375,
-      -0.020551681518554688,
-      -0.01478719711303711,
-      -0.013951778411865234,
-      -0.01464700698852539,
-      -0.015613079071044922,
-      -0.013948440551757812,
-      -0.016277790069580078,
-      -0.010252952575683594,
-      -0.012321949005126953,
-      -0.008607149124145508,
-      -0.012037992477416992,
-      -0.008737802505493164,
-      -0.013904571533203125,
-      -0.010097980499267578,
-      -0.013910055160522461,
-      -0.010562896728515625,
-      -0.018952369689941406,
-      -0.013309478759765625,
-      -0.018421173095703125,
-      -0.025274276733398438,
-      -0.02254462242126465,
-      -0.01696610450744629,
-      -0.03021550178527832,
-      -0.03984832763671875,
-      -0.0401759147644043,
-      -0.030680418014526367,
-      -0.05649089813232422,
-      -0.07359719276428223,
-      -0.09176802635192871,
-      -0.07550787925720215,
-      -0.11891603469848633,
-      -0.10670852661132812,
-      -0.1276233196258545,
-      -0.14898061752319336,
-      -0.1840728521347046,
-      -0.19084644317626953,
-      -0.2173391580581665,
-      -0.2383211851119995,
-      -0.23623204231262207,
-      -0.27347636222839355,
-      -0.28460466861724854,
-      -0.32501447200775146,
-      -0.32063066959381104,
-      -0.34785377979278564,
-      -0.38096821308135986,
-      -0.3693283796310425,
-      -0.39820408821105957,
-      -0.4302417039871216,
-      -0.40181398391723633,
-      -0.4437370300292969,
-      -0.46821415424346924,
-      -0.4569128751754761,
-      -0.4830566644668579,
-      -0.5098972320556641,
-      -0.4775048494338989,
-      -0.4846118688583374,
-      -0.4952908754348755,
-      -0.5258979797363281,
-      -0.5176137685775757,
-      -0.518396258354187,
-      -0.5309580564498901,
-      -0.5087277889251709,
-      -0.5388277769088745
     ],
     "abs_loss": [
-      2.3025481700897217,
-      2.2900214195251465,
-      2.254666566848755,
-      2.190378189086914,
-      2.0967636108398438,
-      2.037475347518921,
-      1.9807038307189941,
-      1.8861985206604004,
-      1.8569782972335815,
-      1.824637532234192,
-      1.8228012323379517,
-      1.818422794342041,
-      1.8206357955932617,
-      1.822617530822754,
-      1.804533839225769,
-      1.8139501810073853,
-      1.793150544166565,
-      1.7828887701034546,
-      1.8064988851547241,
-      1.8026375770568848,
-      1.797040343284607,
-      1.8011388778686523,
-      1.8015633821487427,
-      1.8015815019607544,
-      1.799573302268982,
-      1.7933950424194336,
-      1.805014967918396,
-      1.8085817098617554,
-      1.7863329648971558,
-      1.7919312715530396,
-      1.783199429512024,
-      1.7947336435317993,
-      1.8047763109207153,
-      1.8011966943740845,
-      1.788076400756836,
-      1.7798686027526855,
-      1.7843881845474243,
-      1.7868894338607788,
-      1.7851332426071167,
-      1.7916141748428345,
-      1.7841373682022095,
-      1.7945789098739624,
-      1.7695688009262085,
-      1.8075765371322632,
-      1.8012295961380005,
-      1.8081436157226562,
-      1.7965914011001587,
-      1.7999669313430786,
-      1.7932742834091187,
-      1.7773023843765259,
-      1.7790831327438354,
-      1.7893775701522827,
-      1.7931427955627441,
-      1.8046531677246094,
-      1.7975425720214844,
-      1.7841027975082397,
-      1.7952299118041992,
-      1.8000589609146118,
-      1.7956489324569702,
-      1.7838338613510132,
-      1.7932010889053345,
-      1.785744547843933,
-      1.773536205291748,
-      1.7905513048171997,
-      1.792868971824646,
-      1.8099616765975952,
-      1.801303505897522,
-      1.800384521484375,
-      1.7764925956726074,
-      1.7878471612930298
     ],
     "zipf_loss": [
-      9.207036018371582,
-      8.74245548248291,
-      7.782087326049805,
-      6.414673805236816,
-      5.0151753425598145,
-      3.9485976696014404,
-      3.1862025260925293,
-      2.4078822135925293,
-      2.198904514312744,
-      2.067970037460327,
-      1.9859960079193115,
-      1.928288459777832,
-      1.8869855403900146,
-      1.853148341178894,
-      1.8104872703552246,
-      1.7938334941864014,
-      1.7794156074523926,
-      1.7680517435073853,
-      1.7584320306777954,
-      1.7499982118606567,
-      1.743680715560913,
-      1.7342076301574707,
-      1.730997920036316,
-      1.7279741764068604,
-      1.7251121997833252,
-      1.7227725982666016,
-      1.7214350700378418,
-      1.7198371887207031,
-      1.7171660661697388,
-      1.7158317565917969,
-      1.7149643898010254,
-      1.7142155170440674,
-      1.713447093963623,
-      1.7123608589172363,
-      1.7120893001556396,
-      1.7111916542053223,
-      1.7106807231903076,
-      1.7102984189987183,
-      1.7099019289016724,
-      1.7092549800872803,
-      1.7088669538497925,
-      1.7088780403137207,
-      1.7081632614135742,
-      1.707948923110962,
-      1.7072715759277344,
-      1.7079429626464844,
-      1.7079581022262573,
-      1.707815408706665,
-      1.7075835466384888,
-      1.7081646919250488,
-      1.707925796508789,
-      1.7083090543746948,
-      1.708240270614624,
-      1.7082902193069458,
-      1.708122968673706,
-      1.7083569765090942,
-      1.7080758810043335,
-      1.708160161972046,
-      1.7081109285354614,
-      1.7085528373718262,
-      1.7085543870925903,
-      1.70847487449646,
-      1.7086145877838135,
-      1.7087550163269043,
-      1.708471417427063,
-      1.708816409111023,
-      1.7089366912841797,
-      1.7089341878890991,
-      1.708909034729004,
-      1.7087242603302002
     ],
     "denoise_loss": [],
     "ortho_loss": [
-      0.6665944457054138,
-      0.6689225435256958,
-      0.6665652394294739,
-      0.5638207197189331,
-      0.5186919569969177,
-      0.4937678277492523,
-      0.440208375453949,
-      0.44368138909339905,
-      0.4490883946418762,
-      0.46086809039115906,
-      0.47744476795196533,
-      0.4832924008369446,
-      0.5013949871063232,
-      0.49298223853111267,
-      0.48781320452690125,
-      0.4863438904285431,
-      0.4936782121658325,
-      0.49258145689964294,
-      0.49533185362815857,
-      0.507675290107727,
-      0.5198795795440674,
-      0.49237918853759766,
-      0.5015400648117065,
-      0.4975549578666687,
-      0.4957054555416107,
-      0.49398577213287354,
-      0.507213294506073,
-      0.5123801827430725,
-      0.5207350254058838,
-      0.5060194134712219,
-      0.5115739107131958,
-      0.5285096764564514,
-      0.5283535122871399,
-      0.524570643901825,
-      0.5327265858650208,
-      0.549440324306488,
-      0.5494626760482788,
-      0.558219313621521,
-      0.5633650422096252,
-      0.5702787637710571,
-      0.5571073293685913,
-      0.5566506385803223,
-      0.5707675814628601,
-      0.5843377113342285,
-      0.584276556968689,
-      0.5893025994300842,
-      0.6109833121299744,
-      0.6101059317588806,
-      0.6098511815071106,
-      0.6212247610092163,
-      0.6309865713119507,
-      0.6344567537307739,
-      0.6354795098304749,
-      0.647710919380188,
-      0.6540753841400146,
-      0.6593545079231262,
-      0.6664237976074219,
-      0.6748871803283691,
-      0.6771064400672913,
-      0.6822501420974731,
-      0.6884104013442993,
-      0.6926484704017639,
-      0.6983345150947571,
-      0.7035016417503357,
-      0.7052139639854431,
-      0.7094832062721252,
-      0.7137283682823181,
-      0.7142598628997803,
-      0.7140603065490723,
-      0.716733455657959
     ],
     "lr": [
-      8.376068376068378e-06,
-      1.6923076923076924e-05,
       2e-05,
       2e-05,
       2e-05,
@@ -548,34 +1040,102 @@
       2e-05,
       2e-05,
       2e-05,
-      1.9973899288162407e-05,
-      1.9380701291853413e-05,
-      1.8787503295544426e-05,
-      1.8194305299235432e-05,
-      1.7601107302926442e-05,
-      1.7007909306617455e-05,
-      1.6414711310308464e-05,
-      1.5335090957026102e-05,
-      1.474189296071711e-05,
-      1.4148694964408121e-05,
-      1.355549696809913e-05,
-      1.2962298971790142e-05,
-      1.2369100975481152e-05,
-      1.1775902979172158e-05,
-      1.0696282625889797e-05,
-      1.0103084629580805e-05,
-      9.50988663327182e-06,
-      8.91668863696283e-06,
-      8.323490640653837e-06,
-      7.730292644344845e-06,
-      7.137094648035855e-06,
-      6.057474294753492e-06,
-      5.4642762984445065e-06,
-      4.871078302135514e-06,
-      4.277880305826523e-06,
-      3.684682309517532e-06,
-      3.091484313208541e-06,
-      2.4982863168995496e-06
     ],
     "emb_lr": [],
     "eval_step": [
@@ -588,588 +1148,608 @@
       2696,
       3087,
       3478,
-      3869
     ],
     "eval_accuracy": [
       0.01,
       0.0,
-      0.0,
       0.01,
-      0.0,
-      0.0,
       0.01,
-      0.0,
-      0.0,
       0.01
     ]
   },
-  "final_accuracy": 0.020416666666666666,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.3457142857142857,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.22372881355932203,
-            "count": 295
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 55
           }
         }
       },
       "add_S1": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.19714285714285715,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.25396825396825395,
-            "count": 126
           },
           "SC": {
-            "accuracy": 0.05063291139240506,
-            "count": 79
           },
           "SS": {
-            "accuracy": 0.8571428571428571,
-            "count": 21
           },
           "UC": {
-            "accuracy": 0.12096774193548387,
-            "count": 124
           }
         }
       },
       "add_S2": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.37142857142857144,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.48,
-            "count": 75
           },
           "SC": {
-            "accuracy": 0.08064516129032258,
-            "count": 62
           },
           "SS": {
-            "accuracy": 0.8717948717948718,
-            "count": 39
           },
           "UC": {
-            "accuracy": 0.23423423423423423,
-            "count": 111
           },
           "US": {
-            "accuracy": 0.4603174603174603,
-            "count": 63
           }
         }
       },
       "add_S3": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.3057142857142857,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.5833333333333334,
-            "count": 60
           },
           "SC": {
-            "accuracy": 0.05263157894736842,
-            "count": 57
           },
           "SS": {
-            "accuracy": 0.8947368421052632,
-            "count": 19
           },
           "UC": {
-            "accuracy": 0.16346153846153846,
-            "count": 104
           },
           "US": {
-            "accuracy": 0.3181818181818182,
-            "count": 110
           }
         }
       },
       "add_S4": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.29714285714285715,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.5208333333333334,
-            "count": 48
           },
           "SC": {
-            "accuracy": 0.057692307692307696,
-            "count": 52
           },
           "SS": {
-            "accuracy": 0.7142857142857143,
-            "count": 7
           },
           "UC": {
-            "accuracy": 0.21348314606741572,
-            "count": 89
           },
           "US": {
-            "accuracy": 0.33766233766233766,
-            "count": 154
           }
         }
       },
       "add_S5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.09142857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.5,
-            "count": 50
           },
           "SC": {
-            "accuracy": 0.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.06,
-            "count": 50
           },
           "US": {
-            "accuracy": 0.02,
-            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.12,
-        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.14,
-            "count": 50
           },
           "US": {
-            "accuracy": 0.14,
-            "count": 250
           }
         }
       },
       "add_random": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.19857142857142857,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.2482598607888631,
-            "count": 431
           },
           "SC": {
-            "accuracy": 0.05379746835443038,
-            "count": 316
           },
           "SS": {
-            "accuracy": 0.8461538461538461,
-            "count": 39
           },
           "UC": {
-            "accuracy": 0.17142857142857143,
-            "count": 560
           },
           "US": {
-            "accuracy": 0.46296296296296297,
-            "count": 54
           }
         }
       },
       "add_C1": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.12285714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.172,
-            "count": 250
           },
           "SC": {
             "accuracy": 0.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.0,
-            "count": 50
           }
         }
       },
       "add_C2": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.12285714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.19,
-            "count": 200
           },
           "SC": {
-            "accuracy": 0.02,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.04819277108433735,
-            "count": 83
           },
           "US": {
-            "accuracy": 0.0,
-            "count": 17
           }
         }
       },
       "add_C3": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.13428571428571429,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.24666666666666667,
-            "count": 150
           },
           "SC": {
-            "accuracy": 0.0,
-            "count": 50
-          },
-          "UC": {
             "accuracy": 0.06,
             "count": 100
           },
           "US": {
-            "accuracy": 0.08,
-            "count": 50
           }
         }
       },
       "add_C4": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.12857142857142856,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.35,
-            "count": 100
           },
           "SC": {
-            "accuracy": 0.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.03787878787878788,
-            "count": 132
           },
           "US": {
-            "accuracy": 0.07352941176470588,
-            "count": 68
           }
         }
       },
       "add_C5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.21428571428571427,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.36,
-            "count": 50
           },
           "SC": {
-            "accuracy": 0.04,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.13013698630136986,
-            "count": 146
           },
           "US": {
-            "accuracy": 0.34615384615384615,
-            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.3,
-        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.20105820105820105,
-            "count": 189
           },
           "US": {
-            "accuracy": 0.6036036036036037,
-            "count": 111
           }
         }
       },
       "sub_M0": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.3057142857142857,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.19801980198019803,
-            "count": 303
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 47
           }
         }
       },
       "sub_M1": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.26285714285714284,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.3900709219858156,
-            "count": 141
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 72
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 18
           },
           "UB": {
-            "accuracy": 0.15966386554621848,
-            "count": 119
           }
         }
       },
       "sub_M2": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.38285714285714284,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.6428571428571429,
-            "count": 112
           },
           "MB": {
-            "accuracy": 0.0,
-            "count": 53
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 47
           },
           "UB": {
-            "accuracy": 0.17647058823529413,
-            "count": 85
           },
           "UD": {
-            "accuracy": 0.0,
-            "count": 53
           }
         }
       },
       "sub_M3": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.28,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.6494845360824743,
-            "count": 97
           },
           "MB": {
-            "accuracy": 0.0,
-            "count": 51
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 27
           },
           "UB": {
-            "accuracy": 0.10810810810810811,
-            "count": 74
           },
           "UD": {
-            "accuracy": 0.0,
-            "count": 101
           }
         }
       },
       "sub_M4": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.21142857142857144,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.5,
-            "count": 100
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.48,
-            "count": 50
           },
           "UD": {
-            "accuracy": 0.0,
-            "count": 150
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.18285714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
           },
           "MB": {
-            "accuracy": 0.0,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.28,
-            "count": 50
           },
           "UD": {
-            "accuracy": 0.0,
-            "count": 200
           }
         }
       },
       "sub_random": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.2307142857142857,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.37719298245614036,
-            "count": 570
           },
           "MB": {
-            "accuracy": 0.0,
-            "count": 277
           },
           "ME": {
-            "accuracy": 1.0,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.11677282377919321,
-            "count": 471
           },
           "UD": {
-            "accuracy": 0.0,
-            "count": 29
           }
         }
       },
       "sub_B3": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.18857142857142858,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.3333333333333333,
-            "count": 150
           },
           "MB": {
-            "accuracy": 0.0,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.15841584158415842,
-            "count": 101
           },
           "UD": {
-            "accuracy": 0.0,
-            "count": 49
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.17714285714285713,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.5,
-            "count": 100
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.09917355371900827,
-            "count": 121
           },
           "UD": {
-            "accuracy": 0.0,
-            "count": 79
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.18,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
           },
           "MB": {
-            "accuracy": 0.0,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.08552631578947369,
-            "count": 152
           },
           "UD": {
-            "accuracy": 0.0,
-            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.0,
-      "digit_accuracy": 0.22114285714285714,
-      "total_examples": 1500,
       "n_splits": 24
     }
   },
@@ -1179,569 +1759,569 @@
       "K": 1,
       "mode": "sorl",
       "n_digits": 6,
-      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.08,
-        "digit_accuracy": 0.7114285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.688135593220339,
-            "count": 295
           },
           "SS": {
-            "accuracy": 0.8363636363636363,
-            "count": 55
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.6228571428571429,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.7142857142857143,
-            "count": 126
           },
           "SC": {
-            "accuracy": 0.7215189873417721,
-            "count": 79
           },
           "SS": {
-            "accuracy": 0.8571428571428571,
-            "count": 21
           },
           "UC": {
-            "accuracy": 0.4274193548387097,
-            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.5857142857142857,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.7333333333333333,
-            "count": 75
           },
           "SC": {
-            "accuracy": 0.7096774193548387,
-            "count": 62
           },
           "SS": {
-            "accuracy": 0.7435897435897436,
-            "count": 39
           },
           "UC": {
-            "accuracy": 0.36936936936936937,
-            "count": 111
           },
           "US": {
-            "accuracy": 0.5714285714285714,
-            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.5485714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.7833333333333333,
-            "count": 60
           },
           "SC": {
-            "accuracy": 0.631578947368421,
-            "count": 57
           },
           "SS": {
-            "accuracy": 0.7894736842105263,
-            "count": 19
           },
           "UC": {
-            "accuracy": 0.3942307692307692,
-            "count": 104
           },
           "US": {
-            "accuracy": 0.4818181818181818,
-            "count": 110
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.04,
-        "digit_accuracy": 0.46,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.7708333333333334,
-            "count": 48
           },
           "SC": {
-            "accuracy": 0.6923076923076923,
-            "count": 52
           },
           "SS": {
-            "accuracy": 0.7142857142857143,
-            "count": 7
           },
           "UC": {
-            "accuracy": 0.3258426966292135,
-            "count": 89
           },
           "US": {
-            "accuracy": 0.35064935064935066,
-            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.3314285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.68,
-            "count": 50
           },
           "SC": {
-            "accuracy": 0.72,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.12,
-            "count": 50
           },
           "US": {
-            "accuracy": 0.2,
-            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.09142857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.52,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.0,
-            "count": 50
           },
           "US": {
-            "accuracy": 0.024,
-            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.025,
-        "digit_accuracy": 0.5957142857142858,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.6682134570765661,
-            "count": 431
           },
           "SC": {
-            "accuracy": 0.6930379746835443,
-            "count": 316
           },
           "SS": {
-            "accuracy": 0.8974358974358975,
-            "count": 39
           },
           "UC": {
-            "accuracy": 0.4589285714285714,
-            "count": 560
           },
           "US": {
-            "accuracy": 0.6481481481481481,
-            "count": 54
           }
         }
       },
       "add_C1": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.7228571428571429,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.82,
-            "count": 250
           },
           "SC": {
-            "accuracy": 0.78,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.18,
-            "count": 50
           }
         }
       },
       "add_C2": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.6485714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.81,
-            "count": 200
           },
           "SC": {
-            "accuracy": 0.76,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.20481927710843373,
-            "count": 83
           },
           "US": {
-            "accuracy": 0.5882352941176471,
-            "count": 17
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.04,
-        "digit_accuracy": 0.62,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.8,
-            "count": 150
           },
           "SC": {
-            "accuracy": 0.78,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.28,
-            "count": 100
           },
           "US": {
-            "accuracy": 0.6,
-            "count": 50
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.52,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.76,
-            "count": 100
           },
           "SC": {
-            "accuracy": 0.68,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.15151515151515152,
-            "count": 132
           },
           "US": {
-            "accuracy": 0.7647058823529411,
-            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.5,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.76,
-            "count": 50
           },
           "SC": {
-            "accuracy": 0.72,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.2465753424657534,
-            "count": 146
           },
           "US": {
-            "accuracy": 0.625,
-            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.44571428571428573,
-        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.66,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.37037037037037035,
-            "count": 189
           },
           "US": {
-            "accuracy": 0.4774774774774775,
-            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.06,
-        "digit_accuracy": 0.6371428571428571,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.5808580858085809,
-            "count": 303
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.5828571428571429,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.6453900709219859,
-            "count": 141
           },
           "MB": {
-            "accuracy": 0.4027777777777778,
-            "count": 72
           },
           "ME": {
-            "accuracy": 0.8888888888888888,
-            "count": 18
           },
           "UB": {
-            "accuracy": 0.5714285714285714,
-            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.5885714285714285,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.8035714285714286,
-            "count": 112
           },
           "MB": {
-            "accuracy": 0.33962264150943394,
-            "count": 53
           },
           "ME": {
-            "accuracy": 0.9574468085106383,
-            "count": 47
           },
           "UB": {
-            "accuracy": 0.5058823529411764,
-            "count": 85
           },
           "UD": {
-            "accuracy": 0.18867924528301888,
-            "count": 53
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.44857142857142857,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.7628865979381443,
-            "count": 97
           },
           "MB": {
-            "accuracy": 0.19607843137254902,
-            "count": 51
           },
           "ME": {
-            "accuracy": 0.9629629629629629,
-            "count": 27
           },
           "UB": {
-            "accuracy": 0.5675675675675675,
-            "count": 74
           },
           "UD": {
-            "accuracy": 0.04950495049504951,
-            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.3485714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.62,
-            "count": 100
           },
           "MB": {
-            "accuracy": 0.18,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.78,
-            "count": 50
           },
           "UD": {
-            "accuracy": 0.08,
-            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.28285714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
           },
           "MB": {
-            "accuracy": 0.02,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.74,
-            "count": 50
           },
           "UD": {
-            "accuracy": 0.055,
-            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.025,
-        "digit_accuracy": 0.5657142857142857,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.6385964912280702,
-            "count": 570
           },
           "MB": {
-            "accuracy": 0.45126353790613716,
-            "count": 277
           },
           "ME": {
-            "accuracy": 0.9245283018867925,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.5329087048832272,
-            "count": 471
           },
           "UD": {
-            "accuracy": 0.10344827586206896,
-            "count": 29
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.4542857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.5666666666666667,
-            "count": 150
           },
           "MB": {
-            "accuracy": 0.36,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.5148514851485149,
-            "count": 101
           },
           "UD": {
-            "accuracy": 0.08163265306122448,
-            "count": 49
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.4342857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.6,
-            "count": 100
           },
           "MB": {
-            "accuracy": 0.36,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.4793388429752066,
-            "count": 121
           },
           "UD": {
-            "accuracy": 0.20253164556962025,
-            "count": 79
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.46,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
           },
           "MB": {
-            "accuracy": 0.2,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.4407894736842105,
-            "count": 152
           },
           "UD": {
-            "accuracy": 0.3469387755102041,
-            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.019333333333333334,
-      "digit_accuracy": 0.522952380952381,
-      "total_examples": 1500,
       "n_splits": 24
     }
   },
-  "sorl_overall_accuracy": 0.020416666666666666,
-  "sft_overall_accuracy": 0.0
 }

       3719,
       3769,
       3819,
+      3869,
+      3960,
+      4010,
+      4060,
+      4110,
+      4160,
+      4210,
+      4260,
+      4351,
+      4401,
+      4451,
+      4501,
+      4551,
+      4601,
+      4651,
+      4742,
+      4792,
+      4842,
+      4892,
+      4942,
+      4992,
+      5042,
+      5133,
+      5183,
+      5233,
+      5283,
+      5333,
+      5383,
+      5433,
+      5524,
+      5574,
+      5624,
+      5674,
+      5724,
+      5774,
+      5824,
+      5915,
+      5965,
+      6015,
+      6065,
+      6115,
+      6165,
+      6215,
+      6306,
+      6356,
+      6406,
+      6456,
+      6506,
+      6556,
+      6606,
+      6697,
+      6747,
+      6797,
+      6847,
+      6897,
+      6947,
+      6997,
+      7088,
+      7138,
+      7188,
+      7238,
+      7288,
+      7338,
+      7388,
+      7479,
+      7529,
+      7579,
+      7629,
+      7679,
+      7729,
+      7779
     ],
     "loss": [
+      15.438215255737305,
+      11.698705673217773,
+      11.35198974609375,
+      11.583304405212402,
+      13.52648639678955,
+      14.204401016235352,
+      12.998001098632812,
+      11.166366577148438,
+      10.07100772857666,
+      9.512551307678223,
+      8.862318992614746,
+      8.31992244720459,
+      7.685413837432861,
+      7.327797889709473,
+      6.562270641326904,
+      6.250687599182129,
+      5.861550331115723,
+      5.564685344696045,
+      5.315859794616699,
+      5.033608436584473,
+      4.932528972625732,
+      4.55800724029541,
+      4.5580596923828125,
+      4.372578144073486,
+      4.254790306091309,
+      4.210170745849609,
+      4.063313007354736,
+      4.026899814605713,
+      4.05972957611084,
+      3.9455442428588867,
+      3.8566818237304688,
+      3.797435760498047,
+      3.6889798641204834,
+      3.740785837173462,
+      3.5677337646484375,
+      3.458096504211426,
+      3.2327089309692383,
+      3.0761547088623047,
+      2.7873401641845703,
+      2.8123507499694824,
+      2.591607093811035,
+      2.6727454662323,
+      2.263127088546753,
+      2.2004404067993164,
+      1.9771809577941895,
+      1.6977636814117432,
+      1.3677500486373901,
+      1.4016914367675781,
+      1.1960985660552979,
+      0.7710381150245667,
+      0.9146587252616882,
+      0.6109359264373779,
+      0.4325253963470459,
+      0.3495136499404907,
+      -0.13719046115875244,
+      -0.3947882652282715,
+      -0.37380528450012207,
+      -0.5639680624008179,
+      -0.7665214538574219,
+      -1.1608381271362305,
+      -1.1922835111618042,
+      -1.4211184978485107,
+      -1.504981517791748,
+      -1.635431170463562,
+      -2.1759157180786133,
+      -1.9983330965042114,
+      -2.054861545562744,
+      -2.627793312072754,
+      -2.457620143890381,
+      -3.170637369155884,
+      -2.9656014442443848,
+      -2.9379959106445312,
+      -3.5171995162963867,
+      -3.3045692443847656,
+      -3.2605278491973877,
+      -3.6454877853393555,
+      -3.7729570865631104,
+      -3.7499589920043945,
+      -4.199748516082764,
+      -3.9465160369873047,
+      -4.559499740600586,
+      -4.674802780151367,
+      -4.359200477600098,
+      -4.95430326461792,
+      -5.200588226318359,
+      -5.365489959716797,
+      -5.439723491668701,
+      -5.131085395812988,
+      -5.364423751831055,
+      -5.8407769203186035,
+      -5.78155517578125,
+      -5.790386199951172,
+      -6.290831089019775,
+      -6.478070259094238,
+      -6.362205505371094,
+      -6.496718406677246,
+      -6.589505672454834,
+      -6.66395902633667,
+      -7.2407732009887695,
+      -6.973321914672852,
+      -7.167119979858398,
+      -7.098984718322754,
+      -7.72056770324707,
+      -7.13163948059082,
+      -7.3493571281433105,
+      -7.345783233642578,
+      -7.536464691162109,
+      -7.644423484802246,
+      -7.953723907470703,
+      -7.990867614746094,
+      -7.49429988861084,
+      -7.873161792755127,
+      -8.245901107788086,
+      -7.67487907409668,
+      -8.254423141479492,
+      -8.392303466796875,
+      -8.373635292053223,
+      -8.042476654052734,
+      -8.853918075561523,
+      -8.413291931152344,
+      -8.181249618530273,
+      -8.420297622680664,
+      -8.319875717163086,
+      -9.00948429107666,
+      -8.943597793579102,
+      -8.881410598754883,
+      -8.481571197509766,
+      -8.24708366394043,
+      -8.969070434570312,
+      -9.175048828125,
+      -9.334626197814941,
+      -8.908629417419434,
+      -8.629892349243164,
+      -8.761899948120117,
+      -8.947555541992188,
+      -9.112472534179688,
+      -8.871394157409668,
+      -8.744105339050293,
+      -9.35738754272461,
+      -8.966874122619629
     ],
     "base_loss": [
+      11.9345703125,
+      11.817322731018066,
+      11.699155807495117,
+      11.364814758300781,
+      10.792383193969727,
+      10.061384201049805,
+      9.513914108276367,
+      8.453359603881836,
+      7.739280700683594,
+      7.290415287017822,
+      6.789176940917969,
+      6.299552917480469,
+      5.759676456451416,
+      5.421372413635254,
+      4.712368965148926,
+      4.4211649894714355,
+      4.006752967834473,
+      3.7291598320007324,
+      3.5219593048095703,
+      3.2360665798187256,
+      3.1453914642333984,
+      2.8143575191497803,
+      2.7994096279144287,
+      2.621875047683716,
+      2.5696024894714355,
+      2.534588098526001,
+      2.4512345790863037,
+      2.396812915802002,
+      2.3602030277252197,
+      2.3069894313812256,
+      2.3317739963531494,
+      2.2865636348724365,
+      2.288856267929077,
+      2.304441213607788,
+      2.1969876289367676,
+      2.251948118209839,
+      2.1730990409851074,
+      2.1580147743225098,
+      2.1613924503326416,
+      2.200089693069458,
+      2.163917303085327,
+      2.1799607276916504,
+      2.1372125148773193,
+      2.189011335372925,
+      2.124418258666992,
+      2.116373300552368,
+      2.096065044403076,
+      2.1109848022460938,
+      2.1218228340148926,
+      2.142543077468872,
+      2.032945156097412,
+      2.0584824085235596,
+      2.0884969234466553,
+      2.0404374599456787,
+      2.1102449893951416,
+      2.1072633266448975,
+      2.0765204429626465,
+      2.0333292484283447,
+      2.0203804969787598,
+      2.087628126144409,
+      2.0504987239837646,
+      2.02152681350708,
+      2.0398547649383545,
+      1.9614677429199219,
+      2.066767454147339,
+      2.0372583866119385,
+      1.978596568107605,
+      2.0089972019195557,
+      2.0185484886169434,
+      2.0397589206695557,
+      2.0083587169647217,
+      1.9470815658569336,
+      1.9792274236679077,
+      1.9839227199554443,
+      1.9301831722259521,
+      1.9690905809402466,
+      1.9336986541748047,
+      1.8985034227371216,
+      1.9798091650009155,
+      1.8981035947799683,
+      1.983641505241394,
+      2.005276918411255,
+      1.9155381917953491,
+      1.9187160730361938,
+      1.9617462158203125,
+      1.9528653621673584,
+      1.993743658065796,
+      1.8443126678466797,
+      1.8498674631118774,
+      1.9844404458999634,
+      1.9315986633300781,
+      1.8545875549316406,
+      1.9561283588409424,
+      1.9475451707839966,
+      1.8881018161773682,
+      1.904740333557129,
+      1.900530457496643,
+      1.924454689025879,
+      1.9552252292633057,
+      1.8688642978668213,
+      1.9144586324691772,
+      1.8758822679519653,
+      1.9754207134246826,
+      1.840330958366394,
+      1.850648045539856,
+      1.8594962358474731,
+      1.8436225652694702,
+      1.8632913827896118,
+      1.902077555656433,
+      1.9200162887573242,
+      1.8076122999191284,
+      1.8773219585418701,
+      1.8715859651565552,
+      1.8249441385269165,
+      1.876495599746704,
+      1.8286583423614502,
+      1.8903467655181885,
+      1.8312424421310425,
+      1.9122331142425537,
+      1.8325960636138916,
+      1.8598573207855225,
+      1.8170515298843384,
+      1.8049308061599731,
+      1.8960034847259521,
+      1.9015015363693237,
+      1.9183950424194336,
+      1.8224385976791382,
+      1.7621303796768188,
+      1.877859354019165,
+      1.9046906232833862,
+      1.89964759349823,
+      1.851623773574829,
+      1.782663345336914,
+      1.8026123046875,
+      1.8264520168304443,
+      1.8560417890548706,
+      1.8178356885910034,
+      1.7928167581558228,
+      1.8647464513778687,
+      1.832586646080017
     ],
     "info_loss": [
+      -0.5988006591796875,
+      -0.9409332275390625,
+      -0.927983283996582,
+      -0.79962158203125,
+      -0.4240531921386719,
+      -0.13122177124023438,
+      -0.06601619720458984,
+      -0.02817535400390625,
+      -0.021374225616455078,
+      -0.011267662048339844,
+      -0.014470100402832031,
+      -0.011033058166503906,
+      -0.014579296112060547,
+      -0.012886524200439453,
+      -0.01365804672241211,
+      -0.01392221450805664,
+      -0.00992727279663086,
+      -0.010815858840942383,
+      -0.013843536376953125,
+      -0.012883901596069336,
+      -0.013205766677856445,
+      -0.016824722290039062,
+      -0.015227794647216797,
+      -0.015698909759521484,
+      -0.02181529998779297,
+      -0.022611379623413086,
+      -0.028711318969726562,
+      -0.026511430740356445,
+      -0.01956796646118164,
+      -0.02561187744140625,
+      -0.036810874938964844,
+      -0.03793692588806152,
+      -0.04924178123474121,
+      -0.045455217361450195,
+      -0.05187726020812988,
+      -0.0681753158569336,
+      -0.0828087329864502,
+      -0.09702324867248535,
+      -0.12611079216003418,
+      -0.12749648094177246,
+      -0.14578962326049805,
+      -0.13935232162475586,
+      -0.17599689960479736,
+      -0.187652587890625,
+      -0.20319104194641113,
+      -0.23052692413330078,
+      -0.26147031784057617,
+      -0.2596166133880615,
+      -0.28107666969299316,
+      -0.3258247375488281,
+      -0.300618052482605,
+      -0.33340394496917725,
+      -0.354347825050354,
+      -0.35777151584625244,
+      -0.4135168790817261,
+      -0.43906593322753906,
+      -0.4341762065887451,
+      -0.4487084150314331,
+      -0.46751904487609863,
+      -0.513728141784668,
+      -0.513069748878479,
+      -0.5332159996032715,
+      -0.5433926582336426,
+      -0.5485368967056274,
+      -0.6132192611694336,
+      -0.5926066637039185,
+      -0.5921446084976196,
+      -0.6526178121566772,
+      -0.636623740196228,
+      -0.7098889350891113,
+      -0.6862444877624512,
+      -0.6774629354476929,
+      -0.7385284900665283,
+      -0.7177724838256836,
+      -0.7079417705535889,
+      -0.7501206398010254,
+      -0.759358286857605,
+      -0.7535332441329956,
+      -0.8066447973251343,
+      -0.7729413509368896,
+      -0.842991828918457,
+      -0.8565130233764648,
+      -0.8158915042877197,
+      -0.8756431341171265,
+      -0.9046518802642822,
+      -0.920454740524292,
+      -0.9318876266479492,
+      -0.8859519958496094,
+      -0.9098713994026184,
+      -0.9709377288818359,
+      -0.9593663215637207,
+      -0.9529187083244324,
+      -1.013043761253357,
+      -1.0308873653411865,
+      -1.013200044631958,
+      -1.0285481214523315,
+      -1.0371031761169434,
+      -1.046999216079712,
+      -1.1075186729431152,
+      -1.0722583532333374,
+      -1.0959675312042236,
+      -1.0852978229522705,
+      -1.1574437618255615,
+      -1.0848921537399292,
+      -1.1078517436981201,
+      -1.1082909107208252,
+      -1.1257438659667969,
+      -1.138296127319336,
+      -1.1731445789337158,
+      -1.1784374713897705,
+      -1.1174108982086182,
+      -1.162405252456665,
+      -1.19862699508667,
+      -1.136898159980774,
+      -1.2002137899398804,
+      -1.2089275121688843,
+      -1.2131140232086182,
+      -1.1734609603881836,
+      -1.263019323348999,
+      -1.2105844020843506,
+      -1.1900787353515625,
+      -1.2095270156860352,
+      -1.1981027126312256,
+      -1.2763946056365967,
+      -1.270012617111206,
+      -1.265164852142334,
+      -1.215531587600708,
+      -1.1855628490447998,
+      -1.2688469886779785,
+      -1.2921745777130127,
+      -1.3072056770324707,
+      -1.2597821950912476,
+      -1.2257564067840576,
+      -1.2400943040847778,
+      -1.260976791381836,
+      -1.2807002067565918,
+      -1.252894401550293,
+      -1.236717700958252,
+      -1.3059754371643066,
+      -1.2633001804351807
     ],
     "abs_loss": [
+      2.3030121326446533,
+      2.299145460128784,
+      2.289541244506836,
+      2.273411750793457,
+      2.2070724964141846,
+      2.1113064289093018,
+      2.0303194522857666,
+      1.9397281408309937,
+      1.868037223815918,
+      1.8619756698608398,
+      1.8237711191177368,
+      1.8233033418655396,
+      1.8083871603012085,
+      1.8085123300552368,
+      1.807869553565979,
+      1.8163694143295288,
+      1.8033103942871094,
+      1.8110136985778809,
+      1.789840579032898,
+      1.801681399345398,
+      1.7901602983474731,
+      1.79034423828125,
+      1.817492127418518,
+      1.8122094869613647,
+      1.796120047569275,
+      1.8059641122817993,
+      1.7956987619400024,
+      1.7762395143508911,
+      1.796688437461853,
+      1.8057736158370972,
+      1.8015772104263306,
+      1.7790518999099731,
+      1.8092012405395508,
+      1.7991009950637817,
+      1.7919492721557617,
+      1.787245750427246,
+      1.7877745628356934,
+      1.7972835302352905,
+      1.7926057577133179,
+      1.7955513000488281,
+      1.7774804830551147,
+      1.789096474647522,
+      1.7900733947753906,
+      1.8124494552612305,
+      1.7770217657089233,
+      1.7979646921157837,
+      1.795373797416687,
+      1.8022948503494263,
+      1.7857869863510132,
+      1.7971566915512085,
+      1.8082588911056519,
+      1.7944446802139282,
+      1.8030062913894653,
+      1.7937690019607544,
+      1.8005198240280151,
+      1.804343581199646,
+      1.826321005821228,
+      1.8007465600967407,
+      1.79544198513031,
+      1.7963615655899048,
+      1.7872138023376465,
+      1.798779845237732,
+      1.7960869073867798,
+      1.79216730594635,
+      1.7983245849609375,
+      1.810585379600525,
+      1.7835979461669922,
+      1.8006205558776855,
+      1.8101950883865356,
+      1.7974900007247925,
+      1.7953786849975586,
+      1.8133975267410278,
+      1.803335189819336,
+      1.8109596967697144,
+      1.8093476295471191,
+      1.790522575378418,
+      1.8022232055664062,
+      1.7956289052963257,
+      1.7983678579330444,
+      1.785042405128479,
+      1.8020159006118774,
+      1.7974272966384888,
+      1.7881141901016235,
+      1.7824897766113281,
+      1.7926312685012817,
+      1.8120203018188477,
+      1.8074384927749634,
+      1.7968801259994507,
+      1.8008607625961304,
+      1.798738956451416,
+      1.774499773979187,
+      1.8125265836715698,
+      1.8151473999023438,
+      1.809104561805725,
+      1.7991762161254883,
+      1.8214031457901,
+      1.7966961860656738,
+      1.8064117431640625,
+      1.7899032831192017,
+      1.8015222549438477,
+      1.784250259399414,
+      1.7856217622756958,
+      1.788508415222168,
+      1.7838386297225952,
+      1.7966995239257812,
+      1.7942562103271484,
+      1.804557204246521,
+      1.7753878831863403,
+      1.7882212400436401,
+      1.7564178705215454,
+      1.762564778327942,
+      1.7851663827896118,
+      1.7425185441970825,
+      1.7506004571914673,
+      1.7620363235473633,
+      1.7629045248031616,
+      1.736350178718567,
+      1.721484661102295,
+      1.7447915077209473,
+      1.6947892904281616,
+      1.7224642038345337,
+      1.7099981307983398,
+      1.7195545434951782,
+      1.7268810272216797,
+      1.7024202346801758,
+      1.6876293420791626,
+      1.6866716146469116,
+      1.6718778610229492,
+      1.6505346298217773,
+      1.6802678108215332,
+      1.6462732553482056,
+      1.676039695739746,
+      1.6758023500442505,
+      1.6639236211776733,
+      1.667460560798645,
+      1.6574944257736206,
+      1.6825519800186157,
+      1.6431549787521362,
+      1.6516785621643066,
+      1.6596717834472656
     ],
     "zipf_loss": [
+      9.261350631713867,
+      9.060800552368164,
+      8.703712463378906,
+      7.987364292144775,
+      6.753927707672119,
+      5.244103908538818,
+      3.9412174224853516,
+      2.8007874488830566,
+      2.3586654663085938,
+      2.1486148834228516,
+      2.035465717315674,
+      1.948370337486267,
+      1.8906917572021484,
+      1.85443913936615,
+      1.8056950569152832,
+      1.7871077060699463,
+      1.7737386226654053,
+      1.7625828981399536,
+      1.7533515691757202,
+      1.7462127208709717,
+      1.7401790618896484,
+      1.7328624725341797,
+      1.7291789054870605,
+      1.726471185684204,
+      1.7237286567687988,
+      1.7211002111434937,
+      1.7196215391159058,
+      1.7175772190093994,
+      1.7155373096466064,
+      1.7140963077545166,
+      1.7128589153289795,
+      1.7123363018035889,
+      1.7116212844848633,
+      1.710986614227295,
+      1.710323691368103,
+      1.709177017211914,
+      1.7089197635650635,
+      1.7086440324783325,
+      1.7077951431274414,
+      1.70767080783844,
+      1.7078380584716797,
+      1.7073982954025269,
+      1.706876277923584,
+      1.706709861755371,
+      1.7069709300994873,
+      1.7068631649017334,
+      1.7068507671356201,
+      1.7066432237625122,
+      1.7064636945724487,
+      1.707026720046997,
+      1.7070682048797607,
+      1.7070485353469849,
+      1.707206130027771,
+      1.7074145078659058,
+      1.7076815366744995,
+      1.7081732749938965,
+      1.7088041305541992,
+      1.7097123861312866,
+      1.7087442874908447,
+      1.7091789245605469,
+      1.7091935873031616,
+      1.7096366882324219,
+      1.7094817161560059,
+      1.709253191947937,
+      1.7096768617630005,
+      1.7094162702560425,
+      1.7096279859542847,
+      1.7093257904052734,
+      1.7090497016906738,
+      1.7087442874908447,
+      1.7089468240737915,
+      1.7082123756408691,
+      1.708524227142334,
+      1.7081365585327148,
+      1.7077720165252686,
+      1.707575798034668,
+      1.70670485496521,
+      1.70730721950531,
+      1.707053780555725,
+      1.7062891721725464,
+      1.706575632095337,
+      1.7053077220916748,
+      1.7053639888763428,
+      1.7051633596420288,
+      1.7049205303192139,
+      1.7049903869628906,
+      1.7046650648117065,
+      1.7044339179992676,
+      1.704336404800415,
+      1.704285979270935,
+      1.7030590772628784,
+      1.7029609680175781,
+      1.7019639015197754,
+      1.7023484706878662,
+      1.7017755508422852,
+      1.7018823623657227,
+      1.7013258934020996,
+      1.7009366750717163,
+      1.7001982927322388,
+      1.7002460956573486,
+      1.6996716260910034,
+      1.6995488405227661,
+      1.6995980739593506,
+      1.698567271232605,
+      1.6988425254821777,
+      1.6982042789459229,
+      1.6968954801559448,
+      1.6977074146270752,
+      1.6968213319778442,
+      1.6978480815887451,
+      1.6959397792816162,
+      1.6950517892837524,
+      1.6945316791534424,
+      1.6940975189208984,
+      1.6950159072875977,
+      1.692023754119873,
+      1.693522334098816,
+      1.6887413263320923,
+      1.6895627975463867,
+      1.690476894378662,
+      1.6874343156814575,
+      1.6869207620620728,
+      1.6842659711837769,
+      1.6857703924179077,
+      1.6847844123840332,
+      1.6830801963806152,
+      1.6826388835906982,
+      1.6792271137237549,
+      1.6764867305755615,
+      1.6739792823791504,
+      1.6731560230255127,
+      1.6699650287628174,
+      1.6774280071258545,
+      1.670038104057312,
+      1.66901433467865,
+      1.6727375984191895,
+      1.6714589595794678,
+      1.6659390926361084,
+      1.6724532842636108,
+      1.667574167251587
     ],
     "denoise_loss": [],
     "ortho_loss": [
+      0.7279962301254272,
+      0.6579767465591431,
+      0.6817273497581482,
+      0.6108765006065369,
+      0.579704225063324,
+      0.5299669504165649,
+      0.508258581161499,
+      0.44487300515174866,
+      0.45012974739074707,
+      0.47072339057922363,
+      0.4518543779850006,
+      0.4742773473262787,
+      0.48254117369651794,
+      0.49086296558380127,
+      0.5048026442527771,
+      0.4966145157814026,
+      0.49529483914375305,
+      0.5079429745674133,
+      0.49587127566337585,
+      0.5039945244789124,
+      0.5124256610870361,
+      0.5046111345291138,
+      0.5036980509757996,
+      0.5007200837135315,
+      0.4964256286621094,
+      0.47616976499557495,
+      0.4820389449596405,
+      0.4889011085033417,
+      0.4739861488342285,
+      0.4770560562610626,
+      0.48463118076324463,
+      0.48521560430526733,
+      0.47872859239578247,
+      0.47701188921928406,
+      0.47266486287117004,
+      0.4762444496154785,
+      0.49047499895095825,
+      0.49516916275024414,
+      0.49629321694374084,
+      0.49382537603378296,
+      0.5009201169013977,
+      0.514494776725769,
+      0.5163015723228455,
+      0.5229976177215576,
+      0.5241018533706665,
+      0.5448073744773865,
+      0.5465819835662842,
+      0.5636215806007385,
+      0.5807940363883972,
+      0.6014026999473572,
+      0.6071130633354187,
+      0.6144046187400818,
+      0.6215061545372009,
+      0.6127831339836121,
+      0.6225723624229431,
+      0.635498583316803,
+      0.6355625987052917,
+      0.6451646685600281,
+      0.6524924039840698,
+      0.6498976349830627,
+      0.6616995930671692,
+      0.6724307537078857,
+      0.6730102300643921,
+      0.6778948903083801,
+      0.7103477716445923,
+      0.7236534357070923,
+      0.7376241087913513,
+      0.7405605316162109,
+      0.7441432476043701,
+      0.7507919669151306,
+      0.7573267817497253,
+      0.7659734487533569,
+      0.778541088104248,
+      0.7812742590904236,
+      0.7830356359481812,
+      0.7777599692344666,
+      0.7870175838470459,
+      0.7978265881538391,
+      0.7991683483123779,
+      0.8024166822433472,
+      0.8022057414054871,
+      0.8008230328559875,
+      0.7993347644805908,
+      0.795768678188324,
+      0.7919573783874512,
+      0.7857846617698669,
+      0.785811722278595,
+      0.7959153056144714,
+      0.7985197305679321,
+      0.8080344796180725,
+      0.8078311681747437,
+      0.8130998015403748,
+      0.8166013956069946,
+      0.8189990520477295,
+      0.8234202265739441,
+      0.8193479776382446,
+      0.8200980424880981,
+      0.8206661939620972,
+      0.8180730938911438,
+      0.8158308267593384,
+      0.8163653016090393,
+      0.8168421387672424,
+      0.8173116445541382,
+      0.8186611533164978,
+      0.8203118443489075,
+      0.8226667642593384,
+      0.8240026235580444,
+      0.8218040466308594,
+      0.8227177858352661,
+      0.8246716260910034,
+      0.8230711221694946,
+      0.8251593708992004,
+      0.8275840282440186,
+      0.8295618891716003,
+      0.8331630229949951,
+      0.8328480124473572,
+      0.8344019651412964,
+      0.8350807428359985,
+      0.8389624953269958,
+      0.8427870273590088,
+      0.8432278037071228,
+      0.8458070755004883,
+      0.8469033241271973,
+      0.8485611081123352,
+      0.8497316241264343,
+      0.8513529896736145,
+      0.8531062006950378,
+      0.8545234203338623,
+      0.8543184995651245,
+      0.8548187613487244,
+      0.8546685576438904,
+      0.854476273059845,
+      0.856360673904419,
+      0.8580570220947266,
+      0.8584591746330261,
+      0.858634352684021,
+      0.8592213988304138,
+      0.8611096143722534,
+      0.8617076873779297,
+      0.8623132109642029
     ],
     "lr": [
+      4.188034188034189e-06,
+      8.461538461538462e-06,
+      1.2735042735042738e-05,
+      1.700854700854701e-05,
       2e-05,
       2e-05,
       2e-05,
       2e-05,
       2e-05,
       2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      2e-05,
+      1.9967967308199317e-05,
+      1.967136831004482e-05,
+      1.9374769311890326e-05,
+      1.907817031373583e-05,
+      1.8781571315581332e-05,
+      1.848497231742684e-05,
+      1.7945162140785656e-05,
+      1.7648563142631164e-05,
+      1.735196414447667e-05,
+      1.705536514632217e-05,
+      1.675876614816768e-05,
+      1.6462167150013183e-05,
+      1.6165568151858688e-05,
+      1.5625757975217507e-05,
+      1.532915897706301e-05,
+      1.5032559978908518e-05,
+      1.473596098075402e-05,
+      1.4439361982599525e-05,
+      1.4142762984445033e-05,
+      1.3846163986290536e-05,
+      1.3306353809649356e-05,
+      1.300975481149486e-05,
+      1.2713155813340366e-05,
+      1.2416556815185872e-05,
+      1.2119957817031374e-05,
+      1.182335881887688e-05,
+      1.1526759820722382e-05,
+      1.0986949644081204e-05,
+      1.0690350645926705e-05,
+      1.0393751647772212e-05,
+      1.009715264961772e-05,
+      9.800553651463221e-06,
+      9.50395465330873e-06,
+      9.207355655154231e-06,
+      8.66754547851305e-06,
+      8.370946480358558e-06,
+      8.07434748220406e-06,
+      7.777748484049568e-06,
+      7.481149485895069e-06,
+      7.184550487740576e-06,
+      6.887951489586082e-06,
+      6.3481413129448985e-06,
+      6.051542314790406e-06,
+      5.754943316635908e-06,
+      5.4583443184814145e-06,
+      5.161745320326917e-06,
+      4.865146322172423e-06,
+      4.5685473240179305e-06,
+      4.028737147376746e-06,
+      3.732138149222253e-06,
+      3.435539151067755e-06,
+      3.1389401529132617e-06,
+      2.8423411547587686e-06,
+      2.5457421566042704e-06,
+      2.2491431584497773e-06
     ],
     "emb_lr": [],
     "eval_step": [
       2696,
       3087,
       3478,
+      3869,
+      4260,
+      4651,
+      5042,
+      5433,
+      5824,
+      6215,
+      6606,
+      6997,
+      7388,
+      7779
     ],
     "eval_accuracy": [
+      0.02,
+      0.02,
+      0.02,
+      0.02,
+      0.02,
+      0.02,
       0.01,
       0.0,
       0.01,
       0.01,
+      0.01,
+      0.01,
+      0.01,
+      0.01,
+      0.01,
+      0.01,
+      0.01,
+      0.01,
+      0.01,
       0.01
     ]
   },
+  "final_accuracy": 0.20653846153846153,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
       "K": null,
       "mode": "sft",
       "n_digits": 6,
+      "n_per_split": 100
     },
     "splits": {
       "add_S0": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.3142857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.21487603305785125,
+            "count": 605
           },
           "SS": {
+            "accuracy": 0.9473684210526315,
+            "count": 95
           }
         }
       },
       "add_S1": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.22142857142857142,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.24019607843137256,
+            "count": 204
           },
           "SC": {
+            "accuracy": 0.10650887573964497,
+            "count": 169
           },
           "SS": {
+            "accuracy": 0.8387096774193549,
+            "count": 31
           },
           "UC": {
+            "accuracy": 0.20945945945945946,
+            "count": 296
           }
         }
       },
       "add_S2": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.34285714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.3619631901840491,
+            "count": 163
           },
           "SC": {
+            "accuracy": 0.06923076923076923,
+            "count": 130
           },
           "SS": {
+            "accuracy": 0.5632183908045977,
+            "count": 87
           },
           "UC": {
+            "accuracy": 0.32019704433497537,
+            "count": 203
           },
           "US": {
+            "accuracy": 0.49572649572649574,
+            "count": 117
           }
         }
       },
       "add_S3": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.3271428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.3884297520661157,
+            "count": 121
           },
           "SC": {
+            "accuracy": 0.06611570247933884,
+            "count": 121
           },
           "SS": {
+            "accuracy": 0.7346938775510204,
+            "count": 49
           },
           "UC": {
+            "accuracy": 0.27419354838709675,
+            "count": 186
           },
           "US": {
+            "accuracy": 0.3901345291479821,
+            "count": 223
           }
         }
       },
       "add_S4": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.4042857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.3557692307692308,
+            "count": 104
           },
           "SC": {
+            "accuracy": 0.10377358490566038,
+            "count": 106
           },
           "SS": {
+            "accuracy": 0.5652173913043478,
+            "count": 23
           },
           "UC": {
+            "accuracy": 0.38125,
+            "count": 160
           },
           "US": {
+            "accuracy": 0.5244299674267101,
+            "count": 307
           }
         }
       },
       "add_S5": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.13,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.46,
+            "count": 100
           },
           "SC": {
+            "accuracy": 0.01,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.16,
+            "count": 100
           },
           "US": {
+            "accuracy": 0.07,
+            "count": 400
           }
         }
       },
       "add_S6": {
+        "full_accuracy": 0.1,
+        "digit_accuracy": 0.4857142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
+            "accuracy": 0.1,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.58,
+            "count": 100
           },
           "US": {
+            "accuracy": 0.544,
+            "count": 500
           }
         }
       },
       "add_random": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.2307142857142857,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.25279642058165547,
+            "count": 447
           },
           "SC": {
+            "accuracy": 0.09375,
+            "count": 320
           },
           "SS": {
+            "accuracy": 0.625,
+            "count": 56
           },
           "UC": {
+            "accuracy": 0.23062381852551986,
+            "count": 529
           },
           "US": {
+            "accuracy": 0.4791666666666667,
+            "count": 48
           }
         }
       },
       "add_C1": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.1442857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.178,
+            "count": 500
           },
           "SC": {
             "accuracy": 0.0,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.12,
+            "count": 100
           }
         }
       },
       "add_C2": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.15142857142857144,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.215,
+            "count": 400
           },
           "SC": {
+            "accuracy": 0.01,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.09615384615384616,
+            "count": 156
           },
           "US": {
+            "accuracy": 0.09090909090909091,
+            "count": 44
           }
         }
       },
       "add_C3": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.15428571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.25,
+            "count": 300
           },
           "SC": {
             "accuracy": 0.06,
             "count": 100
           },
+          "UC": {
+            "accuracy": 0.09045226130653267,
+            "count": 199
+          },
           "US": {
+            "accuracy": 0.0891089108910891,
+            "count": 101
           }
         }
       },
       "add_C4": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.18571428571428572,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.34,
+            "count": 200
           },
           "SC": {
+            "accuracy": 0.05,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.10227272727272728,
+            "count": 264
           },
           "US": {
+            "accuracy": 0.22058823529411764,
+            "count": 136
           }
         }
       },
       "add_C5": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.21,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.52,
+            "count": 100
           },
           "SC": {
+            "accuracy": 0.06,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.11935483870967742,
+            "count": 310
           },
           "US": {
+            "accuracy": 0.2736842105263158,
+            "count": 190
           }
         }
       },
       "add_C6": {
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.36428571428571427,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
+            "accuracy": 0.12,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.24324324324324326,
+            "count": 370
           },
           "US": {
+            "accuracy": 0.6652173913043479,
+            "count": 230
           }
         }
       },
       "sub_M0": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.3028571428571429,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.216260162601626,
+            "count": 615
           },
           "ME": {
+            "accuracy": 0.9294117647058824,
+            "count": 85
           }
         }
       },
       "sub_M1": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.23142857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.3904109589041096,
+            "count": 292
           },
           "MB": {
             "accuracy": 0.0,
+            "count": 144
           },
           "ME": {
+            "accuracy": 0.92,
+            "count": 25
           },
           "UB": {
+            "accuracy": 0.10460251046025104,
+            "count": 239
           }
         }
       },
       "sub_M2": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.35714285714285715,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.6113744075829384,
+            "count": 211
           },
           "MB": {
+            "accuracy": 0.02608695652173913,
+            "count": 115
           },
           "ME": {
+            "accuracy": 0.9529411764705882,
+            "count": 85
           },
           "UB": {
+            "accuracy": 0.15469613259668508,
+            "count": 181
           },
           "UD": {
+            "accuracy": 0.08333333333333333,
+            "count": 108
           }
         }
       },
       "sub_M3": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.33714285714285713,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.7430167597765364,
+            "count": 179
           },
           "MB": {
+            "accuracy": 0.009708737864077669,
+            "count": 103
           },
           "ME": {
+            "accuracy": 0.9821428571428571,
+            "count": 56
           },
           "UB": {
+            "accuracy": 0.15436241610738255,
+            "count": 149
           },
           "UD": {
+            "accuracy": 0.11267605633802817,
+            "count": 213
           }
         }
       },
       "sub_M4": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.22285714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.53,
+            "count": 200
           },
           "MB": {
             "accuracy": 0.0,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.33,
+            "count": 100
           },
           "UD": {
+            "accuracy": 0.056666666666666664,
+            "count": 300
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.23857142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
+            "accuracy": 0.03,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.35,
+            "count": 100
           },
           "UD": {
+            "accuracy": 0.0725,
+            "count": 400
           }
         }
       },
       "sub_random": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.24642857142857144,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.39166666666666666,
+            "count": 600
           },
           "MB": {
+            "accuracy": 0.00749063670411985,
+            "count": 267
           },
           "ME": {
+            "accuracy": 0.9433962264150944,
             "count": 53
           },
           "UB": {
+            "accuracy": 0.12984054669703873,
+            "count": 439
           },
           "UD": {
+            "accuracy": 0.024390243902439025,
+            "count": 41
           }
         }
       },
       "sub_B3": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.21142857142857144,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.36,
+            "count": 300
           },
           "MB": {
+            "accuracy": 0.02,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.17258883248730963,
+            "count": 197
           },
           "UD": {
+            "accuracy": 0.038834951456310676,
+            "count": 103
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.22285714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.53,
+            "count": 200
           },
           "MB": {
             "accuracy": 0.0,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.14979757085020243,
+            "count": 247
           },
           "UD": {
+            "accuracy": 0.08496732026143791,
+            "count": 153
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.21857142857142858,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
+            "accuracy": 0.02,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.14093959731543623,
+            "count": 298
           },
           "UD": {
+            "accuracy": 0.04455445544554455,
+            "count": 202
           }
         }
       }
     },
     "summary": {
+      "overall_accuracy": 0.004230769230769231,
+      "digit_accuracy": 0.25884615384615384,
+      "total_examples": 2600,
       "n_splits": 24
     }
   },
       "K": 1,
       "mode": "sorl",
       "n_digits": 6,
+      "n_per_split": 100
     },
     "splits": {
       "add_S0": {
+        "full_accuracy": 0.46,
+        "digit_accuracy": 0.9114285714285715,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.8975206611570248,
+            "count": 605
           },
           "SS": {
+            "accuracy": 1.0,
+            "count": 95
           }
         }
       },
       "add_S1": {
+        "full_accuracy": 0.43,
+        "digit_accuracy": 0.9,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9264705882352942,
+            "count": 204
           },
           "SC": {
+            "accuracy": 0.8875739644970414,
+            "count": 169
           },
           "SS": {
+            "accuracy": 1.0,
+            "count": 31
           },
           "UC": {
+            "accuracy": 0.8783783783783784,
+            "count": 296
           }
         }
       },
       "add_S2": {
+        "full_accuracy": 0.19,
+        "digit_accuracy": 0.82,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.8834355828220859,
+            "count": 163
           },
           "SC": {
+            "accuracy": 0.8692307692307693,
+            "count": 130
           },
           "SS": {
+            "accuracy": 0.9655172413793104,
+            "count": 87
           },
           "UC": {
+            "accuracy": 0.6354679802955665,
+            "count": 203
           },
           "US": {
+            "accuracy": 0.8888888888888888,
+            "count": 117
           }
         }
       },
       "add_S3": {
+        "full_accuracy": 0.09,
+        "digit_accuracy": 0.75,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9173553719008265,
+            "count": 121
           },
           "SC": {
+            "accuracy": 0.8925619834710744,
+            "count": 121
           },
           "SS": {
+            "accuracy": 0.9795918367346939,
+            "count": 49
           },
           "UC": {
+            "accuracy": 0.5645161290322581,
+            "count": 186
           },
           "US": {
+            "accuracy": 0.6860986547085202,
+            "count": 223
           }
         }
       },
       "add_S4": {
+        "full_accuracy": 0.15,
+        "digit_accuracy": 0.6914285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9711538461538461,
+            "count": 104
           },
           "SC": {
+            "accuracy": 0.9245283018867925,
+            "count": 106
           },
           "SS": {
+            "accuracy": 1.0,
+            "count": 23
           },
           "UC": {
+            "accuracy": 0.56875,
+            "count": 160
           },
           "US": {
+            "accuracy": 0.5570032573289903,
+            "count": 307
           }
         }
       },
       "add_S5": {
+        "full_accuracy": 0.11,
+        "digit_accuracy": 0.5214285714285715,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.99,
+            "count": 100
           },
           "SC": {
+            "accuracy": 0.97,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.33,
+            "count": 100
           },
           "US": {
+            "accuracy": 0.34,
+            "count": 400
           }
         }
       },
       "add_S6": {
+        "full_accuracy": 0.14,
+        "digit_accuracy": 0.43142857142857144,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
+            "accuracy": 1.0,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.21,
+            "count": 100
           },
           "US": {
+            "accuracy": 0.362,
+            "count": 500
           }
         }
       },
       "add_random": {
+        "full_accuracy": 0.315,
+        "digit_accuracy": 0.8671428571428571,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.8926174496644296,
+            "count": 447
           },
           "SC": {
+            "accuracy": 0.93125,
+            "count": 320
           },
           "SS": {
+            "accuracy": 0.9464285714285714,
+            "count": 56
           },
           "UC": {
+            "accuracy": 0.8071833648393195,
+            "count": 529
           },
           "US": {
+            "accuracy": 0.7708333333333334,
+            "count": 48
           }
         }
       },
       "add_C1": {
+        "full_accuracy": 0.31,
+        "digit_accuracy": 0.8614285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9,
+            "count": 500
           },
           "SC": {
+            "accuracy": 0.91,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.62,
+            "count": 100
           }
         }
       },
       "add_C2": {
+        "full_accuracy": 0.34,
+        "digit_accuracy": 0.8528571428571429,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.905,
+            "count": 400
           },
           "SC": {
+            "accuracy": 0.92,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.6794871794871795,
+            "count": 156
           },
           "US": {
+            "accuracy": 0.8409090909090909,
+            "count": 44
           }
         }
       },
       "add_C3": {
+        "full_accuracy": 0.23,
+        "digit_accuracy": 0.8328571428571429,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9366666666666666,
+            "count": 300
           },
           "SC": {
+            "accuracy": 0.95,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.6432160804020101,
+            "count": 199
           },
           "US": {
+            "accuracy": 0.7821782178217822,
+            "count": 101
           }
         }
       },
       "add_C4": {
+        "full_accuracy": 0.16,
+        "digit_accuracy": 0.8014285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.945,
+            "count": 200
           },
           "SC": {
+            "accuracy": 0.97,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.6287878787878788,
+            "count": 264
           },
           "US": {
+            "accuracy": 0.8014705882352942,
+            "count": 136
           }
         }
       },
       "add_C5": {
+        "full_accuracy": 0.13,
+        "digit_accuracy": 0.7571428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 1.0,
+            "count": 100
           },
           "SC": {
+            "accuracy": 0.97,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.5806451612903226,
+            "count": 310
           },
           "US": {
+            "accuracy": 0.8052631578947368,
+            "count": 190
           }
         }
       },
       "add_C6": {
+        "full_accuracy": 0.15,
+        "digit_accuracy": 0.7857142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
+            "accuracy": 1.0,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.6891891891891891,
+            "count": 370
           },
           "US": {
+            "accuracy": 0.8478260869565217,
+            "count": 230
           }
         }
       },
       "sub_M0": {
+        "full_accuracy": 0.44,
+        "digit_accuracy": 0.9,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.8878048780487805,
+            "count": 615
           },
           "ME": {
+            "accuracy": 0.9882352941176471,
+            "count": 85
           }
         }
       },
       "sub_M1": {
+        "full_accuracy": 0.22,
+        "digit_accuracy": 0.8314285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.910958904109589,
+            "count": 292
           },
           "MB": {
+            "accuracy": 0.8888888888888888,
+            "count": 144
           },
           "ME": {
+            "accuracy": 1.0,
+            "count": 25
           },
           "UB": {
+            "accuracy": 0.6820083682008368,
+            "count": 239
           }
         }
       },
       "sub_M2": {
+        "full_accuracy": 0.21,
+        "digit_accuracy": 0.8314285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.9241706161137441,
+            "count": 211
           },
           "MB": {
+            "accuracy": 0.9391304347826087,
+            "count": 115
           },
           "ME": {
+            "accuracy": 1.0,
+            "count": 85
           },
           "UB": {
+            "accuracy": 0.5248618784530387,
+            "count": 181
           },
           "UD": {
+            "accuracy": 0.9166666666666666,
+            "count": 108
           }
         }
       },
       "sub_M3": {
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.7014285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.9664804469273743,
+            "count": 179
           },
           "MB": {
+            "accuracy": 0.941747572815534,
+            "count": 103
           },
           "ME": {
+            "accuracy": 0.9821428571428571,
+            "count": 56
           },
           "UB": {
+            "accuracy": 0.44966442953020136,
+            "count": 149
           },
           "UD": {
+            "accuracy": 0.4647887323943662,
+            "count": 213
           }
         }
       },
       "sub_M4": {
+        "full_accuracy": 0.04,
+        "digit_accuracy": 0.63,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.925,
+            "count": 200
           },
           "MB": {
+            "accuracy": 0.95,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.59,
+            "count": 100
           },
           "UD": {
+            "accuracy": 0.34,
+            "count": 300
           }
         }
       },
       "sub_M5": {
+        "full_accuracy": 0.04,
+        "digit_accuracy": 0.5014285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
+            "accuracy": 1.0,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.62,
+            "count": 100
           },
           "UD": {
+            "accuracy": 0.2225,
+            "count": 400
           }
         }
       },
       "sub_random": {
+        "full_accuracy": 0.32,
+        "digit_accuracy": 0.8492857142857143,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.905,
+            "count": 600
           },
           "MB": {
+            "accuracy": 0.9775280898876404,
+            "count": 267
           },
           "ME": {
+            "accuracy": 0.9811320754716981,
             "count": 53
           },
           "UB": {
+            "accuracy": 0.6674259681093394,
+            "count": 439
           },
           "UD": {
+            "accuracy": 0.975609756097561,
+            "count": 41
           }
         }
       },
       "sub_B3": {
+        "full_accuracy": 0.08,
+        "digit_accuracy": 0.7871428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.8666666666666667,
+            "count": 300
           },
           "MB": {
+            "accuracy": 0.95,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.5786802030456852,
+            "count": 197
           },
           "UD": {
+            "accuracy": 0.7961165048543689,
+            "count": 103
           }
         }
       },
       "sub_B4": {
+        "full_accuracy": 0.07,
+        "digit_accuracy": 0.7342857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.895,
+            "count": 200
           },
           "MB": {
+            "accuracy": 0.93,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.5789473684210527,
+            "count": 247
           },
           "UD": {
+            "accuracy": 0.6470588235294118,
+            "count": 153
           }
         }
       },
       "sub_B5": {
+        "full_accuracy": 0.09,
+        "digit_accuracy": 0.7557142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
+            "accuracy": 1.0,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.6375838926174496,
+            "count": 298
           },
           "UD": {
+            "accuracy": 0.6881188118811881,
+            "count": 202
           }
         }
       }
     },
     "summary": {
+      "overall_accuracy": 0.20653846153846153,
+      "digit_accuracy": 0.7701098901098901,
+      "total_examples": 2600,
       "n_splits": 24
     }
   },
+  "sorl_overall_accuracy": 0.20653846153846153,
+  "sft_overall_accuracy": 0.004230769230769231
 }

add_sub_sorl_v1_abs10_K1_25K_2L1H128d/model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:552c3c376c464a283388ace103d56b77840cec872425d931fcab7533aaebd6fd
 size 157702060

 version https://git-lfs.github.com/spec/v1
+oid sha256:dcb5ae69edf2987d564ede203c9db7efacfd8822d3c6cfb5e13932b8c61bf6a6
 size 157702060

add_sub_sorl_v1_abs10_K1_25K_2L1H128d/train_config.json CHANGED Viewed

@@ -20,7 +20,7 @@
   "lr": 2e-05,
   "emb_lr_mult": 1.0,
   "weight_decay": 0.01,
-  "warmup_steps": 117,
   "cooldown_frac": 0.4,
   "max_grad_norm": 1.0,
   "vq_abs_pretrain_steps": 0,
@@ -30,7 +30,7 @@
   "vq_abs_pretrain_target_vectors": 20000,
   "batch_size": 64,
   "gradient_accumulation_steps": 1,
-  "num_epochs": 10,
   "emb_warmup_steps": 0,
   "log_every": 50,
   "eval_every": 390,
@@ -69,16 +69,20 @@
   "no_wandb": false,
   "n_params": 39348864,
   "run_name": "add_sub_sorl_v1_abs10_K1_25K_2L1H128d",
-  "git_commit": "f447da529caceac8c7d256cbb2cd185cbc50feac",
-  "timestamp": "2026-04-12T13:04:02.986587+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "v1",
-  "wandb_run_id": "r858hl5t",
-  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/r858hl5t",
-  "final_accuracy": 0.020416666666666666,
-  "sft_accuracy": 0.0,
   "eval_method": "ArithmeticEvaluator"
 }

   "lr": 2e-05,
   "emb_lr_mult": 1.0,
   "weight_decay": 0.01,
+  "warmup_steps": 234,
   "cooldown_frac": 0.4,
   "max_grad_norm": 1.0,
   "vq_abs_pretrain_steps": 0,
   "vq_abs_pretrain_target_vectors": 20000,
   "batch_size": 64,
   "gradient_accumulation_steps": 1,
+  "num_epochs": 20,
   "emb_warmup_steps": 0,
   "log_every": 50,
   "eval_every": 390,
   "no_wandb": false,
   "n_params": 39348864,
   "run_name": "add_sub_sorl_v1_abs10_K1_25K_2L1H128d",
+  "git_commit": "f835493c19eb98267697007042c9d440cad2afbb",
+  "timestamp": "2026-04-15T18:28:58.775284+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
+  "train_dataset": "fixed_train/train_25K_seed42.pt",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "v1",
+  "wandb_run_id": "imtqvdf8",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/imtqvdf8",
+  "eval_final_dataset": "eval_sets/eval_add_sub_6d_N100_seed42.json",
+  "eval_epoch_dataset": "eval_sets/eval_add_sub_6d_N25_seed42.json",
+  "eval_hf_repo": "thoughtworks/arithmetic-sorl-data",
+  "final_accuracy": 0.20653846153846153,
+  "sft_accuracy": 0.004230769230769231,
   "eval_method": "ArithmeticEvaluator"
 }