Upload add_sub_baseline_10K

add_sub_baseline_10K/metrics.json

@@ -65,196 +65,196 @@
       3100
     ],
     "loss": [
-      10.090532302856445,
-      7.41793155670166,
-      6.670701503753662,
-      5.4355902671813965,
-      4.486183166503906,
-      3.106839418411255,
-      2.181303024291992,
-      1.9472637176513672,
-      1.6879997253417969,
-      1.7242902517318726,
-      1.3950544595718384,
-      0.9311097860336304,
-      0.593478798866272,
-      0.4981410801410675,
-      0.24719169735908508,
-      0.10993476957082748,
-      0.09566618502140045,
-      0.08340698480606079,
-      0.07292894273996353,
-      0.05232112854719162,
-      0.04302288219332695,
-      0.04004356637597084,
-      0.04495292156934738,
-      0.048470307141542435,
-      0.033653225749731064,
-      0.030521346256136894,
-      0.06012653931975365,
-      0.05565373972058296,
-      0.04759713634848595,
-      0.054727476090192795,
-      0.034593965858221054,
-      0.027987422421574593,
-      0.034813184291124344,
-      0.03215349465608597,
-      0.02478446625173092,
-      0.022883405908942223,
-      0.02932637929916382,
-      0.017739301547408104,
-      0.024523159489035606,
-      0.02478833682835102,
-      0.01739186979830265,
-      0.01981291174888611,
-      0.038495369255542755,
-      0.02020339109003544,
-      0.035459987819194794,
-      0.033973682671785355,
-      0.03820496052503586,
-      0.01859247498214245,
-      0.02102138102054596,
-      0.021527009084820747,
-      0.02982836402952671,
-      0.026523862034082413,
-      0.0283384770154953,
-      0.023189542815089226,
-      0.022154804319143295,
-      0.02203175611793995,
-      0.01161875855177641,
-      0.01782594434916973,
-      0.014171228744089603,
-      0.014905942603945732,
-      0.019558599218726158,
-      0.012723379768431187
+      7.930843830108643,
+      5.878052711486816,
+      4.171169757843018,
+      2.4461560249328613,
+      1.9941270351409912,
+      1.885154366493225,
+      1.7704766988754272,
+      1.7200186252593994,
+      1.6930021047592163,
+      1.659714698791504,
+      1.6491907835006714,
+      1.440868616104126,
+      1.2277355194091797,
+      0.9296635389328003,
+      0.727922797203064,
+      0.6531419157981873,
+      0.6338068842887878,
+      0.5315085053443909,
+      0.471783846616745,
+      0.44315391778945923,
+      0.37063688039779663,
+      0.31710734963417053,
+      0.32954543828964233,
+      0.2792213559150696,
+      0.28969308733940125,
+      0.27333250641822815,
+      0.2600690424442291,
+      0.22665680944919586,
+      0.22485388815402985,
+      0.18426764011383057,
+      0.20698437094688416,
+      0.20130981504917145,
+      0.16024114191532135,
+      0.20614077150821686,
+      0.16621388494968414,
+      0.1674460619688034,
+      0.13519535958766937,
+      0.17641237378120422,
+      0.14274020493030548,
+      0.15076635777950287,
+      0.14852766692638397,
+      0.12220268696546555,
+      0.13161125779151917,
+      0.12244851887226105,
+      0.14852531254291534,
+      0.15561267733573914,
+      0.10794571787118912,
+      0.14725656807422638,
+      0.14284299314022064,
+      0.10321654379367828,
+      0.10347577184438705,
+      0.1123846247792244,
+      0.11255372315645218,
+      0.11303780227899551,
+      0.07373049110174179,
+      0.1012827679514885,
+      0.08089525252580643,
+      0.10804768651723862,
+      0.08694573491811752,
+      0.11614620685577393,
+      0.090724878013134,
+      0.11524895578622818
     ],
     "base_loss": [
-      10.090532302856445,
-      7.41793155670166,
-      6.670701503753662,
-      5.4355902671813965,
-      4.486183166503906,
-      3.106839418411255,
-      2.181303024291992,
-      1.9472637176513672,
-      1.6879997253417969,
-      1.7242902517318726,
-      1.3950544595718384,
-      0.9311097860336304,
-      0.593478798866272,
-      0.4981410801410675,
-      0.24719169735908508,
-      0.10993476957082748,
-      0.09566618502140045,
-      0.08340698480606079,
-      0.07292894273996353,
-      0.05232112854719162,
-      0.04302288219332695,
-      0.04004356637597084,
-      0.04495292156934738,
-      0.048470307141542435,
-      0.033653225749731064,
-      0.030521346256136894,
-      0.06012653931975365,
-      0.05565373972058296,
-      0.04759713634848595,
-      0.054727476090192795,
-      0.034593965858221054,
-      0.027987422421574593,
-      0.034813184291124344,
-      0.03215349465608597,
-      0.02478446625173092,
-      0.022883405908942223,
-      0.02932637929916382,
-      0.017739301547408104,
-      0.024523159489035606,
-      0.02478833682835102,
-      0.01739186979830265,
-      0.01981291174888611,
-      0.038495369255542755,
-      0.02020339109003544,
-      0.035459987819194794,
-      0.033973682671785355,
-      0.03820496052503586,
-      0.01859247498214245,
-      0.02102138102054596,
-      0.021527009084820747,
-      0.02982836402952671,
-      0.026523862034082413,
-      0.0283384770154953,
-      0.023189542815089226,
-      0.022154804319143295,
-      0.02203175611793995,
-      0.01161875855177641,
-      0.01782594434916973,
-      0.014171228744089603,
-      0.014905942603945732,
-      0.019558599218726158,
-      0.012723379768431187
+      7.930843830108643,
+      5.878052711486816,
+      4.171169757843018,
+      2.4461560249328613,
+      1.9941270351409912,
+      1.885154366493225,
+      1.7704766988754272,
+      1.7200186252593994,
+      1.6930021047592163,
+      1.659714698791504,
+      1.6491907835006714,
+      1.440868616104126,
+      1.2277355194091797,
+      0.9296635389328003,
+      0.727922797203064,
+      0.6531419157981873,
+      0.6338068842887878,
+      0.5315085053443909,
+      0.471783846616745,
+      0.44315391778945923,
+      0.37063688039779663,
+      0.31710734963417053,
+      0.32954543828964233,
+      0.2792213559150696,
+      0.28969308733940125,
+      0.27333250641822815,
+      0.2600690424442291,
+      0.22665680944919586,
+      0.22485388815402985,
+      0.18426764011383057,
+      0.20698437094688416,
+      0.20130981504917145,
+      0.16024114191532135,
+      0.20614077150821686,
+      0.16621388494968414,
+      0.1674460619688034,
+      0.13519535958766937,
+      0.17641237378120422,
+      0.14274020493030548,
+      0.15076635777950287,
+      0.14852766692638397,
+      0.12220268696546555,
+      0.13161125779151917,
+      0.12244851887226105,
+      0.14852531254291534,
+      0.15561267733573914,
+      0.10794571787118912,
+      0.14725656807422638,
+      0.14284299314022064,
+      0.10321654379367828,
+      0.10347577184438705,
+      0.1123846247792244,
+      0.11255372315645218,
+      0.11303780227899551,
+      0.07373049110174179,
+      0.1012827679514885,
+      0.08089525252580643,
+      0.10804768651723862,
+      0.08694573491811752,
+      0.11614620685577393,
+      0.090724878013134,
+      0.11524895578622818
     ],
     "lr": [
-      6.242038216560511e-06,
-      1.2611464968152866e-05,
-      1.8980891719745225e-05,
-      2.5350318471337578e-05,
-      3.1719745222929934e-05,
-      3.808917197452229e-05,
-      4.445859872611465e-05,
-      5.082802547770701e-05,
-      5.7197452229299365e-05,
-      6.356687898089173e-05,
-      6.993630573248408e-05,
-      7.630573248407644e-05,
-      7.99862055592881e-05,
-      7.984241248831029e-05,
-      7.954287783192742e-05,
-      7.90887724530305e-05,
-      7.848187142213441e-05,
-      7.772454707873448e-05,
-      7.681975975797462e-05,
-      7.57710462188759e-05,
-      7.458250581935905e-05,
-      7.325878449210182e-05,
-      7.180505658386849e-05,
-      7.022700462930083e-05,
-      6.853079713823312e-05,
-      6.672306448335957e-05,
-      6.481087298250779e-05,
-      6.280169727682872e-05,
-      6.070339111287581e-05,
-      5.8524156642783655e-05,
-      5.627251236255051e-05,
-      5.3957259813751526e-05,
-      5.1587449178844164e-05,
-      4.917234390455111e-05,
-      4.672138449160635e-05,
-      4.424415159240753e-05,
-      4.175032856082417e-05,
-      3.924966360055181e-05,
-      3.675193165997228e-05,
-      3.42668962224704e-05,
-      3.180427114156694e-05,
-      2.9373682670051437e-05,
-      2.6984631831541183e-05,
-      2.4646457281553407e-05,
-      2.2368298803264487e-05,
-      2.0159061580649347e-05,
-      1.8027381388654794e-05,
-      1.5981590836476463e-05,
-      1.4029686795892575e-05,
-      1.2179299141974771e-05,
-      1.0437660928367057e-05,
-      8.811580113715755e-06,
-      7.307412949770034e-06,
-      5.9310391351775455e-06,
-      4.687838832097362e-06,
-      3.5826716354707645e-06,
-      2.6198575771580583e-06,
-      1.8031602391947344e-06,
-      1.1357720421765062e-06,
-      6.203017662798872e-07,
-      2.5876435369797334e-07,
-      5.257303235302935e-08
+      1.9600000000000002e-05,
+      3.96e-05,
+      3.9974363901222355e-05,
+      3.9895421087752256e-05,
+      3.976337160140893e-05,
+      3.957856792072718e-05,
+      3.9341503340549716e-05,
+      3.9052810655279495e-05,
+      3.8713260469767256e-05,
+      3.832375914234272e-05,
+      3.788534636548025e-05,
+      3.739919239055685e-05,
+      3.6866594904110466e-05,
+      3.62889755639367e-05,
+      3.566787620427007e-05,
+      3.5004954720179526e-05,
+      3.4301980642163606e-05,
+      3.3560830412758255e-05,
+      3.2783482377765215e-05,
+      3.197201150547096e-05,
+      3.112858384795204e-05,
+      3.0255450759251313e-05,
+      2.9354942885858323e-05,
+      2.842946394553513e-05,
+      2.7481484311093542e-05,
+      2.6513534416250682e-05,
+      2.5528198001164462e-05,
+      2.4528105215678678e-05,
+      2.3515925598687097e-05,
+      2.249436095235672e-05,
+      2.146613813023101e-05,
+      2.043400175846362e-05,
+      1.9400706909611906e-05,
+      1.8369011748545936e-05,
+      1.734167017010322e-05,
+      1.6321424448141397e-05,
+      1.5310997915610664e-05,
+      1.4313087695185074e-05,
+      1.3330357499856637e-05,
+      1.2365430522709616e-05,
+      1.1420882434854255e-05,
+      1.049923451021051e-05,
+      9.602946895493652e-06,
+      8.734412043366293e-06,
+      7.89594832628547e-06,
+      7.089793848091389e-06,
+      6.318100469856501e-06,
+      5.582928065941624e-06,
+      4.886239025591397e-06,
+      4.229893014745887e-06,
+      3.6156420120506306e-06,
+      3.045125632315402e-06,
+      2.5198667499047936e-06,
+      2.0412674337430526e-06,
+      1.610605204783806e-06,
+      1.2290296259345835e-06,
+      8.975592335386451e-07,
+      6.170788186048593e-07,
+      3.883370650428364e-07,
+      2.11944551207528e-07,
+      8.837212008778961e-08,
+      1.79496224893283e-08
     ],
     "eval_step": [
       156,
@@ -301,530 +301,530 @@
       20
     ],
     "eval_accuracy": [
-      0.0,
-      0.0064285714285714285,
-      0.007142857142857143,
-      0.04,
-      0.6357142857142857,
-      0.7271428571428571,
-      0.7485714285714286,
-      0.7471428571428571,
-      0.75,
-      0.7878571428571428,
-      0.7842857142857143,
-      0.7714285714285715,
-      0.8028571428571428,
-      0.8085714285714286,
-      0.8228571428571428,
-      0.8342857142857143,
-      0.8307142857142857,
-      0.8635714285714285,
-      0.8814285714285715,
-      0.8628571428571429
+      0.006666666666666667,
+      0.0022222222222222222,
+      0.01,
+      0.014444444444444444,
+      0.051111111111111114,
+      0.13444444444444445,
+      0.26555555555555554,
+      0.39,
+      0.49666666666666665,
+      0.56,
+      0.6044444444444445,
+      0.6322222222222222,
+      0.6344444444444445,
+      0.6344444444444445,
+      0.6866666666666666,
+      0.6888888888888889,
+      0.7033333333333334,
+      0.7033333333333334,
+      0.7144444444444444,
+      0.71
     ]
   },
-  "final_accuracy": 0.8628571428571429,
+  "final_accuracy": 0.6270833333333333,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 50
+      "n_per_split": 100
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 1.0,
-        "n_examples": 50,
+        "full_accuracy": 0.87,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
-            "count": 295
+            "accuracy": 0.9785123966942149,
+            "count": 605
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 55
+            "count": 95
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 1.0,
-        "n_examples": 50,
+        "full_accuracy": 0.98,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
-            "count": 126
+            "accuracy": 0.9950980392156863,
+            "count": 204
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 79
+            "count": 169
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 21
+            "count": 31
           },
           "UC": {
-            "accuracy": 1.0,
-            "count": 124
+            "accuracy": 0.9966216216216216,
+            "count": 296
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.96,
-        "n_examples": 50,
+        "full_accuracy": 0.7,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
-            "count": 75
+            "accuracy": 0.9877300613496932,
+            "count": 163
           },
           "SC": {
-            "accuracy": 0.967741935483871,
-            "count": 62
+            "accuracy": 0.9692307692307692,
+            "count": 130
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 39
+            "accuracy": 0.9655172413793104,
+            "count": 87
           },
           "UC": {
-            "accuracy": 1.0,
-            "count": 111
+            "accuracy": 0.8669950738916257,
+            "count": 203
           },
           "US": {
             "accuracy": 1.0,
-            "count": 63
+            "count": 117
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.88,
-        "n_examples": 50,
+        "full_accuracy": 0.52,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 60
+            "count": 121
           },
           "SC": {
-            "accuracy": 1.0,
-            "count": 57
+            "accuracy": 0.9917355371900827,
+            "count": 121
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 19
+            "count": 49
           },
           "UC": {
-            "accuracy": 0.9423076923076923,
-            "count": 104
+            "accuracy": 0.7795698924731183,
+            "count": 186
           },
           "US": {
-            "accuracy": 1.0,
-            "count": 110
+            "accuracy": 0.8923766816143498,
+            "count": 223
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.7,
-        "n_examples": 50,
+        "full_accuracy": 0.58,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 48
+            "count": 104
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 52
+            "count": 106
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 7
+            "count": 23
           },
           "UC": {
-            "accuracy": 0.8314606741573034,
-            "count": 89
+            "accuracy": 0.8125,
+            "count": 160
           },
           "US": {
-            "accuracy": 0.9025974025974026,
-            "count": 154
+            "accuracy": 0.8273615635179153,
+            "count": 307
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.58,
-        "n_examples": 50,
+        "full_accuracy": 0.29,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.6,
-            "count": 50
+            "accuracy": 0.41,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.79,
-            "count": 200
+            "accuracy": 0.58,
+            "count": 400
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 1.0,
-        "n_examples": 50,
+        "full_accuracy": 0.56,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 1.0,
-            "count": 50
+            "accuracy": 0.64,
+            "count": 100
           },
           "US": {
-            "accuracy": 1.0,
-            "count": 250
+            "accuracy": 0.688,
+            "count": 500
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.99,
+        "full_accuracy": 0.89,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
-            "count": 431
+            "accuracy": 0.9910514541387024,
+            "count": 447
           },
           "SC": {
-            "accuracy": 0.9968354430379747,
-            "count": 316
+            "accuracy": 0.99375,
+            "count": 320
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 39
+            "count": 56
           },
           "UC": {
-            "accuracy": 0.9982142857142857,
-            "count": 560
+            "accuracy": 0.9716446124763705,
+            "count": 529
           },
           "US": {
-            "accuracy": 1.0,
-            "count": 54
+            "accuracy": 0.9791666666666666,
+            "count": 48
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.88,
-        "n_examples": 50,
+        "full_accuracy": 0.76,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 150
+            "count": 300
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.9423076923076923,
-            "count": 104
+            "accuracy": 0.8808290155440415,
+            "count": 193
           },
           "US": {
-            "accuracy": 1.0,
-            "count": 46
+            "accuracy": 0.9532710280373832,
+            "count": 107
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.86,
-        "n_examples": 50,
+        "full_accuracy": 0.58,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 200
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.943089430894309,
-            "count": 123
+            "accuracy": 0.84375,
+            "count": 256
           },
           "US": {
-            "accuracy": 0.948051948051948,
-            "count": 77
+            "accuracy": 0.9236111111111112,
+            "count": 144
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.86,
-        "n_examples": 50,
+        "full_accuracy": 0.6,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.9545454545454546,
-            "count": 154
+            "accuracy": 0.8823529411764706,
+            "count": 306
           },
           "US": {
-            "accuracy": 0.9895833333333334,
-            "count": 96
+            "accuracy": 0.865979381443299,
+            "count": 194
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.88,
-        "n_examples": 50,
+        "full_accuracy": 0.52,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.967032967032967,
-            "count": 182
+            "accuracy": 0.8797814207650273,
+            "count": 366
           },
           "US": {
-            "accuracy": 0.940677966101695,
-            "count": 118
+            "accuracy": 0.9102564102564102,
+            "count": 234
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 1.0,
-        "n_examples": 50,
+        "full_accuracy": 0.96,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 294
+            "accuracy": 0.9933444259567388,
+            "count": 601
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 56
+            "accuracy": 0.98989898989899,
+            "count": 99
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 1.0,
-        "n_examples": 50,
+        "full_accuracy": 0.93,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 143
+            "accuracy": 0.989247311827957,
+            "count": 279
           },
           "MB": {
-            "accuracy": 1.0,
-            "count": 69
+            "accuracy": 0.9793103448275862,
+            "count": 145
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 15
+            "count": 24
           },
           "UB": {
-            "accuracy": 1.0,
-            "count": 123
+            "accuracy": 0.996031746031746,
+            "count": 252
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 1.0,
-        "n_examples": 50,
+        "full_accuracy": 0.72,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 108
+            "accuracy": 0.9859154929577465,
+            "count": 213
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 52
+            "count": 113
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 52
+            "count": 85
           },
           "UB": {
-            "accuracy": 1.0,
-            "count": 87
+            "accuracy": 0.861878453038674,
+            "count": 181
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 51
+            "count": 108
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.46,
-        "n_examples": 50,
+        "full_accuracy": 0.26,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 94
+            "count": 179
           },
           "MB": {
-            "accuracy": 1.0,
-            "count": 51
+            "accuracy": 0.9805825242718447,
+            "count": 103
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 25
+            "count": 56
           },
           "UB": {
-            "accuracy": 0.6538461538461539,
-            "count": 78
+            "accuracy": 0.46308724832214765,
+            "count": 149
           },
           "UD": {
-            "accuracy": 1.0,
-            "count": 102
+            "accuracy": 0.9859154929577465,
+            "count": 213
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.48,
-        "n_examples": 50,
+        "full_accuracy": 0.0,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 200
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.48,
-            "count": 50
+            "accuracy": 0.26,
+            "count": 100
           },
           "UD": {
-            "accuracy": 0.8333333333333334,
-            "count": 150
+            "accuracy": 0.55,
+            "count": 300
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.36,
-        "n_examples": 50,
+        "full_accuracy": 0.0,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.44,
-            "count": 50
+            "accuracy": 0.25,
+            "count": 100
           },
           "UD": {
-            "accuracy": 0.69,
-            "count": 200
+            "accuracy": 0.3975,
+            "count": 400
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.995,
+        "full_accuracy": 0.905,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 588
+            "accuracy": 0.995,
+            "count": 600
           },
           "MB": {
-            "accuracy": 1.0,
-            "count": 268
+            "accuracy": 0.9775280898876404,
+            "count": 267
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 60
+            "count": 53
           },
           "UB": {
-            "accuracy": 0.9977628635346756,
-            "count": 447
+            "accuracy": 0.9749430523917996,
+            "count": 439
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 37
+            "count": 41
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.92,
-        "n_examples": 50,
+        "full_accuracy": 0.67,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 150
+            "accuracy": 0.98,
+            "count": 300
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.9626168224299065,
-            "count": 107
+            "accuracy": 0.8527918781725888,
+            "count": 197
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 43
+            "count": 103
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.72,
-        "n_examples": 50,
+        "full_accuracy": 0.58,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 100
+            "accuracy": 0.985,
+            "count": 200
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.8771929824561403,
-            "count": 114
+            "accuracy": 0.854251012145749,
+            "count": 247
           },
           "UD": {
-            "accuracy": 0.9302325581395349,
-            "count": 86
+            "accuracy": 0.8758169934640523,
+            "count": 153
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.7,
-        "n_examples": 50,
+        "full_accuracy": 0.42,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.9019607843137255,
-            "count": 153
+            "accuracy": 0.8154362416107382,
+            "count": 298
           },
           "UD": {
-            "accuracy": 0.865979381443299,
-            "count": 97
+            "accuracy": 0.8564356435643564,
+            "count": 202
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.8628571428571429,
-      "total_examples": 1400,
+      "overall_accuracy": 0.6270833333333333,
+      "total_examples": 2400,
       "n_splits": 22
     }
   }

add_sub_baseline_10K/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc2ceca0901e7a6c1a4fdcedf14bf615dda905c45631f688fe16182a3dabbdcd
+oid sha256:83183edc6b1fa903ee491f2ac4fae76e8abb9d76035b9ce79aec6265c267c8ec
 size 650266922

add_sub_baseline_10K/train_config.json

@@ -1,35 +1,84 @@
 {
-  "mode": "baseline",
-  "ops": "add_sub",
-  "n_digits": 6,
-  "n_layer": 2,
-  "n_head": 3,
-  "n_embd": 510,
-  "abs_vocab": 0,
+  "num_rollouts": 4,
   "K": 4,
+  "max_iterations": 2,
+  "memory_span_abs": 1792,
+  "memory_span_traj": 1792,
+  "temperature": 1.0,
+  "ar_search": false,
+  "response_only_abs": false,
   "alpha_info_gain": 10.0,
   "alpha_abs": 0.1,
   "alpha_soft_zipf": 1.0,
+  "alpha_ortho": 0.0,
+  "alpha_anchor": 0.0,
+  "alpha_jacobi": 0.0,
+  "decay": 0.8,
+  "target_vocab_util": 0.8,
+  "min_abs_ppl": 0.0,
+  "zipf_alpha": 1.0,
+  "lr": 4e-05,
+  "emb_lr_mult": 1.0,
+  "weight_decay": 0.01,
+  "warmup_steps": 100,
+  "cooldown_frac": 0.4,
+  "max_grad_norm": 1.0,
+  "vq_abs_pretrain_steps": 0,
+  "vq_abs_pretrain_lr": 0.001,
+  "vq_abs_pretrain_layer": -1,
+  "vq_abs_pretrain_batch_size": 256,
+  "vq_abs_pretrain_target_vectors": 20000,
   "batch_size": 64,
+  "gradient_accumulation_steps": 1,
   "num_epochs": 20,
-  "dataset_size": 10000,
-  "lr": 8e-05,
+  "emb_warmup_steps": 0,
+  "log_every": 50,
+  "eval_every": 156,
+  "save_every": 999999,
+  "eval_samples": 100,
   "output_dir": "ckpt/sweep/add_sub_baseline_10K",
+  "eval_K": 4,
+  "alpha_traj": 0.0,
+  "corrupt_method": "shuffle",
+  "corrupt_ratio": 0.3,
+  "alpha_contrastive": 1.0,
+  "gamma_contrastive": 0.5,
+  "alpha_masked_traj": 0.0,
+  "mask_nl_ratio": 0.3,
+  "mask_nl_mode": "fixed",
+  "mask_nl_fixed_id": 0,
+  "use_ste": true,
+  "n_inner": 1,
+  "random_K": null,
+  "strip_suffix": null,
+  "compress_prefix": null,
+  "random_mem_span": null,
+  "warmup_ratio": 0.03,
+  "beta2": 0.999,
+  "seed": 42,
+  "n_digits": 6,
+  "n_layer": 2,
+  "n_head": 3,
+  "n_embd": 510,
+  "ops": "add_sub",
+  "abs_vocab": 0,
+  "dataset_size": 10000,
+  "mode": "baseline",
   "device": "cuda",
   "push_to_hub": true,
   "no_wandb": false,
   "n_params": 162490082,
   "run_name": "add_sub_baseline_10K",
-  "git_commit": "800625019270114adcda289bbd550c4f1109a514",
-  "timestamp": "2026-04-12T01:58:10.796905+00:00",
+  "git_commit": "78d46f8665a87f4b44bd5894bd34f393f2dea51f",
+  "timestamp": "2026-04-12T08:59:11.045620+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "sft",
-  "wandb_run_id": "04vy23fy",
-  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/04vy23fy",
-  "final_accuracy": 0.8628571428571429,
-  "sft_accuracy": 0.8628571428571429,
+  "wandb_run_id": "7dqmkyo7",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/7dqmkyo7",
+  "final_accuracy": 0.6270833333333333,
+  "sft_accuracy": 0.6270833333333333,
   "eval_method": "ArithmeticEvaluator"
 }