Upload add_sub_baseline_25K_2L1H128d

add_sub_baseline_25K_2L1H128d/metrics.json

@@ -159,320 +159,320 @@
       7800
     ],
     "loss": [
-      11.851262092590332,
-      11.322190284729004,
-      10.84399700164795,
-      10.476851463317871,
-      10.189689636230469,
-      9.838882446289062,
-      9.510722160339355,
-      9.290719032287598,
-      8.962188720703125,
-      8.611163139343262,
-      8.361300468444824,
-      8.08813190460205,
-      7.842682838439941,
-      7.567698001861572,
-      7.269339561462402,
-      7.015201091766357,
-      6.760293006896973,
-      6.50231409072876,
-      6.1420512199401855,
-      5.9626617431640625,
-      5.677587032318115,
-      5.316855430603027,
-      5.1820387840271,
-      4.774725914001465,
-      4.56818962097168,
-      4.361102104187012,
-      4.119393825531006,
-      3.8857994079589844,
-      3.673340320587158,
-      3.468486785888672,
-      3.2346725463867188,
-      3.0731041431427,
-      2.8872439861297607,
-      2.8238704204559326,
-      2.6868231296539307,
-      2.5601580142974854,
-      2.5200085639953613,
-      2.4465887546539307,
-      2.3342435359954834,
-      2.310267686843872,
-      2.2022716999053955,
-      2.2503445148468018,
-      2.2016828060150146,
-      2.1673545837402344,
-      2.134101152420044,
-      2.205885887145996,
-      2.1625630855560303,
-      2.090986967086792,
-      2.111237049102783,
-      2.0769805908203125,
-      2.029428243637085,
-      2.0649588108062744,
-      2.00223970413208,
-      2.041292667388916,
-      1.9394997358322144,
-      1.9607492685317993,
-      1.9083858728408813,
-      1.9098879098892212,
-      1.9169260263442993,
-      1.9707139730453491,
-      1.8977491855621338,
-      1.9636948108673096,
-      1.8278611898422241,
-      1.8901598453521729,
-      1.8808103799819946,
-      1.8042938709259033,
-      1.8307939767837524,
-      1.7672587633132935,
-      1.8975050449371338,
-      1.8720804452896118,
-      1.8167418241500854,
-      1.870581865310669,
-      1.6975057125091553,
-      1.7767753601074219,
-      1.7748700380325317,
-      1.65091073513031,
-      1.70918869972229,
-      1.8009940385818481,
-      1.6945215463638306,
-      1.7233848571777344,
-      1.6666440963745117,
-      1.6989432573318481,
-      1.657471776008606,
-      1.6441662311553955,
-      1.597665548324585,
-      1.5535640716552734,
-      1.5122039318084717,
-      1.5779234170913696,
-      1.5203800201416016,
-      1.5966535806655884,
-      1.5483477115631104,
-      1.4608439207077026,
-      1.493105173110962,
-      1.486636996269226,
-      1.4677892923355103,
-      1.4958831071853638,
-      1.5283870697021484,
-      1.3748594522476196,
-      1.3993589878082275,
-      1.444647192955017,
-      1.4109257459640503,
-      1.4192183017730713,
-      1.3186607360839844,
-      1.314361572265625,
-      1.3938757181167603,
-      1.3351393938064575,
-      1.3570326566696167,
-      1.3236685991287231,
-      1.376649022102356,
-      1.3282345533370972,
-      1.3094770908355713,
-      1.299924612045288,
-      1.3394296169281006,
-      1.2237088680267334,
-      1.267302393913269,
-      1.2889286279678345,
-      1.2387006282806396,
-      1.2579516172409058,
-      1.2616617679595947,
-      1.2706013917922974,
-      1.273461103439331,
-      1.2665029764175415,
-      1.3155572414398193,
-      1.2146965265274048,
-      1.2717746496200562,
-      1.2832603454589844,
-      1.2183363437652588,
-      1.2612063884735107,
-      1.2248671054840088,
-      1.2440804243087769,
-      1.26742684841156,
-      1.2262872457504272,
-      1.2662094831466675,
-      1.2455024719238281,
-      1.2313419580459595,
-      1.251754641532898,
-      1.2320165634155273,
-      1.2252769470214844,
-      1.2007458209991455,
-      1.2372609376907349,
-      1.20767080783844,
-      1.2022647857666016,
-      1.2439204454421997,
-      1.2493950128555298,
-      1.1735715866088867,
-      1.222267508506775,
-      1.2459700107574463,
-      1.2111799716949463,
-      1.1730115413665771,
-      1.1799018383026123,
-      1.217897653579712,
-      1.2276479005813599,
-      1.2698315382003784,
-      1.2276161909103394,
-      1.2415111064910889,
-      1.2021055221557617
+      11.832879066467285,
+      11.36864185333252,
+      10.761643409729004,
+      10.459257125854492,
+      10.175871849060059,
+      9.816800117492676,
+      9.529030799865723,
+      9.260773658752441,
+      9.017507553100586,
+      8.663485527038574,
+      8.371795654296875,
+      8.118546485900879,
+      7.833754539489746,
+      7.534058570861816,
+      7.297992706298828,
+      6.95162296295166,
+      6.682138919830322,
+      6.448472023010254,
+      6.186150550842285,
+      5.851781368255615,
+      5.642083168029785,
+      5.3920135498046875,
+      5.093167781829834,
+      4.796841621398926,
+      4.534151554107666,
+      4.4100446701049805,
+      4.1128153800964355,
+      3.8507931232452393,
+      3.6341888904571533,
+      3.4420320987701416,
+      3.2039170265197754,
+      3.1796298027038574,
+      2.970296621322632,
+      2.8707118034362793,
+      2.7452502250671387,
+      2.64337158203125,
+      2.503178119659424,
+      2.4706099033355713,
+      2.370277166366577,
+      2.268285036087036,
+      2.318305492401123,
+      2.2794229984283447,
+      2.149564504623413,
+      2.1994822025299072,
+      2.2138619422912598,
+      2.154664993286133,
+      2.1253409385681152,
+      2.104158401489258,
+      2.0960659980773926,
+      2.066467523574829,
+      2.087989568710327,
+      2.050243616104126,
+      1.993398666381836,
+      2.037946939468384,
+      1.9801470041275024,
+      2.0205399990081787,
+      2.0452141761779785,
+      1.938254475593567,
+      1.964411973953247,
+      1.9696029424667358,
+      1.9631099700927734,
+      1.9499931335449219,
+      1.9534265995025635,
+      1.930572271347046,
+      1.9394501447677612,
+      1.9909099340438843,
+      1.8681749105453491,
+      1.8672945499420166,
+      1.9460028409957886,
+      1.8758708238601685,
+      1.8508617877960205,
+      1.8957277536392212,
+      1.89703369140625,
+      1.8883601427078247,
+      1.9217368364334106,
+      1.8644115924835205,
+      1.8522149324417114,
+      1.8996679782867432,
+      1.7724077701568604,
+      1.8279880285263062,
+      1.8713520765304565,
+      1.7754944562911987,
+      1.8203290700912476,
+      1.8123414516448975,
+      1.718186616897583,
+      1.7727686166763306,
+      1.7215014696121216,
+      1.6721407175064087,
+      1.7613623142242432,
+      1.7179336547851562,
+      1.773058533668518,
+      1.7030664682388306,
+      1.6822694540023804,
+      1.708242654800415,
+      1.6574541330337524,
+      1.7228068113327026,
+      1.6863057613372803,
+      1.7814959287643433,
+      1.6393948793411255,
+      1.602840781211853,
+      1.6753469705581665,
+      1.6772491931915283,
+      1.6624025106430054,
+      1.6662027835845947,
+      1.6071633100509644,
+      1.6065205335617065,
+      1.6259610652923584,
+      1.6235558986663818,
+      1.545785665512085,
+      1.5571601390838623,
+      1.6024692058563232,
+      1.5312445163726807,
+      1.5064446926116943,
+      1.5606070756912231,
+      1.5643519163131714,
+      1.505636215209961,
+      1.542928695678711,
+      1.5629281997680664,
+      1.5619080066680908,
+      1.5639537572860718,
+      1.4565213918685913,
+      1.4847272634506226,
+      1.502737045288086,
+      1.4859784841537476,
+      1.4928518533706665,
+      1.4829928874969482,
+      1.4601913690567017,
+      1.531474232673645,
+      1.4434651136398315,
+      1.5020925998687744,
+      1.47160804271698,
+      1.4648197889328003,
+      1.5810593366622925,
+      1.4178014993667603,
+      1.4882278442382812,
+      1.4169243574142456,
+      1.434967279434204,
+      1.4791187047958374,
+      1.4924046993255615,
+      1.464010238647461,
+      1.355211853981018,
+      1.4422768354415894,
+      1.4772409200668335,
+      1.4855642318725586,
+      1.3914214372634888,
+      1.5317624807357788,
+      1.4767916202545166,
+      1.5525057315826416,
+      1.4267528057098389,
+      1.3921186923980713,
+      1.4735260009765625,
+      1.4076300859451294,
+      1.4896430969238281,
+      1.440387487411499,
+      1.436928391456604,
+      1.4201313257217407
     ],
     "base_loss": [
-      11.851262092590332,
-      11.322190284729004,
-      10.84399700164795,
-      10.476851463317871,
-      10.189689636230469,
-      9.838882446289062,
-      9.510722160339355,
-      9.290719032287598,
-      8.962188720703125,
-      8.611163139343262,
-      8.361300468444824,
-      8.08813190460205,
-      7.842682838439941,
-      7.567698001861572,
-      7.269339561462402,
-      7.015201091766357,
-      6.760293006896973,
-      6.50231409072876,
-      6.1420512199401855,
-      5.9626617431640625,
-      5.677587032318115,
-      5.316855430603027,
-      5.1820387840271,
-      4.774725914001465,
-      4.56818962097168,
-      4.361102104187012,
-      4.119393825531006,
-      3.8857994079589844,
-      3.673340320587158,
-      3.468486785888672,
-      3.2346725463867188,
-      3.0731041431427,
-      2.8872439861297607,
-      2.8238704204559326,
-      2.6868231296539307,
-      2.5601580142974854,
-      2.5200085639953613,
-      2.4465887546539307,
-      2.3342435359954834,
-      2.310267686843872,
-      2.2022716999053955,
-      2.2503445148468018,
-      2.2016828060150146,
-      2.1673545837402344,
-      2.134101152420044,
-      2.205885887145996,
-      2.1625630855560303,
-      2.090986967086792,
-      2.111237049102783,
-      2.0769805908203125,
-      2.029428243637085,
-      2.0649588108062744,
-      2.00223970413208,
-      2.041292667388916,
-      1.9394997358322144,
-      1.9607492685317993,
-      1.9083858728408813,
-      1.9098879098892212,
-      1.9169260263442993,
-      1.9707139730453491,
-      1.8977491855621338,
-      1.9636948108673096,
-      1.8278611898422241,
-      1.8901598453521729,
-      1.8808103799819946,
-      1.8042938709259033,
-      1.8307939767837524,
-      1.7672587633132935,
-      1.8975050449371338,
-      1.8720804452896118,
-      1.8167418241500854,
-      1.870581865310669,
-      1.6975057125091553,
-      1.7767753601074219,
-      1.7748700380325317,
-      1.65091073513031,
-      1.70918869972229,
-      1.8009940385818481,
-      1.6945215463638306,
-      1.7233848571777344,
-      1.6666440963745117,
-      1.6989432573318481,
-      1.657471776008606,
-      1.6441662311553955,
-      1.597665548324585,
-      1.5535640716552734,
-      1.5122039318084717,
-      1.5779234170913696,
-      1.5203800201416016,
-      1.5966535806655884,
-      1.5483477115631104,
-      1.4608439207077026,
-      1.493105173110962,
-      1.486636996269226,
-      1.4677892923355103,
-      1.4958831071853638,
-      1.5283870697021484,
-      1.3748594522476196,
-      1.3993589878082275,
-      1.444647192955017,
-      1.4109257459640503,
-      1.4192183017730713,
-      1.3186607360839844,
-      1.314361572265625,
-      1.3938757181167603,
-      1.3351393938064575,
-      1.3570326566696167,
-      1.3236685991287231,
-      1.376649022102356,
-      1.3282345533370972,
-      1.3094770908355713,
-      1.299924612045288,
-      1.3394296169281006,
-      1.2237088680267334,
-      1.267302393913269,
-      1.2889286279678345,
-      1.2387006282806396,
-      1.2579516172409058,
-      1.2616617679595947,
-      1.2706013917922974,
-      1.273461103439331,
-      1.2665029764175415,
-      1.3155572414398193,
-      1.2146965265274048,
-      1.2717746496200562,
-      1.2832603454589844,
-      1.2183363437652588,
-      1.2612063884735107,
-      1.2248671054840088,
-      1.2440804243087769,
-      1.26742684841156,
-      1.2262872457504272,
-      1.2662094831466675,
-      1.2455024719238281,
-      1.2313419580459595,
-      1.251754641532898,
-      1.2320165634155273,
-      1.2252769470214844,
-      1.2007458209991455,
-      1.2372609376907349,
-      1.20767080783844,
-      1.2022647857666016,
-      1.2439204454421997,
-      1.2493950128555298,
-      1.1735715866088867,
-      1.222267508506775,
-      1.2459700107574463,
-      1.2111799716949463,
-      1.1730115413665771,
-      1.1799018383026123,
-      1.217897653579712,
-      1.2276479005813599,
-      1.2698315382003784,
-      1.2276161909103394,
-      1.2415111064910889,
-      1.2021055221557617
+      11.832879066467285,
+      11.36864185333252,
+      10.761643409729004,
+      10.459257125854492,
+      10.175871849060059,
+      9.816800117492676,
+      9.529030799865723,
+      9.260773658752441,
+      9.017507553100586,
+      8.663485527038574,
+      8.371795654296875,
+      8.118546485900879,
+      7.833754539489746,
+      7.534058570861816,
+      7.297992706298828,
+      6.95162296295166,
+      6.682138919830322,
+      6.448472023010254,
+      6.186150550842285,
+      5.851781368255615,
+      5.642083168029785,
+      5.3920135498046875,
+      5.093167781829834,
+      4.796841621398926,
+      4.534151554107666,
+      4.4100446701049805,
+      4.1128153800964355,
+      3.8507931232452393,
+      3.6341888904571533,
+      3.4420320987701416,
+      3.2039170265197754,
+      3.1796298027038574,
+      2.970296621322632,
+      2.8707118034362793,
+      2.7452502250671387,
+      2.64337158203125,
+      2.503178119659424,
+      2.4706099033355713,
+      2.370277166366577,
+      2.268285036087036,
+      2.318305492401123,
+      2.2794229984283447,
+      2.149564504623413,
+      2.1994822025299072,
+      2.2138619422912598,
+      2.154664993286133,
+      2.1253409385681152,
+      2.104158401489258,
+      2.0960659980773926,
+      2.066467523574829,
+      2.087989568710327,
+      2.050243616104126,
+      1.993398666381836,
+      2.037946939468384,
+      1.9801470041275024,
+      2.0205399990081787,
+      2.0452141761779785,
+      1.938254475593567,
+      1.964411973953247,
+      1.9696029424667358,
+      1.9631099700927734,
+      1.9499931335449219,
+      1.9534265995025635,
+      1.930572271347046,
+      1.9394501447677612,
+      1.9909099340438843,
+      1.8681749105453491,
+      1.8672945499420166,
+      1.9460028409957886,
+      1.8758708238601685,
+      1.8508617877960205,
+      1.8957277536392212,
+      1.89703369140625,
+      1.8883601427078247,
+      1.9217368364334106,
+      1.8644115924835205,
+      1.8522149324417114,
+      1.8996679782867432,
+      1.7724077701568604,
+      1.8279880285263062,
+      1.8713520765304565,
+      1.7754944562911987,
+      1.8203290700912476,
+      1.8123414516448975,
+      1.718186616897583,
+      1.7727686166763306,
+      1.7215014696121216,
+      1.6721407175064087,
+      1.7613623142242432,
+      1.7179336547851562,
+      1.773058533668518,
+      1.7030664682388306,
+      1.6822694540023804,
+      1.708242654800415,
+      1.6574541330337524,
+      1.7228068113327026,
+      1.6863057613372803,
+      1.7814959287643433,
+      1.6393948793411255,
+      1.602840781211853,
+      1.6753469705581665,
+      1.6772491931915283,
+      1.6624025106430054,
+      1.6662027835845947,
+      1.6071633100509644,
+      1.6065205335617065,
+      1.6259610652923584,
+      1.6235558986663818,
+      1.545785665512085,
+      1.5571601390838623,
+      1.6024692058563232,
+      1.5312445163726807,
+      1.5064446926116943,
+      1.5606070756912231,
+      1.5643519163131714,
+      1.505636215209961,
+      1.542928695678711,
+      1.5629281997680664,
+      1.5619080066680908,
+      1.5639537572860718,
+      1.4565213918685913,
+      1.4847272634506226,
+      1.502737045288086,
+      1.4859784841537476,
+      1.4928518533706665,
+      1.4829928874969482,
+      1.4601913690567017,
+      1.531474232673645,
+      1.4434651136398315,
+      1.5020925998687744,
+      1.47160804271698,
+      1.4648197889328003,
+      1.5810593366622925,
+      1.4178014993667603,
+      1.4882278442382812,
+      1.4169243574142456,
+      1.434967279434204,
+      1.4791187047958374,
+      1.4924046993255615,
+      1.464010238647461,
+      1.355211853981018,
+      1.4422768354415894,
+      1.4772409200668335,
+      1.4855642318725586,
+      1.3914214372634888,
+      1.5317624807357788,
+      1.4767916202545166,
+      1.5525057315826416,
+      1.4267528057098389,
+      1.3921186923980713,
+      1.4735260009765625,
+      1.4076300859451294,
+      1.4896430969238281,
+      1.440387487411499,
+      1.436928391456604,
+      1.4201313257217407
     ],
     "lr": [
       4.188034188034189e-06,
@@ -682,590 +682,590 @@
       0.0,
       0.0,
       0.0,
+      0.003157894736842105,
+      0.0010526315789473684,
       0.0,
       0.0,
-      0.0,
-      0.0033333333333333335,
-      0.006666666666666667,
-      0.008888888888888889,
-      0.01,
-      0.015555555555555555,
-      0.02666666666666667,
-      0.027777777777777776,
-      0.03,
-      0.04,
-      0.03333333333333333,
-      0.04111111111111111,
-      0.03777777777777778
+      0.0010526315789473684,
+      0.002105263157894737,
+      0.010526315789473684,
+      0.00631578947368421,
+      0.002105263157894737,
+      0.004210526315789474,
+      0.010526315789473684,
+      0.008421052631578947,
+      0.011578947368421053,
+      0.009473684210526316,
+      0.009473684210526316
     ]
   },
-  "final_accuracy": 0.03791666666666667,
+  "final_accuracy": 0.011153846153846153,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 50
+      "n_per_split": 100
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.04,
-        "digit_accuracy": 0.6485714285714286,
-        "n_examples": 50,
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.49142857142857144,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.6372881355932203,
-            "count": 295
+            "accuracy": 0.4396694214876033,
+            "count": 605
           },
           "SS": {
-            "accuracy": 0.7090909090909091,
-            "count": 55
+            "accuracy": 0.8210526315789474,
+            "count": 95
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.5885714285714285,
-        "n_examples": 50,
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.4471428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.6349206349206349,
-            "count": 126
+            "accuracy": 0.5245098039215687,
+            "count": 204
           },
           "SC": {
-            "accuracy": 0.620253164556962,
-            "count": 79
+            "accuracy": 0.33136094674556216,
+            "count": 169
           },
           "SS": {
-            "accuracy": 0.6666666666666666,
-            "count": 21
+            "accuracy": 0.6129032258064516,
+            "count": 31
           },
           "UC": {
-            "accuracy": 0.5080645161290323,
-            "count": 124
+            "accuracy": 0.44256756756756754,
+            "count": 296
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.56,
-        "n_examples": 50,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.4957142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.6,
-            "count": 75
+            "accuracy": 0.5460122699386503,
+            "count": 163
           },
           "SC": {
-            "accuracy": 0.46774193548387094,
-            "count": 62
+            "accuracy": 0.38461538461538464,
+            "count": 130
           },
           "SS": {
-            "accuracy": 0.46153846153846156,
-            "count": 39
+            "accuracy": 0.6091954022988506,
+            "count": 87
           },
           "UC": {
-            "accuracy": 0.5405405405405406,
-            "count": 111
+            "accuracy": 0.43349753694581283,
+            "count": 203
           },
           "US": {
-            "accuracy": 0.6984126984126984,
-            "count": 63
+            "accuracy": 0.5726495726495726,
+            "count": 117
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.6085714285714285,
-        "n_examples": 50,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.4685714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.6166666666666667,
-            "count": 60
+            "accuracy": 0.5950413223140496,
+            "count": 121
           },
           "SC": {
-            "accuracy": 0.5964912280701754,
-            "count": 57
+            "accuracy": 0.3305785123966942,
+            "count": 121
           },
           "SS": {
-            "accuracy": 0.42105263157894735,
-            "count": 19
+            "accuracy": 0.673469387755102,
+            "count": 49
           },
           "UC": {
-            "accuracy": 0.47115384615384615,
-            "count": 104
+            "accuracy": 0.3870967741935484,
+            "count": 186
           },
           "US": {
-            "accuracy": 0.7727272727272727,
-            "count": 110
+            "accuracy": 0.4977578475336323,
+            "count": 223
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.08,
-        "digit_accuracy": 0.5942857142857143,
-        "n_examples": 50,
+        "full_accuracy": 0.03,
+        "digit_accuracy": 0.44142857142857145,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.6458333333333334,
-            "count": 48
+            "accuracy": 0.6057692307692307,
+            "count": 104
           },
           "SC": {
-            "accuracy": 0.6730769230769231,
-            "count": 52
+            "accuracy": 0.3584905660377358,
+            "count": 106
           },
           "SS": {
-            "accuracy": 0.8571428571428571,
-            "count": 7
+            "accuracy": 0.7391304347826086,
+            "count": 23
           },
           "UC": {
-            "accuracy": 0.5056179775280899,
-            "count": 89
+            "accuracy": 0.3875,
+            "count": 160
           },
           "US": {
-            "accuracy": 0.5909090909090909,
-            "count": 154
+            "accuracy": 0.4201954397394137,
+            "count": 307
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.16,
-        "digit_accuracy": 0.4685714285714286,
-        "n_examples": 50,
+        "full_accuracy": 0.07,
+        "digit_accuracy": 0.39571428571428574,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.7,
-            "count": 50
+            "accuracy": 0.59,
+            "count": 100
           },
           "SC": {
-            "accuracy": 0.7,
-            "count": 50
+            "accuracy": 0.29,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.38,
-            "count": 50
+            "accuracy": 0.33,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.375,
-            "count": 200
+            "accuracy": 0.39,
+            "count": 400
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.28,
-        "digit_accuracy": 0.3942857142857143,
-        "n_examples": 50,
+        "full_accuracy": 0.13,
+        "digit_accuracy": 0.26,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.5,
-            "count": 50
+            "accuracy": 0.29,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.34,
-            "count": 50
+            "accuracy": 0.25,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.384,
-            "count": 250
+            "accuracy": 0.256,
+            "count": 500
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.5921428571428572,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.43857142857142856,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.5986078886310905,
-            "count": 431
+            "accuracy": 0.4720357941834452,
+            "count": 447
           },
           "SC": {
-            "accuracy": 0.6360759493670886,
-            "count": 316
+            "accuracy": 0.35625,
+            "count": 320
           },
           "SS": {
-            "accuracy": 0.7948717948717948,
-            "count": 39
+            "accuracy": 0.625,
+            "count": 56
           },
           "UC": {
-            "accuracy": 0.5232142857142857,
-            "count": 560
+            "accuracy": 0.42344045368620037,
+            "count": 529
           },
           "US": {
-            "accuracy": 0.8518518518518519,
-            "count": 54
+            "accuracy": 0.625,
+            "count": 48
           }
         }
       },
       "add_C1": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.5714285714285714,
-        "n_examples": 50,
+        "digit_accuracy": 0.4642857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.596,
-            "count": 250
+            "accuracy": 0.526,
+            "count": 500
           },
           "SC": {
-            "accuracy": 0.62,
-            "count": 50
+            "accuracy": 0.28,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.4,
-            "count": 50
+            "accuracy": 0.34,
+            "count": 100
           }
         }
       },
       "add_C2": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.5628571428571428,
-        "n_examples": 50,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.4714285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.64,
-            "count": 200
+            "accuracy": 0.5275,
+            "count": 400
           },
           "SC": {
-            "accuracy": 0.54,
-            "count": 50
+            "accuracy": 0.42,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.3855421686746988,
-            "count": 83
+            "accuracy": 0.3141025641025641,
+            "count": 156
           },
           "US": {
-            "accuracy": 0.5882352941176471,
-            "count": 17
+            "accuracy": 0.6363636363636364,
+            "count": 44
           }
         }
       },
       "add_C3": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.5428571428571428,
-        "n_examples": 50,
+        "digit_accuracy": 0.45571428571428574,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.64,
-            "count": 150
+            "accuracy": 0.5733333333333334,
+            "count": 300
           },
           "SC": {
-            "accuracy": 0.56,
-            "count": 50
+            "accuracy": 0.38,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.35,
-            "count": 100
+            "accuracy": 0.2914572864321608,
+            "count": 199
           },
           "US": {
-            "accuracy": 0.62,
-            "count": 50
+            "accuracy": 0.504950495049505,
+            "count": 101
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.06,
-        "digit_accuracy": 0.5971428571428572,
-        "n_examples": 50,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.48142857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.71,
-            "count": 100
+            "accuracy": 0.65,
+            "count": 200
           },
           "SC": {
-            "accuracy": 0.66,
-            "count": 50
+            "accuracy": 0.38,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.4090909090909091,
-            "count": 132
+            "accuracy": 0.3333333333333333,
+            "count": 264
           },
           "US": {
-            "accuracy": 0.75,
-            "count": 68
+            "accuracy": 0.5955882352941176,
+            "count": 136
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.6428571428571429,
-        "n_examples": 50,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.4928571428571429,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.68,
-            "count": 50
+            "accuracy": 0.7,
+            "count": 100
           },
           "SC": {
-            "accuracy": 0.68,
-            "count": 50
+            "accuracy": 0.43,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.4520547945205479,
-            "count": 146
+            "accuracy": 0.3,
+            "count": 310
           },
           "US": {
-            "accuracy": 0.875,
-            "count": 104
+            "accuracy": 0.7315789473684211,
+            "count": 190
           }
         }
       },
       "add_C6": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.5657142857142857,
-        "n_examples": 50,
+        "digit_accuracy": 0.4957142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.66,
-            "count": 50
+            "accuracy": 0.43,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.5026455026455027,
-            "count": 189
+            "accuracy": 0.3621621621621622,
+            "count": 370
           },
           "US": {
-            "accuracy": 0.6306306306306306,
-            "count": 111
+            "accuracy": 0.7391304347826086,
+            "count": 230
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.04,
-        "digit_accuracy": 0.6571428571428571,
-        "n_examples": 50,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.5628571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.6039603960396039,
-            "count": 303
+            "accuracy": 0.5203252032520326,
+            "count": 615
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 47
+            "accuracy": 0.8705882352941177,
+            "count": 85
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.6542857142857142,
-        "n_examples": 50,
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.45,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.7092198581560284,
-            "count": 141
+            "accuracy": 0.5821917808219178,
+            "count": 292
           },
           "MB": {
-            "accuracy": 0.7083333333333334,
-            "count": 72
+            "accuracy": 0.3541666666666667,
+            "count": 144
           },
           "ME": {
-            "accuracy": 0.9444444444444444,
-            "count": 18
+            "accuracy": 0.92,
+            "count": 25
           },
           "UB": {
-            "accuracy": 0.5126050420168067,
-            "count": 119
+            "accuracy": 0.29707112970711297,
+            "count": 239
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.04,
-        "digit_accuracy": 0.6828571428571428,
-        "n_examples": 50,
+        "full_accuracy": 0.03,
+        "digit_accuracy": 0.5485714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.8214285714285714,
-            "count": 112
+            "accuracy": 0.7488151658767772,
+            "count": 211
           },
           "MB": {
-            "accuracy": 0.7169811320754716,
-            "count": 53
+            "accuracy": 0.26956521739130435,
+            "count": 115
           },
           "ME": {
-            "accuracy": 0.9148936170212766,
-            "count": 47
+            "accuracy": 0.9647058823529412,
+            "count": 85
           },
           "UB": {
-            "accuracy": 0.4470588235294118,
-            "count": 85
+            "accuracy": 0.4530386740331492,
+            "count": 181
           },
           "UD": {
-            "accuracy": 0.5283018867924528,
-            "count": 53
+            "accuracy": 0.28703703703703703,
+            "count": 108
           }
         }
       },
       "sub_M3": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.5085714285714286,
-        "n_examples": 50,
+        "digit_accuracy": 0.43,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.8144329896907216,
-            "count": 97
+            "accuracy": 0.8435754189944135,
+            "count": 179
           },
           "MB": {
-            "accuracy": 0.6274509803921569,
-            "count": 51
+            "accuracy": 0.14563106796116504,
+            "count": 103
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 27
+            "accuracy": 0.9642857142857143,
+            "count": 56
           },
           "UB": {
-            "accuracy": 0.5135135135135135,
-            "count": 74
+            "accuracy": 0.48322147651006714,
+            "count": 149
           },
           "UD": {
-            "accuracy": 0.019801980198019802,
-            "count": 101
+            "accuracy": 0.04225352112676056,
+            "count": 213
           }
         }
       },
       "sub_M4": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.3485714285714286,
-        "n_examples": 50,
+        "digit_accuracy": 0.31857142857142856,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.64,
-            "count": 100
+            "accuracy": 0.705,
+            "count": 200
           },
           "MB": {
-            "accuracy": 0.58,
-            "count": 50
+            "accuracy": 0.08,
+            "count": 100
           },
           "UB": {
             "accuracy": 0.56,
-            "count": 50
+            "count": 100
           },
           "UD": {
-            "accuracy": 0.006666666666666667,
-            "count": 150
+            "accuracy": 0.06,
+            "count": 300
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.34,
-        "n_examples": 50,
+        "digit_accuracy": 0.24428571428571427,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.74,
-            "count": 50
+            "accuracy": 0.0,
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.64,
-            "count": 50
+            "accuracy": 0.71,
+            "count": 100
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 200
+            "count": 400
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.035,
-        "digit_accuracy": 0.6064285714285714,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.4635714285714286,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.6543859649122807,
-            "count": 570
+            "accuracy": 0.5716666666666667,
+            "count": 600
           },
           "MB": {
-            "accuracy": 0.7075812274368231,
-            "count": 277
+            "accuracy": 0.3258426966292135,
+            "count": 267
           },
           "ME": {
-            "accuracy": 0.9245283018867925,
+            "accuracy": 0.8113207547169812,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.46709129511677283,
-            "count": 471
+            "accuracy": 0.35990888382687924,
+            "count": 439
           },
           "UD": {
-            "accuracy": 0.3793103448275862,
-            "count": 29
+            "accuracy": 0.43902439024390244,
+            "count": 41
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.5485714285714286,
-        "n_examples": 50,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.38857142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.6333333333333333,
-            "count": 150
+            "accuracy": 0.57,
+            "count": 300
           },
           "MB": {
-            "accuracy": 0.82,
-            "count": 50
+            "accuracy": 0.22,
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.45544554455445546,
-            "count": 101
+            "accuracy": 0.3350253807106599,
+            "count": 197
           },
           "UD": {
-            "accuracy": 0.20408163265306123,
-            "count": 49
+            "accuracy": 0.1262135922330097,
+            "count": 103
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.52,
-        "n_examples": 50,
+        "digit_accuracy": 0.3742857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.77,
-            "count": 100
+            "accuracy": 0.69,
+            "count": 200
           },
           "MB": {
-            "accuracy": 0.78,
-            "count": 50
+            "accuracy": 0.18,
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.4214876033057851,
-            "count": 121
+            "accuracy": 0.3441295546558704,
+            "count": 247
           },
           "UD": {
-            "accuracy": 0.189873417721519,
-            "count": 79
+            "accuracy": 0.13725490196078433,
+            "count": 153
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.5057142857142857,
-        "n_examples": 50,
+        "digit_accuracy": 0.35714285714285715,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.76,
-            "count": 50
+            "accuracy": 0.17,
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.46710526315789475,
-            "count": 152
+            "accuracy": 0.38926174496644295,
+            "count": 298
           },
           "UD": {
-            "accuracy": 0.1836734693877551,
-            "count": 98
+            "accuracy": 0.08415841584158416,
+            "count": 202
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.034,
-      "digit_accuracy": 0.5633333333333334,
-      "total_examples": 1500,
+      "overall_accuracy": 0.011153846153846153,
+      "digit_accuracy": 0.43615384615384617,
+      "total_examples": 2600,
       "n_splits": 24
     }
   }

add_sub_baseline_25K_2L1H128d/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6dfc7feee43bb81e3190bd7c2981cb84962bbd16d1ec47f74806d132e2a2441d
+oid sha256:2da20f6f58edc03c2d50da20dd2e8f1236b5943d7843ea3a20205d1d47e945ab
 size 157692826

add_sub_baseline_25K_2L1H128d/train_config.json

@@ -69,16 +69,20 @@
   "no_wandb": false,
   "n_params": 39346560,
   "run_name": "add_sub_baseline_25K_2L1H128d",
-  "git_commit": "f447da529caceac8c7d256cbb2cd185cbc50feac",
-  "timestamp": "2026-04-12T18:09:34.520594+00:00",
+  "git_commit": "1d5a160e16a5070d61b881494e832aa88149b15c",
+  "timestamp": "2026-04-15T04:56:24.356457+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
+  "train_dataset": "fixed_train/train_25K_seed42.pt",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "sft",
-  "wandb_run_id": "5l0ckmy9",
-  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/5l0ckmy9",
-  "final_accuracy": 0.03791666666666667,
-  "sft_accuracy": 0.03791666666666667,
+  "wandb_run_id": "rw1bb80x",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/rw1bb80x",
+  "eval_final_dataset": "eval_sets/eval_add_sub_6d_N100_seed42.json",
+  "eval_epoch_dataset": "eval_sets/eval_add_sub_6d_N25_seed42.json",
+  "eval_hf_repo": "thoughtworks/arithmetic-sorl-data",
+  "final_accuracy": 0.011153846153846153,
+  "sft_accuracy": 0.011153846153846153,
   "eval_method": "ArithmeticEvaluator"
 }