Upload add_sub_baseline_25K_1L2H256d

add_sub_baseline_25K_1L2H256d/metrics.json

@@ -159,320 +159,320 @@
       7800
     ],
     "loss": [
-      11.671636581420898,
-      10.659524917602539,
-      9.829411506652832,
-      9.199652671813965,
-      8.631672859191895,
-      7.977789878845215,
-      7.31129789352417,
-      6.896409034729004,
-      6.216288089752197,
-      5.601308345794678,
-      5.060961723327637,
-      4.576255798339844,
-      4.051487922668457,
-      3.5682461261749268,
-      3.152874231338501,
-      2.812526226043701,
-      2.522318124771118,
-      2.3712620735168457,
-      2.1769790649414062,
-      2.142094135284424,
-      2.0678162574768066,
-      1.915800929069519,
-      2.0273938179016113,
-      1.892726182937622,
-      1.8822896480560303,
-      1.9177898168563843,
-      1.9286824464797974,
-      1.8752065896987915,
-      1.8638038635253906,
-      1.8708428144454956,
-      1.8120969533920288,
-      1.827845811843872,
-      1.7674380540847778,
-      1.86089026927948,
-      1.7976375818252563,
-      1.7858340740203857,
-      1.8120166063308716,
-      1.7935630083084106,
-      1.8080732822418213,
-      1.772081971168518,
-      1.7113384008407593,
-      1.8078746795654297,
-      1.7796181440353394,
-      1.7605429887771606,
-      1.749908685684204,
-      1.8526047468185425,
-      1.809884786605835,
-      1.744711995124817,
-      1.7835290431976318,
-      1.7533817291259766,
-      1.7311333417892456,
-      1.7385872602462769,
-      1.7065470218658447,
-      1.7380945682525635,
-      1.6556047201156616,
-      1.6597269773483276,
-      1.5854241847991943,
-      1.6379982233047485,
-      1.636677861213684,
-      1.708864450454712,
-      1.61447274684906,
-      1.7286317348480225,
-      1.5406564474105835,
-      1.5738743543624878,
-      1.599366307258606,
-      1.551311731338501,
-      1.5821548700332642,
-      1.4954397678375244,
-      1.6357351541519165,
-      1.610660433769226,
-      1.5579359531402588,
-      1.585968017578125,
-      1.4483888149261475,
-      1.5140514373779297,
-      1.4171068668365479,
-      1.3210866451263428,
-      1.3247480392456055,
-      1.3959987163543701,
-      1.2662498950958252,
-      1.314728856086731,
-      1.2179921865463257,
-      1.162067174911499,
-      1.156597375869751,
-      1.180772066116333,
-      1.093071460723877,
-      1.0152738094329834,
-      0.9924407601356506,
-      1.028732180595398,
-      1.0052201747894287,
-      1.0063453912734985,
-      0.9981479644775391,
-      0.8756313920021057,
-      0.9030128717422485,
-      0.9033327102661133,
-      0.9253665804862976,
-      0.9430330991744995,
-      0.9484753608703613,
-      0.8658924698829651,
-      0.8693656921386719,
-      0.8428267240524292,
-      0.8409420847892761,
-      0.8913314938545227,
-      0.7889167070388794,
-      0.786769449710846,
-      0.8332729339599609,
-      0.8155103325843811,
-      0.8029834628105164,
-      0.7777830362319946,
-      0.8465943932533264,
-      0.7783411741256714,
-      0.7732486724853516,
-      0.7279804944992065,
-      0.7854341864585876,
-      0.6857983469963074,
-      0.7444823384284973,
-      0.7447658181190491,
-      0.7315057516098022,
-      0.7095687985420227,
-      0.7285814881324768,
-      0.7516853213310242,
-      0.7119408845901489,
-      0.7268796563148499,
-      0.7778827548027039,
-      0.7403841018676758,
-      0.7479632496833801,
-      0.7538272738456726,
-      0.6861394643783569,
-      0.7120760679244995,
-      0.701282799243927,
-      0.6962236762046814,
-      0.7168822884559631,
-      0.7258835434913635,
-      0.7194567918777466,
-      0.702591061592102,
-      0.7357982993125916,
-      0.7088969349861145,
-      0.6972709894180298,
-      0.7118088603019714,
-      0.6672061085700989,
-      0.7101794481277466,
-      0.704630434513092,
-      0.678624153137207,
-      0.7467736005783081,
-      0.7612290978431702,
-      0.6677433848381042,
-      0.7306165099143982,
-      0.6715617179870605,
-      0.6923214197158813,
-      0.6501680016517639,
-      0.6605051755905151,
-      0.6703984141349792,
-      0.7571574449539185,
-      0.7139632105827332,
-      0.74234539270401,
-      0.776197612285614,
-      0.6756425499916077
+      11.67835807800293,
+      10.719292640686035,
+      9.762801170349121,
+      9.028444290161133,
+      8.50659465789795,
+      7.853487491607666,
+      7.278311729431152,
+      6.883544921875,
+      6.277066230773926,
+      5.69489049911499,
+      5.099170207977295,
+      4.710115909576416,
+      4.105507850646973,
+      3.5853869915008545,
+      3.1561012268066406,
+      2.732867956161499,
+      2.5105342864990234,
+      2.380223035812378,
+      2.1963541507720947,
+      2.0700221061706543,
+      2.055406332015991,
+      1.9924981594085693,
+      1.9652398824691772,
+      1.9360522031784058,
+      1.9294606447219849,
+      1.9191540479660034,
+      1.884749174118042,
+      1.8851999044418335,
+      1.875937819480896,
+      1.8560281991958618,
+      1.820598840713501,
+      1.7985869646072388,
+      1.8134841918945312,
+      1.8033429384231567,
+      1.843862771987915,
+      1.8056700229644775,
+      1.7657887935638428,
+      1.771441102027893,
+      1.7327417135238647,
+      1.7670358419418335,
+      1.7790080308914185,
+      1.7342703342437744,
+      1.8182449340820312,
+      1.758160948753357,
+      1.8148043155670166,
+      1.7672239542007446,
+      1.759755253791809,
+      1.6406434774398804,
+      1.6834887266159058,
+      1.6384862661361694,
+      1.669707179069519,
+      1.6000508069992065,
+      1.7436786890029907,
+      1.712514877319336,
+      1.666881799697876,
+      1.7062987089157104,
+      1.604015588760376,
+      1.6764469146728516,
+      1.584958791732788,
+      1.647741675376892,
+      1.6784383058547974,
+      1.5880528688430786,
+      1.6094725131988525,
+      1.5886987447738647,
+      1.5662020444869995,
+      1.6310920715332031,
+      1.5743287801742554,
+      1.5807675123214722,
+      1.537028431892395,
+      1.5652409791946411,
+      1.475896954536438,
+      1.5633798837661743,
+      1.318359136581421,
+      1.489592432975769,
+      1.4318926334381104,
+      1.295494794845581,
+      1.4096946716308594,
+      1.255828857421875,
+      1.237991452217102,
+      1.1837003231048584,
+      1.2067559957504272,
+      1.183036208152771,
+      1.0562609434127808,
+      1.0683873891830444,
+      1.002286672592163,
+      1.0516374111175537,
+      1.0503524541854858,
+      1.0116995573043823,
+      0.9874834418296814,
+      0.9922329783439636,
+      1.0121840238571167,
+      0.9221497774124146,
+      0.9718131422996521,
+      0.9580082893371582,
+      0.9458693265914917,
+      0.8837714791297913,
+      0.852050244808197,
+      0.8929813504219055,
+      0.863052248954773,
+      0.8303223848342896,
+      0.912526547908783,
+      0.8493172526359558,
+      0.836484432220459,
+      0.8957289457321167,
+      0.8345073461532593,
+      0.8347916007041931,
+      0.8315057754516602,
+      0.8784408569335938,
+      0.774987518787384,
+      0.7976661324501038,
+      0.7938314080238342,
+      0.7878810167312622,
+      0.7915247082710266,
+      0.7997151613235474,
+      0.7489715218544006,
+      0.7730088829994202,
+      0.8028727769851685,
+      0.8076448440551758,
+      0.7823832631111145,
+      0.762302815914154,
+      0.714794933795929,
+      0.7424928545951843,
+      0.7266560196876526,
+      0.734574019908905,
+      0.7426309585571289,
+      0.7433751821517944,
+      0.6983920335769653,
+      0.76390141248703,
+      0.7495347857475281,
+      0.783293604850769,
+      0.7293379902839661,
+      0.7455155253410339,
+      0.7366787791252136,
+      0.7058432698249817,
+      0.6945399045944214,
+      0.7149715423583984,
+      0.7223644256591797,
+      0.742652416229248,
+      0.7686190009117126,
+      0.7120343446731567,
+      0.6914497017860413,
+      0.7294840812683105,
+      0.6813774108886719,
+      0.7342352271080017,
+      0.7158920168876648,
+      0.7304601073265076,
+      0.7657988667488098,
+      0.7277460098266602,
+      0.7255235314369202,
+      0.6857752799987793,
+      0.7248324155807495,
+      0.7265051603317261,
+      0.7114661335945129,
+      0.7851969599723816,
+      0.7334709167480469,
+      0.7515817284584045
     ],
     "base_loss": [
-      11.671636581420898,
-      10.659524917602539,
-      9.829411506652832,
-      9.199652671813965,
-      8.631672859191895,
-      7.977789878845215,
-      7.31129789352417,
-      6.896409034729004,
-      6.216288089752197,
-      5.601308345794678,
-      5.060961723327637,
-      4.576255798339844,
-      4.051487922668457,
-      3.5682461261749268,
-      3.152874231338501,
-      2.812526226043701,
-      2.522318124771118,
-      2.3712620735168457,
-      2.1769790649414062,
-      2.142094135284424,
-      2.0678162574768066,
-      1.915800929069519,
-      2.0273938179016113,
-      1.892726182937622,
-      1.8822896480560303,
-      1.9177898168563843,
-      1.9286824464797974,
-      1.8752065896987915,
-      1.8638038635253906,
-      1.8708428144454956,
-      1.8120969533920288,
-      1.827845811843872,
-      1.7674380540847778,
-      1.86089026927948,
-      1.7976375818252563,
-      1.7858340740203857,
-      1.8120166063308716,
-      1.7935630083084106,
-      1.8080732822418213,
-      1.772081971168518,
-      1.7113384008407593,
-      1.8078746795654297,
-      1.7796181440353394,
-      1.7605429887771606,
-      1.749908685684204,
-      1.8526047468185425,
-      1.809884786605835,
-      1.744711995124817,
-      1.7835290431976318,
-      1.7533817291259766,
-      1.7311333417892456,
-      1.7385872602462769,
-      1.7065470218658447,
-      1.7380945682525635,
-      1.6556047201156616,
-      1.6597269773483276,
-      1.5854241847991943,
-      1.6379982233047485,
-      1.636677861213684,
-      1.708864450454712,
-      1.61447274684906,
-      1.7286317348480225,
-      1.5406564474105835,
-      1.5738743543624878,
-      1.599366307258606,
-      1.551311731338501,
-      1.5821548700332642,
-      1.4954397678375244,
-      1.6357351541519165,
-      1.610660433769226,
-      1.5579359531402588,
-      1.585968017578125,
-      1.4483888149261475,
-      1.5140514373779297,
-      1.4171068668365479,
-      1.3210866451263428,
-      1.3247480392456055,
-      1.3959987163543701,
-      1.2662498950958252,
-      1.314728856086731,
-      1.2179921865463257,
-      1.162067174911499,
-      1.156597375869751,
-      1.180772066116333,
-      1.093071460723877,
-      1.0152738094329834,
-      0.9924407601356506,
-      1.028732180595398,
-      1.0052201747894287,
-      1.0063453912734985,
-      0.9981479644775391,
-      0.8756313920021057,
-      0.9030128717422485,
-      0.9033327102661133,
-      0.9253665804862976,
-      0.9430330991744995,
-      0.9484753608703613,
-      0.8658924698829651,
-      0.8693656921386719,
-      0.8428267240524292,
-      0.8409420847892761,
-      0.8913314938545227,
-      0.7889167070388794,
-      0.786769449710846,
-      0.8332729339599609,
-      0.8155103325843811,
-      0.8029834628105164,
-      0.7777830362319946,
-      0.8465943932533264,
-      0.7783411741256714,
-      0.7732486724853516,
-      0.7279804944992065,
-      0.7854341864585876,
-      0.6857983469963074,
-      0.7444823384284973,
-      0.7447658181190491,
-      0.7315057516098022,
-      0.7095687985420227,
-      0.7285814881324768,
-      0.7516853213310242,
-      0.7119408845901489,
-      0.7268796563148499,
-      0.7778827548027039,
-      0.7403841018676758,
-      0.7479632496833801,
-      0.7538272738456726,
-      0.6861394643783569,
-      0.7120760679244995,
-      0.701282799243927,
-      0.6962236762046814,
-      0.7168822884559631,
-      0.7258835434913635,
-      0.7194567918777466,
-      0.702591061592102,
-      0.7357982993125916,
-      0.7088969349861145,
-      0.6972709894180298,
-      0.7118088603019714,
-      0.6672061085700989,
-      0.7101794481277466,
-      0.704630434513092,
-      0.678624153137207,
-      0.7467736005783081,
-      0.7612290978431702,
-      0.6677433848381042,
-      0.7306165099143982,
-      0.6715617179870605,
-      0.6923214197158813,
-      0.6501680016517639,
-      0.6605051755905151,
-      0.6703984141349792,
-      0.7571574449539185,
-      0.7139632105827332,
-      0.74234539270401,
-      0.776197612285614,
-      0.6756425499916077
+      11.67835807800293,
+      10.719292640686035,
+      9.762801170349121,
+      9.028444290161133,
+      8.50659465789795,
+      7.853487491607666,
+      7.278311729431152,
+      6.883544921875,
+      6.277066230773926,
+      5.69489049911499,
+      5.099170207977295,
+      4.710115909576416,
+      4.105507850646973,
+      3.5853869915008545,
+      3.1561012268066406,
+      2.732867956161499,
+      2.5105342864990234,
+      2.380223035812378,
+      2.1963541507720947,
+      2.0700221061706543,
+      2.055406332015991,
+      1.9924981594085693,
+      1.9652398824691772,
+      1.9360522031784058,
+      1.9294606447219849,
+      1.9191540479660034,
+      1.884749174118042,
+      1.8851999044418335,
+      1.875937819480896,
+      1.8560281991958618,
+      1.820598840713501,
+      1.7985869646072388,
+      1.8134841918945312,
+      1.8033429384231567,
+      1.843862771987915,
+      1.8056700229644775,
+      1.7657887935638428,
+      1.771441102027893,
+      1.7327417135238647,
+      1.7670358419418335,
+      1.7790080308914185,
+      1.7342703342437744,
+      1.8182449340820312,
+      1.758160948753357,
+      1.8148043155670166,
+      1.7672239542007446,
+      1.759755253791809,
+      1.6406434774398804,
+      1.6834887266159058,
+      1.6384862661361694,
+      1.669707179069519,
+      1.6000508069992065,
+      1.7436786890029907,
+      1.712514877319336,
+      1.666881799697876,
+      1.7062987089157104,
+      1.604015588760376,
+      1.6764469146728516,
+      1.584958791732788,
+      1.647741675376892,
+      1.6784383058547974,
+      1.5880528688430786,
+      1.6094725131988525,
+      1.5886987447738647,
+      1.5662020444869995,
+      1.6310920715332031,
+      1.5743287801742554,
+      1.5807675123214722,
+      1.537028431892395,
+      1.5652409791946411,
+      1.475896954536438,
+      1.5633798837661743,
+      1.318359136581421,
+      1.489592432975769,
+      1.4318926334381104,
+      1.295494794845581,
+      1.4096946716308594,
+      1.255828857421875,
+      1.237991452217102,
+      1.1837003231048584,
+      1.2067559957504272,
+      1.183036208152771,
+      1.0562609434127808,
+      1.0683873891830444,
+      1.002286672592163,
+      1.0516374111175537,
+      1.0503524541854858,
+      1.0116995573043823,
+      0.9874834418296814,
+      0.9922329783439636,
+      1.0121840238571167,
+      0.9221497774124146,
+      0.9718131422996521,
+      0.9580082893371582,
+      0.9458693265914917,
+      0.8837714791297913,
+      0.852050244808197,
+      0.8929813504219055,
+      0.863052248954773,
+      0.8303223848342896,
+      0.912526547908783,
+      0.8493172526359558,
+      0.836484432220459,
+      0.8957289457321167,
+      0.8345073461532593,
+      0.8347916007041931,
+      0.8315057754516602,
+      0.8784408569335938,
+      0.774987518787384,
+      0.7976661324501038,
+      0.7938314080238342,
+      0.7878810167312622,
+      0.7915247082710266,
+      0.7997151613235474,
+      0.7489715218544006,
+      0.7730088829994202,
+      0.8028727769851685,
+      0.8076448440551758,
+      0.7823832631111145,
+      0.762302815914154,
+      0.714794933795929,
+      0.7424928545951843,
+      0.7266560196876526,
+      0.734574019908905,
+      0.7426309585571289,
+      0.7433751821517944,
+      0.6983920335769653,
+      0.76390141248703,
+      0.7495347857475281,
+      0.783293604850769,
+      0.7293379902839661,
+      0.7455155253410339,
+      0.7366787791252136,
+      0.7058432698249817,
+      0.6945399045944214,
+      0.7149715423583984,
+      0.7223644256591797,
+      0.742652416229248,
+      0.7686190009117126,
+      0.7120343446731567,
+      0.6914497017860413,
+      0.7294840812683105,
+      0.6813774108886719,
+      0.7342352271080017,
+      0.7158920168876648,
+      0.7304601073265076,
+      0.7657988667488098,
+      0.7277460098266602,
+      0.7255235314369202,
+      0.6857752799987793,
+      0.7248324155807495,
+      0.7265051603317261,
+      0.7114661335945129,
+      0.7851969599723816,
+      0.7334709167480469,
+      0.7515817284584045
     ],
     "lr": [
       4.188034188034189e-06,
@@ -677,595 +677,595 @@
       20
     ],
     "eval_accuracy": [
+      0.002105263157894737,
       0.0,
-      0.0,
-      0.0077777777777777776,
-      0.011111111111111112,
-      0.0033333333333333335,
-      0.006666666666666667,
-      0.005555555555555556,
-      0.0033333333333333335,
-      0.01,
-      0.015555555555555555,
-      0.024444444444444446,
-      0.03333333333333333,
-      0.056666666666666664,
-      0.06,
-      0.05555555555555555,
-      0.05555555555555555,
-      0.057777777777777775,
-      0.056666666666666664,
-      0.058888888888888886,
-      0.06444444444444444
+      0.010526315789473684,
+      0.004210526315789474,
+      0.00631578947368421,
+      0.004210526315789474,
+      0.009473684210526316,
+      0.008421052631578947,
+      0.015789473684210527,
+      0.018947368421052633,
+      0.037894736842105266,
+      0.04842105263157895,
+      0.034736842105263156,
+      0.05263157894736842,
+      0.07263157894736842,
+      0.05052631578947368,
+      0.06421052631578947,
+      0.05157894736842105,
+      0.05578947368421053,
+      0.06526315789473684
     ]
   },
-  "final_accuracy": 0.06916666666666667,
+  "final_accuracy": 0.07653846153846154,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 50
+      "n_per_split": 100
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.48,
-        "digit_accuracy": 0.8971428571428571,
-        "n_examples": 50,
+        "full_accuracy": 0.4,
+        "digit_accuracy": 0.8628571428571429,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.8813559322033898,
-            "count": 295
+            "accuracy": 0.8495867768595041,
+            "count": 605
           },
           "SS": {
-            "accuracy": 0.9818181818181818,
-            "count": 55
+            "accuracy": 0.9473684210526315,
+            "count": 95
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.06,
-        "digit_accuracy": 0.6514285714285715,
-        "n_examples": 50,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.64,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.8968253968253969,
-            "count": 126
+            "accuracy": 0.8823529411764706,
+            "count": 204
           },
           "SC": {
-            "accuracy": 0.7848101265822784,
-            "count": 79
+            "accuracy": 0.7455621301775148,
+            "count": 169
           },
           "SS": {
-            "accuracy": 0.7619047619047619,
-            "count": 21
+            "accuracy": 0.8709677419354839,
+            "count": 31
           },
           "UC": {
-            "accuracy": 0.29838709677419356,
-            "count": 124
+            "accuracy": 0.3885135135135135,
+            "count": 296
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.08,
-        "digit_accuracy": 0.64,
-        "n_examples": 50,
+        "full_accuracy": 0.09,
+        "digit_accuracy": 0.6614285714285715,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.8666666666666667,
-            "count": 75
+            "accuracy": 0.9202453987730062,
+            "count": 163
           },
           "SC": {
-            "accuracy": 0.7096774193548387,
-            "count": 62
+            "accuracy": 0.7230769230769231,
+            "count": 130
           },
           "SS": {
-            "accuracy": 0.8461538461538461,
-            "count": 39
+            "accuracy": 0.8390804597701149,
+            "count": 87
           },
           "UC": {
-            "accuracy": 0.44144144144144143,
-            "count": 111
+            "accuracy": 0.4236453201970443,
+            "count": 203
           },
           "US": {
-            "accuracy": 0.5238095238095238,
-            "count": 63
+            "accuracy": 0.5128205128205128,
+            "count": 117
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.08,
-        "digit_accuracy": 0.5142857142857142,
-        "n_examples": 50,
+        "full_accuracy": 0.07,
+        "digit_accuracy": 0.49714285714285716,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9333333333333333,
-            "count": 60
+            "accuracy": 0.9256198347107438,
+            "count": 121
           },
           "SC": {
-            "accuracy": 0.7543859649122807,
-            "count": 57
+            "accuracy": 0.6611570247933884,
+            "count": 121
           },
           "SS": {
-            "accuracy": 0.8421052631578947,
-            "count": 19
+            "accuracy": 0.9387755102040817,
+            "count": 49
           },
           "UC": {
-            "accuracy": 0.375,
-            "count": 104
+            "accuracy": 0.34946236559139787,
+            "count": 186
           },
           "US": {
-            "accuracy": 0.23636363636363636,
-            "count": 110
+            "accuracy": 0.20179372197309417,
+            "count": 223
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.06,
-        "digit_accuracy": 0.44857142857142857,
-        "n_examples": 50,
+        "full_accuracy": 0.11,
+        "digit_accuracy": 0.48428571428571426,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9583333333333334,
-            "count": 48
+            "accuracy": 0.9230769230769231,
+            "count": 104
           },
           "SC": {
-            "accuracy": 0.8076923076923077,
-            "count": 52
+            "accuracy": 0.6981132075471698,
+            "count": 106
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 7
+            "accuracy": 0.8695652173913043,
+            "count": 23
           },
           "UC": {
-            "accuracy": 0.3595505617977528,
-            "count": 89
+            "accuracy": 0.49375,
+            "count": 160
           },
           "US": {
-            "accuracy": 0.19480519480519481,
-            "count": 154
+            "accuracy": 0.2280130293159609,
+            "count": 307
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.1,
-        "digit_accuracy": 0.37714285714285717,
-        "n_examples": 50,
+        "full_accuracy": 0.13,
+        "digit_accuracy": 0.37,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 0.98,
-            "count": 50
+            "count": 100
           },
           "SC": {
-            "accuracy": 0.56,
-            "count": 50
+            "accuracy": 0.55,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.34,
-            "count": 50
+            "accuracy": 0.32,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.19,
-            "count": 200
+            "accuracy": 0.185,
+            "count": 400
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.28,
-        "digit_accuracy": 0.4342857142857143,
-        "n_examples": 50,
+        "full_accuracy": 0.19,
+        "digit_accuracy": 0.3342857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.6,
-            "count": 50
+            "accuracy": 0.64,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.46,
-            "count": 50
+            "accuracy": 0.32,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.396,
-            "count": 250
+            "accuracy": 0.276,
+            "count": 500
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.05,
-        "digit_accuracy": 0.635,
+        "full_accuracy": 0.035,
+        "digit_accuracy": 0.6421428571428571,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.8723897911832946,
-            "count": 431
+            "accuracy": 0.8702460850111857,
+            "count": 447
           },
           "SC": {
-            "accuracy": 0.7974683544303798,
-            "count": 316
+            "accuracy": 0.771875,
+            "count": 320
           },
           "SS": {
-            "accuracy": 0.9230769230769231,
-            "count": 39
+            "accuracy": 0.9464285714285714,
+            "count": 56
           },
           "UC": {
-            "accuracy": 0.36607142857142855,
-            "count": 560
+            "accuracy": 0.3648393194706994,
+            "count": 529
           },
           "US": {
-            "accuracy": 0.37037037037037035,
-            "count": 54
+            "accuracy": 0.3541666666666667,
+            "count": 48
           }
         }
       },
       "add_C1": {
         "full_accuracy": 0.16,
-        "digit_accuracy": 0.8114285714285714,
-        "n_examples": 50,
+        "digit_accuracy": 0.8028571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.928,
-            "count": 250
+            "accuracy": 0.912,
+            "count": 500
           },
           "SC": {
-            "accuracy": 0.78,
-            "count": 50
+            "accuracy": 0.71,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.26,
-            "count": 50
+            "accuracy": 0.35,
+            "count": 100
           }
         }
       },
       "add_C2": {
-        "full_accuracy": 0.08,
-        "digit_accuracy": 0.74,
-        "n_examples": 50,
+        "full_accuracy": 0.07,
+        "digit_accuracy": 0.7271428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.94,
-            "count": 200
+            "accuracy": 0.91,
+            "count": 400
           },
           "SC": {
-            "accuracy": 0.74,
-            "count": 50
+            "accuracy": 0.7,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.3132530120481928,
-            "count": 83
+            "accuracy": 0.3525641025641026,
+            "count": 156
           },
           "US": {
-            "accuracy": 0.47058823529411764,
-            "count": 17
+            "accuracy": 0.45454545454545453,
+            "count": 44
           }
         }
       },
       "add_C3": {
         "full_accuracy": 0.02,
-        "digit_accuracy": 0.6457142857142857,
-        "n_examples": 50,
+        "digit_accuracy": 0.6271428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9133333333333333,
-            "count": 150
+            "accuracy": 0.89,
+            "count": 300
           },
           "SC": {
-            "accuracy": 0.8,
-            "count": 50
+            "accuracy": 0.74,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.33,
-            "count": 100
+            "accuracy": 0.3417085427135678,
+            "count": 199
           },
           "US": {
-            "accuracy": 0.32,
-            "count": 50
+            "accuracy": 0.297029702970297,
+            "count": 101
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.5885714285714285,
-        "n_examples": 50,
+        "full_accuracy": 0.03,
+        "digit_accuracy": 0.55,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 0.89,
+            "count": 200
           },
           "SC": {
-            "accuracy": 0.76,
-            "count": 50
+            "accuracy": 0.69,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.3484848484848485,
-            "count": 132
+            "accuracy": 0.32954545454545453,
+            "count": 264
           },
           "US": {
-            "accuracy": 0.3382352941176471,
-            "count": 68
+            "accuracy": 0.375,
+            "count": 136
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.4657142857142857,
-        "n_examples": 50,
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.45714285714285713,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.94,
-            "count": 50
+            "accuracy": 0.92,
+            "count": 100
           },
           "SC": {
-            "accuracy": 0.72,
-            "count": 50
+            "accuracy": 0.71,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.3835616438356164,
-            "count": 146
+            "accuracy": 0.33225806451612905,
+            "count": 310
           },
           "US": {
-            "accuracy": 0.23076923076923078,
-            "count": 104
+            "accuracy": 0.28421052631578947,
+            "count": 190
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.44571428571428573,
-        "n_examples": 50,
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.43,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.64,
-            "count": 50
+            "accuracy": 0.65,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.4444444444444444,
-            "count": 189
+            "accuracy": 0.41081081081081083,
+            "count": 370
           },
           "US": {
-            "accuracy": 0.36036036036036034,
-            "count": 111
+            "accuracy": 0.3652173913043478,
+            "count": 230
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.26,
-        "digit_accuracy": 0.8657142857142858,
-        "n_examples": 50,
+        "full_accuracy": 0.49,
+        "digit_accuracy": 0.8942857142857142,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.8481848184818482,
-            "count": 303
+            "accuracy": 0.8845528455284553,
+            "count": 615
           },
           "ME": {
-            "accuracy": 0.9787234042553191,
-            "count": 47
+            "accuracy": 0.9647058823529412,
+            "count": 85
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.6371428571428571,
-        "n_examples": 50,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.6257142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.8652482269503546,
-            "count": 141
+            "accuracy": 0.8561643835616438,
+            "count": 292
           },
           "MB": {
-            "accuracy": 0.6527777777777778,
-            "count": 72
+            "accuracy": 0.6319444444444444,
+            "count": 144
           },
           "ME": {
-            "accuracy": 0.9444444444444444,
-            "count": 18
+            "accuracy": 0.88,
+            "count": 25
           },
           "UB": {
-            "accuracy": 0.31092436974789917,
-            "count": 119
+            "accuracy": 0.3138075313807531,
+            "count": 239
           }
         }
       },
       "sub_M2": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.62,
-        "n_examples": 50,
+        "digit_accuracy": 0.5928571428571429,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9285714285714286,
-            "count": 112
+            "accuracy": 0.95260663507109,
+            "count": 211
           },
           "MB": {
-            "accuracy": 0.6981132075471698,
-            "count": 53
+            "accuracy": 0.6,
+            "count": 115
           },
           "ME": {
-            "accuracy": 0.9361702127659575,
-            "count": 47
+            "accuracy": 0.9529411764705882,
+            "count": 85
           },
           "UB": {
-            "accuracy": 0.2823529411764706,
-            "count": 85
+            "accuracy": 0.2541436464088398,
+            "count": 181
           },
           "UD": {
-            "accuracy": 0.1509433962264151,
-            "count": 53
+            "accuracy": 0.16666666666666666,
+            "count": 108
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.48857142857142855,
-        "n_examples": 50,
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.48428571428571426,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9381443298969072,
-            "count": 97
+            "accuracy": 0.9720670391061452,
+            "count": 179
           },
           "MB": {
-            "accuracy": 0.5686274509803921,
-            "count": 51
+            "accuracy": 0.5048543689320388,
+            "count": 103
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 27
+            "accuracy": 0.9464285714285714,
+            "count": 56
           },
           "UB": {
-            "accuracy": 0.1891891891891892,
-            "count": 74
+            "accuracy": 0.28187919463087246,
+            "count": 149
           },
           "UD": {
-            "accuracy": 0.09900990099009901,
-            "count": 101
+            "accuracy": 0.08450704225352113,
+            "count": 213
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.4342857142857143,
-        "n_examples": 50,
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.4085714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.88,
-            "count": 100
+            "accuracy": 0.85,
+            "count": 200
           },
           "MB": {
-            "accuracy": 0.68,
-            "count": 50
+            "accuracy": 0.71,
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.4,
-            "count": 50
+            "accuracy": 0.28,
+            "count": 100
           },
           "UD": {
-            "accuracy": 0.06666666666666667,
-            "count": 150
+            "accuracy": 0.056666666666666664,
+            "count": 300
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.30857142857142855,
-        "n_examples": 50,
+        "digit_accuracy": 0.3157142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.78,
-            "count": 50
+            "accuracy": 0.7,
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.38,
-            "count": 50
+            "accuracy": 0.41,
+            "count": 100
           },
           "UD": {
-            "accuracy": 0.0,
-            "count": 200
+            "accuracy": 0.025,
+            "count": 400
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.62,
+        "full_accuracy": 0.035,
+        "digit_accuracy": 0.6085714285714285,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.8807017543859649,
-            "count": 570
+            "accuracy": 0.87,
+            "count": 600
           },
           "MB": {
-            "accuracy": 0.6534296028880866,
-            "count": 277
+            "accuracy": 0.5767790262172284,
+            "count": 267
           },
           "ME": {
-            "accuracy": 0.9245283018867925,
+            "accuracy": 0.9811320754716981,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.2823779193205945,
-            "count": 471
+            "accuracy": 0.275626423690205,
+            "count": 439
           },
           "UD": {
-            "accuracy": 0.10344827586206896,
-            "count": 29
+            "accuracy": 0.07317073170731707,
+            "count": 41
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.0,
-        "digit_accuracy": 0.5657142857142857,
-        "n_examples": 50,
+        "full_accuracy": 0.01,
+        "digit_accuracy": 0.5728571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.8733333333333333,
-            "count": 150
+            "accuracy": 0.8566666666666667,
+            "count": 300
           },
           "MB": {
-            "accuracy": 0.68,
-            "count": 50
+            "accuracy": 0.75,
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.21782178217821782,
-            "count": 101
+            "accuracy": 0.28426395939086296,
+            "count": 197
           },
           "UD": {
-            "accuracy": 0.22448979591836735,
-            "count": 49
+            "accuracy": 0.1262135922330097,
+            "count": 103
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.52,
-        "n_examples": 50,
+        "digit_accuracy": 0.48428571428571426,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.96,
-            "count": 100
+            "accuracy": 0.905,
+            "count": 200
           },
           "MB": {
             "accuracy": 0.74,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.30578512396694213,
-            "count": 121
+            "accuracy": 0.27125506072874495,
+            "count": 247
           },
           "UD": {
-            "accuracy": 0.1518987341772152,
-            "count": 79
+            "accuracy": 0.1111111111111111,
+            "count": 153
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.4142857142857143,
-        "n_examples": 50,
+        "digit_accuracy": 0.4014285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.76,
-            "count": 50
+            "accuracy": 0.69,
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.2565789473684211,
-            "count": 152
+            "accuracy": 0.28187919463087246,
+            "count": 298
           },
           "UD": {
-            "accuracy": 0.1836734693877551,
-            "count": 98
+            "accuracy": 0.13861386138613863,
+            "count": 202
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.06533333333333333,
-      "digit_accuracy": 0.5842857142857143,
-      "total_examples": 1500,
+      "overall_accuracy": 0.07653846153846154,
+      "digit_accuracy": 0.5660439560439561,
+      "total_examples": 2600,
       "n_splits": 24
     }
   }

add_sub_baseline_25K_1L2H256d/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2419df932e118212192de7abd45dde4a7ef65934c47e2f930ab92a12790c2905
+oid sha256:3fb5c0fa94196e6867184d451f1f6bfc8c18f863c5aacd3016a5f2039f86c3c1
 size 315072674

add_sub_baseline_25K_1L2H256d/train_config.json

@@ -69,16 +69,20 @@
   "no_wandb": false,
   "n_params": 78691840,
   "run_name": "add_sub_baseline_25K_1L2H256d",
-  "git_commit": "f447da529caceac8c7d256cbb2cd185cbc50feac",
-  "timestamp": "2026-04-12T18:21:44.884269+00:00",
+  "git_commit": "f835493c19eb98267697007042c9d440cad2afbb",
+  "timestamp": "2026-04-15T11:19:56.467154+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
+  "train_dataset": "fixed_train/train_25K_seed42.pt",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "sft",
-  "wandb_run_id": "122jv7wp",
-  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/122jv7wp",
-  "final_accuracy": 0.06916666666666667,
-  "sft_accuracy": 0.06916666666666667,
+  "wandb_run_id": "0dg4rvu7",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/0dg4rvu7",
+  "eval_final_dataset": "eval_sets/eval_add_sub_6d_N100_seed42.json",
+  "eval_epoch_dataset": "eval_sets/eval_add_sub_6d_N25_seed42.json",
+  "eval_hf_repo": "thoughtworks/arithmetic-sorl-data",
+  "final_accuracy": 0.07653846153846154,
+  "sft_accuracy": 0.07653846153846154,
   "eval_method": "ArithmeticEvaluator"
 }