Upload add_sub_baseline_50K_2L1H128d

Browse files

Files changed (3) hide show

add_sub_baseline_50K_2L1H128d/metrics.json +897 -897
add_sub_baseline_50K_2L1H128d/model.safetensors +1 -1
add_sub_baseline_50K_2L1H128d/train_config.json +10 -6

add_sub_baseline_50K_2L1H128d/metrics.json CHANGED Viewed

@@ -315,632 +315,632 @@
       15600
     ],
     "loss": [
-      11.93499755859375,
-      11.65730094909668,
-      11.293171882629395,
-      10.867568969726562,
-      10.623129844665527,
-      10.355392456054688,
-      10.099054336547852,
-      9.930326461791992,
-      9.612565994262695,
-      9.24365234375,
-      9.00301456451416,
-      8.693142890930176,
-      8.47545051574707,
-      8.173990249633789,
-      7.881795406341553,
-      7.611324310302734,
-      7.3448920249938965,
-      7.075301170349121,
-      6.706723213195801,
-      6.513725280761719,
-      6.21718168258667,
-      5.842379570007324,
-      5.690333366394043,
-      5.309576511383057,
-      5.099353313446045,
-      4.7732625007629395,
-      4.531905651092529,
-      4.2372260093688965,
-      4.069974899291992,
-      3.8053746223449707,
-      3.5196328163146973,
-      3.335747241973877,
-      3.109636068344116,
-      3.0120599269866943,
-      2.8363168239593506,
-      2.676023483276367,
-      2.6105294227600098,
-      2.5148777961730957,
-      2.3828721046447754,
-      2.345553159713745,
-      2.2795569896698,
-      2.2441999912261963,
-      2.212557792663574,
-      2.105844497680664,
-      2.151437520980835,
-      2.183159351348877,
-      2.157996416091919,
-      2.0764122009277344,
-      2.0949976444244385,
-      2.0614359378814697,
-      2.0152029991149902,
-      2.042832374572754,
-      1.985764741897583,
-      2.0169003009796143,
-      1.9234191179275513,
-      1.98187255859375,
-      1.9020922183990479,
-      1.947366714477539,
-      1.912832498550415,
-      1.9788919687271118,
-      1.8701711893081665,
-      1.9873000383377075,
-      1.8384500741958618,
-      1.864878535270691,
-      1.8773438930511475,
-      1.814258337020874,
-      1.8422654867172241,
-      1.7905521392822266,
-      1.9188811779022217,
-      1.9195749759674072,
-      1.840183973312378,
-      1.9478784799575806,
-      1.7651938199996948,
-      1.8026297092437744,
-      1.8211473226547241,
-      1.6967977285385132,
-      1.774262547492981,
-      1.8092931509017944,
-      1.7626827955245972,
-      1.7678852081298828,
-      1.7257096767425537,
-      1.6912577152252197,
-      1.705618143081665,
-      1.7209644317626953,
-      1.6141128540039062,
-      1.5833446979522705,
-      1.6016104221343994,
-      1.6115186214447021,
-      1.4938329458236694,
-      1.5911413431167603,
-      1.4935094118118286,
-      1.4679266214370728,
-      1.4957019090652466,
-      1.446588158607483,
-      1.3904730081558228,
-      1.4148086309432983,
-      1.4284785985946655,
-      1.2719846963882446,
-      1.2865592241287231,
-      1.2901942729949951,
-      1.2603648900985718,
-      1.305459976196289,
-      1.1517776250839233,
-      1.1661006212234497,
-      1.1481062173843384,
-      1.1654253005981445,
-      1.143684983253479,
-      1.1574296951293945,
-      1.104051113128662,
-      1.1066854000091553,
-      1.0712209939956665,
-      1.0240689516067505,
-      1.0762532949447632,
-      0.9777925610542297,
-      1.0075340270996094,
-      1.0130956172943115,
-      0.9441969990730286,
-      0.9728404879570007,
-      0.9550210237503052,
-      0.9782089591026306,
-      0.9868655204772949,
-      0.9737717509269714,
-      0.9871805906295776,
-      0.8989226222038269,
-      0.9575096368789673,
-      0.9029489159584045,
-      0.8649100065231323,
-      0.8886522650718689,
-      0.8820281028747559,
-      0.8742324113845825,
-      0.88325035572052,
-      0.8407949805259705,
-      0.8328596353530884,
-      0.8495661616325378,
-      0.7535262107849121,
-      0.7798505425453186,
-      0.7734178900718689,
-      0.8000691533088684,
-      0.7600011825561523,
-      0.7487917542457581,
-      0.7534374594688416,
-      0.7358126044273376,
-      0.7776135206222534,
-      0.7900133728981018,
-      0.726458728313446,
-      0.7606948018074036,
-      0.7421143651008606,
-      0.7261181473731995,
-      0.7051188349723816,
-      0.7383636236190796,
-      0.6680508852005005,
-      0.680087685585022,
-      0.7639477849006653,
-      0.6837146878242493,
-      0.6702830195426941,
-      0.667018473148346,
-      0.6458178758621216,
-      0.626135528087616,
-      0.6087480187416077,
-      0.6776902079582214,
-      0.6297626495361328,
-      0.6746311187744141,
-      0.6392488479614258,
-      0.6054442524909973,
-      0.6305686235427856,
-      0.65091472864151,
-      0.6684236526489258,
-      0.630896270275116,
-      0.5898188352584839,
-      0.5816236138343811,
-      0.6031948924064636,
-      0.5798562169075012,
-      0.5800939202308655,
-      0.6128515601158142,
-      0.5971966981887817,
-      0.5636686086654663,
-      0.6101219058036804,
-      0.5626725554466248,
-      0.5759359002113342,
-      0.5579383969306946,
-      0.5766159892082214,
-      0.563798725605011,
-      0.5380284190177917,
-      0.5644034147262573,
-      0.5314499735832214,
-      0.549543559551239,
-      0.5258645415306091,
-      0.5414046049118042,
-      0.5449110269546509,
-      0.5177078247070312,
-      0.5357903242111206,
-      0.5383039116859436,
-      0.5342908501625061,
-      0.5052288770675659,
-      0.4803391993045807,
-      0.5331531167030334,
-      0.5584177374839783,
-      0.5099272131919861,
-      0.5030084252357483,
-      0.5107883810997009,
-      0.5074604749679565,
-      0.474060595035553,
-      0.5217799544334412,
-      0.5024420022964478,
-      0.5018254518508911,
-      0.48899394273757935,
-      0.5179036855697632,
-      0.4651453495025635,
-      0.4600445628166199,
-      0.46727508306503296,
-      0.4327971637248993,
-      0.4529948830604553,
-      0.46315622329711914,
-      0.48899713158607483,
-      0.4627113342285156,
-      0.45527729392051697,
-      0.47023242712020874,
-      0.4694068729877472,
-      0.4536097049713135,
-      0.46996352076530457,
-      0.4478769302368164,
-      0.5023384690284729,
-      0.4848630130290985,
-      0.44978800415992737,
-      0.4636957347393036,
-      0.46273866295814514,
-      0.4170573651790619,
-      0.45214805006980896,
-      0.4561144709587097,
-      0.4628432095050812,
-      0.46510520577430725,
-      0.5094900131225586,
-      0.4306984841823578,
-      0.4236433207988739,
-      0.41763633489608765,
-      0.4181102216243744,
-      0.41568782925605774,
-      0.43321552872657776,
-      0.42102813720703125,
-      0.44086867570877075,
-      0.4437388777732849,
-      0.42119714617729187,
-      0.4386158585548401,
-      0.4230058491230011,
-      0.42731285095214844,
-      0.43081793189048767,
-      0.40430232882499695,
-      0.39149078726768494,
-      0.4208660125732422,
-      0.41456151008605957,
-      0.4628043472766876,
-      0.45343923568725586,
-      0.4034596383571625,
-      0.43333905935287476,
-      0.4089009463787079,
-      0.428451269865036,
-      0.44539371132850647,
-      0.3964701294898987,
-      0.363548219203949,
-      0.41381707787513733,
-      0.41168859601020813,
-      0.3732242286205292,
-      0.4289732575416565,
-      0.4374411404132843,
-      0.4020676016807556,
-      0.39040374755859375,
-      0.435672789812088,
-      0.4191468358039856,
-      0.43827033042907715,
-      0.42149630188941956,
-      0.4028854966163635,
-      0.42512935400009155,
-      0.3935700058937073,
-      0.4023810029029846,
-      0.4407925307750702,
-      0.4217481017112732,
-      0.4327377676963806,
-      0.3848411738872528,
-      0.38072606921195984,
-      0.42019957304000854,
-      0.4077237546443939,
-      0.4318618178367615,
-      0.41669175028800964,
-      0.42001357674598694,
-      0.41785627603530884,
-      0.43324536085128784,
-      0.44770950078964233,
-      0.40982261300086975,
-      0.41195228695869446,
-      0.4122852385044098,
-      0.4267185628414154,
-      0.3932749927043915,
-      0.42152076959609985,
-      0.40566104650497437,
-      0.3950228691101074,
-      0.41320866346359253,
-      0.3923717141151428,
-      0.4263770282268524,
-      0.399299293756485,
-      0.37513676285743713,
-      0.3975963592529297,
-      0.3869399428367615,
-      0.3949268162250519,
-      0.43800851702690125,
-      0.38830724358558655,
-      0.4024786055088043,
-      0.3731174170970917,
-      0.3714914321899414,
-      0.39834064245224,
-      0.39818379282951355,
-      0.39613842964172363,
-      0.3969874978065491
     ],
     "base_loss": [
-      11.93499755859375,
-      11.65730094909668,
-      11.293171882629395,
-      10.867568969726562,
-      10.623129844665527,
-      10.355392456054688,
-      10.099054336547852,
-      9.930326461791992,
-      9.612565994262695,
-      9.24365234375,
-      9.00301456451416,
-      8.693142890930176,
-      8.47545051574707,
-      8.173990249633789,
-      7.881795406341553,
-      7.611324310302734,
-      7.3448920249938965,
-      7.075301170349121,
-      6.706723213195801,
-      6.513725280761719,
-      6.21718168258667,
-      5.842379570007324,
-      5.690333366394043,
-      5.309576511383057,
-      5.099353313446045,
-      4.7732625007629395,
-      4.531905651092529,
-      4.2372260093688965,
-      4.069974899291992,
-      3.8053746223449707,
-      3.5196328163146973,
-      3.335747241973877,
-      3.109636068344116,
-      3.0120599269866943,
-      2.8363168239593506,
-      2.676023483276367,
-      2.6105294227600098,
-      2.5148777961730957,
-      2.3828721046447754,
-      2.345553159713745,
-      2.2795569896698,
-      2.2441999912261963,
-      2.212557792663574,
-      2.105844497680664,
-      2.151437520980835,
-      2.183159351348877,
-      2.157996416091919,
-      2.0764122009277344,
-      2.0949976444244385,
-      2.0614359378814697,
-      2.0152029991149902,
-      2.042832374572754,
-      1.985764741897583,
-      2.0169003009796143,
-      1.9234191179275513,
-      1.98187255859375,
-      1.9020922183990479,
-      1.947366714477539,
-      1.912832498550415,
-      1.9788919687271118,
-      1.8701711893081665,
-      1.9873000383377075,
-      1.8384500741958618,
-      1.864878535270691,
-      1.8773438930511475,
-      1.814258337020874,
-      1.8422654867172241,
-      1.7905521392822266,
-      1.9188811779022217,
-      1.9195749759674072,
-      1.840183973312378,
-      1.9478784799575806,
-      1.7651938199996948,
-      1.8026297092437744,
-      1.8211473226547241,
-      1.6967977285385132,
-      1.774262547492981,
-      1.8092931509017944,
-      1.7626827955245972,
-      1.7678852081298828,
-      1.7257096767425537,
-      1.6912577152252197,
-      1.705618143081665,
-      1.7209644317626953,
-      1.6141128540039062,
-      1.5833446979522705,
-      1.6016104221343994,
-      1.6115186214447021,
-      1.4938329458236694,
-      1.5911413431167603,
-      1.4935094118118286,
-      1.4679266214370728,
-      1.4957019090652466,
-      1.446588158607483,
-      1.3904730081558228,
-      1.4148086309432983,
-      1.4284785985946655,
-      1.2719846963882446,
-      1.2865592241287231,
-      1.2901942729949951,
-      1.2603648900985718,
-      1.305459976196289,
-      1.1517776250839233,
-      1.1661006212234497,
-      1.1481062173843384,
-      1.1654253005981445,
-      1.143684983253479,
-      1.1574296951293945,
-      1.104051113128662,
-      1.1066854000091553,
-      1.0712209939956665,
-      1.0240689516067505,
-      1.0762532949447632,
-      0.9777925610542297,
-      1.0075340270996094,
-      1.0130956172943115,
-      0.9441969990730286,
-      0.9728404879570007,
-      0.9550210237503052,
-      0.9782089591026306,
-      0.9868655204772949,
-      0.9737717509269714,
-      0.9871805906295776,
-      0.8989226222038269,
-      0.9575096368789673,
-      0.9029489159584045,
-      0.8649100065231323,
-      0.8886522650718689,
-      0.8820281028747559,
-      0.8742324113845825,
-      0.88325035572052,
-      0.8407949805259705,
-      0.8328596353530884,
-      0.8495661616325378,
-      0.7535262107849121,
-      0.7798505425453186,
-      0.7734178900718689,
-      0.8000691533088684,
-      0.7600011825561523,
-      0.7487917542457581,
-      0.7534374594688416,
-      0.7358126044273376,
-      0.7776135206222534,
-      0.7900133728981018,
-      0.726458728313446,
-      0.7606948018074036,
-      0.7421143651008606,
-      0.7261181473731995,
-      0.7051188349723816,
-      0.7383636236190796,
-      0.6680508852005005,
-      0.680087685585022,
-      0.7639477849006653,
-      0.6837146878242493,
-      0.6702830195426941,
-      0.667018473148346,
-      0.6458178758621216,
-      0.626135528087616,
-      0.6087480187416077,
-      0.6776902079582214,
-      0.6297626495361328,
-      0.6746311187744141,
-      0.6392488479614258,
-      0.6054442524909973,
-      0.6305686235427856,
-      0.65091472864151,
-      0.6684236526489258,
-      0.630896270275116,
-      0.5898188352584839,
-      0.5816236138343811,
-      0.6031948924064636,
-      0.5798562169075012,
-      0.5800939202308655,
-      0.6128515601158142,
-      0.5971966981887817,
-      0.5636686086654663,
-      0.6101219058036804,
-      0.5626725554466248,
-      0.5759359002113342,
-      0.5579383969306946,
-      0.5766159892082214,
-      0.563798725605011,
-      0.5380284190177917,
-      0.5644034147262573,
-      0.5314499735832214,
-      0.549543559551239,
-      0.5258645415306091,
-      0.5414046049118042,
-      0.5449110269546509,
-      0.5177078247070312,
-      0.5357903242111206,
-      0.5383039116859436,
-      0.5342908501625061,
-      0.5052288770675659,
-      0.4803391993045807,
-      0.5331531167030334,
-      0.5584177374839783,
-      0.5099272131919861,
-      0.5030084252357483,
-      0.5107883810997009,
-      0.5074604749679565,
-      0.474060595035553,
-      0.5217799544334412,
-      0.5024420022964478,
-      0.5018254518508911,
-      0.48899394273757935,
-      0.5179036855697632,
-      0.4651453495025635,
-      0.4600445628166199,
-      0.46727508306503296,
-      0.4327971637248993,
-      0.4529948830604553,
-      0.46315622329711914,
-      0.48899713158607483,
-      0.4627113342285156,
-      0.45527729392051697,
-      0.47023242712020874,
-      0.4694068729877472,
-      0.4536097049713135,
-      0.46996352076530457,
-      0.4478769302368164,
-      0.5023384690284729,
-      0.4848630130290985,
-      0.44978800415992737,
-      0.4636957347393036,
-      0.46273866295814514,
-      0.4170573651790619,
-      0.45214805006980896,
-      0.4561144709587097,
-      0.4628432095050812,
-      0.46510520577430725,
-      0.5094900131225586,
-      0.4306984841823578,
-      0.4236433207988739,
-      0.41763633489608765,
-      0.4181102216243744,
-      0.41568782925605774,
-      0.43321552872657776,
-      0.42102813720703125,
-      0.44086867570877075,
-      0.4437388777732849,
-      0.42119714617729187,
-      0.4386158585548401,
-      0.4230058491230011,
-      0.42731285095214844,
-      0.43081793189048767,
-      0.40430232882499695,
-      0.39149078726768494,
-      0.4208660125732422,
-      0.41456151008605957,
-      0.4628043472766876,
-      0.45343923568725586,
-      0.4034596383571625,
-      0.43333905935287476,
-      0.4089009463787079,
-      0.428451269865036,
-      0.44539371132850647,
-      0.3964701294898987,
-      0.363548219203949,
-      0.41381707787513733,
-      0.41168859601020813,
-      0.3732242286205292,
-      0.4289732575416565,
-      0.4374411404132843,
-      0.4020676016807556,
-      0.39040374755859375,
-      0.435672789812088,
-      0.4191468358039856,
-      0.43827033042907715,
-      0.42149630188941956,
-      0.4028854966163635,
-      0.42512935400009155,
-      0.3935700058937073,
-      0.4023810029029846,
-      0.4407925307750702,
-      0.4217481017112732,
-      0.4327377676963806,
-      0.3848411738872528,
-      0.38072606921195984,
-      0.42019957304000854,
-      0.4077237546443939,
-      0.4318618178367615,
-      0.41669175028800964,
-      0.42001357674598694,
-      0.41785627603530884,
-      0.43324536085128784,
-      0.44770950078964233,
-      0.40982261300086975,
-      0.41195228695869446,
-      0.4122852385044098,
-      0.4267185628414154,
-      0.3932749927043915,
-      0.42152076959609985,
-      0.40566104650497437,
-      0.3950228691101074,
-      0.41320866346359253,
-      0.3923717141151428,
-      0.4263770282268524,
-      0.399299293756485,
-      0.37513676285743713,
-      0.3975963592529297,
-      0.3869399428367615,
-      0.3949268162250519,
-      0.43800851702690125,
-      0.38830724358558655,
-      0.4024786055088043,
-      0.3731174170970917,
-      0.3714914321899414,
-      0.39834064245224,
-      0.39818379282951355,
-      0.39613842964172363,
-      0.3969874978065491
     ],
     "lr": [
       2.0940170940170946e-06,
@@ -1304,592 +1304,592 @@
       0.0,
       0.0,
       0.0,
-      0.0011111111111111111,
-      0.0022222222222222222,
-      0.014444444444444444,
-      0.052222222222222225,
-      0.051111111111111114,
-      0.08333333333333333,
-      0.10888888888888888,
-      0.11222222222222222,
-      0.1288888888888889,
-      0.13777777777777778,
-      0.17,
-      0.15555555555555556,
-      0.18444444444444444,
-      0.2088888888888889,
-      0.20222222222222222,
-      0.21777777777777776,
-      0.21444444444444444
     ]
   },
-  "final_accuracy": 0.1925,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.48,
-        "digit_accuracy": 0.9114285714285715,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.8983050847457628,
-            "count": 295
           },
           "SS": {
-            "accuracy": 0.9818181818181818,
-            "count": 55
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.3,
-        "digit_accuracy": 0.8371428571428572,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9047619047619048,
-            "count": 126
           },
           "SC": {
-            "accuracy": 0.8987341772151899,
-            "count": 79
           },
           "SS": {
-            "accuracy": 0.9523809523809523,
-            "count": 21
           },
           "UC": {
-            "accuracy": 0.7096774193548387,
-            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.2,
-        "digit_accuracy": 0.8114285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.8266666666666667,
-            "count": 75
           },
           "SC": {
-            "accuracy": 0.8870967741935484,
-            "count": 62
           },
           "SS": {
-            "accuracy": 0.8461538461538461,
-            "count": 39
           },
           "UC": {
-            "accuracy": 0.6396396396396397,
-            "count": 111
           },
           "US": {
-            "accuracy": 1.0,
-            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.18,
-        "digit_accuracy": 0.8228571428571428,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9666666666666667,
-            "count": 60
           },
           "SC": {
-            "accuracy": 0.8947368421052632,
-            "count": 57
           },
           "SS": {
-            "accuracy": 0.8421052631578947,
-            "count": 19
           },
           "UC": {
-            "accuracy": 0.7115384615384616,
-            "count": 104
           },
           "US": {
-            "accuracy": 0.8090909090909091,
-            "count": 110
           }
         }
       },
       "add_S4": {
         "full_accuracy": 0.24,
-        "digit_accuracy": 0.6914285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9375,
-            "count": 48
           },
           "SC": {
-            "accuracy": 0.9423076923076923,
-            "count": 52
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 7
           },
           "UC": {
-            "accuracy": 0.6067415730337079,
-            "count": 89
           },
           "US": {
-            "accuracy": 0.564935064935065,
-            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.1,
-        "digit_accuracy": 0.5257142857142857,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
-            "count": 50
           },
           "SC": {
-            "accuracy": 1.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.42,
-            "count": 50
           },
           "US": {
-            "accuracy": 0.315,
-            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.34,
-        "digit_accuracy": 0.5742857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 1.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.46,
-            "count": 50
           },
           "US": {
-            "accuracy": 0.512,
-            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.25,
-        "digit_accuracy": 0.8514285714285714,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.91415313225058,
-            "count": 431
           },
           "SC": {
-            "accuracy": 0.9493670886075949,
-            "count": 316
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 39
           },
           "UC": {
-            "accuracy": 0.7303571428571428,
-            "count": 560
           },
           "US": {
-            "accuracy": 0.9259259259259259,
-            "count": 54
           }
         }
       },
       "add_C1": {
-        "full_accuracy": 0.26,
-        "digit_accuracy": 0.8657142857142858,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.928,
-            "count": 250
           },
           "SC": {
-            "accuracy": 0.96,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.46,
-            "count": 50
           }
         }
       },
       "add_C2": {
-        "full_accuracy": 0.24,
-        "digit_accuracy": 0.8371428571428572,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.945,
-            "count": 200
           },
           "SC": {
-            "accuracy": 0.94,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.5180722891566265,
-            "count": 83
           },
           "US": {
-            "accuracy": 0.8235294117647058,
-            "count": 17
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.14,
         "digit_accuracy": 0.8057142857142857,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9533333333333334,
-            "count": 150
           },
           "SC": {
-            "accuracy": 1.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.5,
-            "count": 100
           },
           "US": {
-            "accuracy": 0.78,
-            "count": 50
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.18,
-        "digit_accuracy": 0.82,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.98,
-            "count": 100
           },
           "SC": {
-            "accuracy": 0.98,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.6060606060606061,
-            "count": 132
           },
           "US": {
-            "accuracy": 0.8823529411764706,
-            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.16,
         "digit_accuracy": 0.7942857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.98,
-            "count": 50
           },
           "SC": {
-            "accuracy": 1.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.6438356164383562,
-            "count": 146
           },
           "US": {
-            "accuracy": 0.8173076923076923,
-            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.14,
-        "digit_accuracy": 0.7628571428571429,
-        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.98,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.6931216931216931,
-            "count": 189
           },
           "US": {
-            "accuracy": 0.7837837837837838,
-            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.48,
-        "digit_accuracy": 0.8971428571428571,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.8811881188118812,
-            "count": 303
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.22,
-        "digit_accuracy": 0.8314285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9361702127659575,
-            "count": 141
           },
           "MB": {
-            "accuracy": 0.9583333333333334,
-            "count": 72
           },
           "ME": {
-            "accuracy": 0.8888888888888888,
-            "count": 18
           },
           "UB": {
-            "accuracy": 0.6218487394957983,
-            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.14,
-        "digit_accuracy": 0.8171428571428572,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9464285714285714,
-            "count": 112
           },
           "MB": {
-            "accuracy": 0.9245283018867925,
-            "count": 53
           },
           "ME": {
-            "accuracy": 0.9148936170212766,
-            "count": 47
           },
           "UB": {
-            "accuracy": 0.4588235294117647,
-            "count": 85
           },
           "UD": {
-            "accuracy": 0.9245283018867925,
-            "count": 53
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.02,
-        "digit_accuracy": 0.7142857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9278350515463918,
-            "count": 97
           },
           "MB": {
-            "accuracy": 0.9607843137254902,
-            "count": 51
           },
           "ME": {
-            "accuracy": 0.9629629629629629,
-            "count": 27
           },
           "UB": {
-            "accuracy": 0.4594594594594595,
-            "count": 74
           },
           "UD": {
-            "accuracy": 0.504950495049505,
-            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.08,
-        "digit_accuracy": 0.6542857142857142,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.96,
-            "count": 100
           },
           "MB": {
-            "accuracy": 0.94,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.56,
-            "count": 50
           },
           "UD": {
-            "accuracy": 0.38666666666666666,
-            "count": 150
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.04,
-        "digit_accuracy": 0.5314285714285715,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
           },
           "MB": {
-            "accuracy": 1.0,
-            "count": 50
           },
           "UB": {
             "accuracy": 0.56,
-            "count": 50
           },
           "UD": {
-            "accuracy": 0.29,
-            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.31,
-        "digit_accuracy": 0.8514285714285714,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9526315789473684,
-            "count": 570
           },
           "MB": {
-            "accuracy": 0.9494584837545126,
-            "count": 277
           },
           "ME": {
             "accuracy": 0.9811320754716981,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.6475583864118896,
-            "count": 471
           },
           "UD": {
-            "accuracy": 1.0,
-            "count": 29
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.06,
-        "digit_accuracy": 0.7942857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9666666666666667,
-            "count": 150
           },
           "MB": {
-            "accuracy": 0.98,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.48514851485148514,
-            "count": 101
           },
           "UD": {
-            "accuracy": 0.7142857142857143,
-            "count": 49
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.08,
-        "digit_accuracy": 0.7171428571428572,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 100
           },
           "MB": {
-            "accuracy": 0.98,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.45454545454545453,
-            "count": 121
           },
           "UD": {
-            "accuracy": 0.5949367088607594,
-            "count": 79
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.06,
-        "digit_accuracy": 0.7057142857142857,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.4934210526315789,
-            "count": 152
           },
           "UD": {
-            "accuracy": 0.7346938775510204,
-            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.21266666666666667,
-      "digit_accuracy": 0.7844761904761904,
-      "total_examples": 1500,
       "n_splits": 24
     }
   }

       15600
     ],
     "loss": [
+      11.902360916137695,
+      11.666987419128418,
+      11.278733253479004,
+      10.870060920715332,
+      10.583491325378418,
+      10.357315063476562,
+      10.140096664428711,
+      9.8663911819458,
+      9.547391891479492,
+      9.399455070495605,
+      9.050603866577148,
+      8.765486717224121,
+      8.510666847229004,
+      8.226161003112793,
+      7.946071147918701,
+      7.7140021324157715,
+      7.339910984039307,
+      7.013484477996826,
+      6.772124767303467,
+      6.514701843261719,
+      6.206382751464844,
+      5.888983726501465,
+      5.631106853485107,
+      5.346935272216797,
+      5.030400276184082,
+      4.801684379577637,
+      4.544530868530273,
+      4.25732946395874,
+      4.024832725524902,
+      3.7788538932800293,
+      3.5896661281585693,
+      3.4343185424804688,
+      3.1499485969543457,
+      2.975759983062744,
+      2.84167218208313,
+      2.7719228267669678,
+      2.685962677001953,
+      2.4699745178222656,
+      2.415933609008789,
+      2.3947372436523438,
+      2.274650812149048,
+      2.2390007972717285,
+      2.2176783084869385,
+      2.1771786212921143,
+      2.1928083896636963,
+      2.131913661956787,
+      2.166004180908203,
+      2.091320037841797,
+      2.138065814971924,
+      2.059041738510132,
+      2.0520567893981934,
+      2.069549322128296,
+      2.1000163555145264,
+      2.0322933197021484,
+      2.002192497253418,
+      1.993998646736145,
+      1.9147483110427856,
+      2.025142192840576,
+      1.9972620010375977,
+      1.9602936506271362,
+      1.9629830121994019,
+      1.9658845663070679,
+      1.908106803894043,
+      1.8817909955978394,
+      1.9339027404785156,
+      1.9608190059661865,
+      1.8899195194244385,
+      1.8446751832962036,
+      1.8116694688796997,
+      1.8471367359161377,
+      1.8834497928619385,
+      1.8279780149459839,
+      1.7376059293746948,
+      1.8068692684173584,
+      1.8083994388580322,
+      1.7907131910324097,
+      1.7131086587905884,
+      1.6682668924331665,
+      1.7329654693603516,
+      1.612639307975769,
+      1.6369705200195312,
+      1.5567694902420044,
+      1.6371721029281616,
+      1.5882548093795776,
+      1.5648012161254883,
+      1.4990679025650024,
+      1.486085057258606,
+      1.4619702100753784,
+      1.492026686668396,
+      1.431452989578247,
+      1.4061998128890991,
+      1.4255949258804321,
+      1.369724988937378,
+      1.3176831007003784,
+      1.353609323501587,
+      1.287545919418335,
+      1.2904835939407349,
+      1.237708330154419,
+      1.2732247114181519,
+      1.2794243097305298,
+      1.232486605644226,
+      1.2047475576400757,
+      1.1985729932785034,
+      1.175077199935913,
+      1.152284026145935,
+      1.1384241580963135,
+      1.134267807006836,
+      1.157617211341858,
+      1.0927313566207886,
+      1.1036320924758911,
+      1.036346197128296,
+      1.1012026071548462,
+      1.0513644218444824,
+      1.082848310470581,
+      1.0133121013641357,
+      1.0115025043487549,
+      0.9968132376670837,
+      1.0016825199127197,
+      0.9806528687477112,
+      1.0336967706680298,
+      0.9553194642066956,
+      0.9541217684745789,
+      0.9729284644126892,
+      0.9127997159957886,
+      0.8899979591369629,
+      0.9080758094787598,
+      0.9263311624526978,
+      0.8968016505241394,
+      0.9046673774719238,
+      0.9198164343833923,
+      0.880632758140564,
+      0.858092188835144,
+      0.8669767379760742,
+      0.8270903825759888,
+      0.8441628813743591,
+      0.8286030888557434,
+      0.8017106056213379,
+      0.8334652185440063,
+      0.7973408102989197,
+      0.7861750721931458,
+      0.7986087799072266,
+      0.7349762320518494,
+      0.7474499940872192,
+      0.7652159333229065,
+      0.7694664001464844,
+      0.7698801159858704,
+      0.7689107060432434,
+      0.7688385844230652,
+      0.7331604361534119,
+      0.739378035068512,
+      0.7268163561820984,
+      0.6888265013694763,
+      0.7279484868049622,
+      0.6972444653511047,
+      0.6783798933029175,
+      0.7007697224617004,
+      0.6984170079231262,
+      0.7051367163658142,
+      0.6599271893501282,
+      0.6801080703735352,
+      0.6596269607543945,
+      0.6970394849777222,
+      0.6735067367553711,
+      0.6091650724411011,
+      0.5997785329818726,
+      0.6572319865226746,
+      0.6416034698486328,
+      0.6416297554969788,
+      0.6632289290428162,
+      0.6143150329589844,
+      0.6194843053817749,
+      0.5911574959754944,
+      0.5913539528846741,
+      0.6205930709838867,
+      0.6280899047851562,
+      0.5930410027503967,
+      0.5673736929893494,
+      0.5849844813346863,
+      0.573549747467041,
+      0.5737026333808899,
+      0.5761013031005859,
+      0.5887518525123596,
+      0.5787922739982605,
+      0.5531727075576782,
+      0.5395235419273376,
+      0.5347995162010193,
+      0.5664406418800354,
+      0.535719633102417,
+      0.5654853582382202,
+      0.5577755570411682,
+      0.5406196713447571,
+      0.5438421368598938,
+      0.5191155076026917,
+      0.5238958597183228,
+      0.5294751524925232,
+      0.5278587341308594,
+      0.5264574289321899,
+      0.526719868183136,
+      0.5218141674995422,
+      0.4832751452922821,
+      0.5143148303031921,
+      0.5116984248161316,
+      0.5009939670562744,
+      0.5164638161659241,
+      0.4878538250923157,
+      0.49269673228263855,
+      0.5037528276443481,
+      0.4777663052082062,
+      0.487152099609375,
+      0.46394041180610657,
+      0.4958875775337219,
+      0.5004867911338806,
+      0.4617388844490051,
+      0.4889770448207855,
+      0.46261903643608093,
+      0.44640398025512695,
+      0.4764914810657501,
+      0.49255529046058655,
+      0.4767415225505829,
+      0.4745542109012604,
+      0.44122710824012756,
+      0.44721779227256775,
+      0.4414359927177429,
+      0.4407655894756317,
+      0.4724572002887726,
+      0.46291184425354004,
+      0.4877897799015045,
+      0.46290403604507446,
+      0.4350273907184601,
+      0.4549728035926819,
+      0.4433819353580475,
+      0.459841787815094,
+      0.4417283535003662,
+      0.4167630970478058,
+      0.4117043912410736,
+      0.4316425919532776,
+      0.43041709065437317,
+      0.45717042684555054,
+      0.4303068518638611,
+      0.40953192114830017,
+      0.425926148891449,
+      0.3955731987953186,
+      0.42986229062080383,
+      0.4956095814704895,
+      0.4266194701194763,
+      0.4557635188102722,
+      0.44420650601387024,
+      0.41897672414779663,
+      0.43166255950927734,
+      0.4073420464992523,
+      0.4340267777442932,
+      0.43941253423690796,
+      0.41659075021743774,
+      0.4046151340007782,
+      0.4280521869659424,
+      0.42428717017173767,
+      0.4402787685394287,
+      0.4167115390300751,
+      0.4264659285545349,
+      0.4104572832584381,
+      0.41204768419265747,
+      0.44351884722709656,
+      0.4115920066833496,
+      0.41794463992118835,
+      0.39936190843582153,
+      0.4187738001346588,
+      0.40758633613586426,
+      0.40320804715156555,
+      0.4426390826702118,
+      0.42466142773628235,
+      0.4063023626804352,
+      0.4364185929298401,
+      0.3955465853214264,
+      0.39736407995224,
+      0.4157145917415619,
+      0.3955707252025604,
+      0.40614697337150574,
+      0.4296145737171173,
+      0.4289790689945221,
+      0.4248588979244232,
+      0.40194225311279297,
+      0.3777453601360321,
+      0.41459909081459045,
+      0.4123244881629944,
+      0.39063140749931335,
+      0.3955395221710205,
+      0.4094971716403961,
+      0.43402963876724243,
+      0.39580103754997253,
+      0.41646233201026917,
+      0.397787481546402,
+      0.39125150442123413,
+      0.4029674530029297,
+      0.4123210310935974,
+      0.3969133794307709,
+      0.41159409284591675,
+      0.36812132596969604,
+      0.40473809838294983,
+      0.4111975133419037,
+      0.420559823513031,
+      0.39803218841552734,
+      0.38535213470458984,
+      0.4098115563392639,
+      0.40594691038131714,
+      0.41488656401634216,
+      0.3916519582271576,
+      0.4265446066856384,
+      0.39961308240890503,
+      0.3947022259235382,
+      0.4145981967449188,
+      0.3922405540943146,
+      0.3995726406574249
     ],
     "base_loss": [
+      11.902360916137695,
+      11.666987419128418,
+      11.278733253479004,
+      10.870060920715332,
+      10.583491325378418,
+      10.357315063476562,
+      10.140096664428711,
+      9.8663911819458,
+      9.547391891479492,
+      9.399455070495605,
+      9.050603866577148,
+      8.765486717224121,
+      8.510666847229004,
+      8.226161003112793,
+      7.946071147918701,
+      7.7140021324157715,
+      7.339910984039307,
+      7.013484477996826,
+      6.772124767303467,
+      6.514701843261719,
+      6.206382751464844,
+      5.888983726501465,
+      5.631106853485107,
+      5.346935272216797,
+      5.030400276184082,
+      4.801684379577637,
+      4.544530868530273,
+      4.25732946395874,
+      4.024832725524902,
+      3.7788538932800293,
+      3.5896661281585693,
+      3.4343185424804688,
+      3.1499485969543457,
+      2.975759983062744,
+      2.84167218208313,
+      2.7719228267669678,
+      2.685962677001953,
+      2.4699745178222656,
+      2.415933609008789,
+      2.3947372436523438,
+      2.274650812149048,
+      2.2390007972717285,
+      2.2176783084869385,
+      2.1771786212921143,
+      2.1928083896636963,
+      2.131913661956787,
+      2.166004180908203,
+      2.091320037841797,
+      2.138065814971924,
+      2.059041738510132,
+      2.0520567893981934,
+      2.069549322128296,
+      2.1000163555145264,
+      2.0322933197021484,
+      2.002192497253418,
+      1.993998646736145,
+      1.9147483110427856,
+      2.025142192840576,
+      1.9972620010375977,
+      1.9602936506271362,
+      1.9629830121994019,
+      1.9658845663070679,
+      1.908106803894043,
+      1.8817909955978394,
+      1.9339027404785156,
+      1.9608190059661865,
+      1.8899195194244385,
+      1.8446751832962036,
+      1.8116694688796997,
+      1.8471367359161377,
+      1.8834497928619385,
+      1.8279780149459839,
+      1.7376059293746948,
+      1.8068692684173584,
+      1.8083994388580322,
+      1.7907131910324097,
+      1.7131086587905884,
+      1.6682668924331665,
+      1.7329654693603516,
+      1.612639307975769,
+      1.6369705200195312,
+      1.5567694902420044,
+      1.6371721029281616,
+      1.5882548093795776,
+      1.5648012161254883,
+      1.4990679025650024,
+      1.486085057258606,
+      1.4619702100753784,
+      1.492026686668396,
+      1.431452989578247,
+      1.4061998128890991,
+      1.4255949258804321,
+      1.369724988937378,
+      1.3176831007003784,
+      1.353609323501587,
+      1.287545919418335,
+      1.2904835939407349,
+      1.237708330154419,
+      1.2732247114181519,
+      1.2794243097305298,
+      1.232486605644226,
+      1.2047475576400757,
+      1.1985729932785034,
+      1.175077199935913,
+      1.152284026145935,
+      1.1384241580963135,
+      1.134267807006836,
+      1.157617211341858,
+      1.0927313566207886,
+      1.1036320924758911,
+      1.036346197128296,
+      1.1012026071548462,
+      1.0513644218444824,
+      1.082848310470581,
+      1.0133121013641357,
+      1.0115025043487549,
+      0.9968132376670837,
+      1.0016825199127197,
+      0.9806528687477112,
+      1.0336967706680298,
+      0.9553194642066956,
+      0.9541217684745789,
+      0.9729284644126892,
+      0.9127997159957886,
+      0.8899979591369629,
+      0.9080758094787598,
+      0.9263311624526978,
+      0.8968016505241394,
+      0.9046673774719238,
+      0.9198164343833923,
+      0.880632758140564,
+      0.858092188835144,
+      0.8669767379760742,
+      0.8270903825759888,
+      0.8441628813743591,
+      0.8286030888557434,
+      0.8017106056213379,
+      0.8334652185440063,
+      0.7973408102989197,
+      0.7861750721931458,
+      0.7986087799072266,
+      0.7349762320518494,
+      0.7474499940872192,
+      0.7652159333229065,
+      0.7694664001464844,
+      0.7698801159858704,
+      0.7689107060432434,
+      0.7688385844230652,
+      0.7331604361534119,
+      0.739378035068512,
+      0.7268163561820984,
+      0.6888265013694763,
+      0.7279484868049622,
+      0.6972444653511047,
+      0.6783798933029175,
+      0.7007697224617004,
+      0.6984170079231262,
+      0.7051367163658142,
+      0.6599271893501282,
+      0.6801080703735352,
+      0.6596269607543945,
+      0.6970394849777222,
+      0.6735067367553711,
+      0.6091650724411011,
+      0.5997785329818726,
+      0.6572319865226746,
+      0.6416034698486328,
+      0.6416297554969788,
+      0.6632289290428162,
+      0.6143150329589844,
+      0.6194843053817749,
+      0.5911574959754944,
+      0.5913539528846741,
+      0.6205930709838867,
+      0.6280899047851562,
+      0.5930410027503967,
+      0.5673736929893494,
+      0.5849844813346863,
+      0.573549747467041,
+      0.5737026333808899,
+      0.5761013031005859,
+      0.5887518525123596,
+      0.5787922739982605,
+      0.5531727075576782,
+      0.5395235419273376,
+      0.5347995162010193,
+      0.5664406418800354,
+      0.535719633102417,
+      0.5654853582382202,
+      0.5577755570411682,
+      0.5406196713447571,
+      0.5438421368598938,
+      0.5191155076026917,
+      0.5238958597183228,
+      0.5294751524925232,
+      0.5278587341308594,
+      0.5264574289321899,
+      0.526719868183136,
+      0.5218141674995422,
+      0.4832751452922821,
+      0.5143148303031921,
+      0.5116984248161316,
+      0.5009939670562744,
+      0.5164638161659241,
+      0.4878538250923157,
+      0.49269673228263855,
+      0.5037528276443481,
+      0.4777663052082062,
+      0.487152099609375,
+      0.46394041180610657,
+      0.4958875775337219,
+      0.5004867911338806,
+      0.4617388844490051,
+      0.4889770448207855,
+      0.46261903643608093,
+      0.44640398025512695,
+      0.4764914810657501,
+      0.49255529046058655,
+      0.4767415225505829,
+      0.4745542109012604,
+      0.44122710824012756,
+      0.44721779227256775,
+      0.4414359927177429,
+      0.4407655894756317,
+      0.4724572002887726,
+      0.46291184425354004,
+      0.4877897799015045,
+      0.46290403604507446,
+      0.4350273907184601,
+      0.4549728035926819,
+      0.4433819353580475,
+      0.459841787815094,
+      0.4417283535003662,
+      0.4167630970478058,
+      0.4117043912410736,
+      0.4316425919532776,
+      0.43041709065437317,
+      0.45717042684555054,
+      0.4303068518638611,
+      0.40953192114830017,
+      0.425926148891449,
+      0.3955731987953186,
+      0.42986229062080383,
+      0.4956095814704895,
+      0.4266194701194763,
+      0.4557635188102722,
+      0.44420650601387024,
+      0.41897672414779663,
+      0.43166255950927734,
+      0.4073420464992523,
+      0.4340267777442932,
+      0.43941253423690796,
+      0.41659075021743774,
+      0.4046151340007782,
+      0.4280521869659424,
+      0.42428717017173767,
+      0.4402787685394287,
+      0.4167115390300751,
+      0.4264659285545349,
+      0.4104572832584381,
+      0.41204768419265747,
+      0.44351884722709656,
+      0.4115920066833496,
+      0.41794463992118835,
+      0.39936190843582153,
+      0.4187738001346588,
+      0.40758633613586426,
+      0.40320804715156555,
+      0.4426390826702118,
+      0.42466142773628235,
+      0.4063023626804352,
+      0.4364185929298401,
+      0.3955465853214264,
+      0.39736407995224,
+      0.4157145917415619,
+      0.3955707252025604,
+      0.40614697337150574,
+      0.4296145737171173,
+      0.4289790689945221,
+      0.4248588979244232,
+      0.40194225311279297,
+      0.3777453601360321,
+      0.41459909081459045,
+      0.4123244881629944,
+      0.39063140749931335,
+      0.3955395221710205,
+      0.4094971716403961,
+      0.43402963876724243,
+      0.39580103754997253,
+      0.41646233201026917,
+      0.397787481546402,
+      0.39125150442123413,
+      0.4029674530029297,
+      0.4123210310935974,
+      0.3969133794307709,
+      0.41159409284591675,
+      0.36812132596969604,
+      0.40473809838294983,
+      0.4111975133419037,
+      0.420559823513031,
+      0.39803218841552734,
+      0.38535213470458984,
+      0.4098115563392639,
+      0.40594691038131714,
+      0.41488656401634216,
+      0.3916519582271576,
+      0.4265446066856384,
+      0.39961308240890503,
+      0.3947022259235382,
+      0.4145981967449188,
+      0.3922405540943146,
+      0.3995726406574249
     ],
     "lr": [
       2.0940170940170946e-06,
       0.0,
       0.0,
       0.0,
+      0.0,
+      0.002105263157894737,
+      0.04,
+      0.06631578947368422,
+      0.08421052631578947,
+      0.06736842105263158,
+      0.1,
+      0.12842105263157894,
+      0.16210526315789472,
+      0.2168421052631579,
+      0.19157894736842104,
+      0.2305263157894737,
+      0.2326315789473684,
+      0.24421052631578946,
+      0.24947368421052632,
+      0.26210526315789473,
+      0.26
     ]
   },
+  "final_accuracy": 0.22615384615384615,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
       "K": null,
       "mode": "sft",
       "n_digits": 6,
+      "n_per_split": 100
     },
     "splits": {
       "add_S0": {
+        "full_accuracy": 0.3,
+        "digit_accuracy": 0.8757142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.8611570247933884,
+            "count": 605
           },
           "SS": {
+            "accuracy": 0.968421052631579,
+            "count": 95
           }
         }
       },
       "add_S1": {
+        "full_accuracy": 0.41,
+        "digit_accuracy": 0.8757142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9264705882352942,
+            "count": 204
           },
           "SC": {
+            "accuracy": 0.9230769230769231,
+            "count": 169
           },
           "SS": {
+            "accuracy": 1.0,
+            "count": 31
           },
           "UC": {
+            "accuracy": 0.8006756756756757,
+            "count": 296
           }
         }
       },
       "add_S2": {
+        "full_accuracy": 0.23,
+        "digit_accuracy": 0.8414285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9079754601226994,
+            "count": 163
           },
           "SC": {
+            "accuracy": 0.8846153846153846,
+            "count": 130
           },
           "SS": {
+            "accuracy": 0.8390804597701149,
+            "count": 87
           },
           "UC": {
+            "accuracy": 0.6896551724137931,
+            "count": 203
           },
           "US": {
+            "accuracy": 0.9658119658119658,
+            "count": 117
           }
         }
       },
       "add_S3": {
+        "full_accuracy": 0.21,
+        "digit_accuracy": 0.8185714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9586776859504132,
+            "count": 121
           },
           "SC": {
+            "accuracy": 0.8677685950413223,
+            "count": 121
           },
           "SS": {
+            "accuracy": 0.8979591836734694,
+            "count": 49
           },
           "UC": {
+            "accuracy": 0.7204301075268817,
+            "count": 186
           },
           "US": {
+            "accuracy": 0.7802690582959642,
+            "count": 223
           }
         }
       },
       "add_S4": {
         "full_accuracy": 0.24,
+        "digit_accuracy": 0.7185714285714285,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9615384615384616,
+            "count": 104
           },
           "SC": {
+            "accuracy": 0.9056603773584906,
+            "count": 106
           },
           "SS": {
+            "accuracy": 0.8695652173913043,
+            "count": 23
           },
           "UC": {
+            "accuracy": 0.6625,
+            "count": 160
           },
           "US": {
+            "accuracy": 0.5895765472312704,
+            "count": 307
           }
         }
       },
       "add_S5": {
+        "full_accuracy": 0.13,
+        "digit_accuracy": 0.5528571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.96,
+            "count": 100
           },
           "SC": {
+            "accuracy": 0.95,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.41,
+            "count": 100
           },
           "US": {
+            "accuracy": 0.3875,
+            "count": 400
           }
         }
       },
       "add_S6": {
+        "full_accuracy": 0.49,
+        "digit_accuracy": 0.6614285714285715,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
+            "accuracy": 0.99,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.58,
+            "count": 100
           },
           "US": {
+            "accuracy": 0.612,
+            "count": 500
           }
         }
       },
       "add_random": {
+        "full_accuracy": 0.33,
+        "digit_accuracy": 0.8578571428571429,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.8859060402684564,
+            "count": 447
           },
           "SC": {
+            "accuracy": 0.928125,
+            "count": 320
           },
           "SS": {
+            "accuracy": 0.9464285714285714,
+            "count": 56
           },
           "UC": {
+            "accuracy": 0.7863894139886578,
+            "count": 529
           },
           "US": {
+            "accuracy": 0.8125,
+            "count": 48
           }
         }
       },
       "add_C1": {
+        "full_accuracy": 0.33,
+        "digit_accuracy": 0.8542857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.886,
+            "count": 500
           },
           "SC": {
+            "accuracy": 0.95,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.6,
+            "count": 100
           }
         }
       },
       "add_C2": {
+        "full_accuracy": 0.3,
+        "digit_accuracy": 0.8442857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.905,
+            "count": 400
           },
           "SC": {
+            "accuracy": 0.97,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.6089743589743589,
+            "count": 156
           },
           "US": {
+            "accuracy": 0.8409090909090909,
+            "count": 44
           }
         }
       },
       "add_C3": {
+        "full_accuracy": 0.23,
         "digit_accuracy": 0.8057142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.8866666666666667,
+            "count": 300
           },
           "SC": {
+            "accuracy": 0.92,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.6180904522613065,
+            "count": 199
           },
           "US": {
+            "accuracy": 0.8217821782178217,
+            "count": 101
           }
         }
       },
       "add_C4": {
+        "full_accuracy": 0.19,
+        "digit_accuracy": 0.79,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.91,
+            "count": 200
           },
           "SC": {
+            "accuracy": 0.94,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.6477272727272727,
+            "count": 264
           },
           "US": {
+            "accuracy": 0.7794117647058824,
+            "count": 136
           }
         }
       },
       "add_C5": {
+        "full_accuracy": 0.17,
         "digit_accuracy": 0.7942857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.97,
+            "count": 100
           },
           "SC": {
+            "accuracy": 0.96,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.667741935483871,
+            "count": 310
           },
           "US": {
+            "accuracy": 0.8210526315789474,
+            "count": 190
           }
         }
       },
       "add_C6": {
+        "full_accuracy": 0.23,
+        "digit_accuracy": 0.8228571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
+            "accuracy": 1.0,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.7594594594594595,
+            "count": 370
           },
           "US": {
+            "accuracy": 0.8478260869565217,
+            "count": 230
           }
         }
       },
       "sub_M0": {
+        "full_accuracy": 0.45,
+        "digit_accuracy": 0.9028571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.8894308943089431,
+            "count": 615
           },
           "ME": {
             "accuracy": 1.0,
+            "count": 85
           }
         }
       },
       "sub_M1": {
+        "full_accuracy": 0.26,
+        "digit_accuracy": 0.8457142857142858,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.9143835616438356,
+            "count": 292
           },
           "MB": {
+            "accuracy": 0.9166666666666666,
+            "count": 144
           },
           "ME": {
+            "accuracy": 1.0,
+            "count": 25
           },
           "UB": {
+            "accuracy": 0.702928870292887,
+            "count": 239
           }
         }
       },
       "sub_M2": {
+        "full_accuracy": 0.2,
+        "digit_accuracy": 0.8342857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.9004739336492891,
+            "count": 211
           },
           "MB": {
+            "accuracy": 0.9304347826086956,
+            "count": 115
           },
           "ME": {
+            "accuracy": 0.9882352941176471,
+            "count": 85
           },
           "UB": {
+            "accuracy": 0.5469613259668509,
+            "count": 181
           },
           "UD": {
+            "accuracy": 0.9629629629629629,
+            "count": 108
           }
         }
       },
       "sub_M3": {
+        "full_accuracy": 0.05,
+        "digit_accuracy": 0.74,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.9553072625698324,
+            "count": 179
           },
           "MB": {
+            "accuracy": 0.970873786407767,
+            "count": 103
           },
           "ME": {
+            "accuracy": 0.9464285714285714,
+            "count": 56
           },
           "UB": {
+            "accuracy": 0.4966442953020134,
+            "count": 149
           },
           "UD": {
+            "accuracy": 0.5633802816901409,
+            "count": 213
           }
         }
       },
       "sub_M4": {
+        "full_accuracy": 0.05,
+        "digit_accuracy": 0.6385714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.965,
+            "count": 200
           },
           "MB": {
+            "accuracy": 0.96,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.57,
+            "count": 100
           },
           "UD": {
+            "accuracy": 0.33666666666666667,
+            "count": 300
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.04,
+        "digit_accuracy": 0.5,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
+            "accuracy": 0.96,
+            "count": 100
           },
           "UB": {
             "accuracy": 0.56,
+            "count": 100
           },
           "UD": {
+            "accuracy": 0.245,
+            "count": 400
           }
         }
       },
       "sub_random": {
+        "full_accuracy": 0.24,
+        "digit_accuracy": 0.8371428571428572,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.9083333333333333,
+            "count": 600
           },
           "MB": {
+            "accuracy": 0.9363295880149812,
+            "count": 267
           },
           "ME": {
             "accuracy": 0.9811320754716981,
             "count": 53
           },
           "UB": {
+            "accuracy": 0.6583143507972665,
+            "count": 439
           },
           "UD": {
+            "accuracy": 0.8780487804878049,
+            "count": 41
           }
         }
       },
       "sub_B3": {
+        "full_accuracy": 0.13,
+        "digit_accuracy": 0.8028571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.94,
+            "count": 300
           },
           "MB": {
+            "accuracy": 0.95,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.5583756345177665,
+            "count": 197
           },
           "UD": {
+            "accuracy": 0.7281553398058253,
+            "count": 103
           }
         }
       },
       "sub_B4": {
+        "full_accuracy": 0.05,
+        "digit_accuracy": 0.72,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.955,
+            "count": 200
           },
           "MB": {
+            "accuracy": 0.96,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.48582995951417,
+            "count": 247
           },
           "UD": {
+            "accuracy": 0.6339869281045751,
+            "count": 153
           }
         }
       },
       "sub_B5": {
+        "full_accuracy": 0.05,
+        "digit_accuracy": 0.6914285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.49328859060402686,
+            "count": 298
           },
           "UD": {
+            "accuracy": 0.6782178217821783,
+            "count": 202
           }
         }
       }
     },
     "summary": {
+      "overall_accuracy": 0.22615384615384615,
+      "digit_accuracy": 0.7815384615384615,
+      "total_examples": 2600,
       "n_splits": 24
     }
   }

add_sub_baseline_50K_2L1H128d/model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f042fb5952a5a3d094f33db76e3792f60cc38e235d390fab794cf6dc4983dad2
 size 157692826

 version https://git-lfs.github.com/spec/v1
+oid sha256:d6aa50140e3dc999e70ccc8a0ebc11a0caa554693342acc49ad8f60da90d1dd5
 size 157692826

add_sub_baseline_50K_2L1H128d/train_config.json CHANGED Viewed

@@ -69,16 +69,20 @@
   "no_wandb": false,
   "n_params": 39346560,
   "run_name": "add_sub_baseline_50K_2L1H128d",
-  "git_commit": "17e935f460a7f9595b705c1d614101a6b0e520f7",
-  "timestamp": "2026-04-14T04:37:53.146117+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "sft",
-  "wandb_run_id": "gvx3cabo",
-  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/gvx3cabo",
-  "final_accuracy": 0.1925,
-  "sft_accuracy": 0.1925,
   "eval_method": "ArithmeticEvaluator"
 }

   "no_wandb": false,
   "n_params": 39346560,
   "run_name": "add_sub_baseline_50K_2L1H128d",
+  "git_commit": "f835493c19eb98267697007042c9d440cad2afbb",
+  "timestamp": "2026-04-15T09:22:35.671141+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
+  "train_dataset": "fixed_train/train_50K_seed42.pt",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "sft",
+  "wandb_run_id": "wnsxa3xn",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/wnsxa3xn",
+  "eval_final_dataset": "eval_sets/eval_add_sub_6d_N100_seed42.json",
+  "eval_epoch_dataset": "eval_sets/eval_add_sub_6d_N25_seed42.json",
+  "eval_hf_repo": "thoughtworks/arithmetic-sorl-data",
+  "final_accuracy": 0.22615384615384615,
+  "sft_accuracy": 0.22615384615384615,
   "eval_method": "ArithmeticEvaluator"
 }