Upload add_sub_baseline_50K_1L3H510d

Browse files

Files changed (3) hide show

add_sub_baseline_50K_1L3H510d/metrics.json +880 -880
add_sub_baseline_50K_1L3H510d/model.safetensors +1 -1
add_sub_baseline_50K_1L3H510d/train_config.json +10 -6

add_sub_baseline_50K_1L3H510d/metrics.json CHANGED Viewed

@@ -315,632 +315,632 @@
       15600
     ],
     "loss": [
-      10.19568157196045,
-      7.192863464355469,
-      6.217032432556152,
-      4.67427921295166,
-      2.955170154571533,
-      2.0684783458709717,
-      1.879281759262085,
-      1.9268118143081665,
-      1.838002324104309,
-      1.6958969831466675,
-      1.6997877359390259,
-      1.5733033418655396,
-      1.5437490940093994,
-      1.147268295288086,
-      0.8210930228233337,
-      0.7633241415023804,
-      0.7406877279281616,
-      0.7202191352844238,
-      0.5617724061012268,
-      0.5543747544288635,
-      0.5196675658226013,
-      0.515282154083252,
-      0.541344404220581,
-      0.4584448039531708,
-      0.43351513147354126,
-      0.43640443682670593,
-      0.40899521112442017,
-      0.3735989034175873,
-      0.36980319023132324,
-      0.38705509901046753,
-      0.34830760955810547,
-      0.3112049400806427,
-      0.38858872652053833,
-      0.3346550166606903,
-      0.3554438054561615,
-      0.31374162435531616,
-      0.3362943232059479,
-      0.3253510296344757,
-      0.3735771179199219,
-      0.31208163499832153,
-      0.2946203649044037,
-      0.28192415833473206,
-      0.37054675817489624,
-      0.2701527178287506,
-      0.35655704140663147,
-      0.29881080985069275,
-      0.2859213054180145,
-      0.28949055075645447,
-      0.28196626901626587,
-      0.23745205998420715,
-      0.3055541217327118,
-      0.32069405913352966,
-      0.2612220048904419,
-      0.2794129252433777,
-      0.35670334100723267,
-      0.2686660587787628,
-      0.2621057331562042,
-      0.2519742548465729,
-      0.2593633234500885,
-      0.23556630313396454,
-      0.2755930423736572,
-      0.23948059976100922,
-      0.28940704464912415,
-      0.23039786517620087,
-      0.21981647610664368,
-      0.2917427718639374,
-      0.2732771635055542,
-      0.24072220921516418,
-      0.2149968147277832,
-      0.22879359126091003,
-      0.23625613749027252,
-      0.2409660965204239,
-      0.2549420893192291,
-      0.19799569249153137,
-      0.23658190667629242,
-      0.25520455837249756,
-      0.2438141405582428,
-      0.21446263790130615,
-      0.18694305419921875,
-      0.21734251081943512,
-      0.23308448493480682,
-      0.25597816705703735,
-      0.2303375005722046,
-      0.22810396552085876,
-      0.2421826273202896,
-      0.24269063770771027,
-      0.19383986294269562,
-      0.19238220155239105,
-      0.23637458682060242,
-      0.2113189697265625,
-      0.18594272434711456,
-      0.2047307789325714,
-      0.22051747143268585,
-      0.2338298112154007,
-      0.18495337665081024,
-      0.31769758462905884,
-      0.17948535084724426,
-      0.16956846415996552,
-      0.18370643258094788,
-      0.21728311479091644,
-      0.16529808938503265,
-      0.1880127638578415,
-      0.1927148550748825,
-      0.17977656424045563,
-      0.19032812118530273,
-      0.225495383143425,
-      0.18763747811317444,
-      0.15616856515407562,
-      0.2110285460948944,
-      0.1917344182729721,
-      0.19217932224273682,
-      0.18634091317653656,
-      0.17289797961711884,
-      0.18017618358135223,
-      0.16975148022174835,
-      0.20279523730278015,
-      0.18793119490146637,
-      0.18208105862140656,
-      0.2210826873779297,
-      0.1848507970571518,
-      0.175362229347229,
-      0.2338799387216568,
-      0.15665939450263977,
-      0.1710033118724823,
-      0.22897052764892578,
-      0.19976939260959625,
-      0.17123167216777802,
-      0.17207647860050201,
-      0.16104988753795624,
-      0.17151372134685516,
-      0.20540104806423187,
-      0.16327939927577972,
-      0.18736600875854492,
-      0.15778745710849762,
-      0.1724669635295868,
-      0.22603313624858856,
-      0.15962539613246918,
-      0.17842444777488708,
-      0.2059607207775116,
-      0.16246894001960754,
-      0.1688106805086136,
-      0.15051047503948212,
-      0.17147870361804962,
-      0.1596122831106186,
-      0.16684506833553314,
-      0.17243006825447083,
-      0.18031249940395355,
-      0.15410535037517548,
-      0.14030024409294128,
-      0.17812776565551758,
-      0.15372362732887268,
-      0.16936704516410828,
-      0.16118179261684418,
-      0.15867330133914948,
-      0.17409607768058777,
-      0.16555775701999664,
-      0.14979203045368195,
-      0.11939563602209091,
-      0.17306402325630188,
-      0.19467274844646454,
-      0.1610839068889618,
-      0.14897356927394867,
-      0.13533374667167664,
-      0.1751706600189209,
-      0.14803266525268555,
-      0.14225070178508759,
-      0.1285431832075119,
-      0.15326274931430817,
-      0.12212397158145905,
-      0.15423189103603363,
-      0.1676071733236313,
-      0.17005637288093567,
-      0.13811911642551422,
-      0.1578800231218338,
-      0.1245812326669693,
-      0.14160023629665375,
-      0.1362280547618866,
-      0.13950759172439575,
-      0.13266520202159882,
-      0.12251361459493637,
-      0.1468406617641449,
-      0.13180361688137054,
-      0.14409062266349792,
-      0.10714907199144363,
-      0.15004582703113556,
-      0.1443106085062027,
-      0.16568748652935028,
-      0.13185156881809235,
-      0.1419181525707245,
-      0.12822316586971283,
-      0.13271304965019226,
-      0.14806745946407318,
-      0.12749268114566803,
-      0.14777174592018127,
-      0.12936027348041534,
-      0.16480930149555206,
-      0.1283726692199707,
-      0.17376258969306946,
-      0.14537104964256287,
-      0.17024172842502594,
-      0.10525000840425491,
-      0.13460473716259003,
-      0.1136106625199318,
-      0.14334174990653992,
-      0.12381266057491302,
-      0.11635736376047134,
-      0.1619941145181656,
-      0.11771261692047119,
-      0.1627335399389267,
-      0.13971097767353058,
-      0.13618090748786926,
-      0.13798284530639648,
-      0.1138051375746727,
-      0.10376495867967606,
-      0.1188473179936409,
-      0.14848637580871582,
-      0.12351793795824051,
-      0.1291089653968811,
-      0.14152243733406067,
-      0.120426245033741,
-      0.12951800227165222,
-      0.1462240368127823,
-      0.12321526557207108,
-      0.11646594107151031,
-      0.16523265838623047,
-      0.09980458766222,
-      0.11321472376585007,
-      0.11211931705474854,
-      0.11767058819532394,
-      0.12701120972633362,
-      0.13879325985908508,
-      0.15463045239448547,
-      0.0981360599398613,
-      0.0952308177947998,
-      0.11886616051197052,
-      0.11998490244150162,
-      0.1317901909351349,
-      0.12052290141582489,
-      0.11561640352010727,
-      0.1148061528801918,
-      0.11529232561588287,
-      0.1109485775232315,
-      0.08685992658138275,
-      0.10863905400037766,
-      0.10214114189147949,
-      0.11502943933010101,
-      0.10768144577741623,
-      0.12711675465106964,
-      0.08342073112726212,
-      0.11724580824375153,
-      0.11047422140836716,
-      0.1387486755847931,
-      0.11161993443965912,
-      0.10834724456071854,
-      0.10992390662431717,
-      0.09308646619319916,
-      0.09626653045415878,
-      0.0924919992685318,
-      0.11180823296308517,
-      0.10675805807113647,
-      0.1210842952132225,
-      0.11283500492572784,
-      0.10502934455871582,
-      0.09572649002075195,
-      0.10792434215545654,
-      0.10839086771011353,
-      0.13045024871826172,
-      0.1423415094614029,
-      0.1113070398569107,
-      0.11313152313232422,
-      0.12788374722003937,
-      0.10165081918239594,
-      0.08276218920946121,
-      0.10745533555746078,
-      0.10156644880771637,
-      0.09847947210073471,
-      0.13116250932216644,
-      0.11027399450540543,
-      0.10681574791669846,
-      0.11552350223064423,
-      0.08603107184171677,
-      0.10476719588041306,
-      0.09377549588680267,
-      0.11473890393972397,
-      0.12605629861354828,
-      0.1027151346206665,
-      0.10792066156864166,
-      0.08990789949893951,
-      0.09799478948116302,
-      0.07296454906463623,
-      0.12055801600217819,
-      0.12072388827800751,
-      0.09598750621080399,
-      0.10575366765260696,
-      0.08751322329044342,
-      0.09683708101511002,
-      0.10606545954942703,
-      0.09735989570617676,
-      0.08438508957624435,
-      0.12220897525548935,
-      0.10032238811254501,
-      0.08653612434864044,
-      0.09255076199769974,
-      0.12790820002555847,
-      0.0972944125533104,
-      0.09457828849554062,
-      0.08459220826625824,
-      0.12037122994661331,
-      0.10575146228075027,
-      0.09452326595783234,
-      0.09958111494779587,
-      0.11665801703929901
     ],
     "base_loss": [
-      10.19568157196045,
-      7.192863464355469,
-      6.217032432556152,
-      4.67427921295166,
-      2.955170154571533,
-      2.0684783458709717,
-      1.879281759262085,
-      1.9268118143081665,
-      1.838002324104309,
-      1.6958969831466675,
-      1.6997877359390259,
-      1.5733033418655396,
-      1.5437490940093994,
-      1.147268295288086,
-      0.8210930228233337,
-      0.7633241415023804,
-      0.7406877279281616,
-      0.7202191352844238,
-      0.5617724061012268,
-      0.5543747544288635,
-      0.5196675658226013,
-      0.515282154083252,
-      0.541344404220581,
-      0.4584448039531708,
-      0.43351513147354126,
-      0.43640443682670593,
-      0.40899521112442017,
-      0.3735989034175873,
-      0.36980319023132324,
-      0.38705509901046753,
-      0.34830760955810547,
-      0.3112049400806427,
-      0.38858872652053833,
-      0.3346550166606903,
-      0.3554438054561615,
-      0.31374162435531616,
-      0.3362943232059479,
-      0.3253510296344757,
-      0.3735771179199219,
-      0.31208163499832153,
-      0.2946203649044037,
-      0.28192415833473206,
-      0.37054675817489624,
-      0.2701527178287506,
-      0.35655704140663147,
-      0.29881080985069275,
-      0.2859213054180145,
-      0.28949055075645447,
-      0.28196626901626587,
-      0.23745205998420715,
-      0.3055541217327118,
-      0.32069405913352966,
-      0.2612220048904419,
-      0.2794129252433777,
-      0.35670334100723267,
-      0.2686660587787628,
-      0.2621057331562042,
-      0.2519742548465729,
-      0.2593633234500885,
-      0.23556630313396454,
-      0.2755930423736572,
-      0.23948059976100922,
-      0.28940704464912415,
-      0.23039786517620087,
-      0.21981647610664368,
-      0.2917427718639374,
-      0.2732771635055542,
-      0.24072220921516418,
-      0.2149968147277832,
-      0.22879359126091003,
-      0.23625613749027252,
-      0.2409660965204239,
-      0.2549420893192291,
-      0.19799569249153137,
-      0.23658190667629242,
-      0.25520455837249756,
-      0.2438141405582428,
-      0.21446263790130615,
-      0.18694305419921875,
-      0.21734251081943512,
-      0.23308448493480682,
-      0.25597816705703735,
-      0.2303375005722046,
-      0.22810396552085876,
-      0.2421826273202896,
-      0.24269063770771027,
-      0.19383986294269562,
-      0.19238220155239105,
-      0.23637458682060242,
-      0.2113189697265625,
-      0.18594272434711456,
-      0.2047307789325714,
-      0.22051747143268585,
-      0.2338298112154007,
-      0.18495337665081024,
-      0.31769758462905884,
-      0.17948535084724426,
-      0.16956846415996552,
-      0.18370643258094788,
-      0.21728311479091644,
-      0.16529808938503265,
-      0.1880127638578415,
-      0.1927148550748825,
-      0.17977656424045563,
-      0.19032812118530273,
-      0.225495383143425,
-      0.18763747811317444,
-      0.15616856515407562,
-      0.2110285460948944,
-      0.1917344182729721,
-      0.19217932224273682,
-      0.18634091317653656,
-      0.17289797961711884,
-      0.18017618358135223,
-      0.16975148022174835,
-      0.20279523730278015,
-      0.18793119490146637,
-      0.18208105862140656,
-      0.2210826873779297,
-      0.1848507970571518,
-      0.175362229347229,
-      0.2338799387216568,
-      0.15665939450263977,
-      0.1710033118724823,
-      0.22897052764892578,
-      0.19976939260959625,
-      0.17123167216777802,
-      0.17207647860050201,
-      0.16104988753795624,
-      0.17151372134685516,
-      0.20540104806423187,
-      0.16327939927577972,
-      0.18736600875854492,
-      0.15778745710849762,
-      0.1724669635295868,
-      0.22603313624858856,
-      0.15962539613246918,
-      0.17842444777488708,
-      0.2059607207775116,
-      0.16246894001960754,
-      0.1688106805086136,
-      0.15051047503948212,
-      0.17147870361804962,
-      0.1596122831106186,
-      0.16684506833553314,
-      0.17243006825447083,
-      0.18031249940395355,
-      0.15410535037517548,
-      0.14030024409294128,
-      0.17812776565551758,
-      0.15372362732887268,
-      0.16936704516410828,
-      0.16118179261684418,
-      0.15867330133914948,
-      0.17409607768058777,
-      0.16555775701999664,
-      0.14979203045368195,
-      0.11939563602209091,
-      0.17306402325630188,
-      0.19467274844646454,
-      0.1610839068889618,
-      0.14897356927394867,
-      0.13533374667167664,
-      0.1751706600189209,
-      0.14803266525268555,
-      0.14225070178508759,
-      0.1285431832075119,
-      0.15326274931430817,
-      0.12212397158145905,
-      0.15423189103603363,
-      0.1676071733236313,
-      0.17005637288093567,
-      0.13811911642551422,
-      0.1578800231218338,
-      0.1245812326669693,
-      0.14160023629665375,
-      0.1362280547618866,
-      0.13950759172439575,
-      0.13266520202159882,
-      0.12251361459493637,
-      0.1468406617641449,
-      0.13180361688137054,
-      0.14409062266349792,
-      0.10714907199144363,
-      0.15004582703113556,
-      0.1443106085062027,
-      0.16568748652935028,
-      0.13185156881809235,
-      0.1419181525707245,
-      0.12822316586971283,
-      0.13271304965019226,
-      0.14806745946407318,
-      0.12749268114566803,
-      0.14777174592018127,
-      0.12936027348041534,
-      0.16480930149555206,
-      0.1283726692199707,
-      0.17376258969306946,
-      0.14537104964256287,
-      0.17024172842502594,
-      0.10525000840425491,
-      0.13460473716259003,
-      0.1136106625199318,
-      0.14334174990653992,
-      0.12381266057491302,
-      0.11635736376047134,
-      0.1619941145181656,
-      0.11771261692047119,
-      0.1627335399389267,
-      0.13971097767353058,
-      0.13618090748786926,
-      0.13798284530639648,
-      0.1138051375746727,
-      0.10376495867967606,
-      0.1188473179936409,
-      0.14848637580871582,
-      0.12351793795824051,
-      0.1291089653968811,
-      0.14152243733406067,
-      0.120426245033741,
-      0.12951800227165222,
-      0.1462240368127823,
-      0.12321526557207108,
-      0.11646594107151031,
-      0.16523265838623047,
-      0.09980458766222,
-      0.11321472376585007,
-      0.11211931705474854,
-      0.11767058819532394,
-      0.12701120972633362,
-      0.13879325985908508,
-      0.15463045239448547,
-      0.0981360599398613,
-      0.0952308177947998,
-      0.11886616051197052,
-      0.11998490244150162,
-      0.1317901909351349,
-      0.12052290141582489,
-      0.11561640352010727,
-      0.1148061528801918,
-      0.11529232561588287,
-      0.1109485775232315,
-      0.08685992658138275,
-      0.10863905400037766,
-      0.10214114189147949,
-      0.11502943933010101,
-      0.10768144577741623,
-      0.12711675465106964,
-      0.08342073112726212,
-      0.11724580824375153,
-      0.11047422140836716,
-      0.1387486755847931,
-      0.11161993443965912,
-      0.10834724456071854,
-      0.10992390662431717,
-      0.09308646619319916,
-      0.09626653045415878,
-      0.0924919992685318,
-      0.11180823296308517,
-      0.10675805807113647,
-      0.1210842952132225,
-      0.11283500492572784,
-      0.10502934455871582,
-      0.09572649002075195,
-      0.10792434215545654,
-      0.10839086771011353,
-      0.13045024871826172,
-      0.1423415094614029,
-      0.1113070398569107,
-      0.11313152313232422,
-      0.12788374722003937,
-      0.10165081918239594,
-      0.08276218920946121,
-      0.10745533555746078,
-      0.10156644880771637,
-      0.09847947210073471,
-      0.13116250932216644,
-      0.11027399450540543,
-      0.10681574791669846,
-      0.11552350223064423,
-      0.08603107184171677,
-      0.10476719588041306,
-      0.09377549588680267,
-      0.11473890393972397,
-      0.12605629861354828,
-      0.1027151346206665,
-      0.10792066156864166,
-      0.08990789949893951,
-      0.09799478948116302,
-      0.07296454906463623,
-      0.12055801600217819,
-      0.12072388827800751,
-      0.09598750621080399,
-      0.10575366765260696,
-      0.08751322329044342,
-      0.09683708101511002,
-      0.10606545954942703,
-      0.09735989570617676,
-      0.08438508957624435,
-      0.12220897525548935,
-      0.10032238811254501,
-      0.08653612434864044,
-      0.09255076199769974,
-      0.12790820002555847,
-      0.0972944125533104,
-      0.09457828849554062,
-      0.08459220826625824,
-      0.12037122994661331,
-      0.10575146228075027,
-      0.09452326595783234,
-      0.09958111494779587,
-      0.11665801703929901
     ],
     "lr": [
       8.376068376068378e-06,
@@ -1301,595 +1301,595 @@
       20
     ],
     "eval_accuracy": [
-      0.021111111111111112,
-      0.24,
-      0.3611111111111111,
-      0.44666666666666666,
-      0.4588888888888889,
-      0.5144444444444445,
-      0.5033333333333333,
-      0.5022222222222222,
-      0.5611111111111111,
-      0.57,
-      0.5722222222222222,
-      0.6244444444444445,
-      0.6166666666666667,
-      0.6211111111111111,
-      0.6455555555555555,
-      0.6711111111111111,
-      0.6311111111111111,
-      0.6322222222222222,
-      0.6533333333333333,
-      0.6666666666666666
     ]
   },
-  "final_accuracy": 0.54125,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.92,
-        "digit_accuracy": 0.9885714285714285,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9864406779661017,
-            "count": 295
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 55
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.88,
-        "digit_accuracy": 0.9828571428571429,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
-            "count": 126
           },
           "SC": {
-            "accuracy": 0.9746835443037974,
-            "count": 79
           },
           "SS": {
-            "accuracy": 0.9523809523809523,
-            "count": 21
           },
           "UC": {
-            "accuracy": 0.9758064516129032,
-            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.64,
-        "digit_accuracy": 0.9428571428571428,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9866666666666667,
-            "count": 75
           },
           "SC": {
-            "accuracy": 0.9516129032258065,
-            "count": 62
           },
           "SS": {
-            "accuracy": 0.9743589743589743,
-            "count": 39
           },
           "UC": {
-            "accuracy": 0.8648648648648649,
-            "count": 111
           },
           "US": {
             "accuracy": 1.0,
-            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.44,
-        "digit_accuracy": 0.8771428571428571,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
-            "count": 60
           },
           "SC": {
-            "accuracy": 1.0,
-            "count": 57
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 19
           },
           "UC": {
-            "accuracy": 0.7692307692307693,
-            "count": 104
           },
           "US": {
-            "accuracy": 0.8272727272727273,
-            "count": 110
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.22,
-        "digit_accuracy": 0.74,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 48
           },
           "SC": {
-            "accuracy": 0.9807692307692307,
-            "count": 52
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 7
           },
           "UC": {
-            "accuracy": 0.6404494382022472,
-            "count": 89
           },
           "US": {
-            "accuracy": 0.6233766233766234,
-            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.18,
         "digit_accuracy": 0.5971428571428572,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.42,
-            "count": 50
           },
           "US": {
-            "accuracy": 0.44,
-            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.5,
-        "digit_accuracy": 0.6485714285714286,
-        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.54,
-            "count": 50
           },
           "US": {
-            "accuracy": 0.6,
-            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.89,
-        "digit_accuracy": 0.9814285714285714,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
-            "count": 431
           },
           "SC": {
-            "accuracy": 1.0,
-            "count": 316
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 39
           },
           "UC": {
-            "accuracy": 0.9607142857142857,
-            "count": 560
           },
           "US": {
-            "accuracy": 0.9259259259259259,
-            "count": 54
           }
         }
       },
       "add_C1": {
-        "full_accuracy": 0.94,
-        "digit_accuracy": 0.9914285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.996,
-            "count": 250
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.96,
-            "count": 50
           }
         }
       },
       "add_C2": {
-        "full_accuracy": 0.84,
-        "digit_accuracy": 0.9742857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
-            "count": 200
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.9036144578313253,
-            "count": 83
           },
           "US": {
-            "accuracy": 0.9411764705882353,
-            "count": 17
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.62,
-        "digit_accuracy": 0.9228571428571428,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9933333333333333,
-            "count": 150
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.85,
-            "count": 100
           },
           "US": {
-            "accuracy": 0.78,
-            "count": 50
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.66,
-        "digit_accuracy": 0.9257142857142857,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.99,
-            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.8787878787878788,
-            "count": 132
           },
           "US": {
-            "accuracy": 0.8676470588235294,
-            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.54,
-        "digit_accuracy": 0.8742857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.8493150684931506,
-            "count": 146
           },
           "US": {
-            "accuracy": 0.7884615384615384,
-            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.5,
-        "digit_accuracy": 0.8771428571428571,
-        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UC": {
-            "accuracy": 0.8571428571428571,
-            "count": 189
           },
           "US": {
-            "accuracy": 0.8558558558558559,
-            "count": 111
           }
         }
       },
       "sub_M0": {
         "full_accuracy": 0.9,
         "digit_accuracy": 0.9857142857142858,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9834983498349835,
-            "count": 303
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.86,
-        "digit_accuracy": 0.98,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9858156028368794,
-            "count": 141
           },
           "MB": {
             "accuracy": 0.9861111111111112,
-            "count": 72
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 18
           },
           "UB": {
-            "accuracy": 0.9663865546218487,
-            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.56,
-        "digit_accuracy": 0.9257142857142857,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9732142857142857,
-            "count": 112
           },
           "MB": {
-            "accuracy": 0.9622641509433962,
-            "count": 53
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 47
           },
           "UB": {
-            "accuracy": 0.7647058823529411,
-            "count": 85
           },
           "UD": {
-            "accuracy": 0.9811320754716981,
-            "count": 53
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.22,
-        "digit_accuracy": 0.8457142857142858,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.979381443298969,
-            "count": 97
           },
           "MB": {
-            "accuracy": 0.9803921568627451,
-            "count": 51
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 27
           },
           "UB": {
-            "accuracy": 0.5945945945945946,
-            "count": 74
           },
           "UD": {
-            "accuracy": 0.7920792079207921,
-            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.04,
-        "digit_accuracy": 0.6771428571428572,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.4,
-            "count": 50
           },
           "UD": {
-            "accuracy": 0.44666666666666666,
-            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.1,
-        "digit_accuracy": 0.5771428571428572,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.58,
-            "count": 50
           },
           "UD": {
-            "accuracy": 0.365,
-            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.87,
-        "digit_accuracy": 0.98,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9982456140350877,
-            "count": 570
           },
           "MB": {
-            "accuracy": 0.9819494584837545,
-            "count": 277
           },
           "ME": {
             "accuracy": 1.0,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.9554140127388535,
-            "count": 471
           },
           "UD": {
-            "accuracy": 0.9655172413793104,
-            "count": 29
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.52,
-        "digit_accuracy": 0.9142857142857143,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 150
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.801980198019802,
-            "count": 101
           },
           "UD": {
-            "accuracy": 0.7959183673469388,
-            "count": 49
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.34,
-        "digit_accuracy": 0.8485714285714285,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.7603305785123967,
-            "count": 121
           },
           "UD": {
-            "accuracy": 0.6962025316455697,
-            "count": 79
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.22,
-        "digit_accuracy": 0.8114285714285714,
-        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
           },
           "UB": {
-            "accuracy": 0.75,
-            "count": 152
           },
           "UD": {
-            "accuracy": 0.7142857142857143,
-            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.6226666666666667,
-      "digit_accuracy": 0.8917142857142857,
-      "total_examples": 1500,
       "n_splits": 24
     }
   }

       15600
     ],
     "loss": [
+      10.278651237487793,
+      7.389064311981201,
+      6.051224231719971,
+      4.550988674163818,
+      2.852480173110962,
+      2.1066410541534424,
+      1.9148751497268677,
+      1.843735933303833,
+      1.8798506259918213,
+      1.8511418104171753,
+      1.7925702333450317,
+      1.6850757598876953,
+      1.6945250034332275,
+      1.5783421993255615,
+      1.089109182357788,
+      0.7692533135414124,
+      0.7096071243286133,
+      0.6046804785728455,
+      0.6018385291099548,
+      0.5790620446205139,
+      0.5549424290657043,
+      0.5272607803344727,
+      0.46060559153556824,
+      0.4667455852031708,
+      0.47789812088012695,
+      0.4029836058616638,
+      0.47773483395576477,
+      0.440927654504776,
+      0.39726462960243225,
+      0.383687287569046,
+      0.4488619863986969,
+      0.3516424298286438,
+      0.32863950729370117,
+      0.327939510345459,
+      0.31499478220939636,
+      0.3476874530315399,
+      0.35759884119033813,
+      0.29587382078170776,
+      0.3376016914844513,
+      0.2916772663593292,
+      0.2944425642490387,
+      0.2793613076210022,
+      0.30821874737739563,
+      0.30350440740585327,
+      0.30654028058052063,
+      0.25445234775543213,
+      0.293817937374115,
+      0.2751149535179138,
+      0.2549825608730316,
+      0.27168112993240356,
+      0.29966264963150024,
+      0.2999582886695862,
+      0.2856738567352295,
+      0.2571064531803131,
+      0.2718660533428192,
+      0.26083821058273315,
+      0.2872292399406433,
+      0.23113496601581573,
+      0.24555249512195587,
+      0.2553543150424957,
+      0.22126071155071259,
+      0.24801431596279144,
+      0.22513781487941742,
+      0.21824850142002106,
+      0.32301944494247437,
+      0.2092411071062088,
+      0.23654519021511078,
+      0.22970826923847198,
+      0.196001797914505,
+      0.19658486545085907,
+      0.19009140133857727,
+      0.254011332988739,
+      0.20502305030822754,
+      0.23104675114154816,
+      0.21880166232585907,
+      0.24527417123317719,
+      0.2686038315296173,
+      0.2016792744398117,
+      0.17031656205654144,
+      0.20333679020404816,
+      0.21816182136535645,
+      0.21064119040966034,
+      0.19884170591831207,
+      0.19956159591674805,
+      0.18471239507198334,
+      0.15811093151569366,
+      0.21883070468902588,
+      0.1914089471101761,
+      0.1655077189207077,
+      0.22221115231513977,
+      0.14605960249900818,
+      0.21837235987186432,
+      0.1704948991537094,
+      0.1736709624528885,
+      0.2253033071756363,
+      0.18853184580802917,
+      0.20015767216682434,
+      0.19018162786960602,
+      0.1817663013935089,
+      0.21601822972297668,
+      0.1782209873199463,
+      0.1798601895570755,
+      0.17680145800113678,
+      0.16178666055202484,
+      0.15822872519493103,
+      0.21383854746818542,
+      0.17432792484760284,
+      0.15497128665447235,
+      0.14949870109558105,
+      0.1623925417661667,
+      0.178536057472229,
+      0.1496189385652542,
+      0.1535705029964447,
+      0.15259061753749847,
+      0.20757174491882324,
+      0.189642533659935,
+      0.16320088505744934,
+      0.19857345521450043,
+      0.17532989382743835,
+      0.18625964224338531,
+      0.1518552303314209,
+      0.15830935537815094,
+      0.16554827988147736,
+      0.1885346621274948,
+      0.19510552287101746,
+      0.1609521359205246,
+      0.22554394602775574,
+      0.17842698097229004,
+      0.16276274621486664,
+      0.1613040417432785,
+      0.13027097284793854,
+      0.16238419711589813,
+      0.14015449583530426,
+      0.17456690967082977,
+      0.1481691151857376,
+      0.1879999339580536,
+      0.17365124821662903,
+      0.2072172909975052,
+      0.13534089922904968,
+      0.12938502430915833,
+      0.16615700721740723,
+      0.1711844503879547,
+      0.1538037359714508,
+      0.17003300786018372,
+      0.1528840959072113,
+      0.14285284280776978,
+      0.1634855419397354,
+      0.13300959765911102,
+      0.16810348629951477,
+      0.13577891886234283,
+      0.1253504455089569,
+      0.13998855650424957,
+      0.1808578222990036,
+      0.14750854671001434,
+      0.15061943233013153,
+      0.15485039353370667,
+      0.15015456080436707,
+      0.12394087016582489,
+      0.13308438658714294,
+      0.12737306952476501,
+      0.10817860811948776,
+      0.1835630238056183,
+      0.1271446943283081,
+      0.17026567459106445,
+      0.14795377850532532,
+      0.15456995368003845,
+      0.12015783041715622,
+      0.14983054995536804,
+      0.15613238513469696,
+      0.1362493485212326,
+      0.11746997386217117,
+      0.12015391886234283,
+      0.14974607527256012,
+      0.12486754357814789,
+      0.1347617208957672,
+      0.13983528316020966,
+      0.14421994984149933,
+      0.15558600425720215,
+      0.1395520716905594,
+      0.12529298663139343,
+      0.14551745355129242,
+      0.16064704954624176,
+      0.123051218688488,
+      0.14231589436531067,
+      0.15310995280742645,
+      0.12945358455181122,
+      0.15963591635227203,
+      0.17953287065029144,
+      0.09377244859933853,
+      0.1109309270977974,
+      0.10077396780252457,
+      0.11636873334646225,
+      0.14231428503990173,
+      0.13868117332458496,
+      0.1199658066034317,
+      0.10137726366519928,
+      0.12408987432718277,
+      0.12158029526472092,
+      0.15156243741512299,
+      0.1056785136461258,
+      0.11238741874694824,
+      0.13356435298919678,
+      0.1199871078133583,
+      0.11451991647481918,
+      0.11946208029985428,
+      0.136271134018898,
+      0.11361639946699142,
+      0.10386927425861359,
+      0.10679834336042404,
+      0.09268485009670258,
+      0.13429896533489227,
+      0.12343748658895493,
+      0.1442834585905075,
+      0.12418247759342194,
+      0.13955631852149963,
+      0.09388206154108047,
+      0.11501727253198624,
+      0.1351633220911026,
+      0.13560284674167633,
+      0.12844859063625336,
+      0.09430808573961258,
+      0.11472094804048538,
+      0.11257084459066391,
+      0.11793804168701172,
+      0.10530801862478256,
+      0.13184157013893127,
+      0.08044975250959396,
+      0.10456537455320358,
+      0.15832646191120148,
+      0.10107123851776123,
+      0.13520397245883942,
+      0.11445610970258713,
+      0.11855047196149826,
+      0.13093633949756622,
+      0.10899967700242996,
+      0.10174088180065155,
+      0.12345916777849197,
+      0.1107829362154007,
+      0.1003185361623764,
+      0.1176886186003685,
+      0.09605075418949127,
+      0.12465865164995193,
+      0.09835167974233627,
+      0.1037013903260231,
+      0.11466304212808609,
+      0.11380521208047867,
+      0.13064700365066528,
+      0.13941724598407745,
+      0.08339516073465347,
+      0.11133090406656265,
+      0.10470746457576752,
+      0.11295027285814285,
+      0.11331796646118164,
+      0.09551217406988144,
+      0.11493296921253204,
+      0.09893112629652023,
+      0.11825107038021088,
+      0.12228565663099289,
+      0.10353125631809235,
+      0.10824307054281235,
+      0.10025204718112946,
+      0.07499944418668747,
+      0.09401531517505646,
+      0.0934530571103096,
+      0.11390485614538193,
+      0.10727037489414215,
+      0.09588678926229477,
+      0.0825532078742981,
+      0.10723954439163208,
+      0.1088128611445427,
+      0.11156749725341797,
+      0.09598823636770248,
+      0.08453374356031418,
+      0.10130847245454788,
+      0.09620478004217148,
+      0.11045535653829575,
+      0.10323013365268707,
+      0.13056471943855286,
+      0.08738001435995102,
+      0.11891673505306244,
+      0.09096094220876694,
+      0.08709857612848282,
+      0.09677024930715561,
+      0.08751770108938217,
+      0.10058047622442245,
+      0.09135355800390244,
+      0.1054212898015976,
+      0.09947032481431961,
+      0.10693687945604324,
+      0.0917934849858284,
+      0.10931998491287231,
+      0.09929491579532623,
+      0.11224542558193207,
+      0.11092674732208252,
+      0.09260962903499603,
+      0.12740318477153778,
+      0.11285065114498138,
+      0.10502947121858597,
+      0.07829917967319489,
+      0.1043354719877243,
+      0.10123976320028305,
+      0.08939259499311447,
+      0.11151234060525894,
+      0.13105222582817078,
+      0.10120860487222672,
+      0.06924744695425034,
+      0.10668231546878815,
+      0.09256572276353836,
+      0.08223498612642288,
+      0.09994925558567047,
+      0.08328042924404144,
+      0.0906481072306633
     ],
     "base_loss": [
+      10.278651237487793,
+      7.389064311981201,
+      6.051224231719971,
+      4.550988674163818,
+      2.852480173110962,
+      2.1066410541534424,
+      1.9148751497268677,
+      1.843735933303833,
+      1.8798506259918213,
+      1.8511418104171753,
+      1.7925702333450317,
+      1.6850757598876953,
+      1.6945250034332275,
+      1.5783421993255615,
+      1.089109182357788,
+      0.7692533135414124,
+      0.7096071243286133,
+      0.6046804785728455,
+      0.6018385291099548,
+      0.5790620446205139,
+      0.5549424290657043,
+      0.5272607803344727,
+      0.46060559153556824,
+      0.4667455852031708,
+      0.47789812088012695,
+      0.4029836058616638,
+      0.47773483395576477,
+      0.440927654504776,
+      0.39726462960243225,
+      0.383687287569046,
+      0.4488619863986969,
+      0.3516424298286438,
+      0.32863950729370117,
+      0.327939510345459,
+      0.31499478220939636,
+      0.3476874530315399,
+      0.35759884119033813,
+      0.29587382078170776,
+      0.3376016914844513,
+      0.2916772663593292,
+      0.2944425642490387,
+      0.2793613076210022,
+      0.30821874737739563,
+      0.30350440740585327,
+      0.30654028058052063,
+      0.25445234775543213,
+      0.293817937374115,
+      0.2751149535179138,
+      0.2549825608730316,
+      0.27168112993240356,
+      0.29966264963150024,
+      0.2999582886695862,
+      0.2856738567352295,
+      0.2571064531803131,
+      0.2718660533428192,
+      0.26083821058273315,
+      0.2872292399406433,
+      0.23113496601581573,
+      0.24555249512195587,
+      0.2553543150424957,
+      0.22126071155071259,
+      0.24801431596279144,
+      0.22513781487941742,
+      0.21824850142002106,
+      0.32301944494247437,
+      0.2092411071062088,
+      0.23654519021511078,
+      0.22970826923847198,
+      0.196001797914505,
+      0.19658486545085907,
+      0.19009140133857727,
+      0.254011332988739,
+      0.20502305030822754,
+      0.23104675114154816,
+      0.21880166232585907,
+      0.24527417123317719,
+      0.2686038315296173,
+      0.2016792744398117,
+      0.17031656205654144,
+      0.20333679020404816,
+      0.21816182136535645,
+      0.21064119040966034,
+      0.19884170591831207,
+      0.19956159591674805,
+      0.18471239507198334,
+      0.15811093151569366,
+      0.21883070468902588,
+      0.1914089471101761,
+      0.1655077189207077,
+      0.22221115231513977,
+      0.14605960249900818,
+      0.21837235987186432,
+      0.1704948991537094,
+      0.1736709624528885,
+      0.2253033071756363,
+      0.18853184580802917,
+      0.20015767216682434,
+      0.19018162786960602,
+      0.1817663013935089,
+      0.21601822972297668,
+      0.1782209873199463,
+      0.1798601895570755,
+      0.17680145800113678,
+      0.16178666055202484,
+      0.15822872519493103,
+      0.21383854746818542,
+      0.17432792484760284,
+      0.15497128665447235,
+      0.14949870109558105,
+      0.1623925417661667,
+      0.178536057472229,
+      0.1496189385652542,
+      0.1535705029964447,
+      0.15259061753749847,
+      0.20757174491882324,
+      0.189642533659935,
+      0.16320088505744934,
+      0.19857345521450043,
+      0.17532989382743835,
+      0.18625964224338531,
+      0.1518552303314209,
+      0.15830935537815094,
+      0.16554827988147736,
+      0.1885346621274948,
+      0.19510552287101746,
+      0.1609521359205246,
+      0.22554394602775574,
+      0.17842698097229004,
+      0.16276274621486664,
+      0.1613040417432785,
+      0.13027097284793854,
+      0.16238419711589813,
+      0.14015449583530426,
+      0.17456690967082977,
+      0.1481691151857376,
+      0.1879999339580536,
+      0.17365124821662903,
+      0.2072172909975052,
+      0.13534089922904968,
+      0.12938502430915833,
+      0.16615700721740723,
+      0.1711844503879547,
+      0.1538037359714508,
+      0.17003300786018372,
+      0.1528840959072113,
+      0.14285284280776978,
+      0.1634855419397354,
+      0.13300959765911102,
+      0.16810348629951477,
+      0.13577891886234283,
+      0.1253504455089569,
+      0.13998855650424957,
+      0.1808578222990036,
+      0.14750854671001434,
+      0.15061943233013153,
+      0.15485039353370667,
+      0.15015456080436707,
+      0.12394087016582489,
+      0.13308438658714294,
+      0.12737306952476501,
+      0.10817860811948776,
+      0.1835630238056183,
+      0.1271446943283081,
+      0.17026567459106445,
+      0.14795377850532532,
+      0.15456995368003845,
+      0.12015783041715622,
+      0.14983054995536804,
+      0.15613238513469696,
+      0.1362493485212326,
+      0.11746997386217117,
+      0.12015391886234283,
+      0.14974607527256012,
+      0.12486754357814789,
+      0.1347617208957672,
+      0.13983528316020966,
+      0.14421994984149933,
+      0.15558600425720215,
+      0.1395520716905594,
+      0.12529298663139343,
+      0.14551745355129242,
+      0.16064704954624176,
+      0.123051218688488,
+      0.14231589436531067,
+      0.15310995280742645,
+      0.12945358455181122,
+      0.15963591635227203,
+      0.17953287065029144,
+      0.09377244859933853,
+      0.1109309270977974,
+      0.10077396780252457,
+      0.11636873334646225,
+      0.14231428503990173,
+      0.13868117332458496,
+      0.1199658066034317,
+      0.10137726366519928,
+      0.12408987432718277,
+      0.12158029526472092,
+      0.15156243741512299,
+      0.1056785136461258,
+      0.11238741874694824,
+      0.13356435298919678,
+      0.1199871078133583,
+      0.11451991647481918,
+      0.11946208029985428,
+      0.136271134018898,
+      0.11361639946699142,
+      0.10386927425861359,
+      0.10679834336042404,
+      0.09268485009670258,
+      0.13429896533489227,
+      0.12343748658895493,
+      0.1442834585905075,
+      0.12418247759342194,
+      0.13955631852149963,
+      0.09388206154108047,
+      0.11501727253198624,
+      0.1351633220911026,
+      0.13560284674167633,
+      0.12844859063625336,
+      0.09430808573961258,
+      0.11472094804048538,
+      0.11257084459066391,
+      0.11793804168701172,
+      0.10530801862478256,
+      0.13184157013893127,
+      0.08044975250959396,
+      0.10456537455320358,
+      0.15832646191120148,
+      0.10107123851776123,
+      0.13520397245883942,
+      0.11445610970258713,
+      0.11855047196149826,
+      0.13093633949756622,
+      0.10899967700242996,
+      0.10174088180065155,
+      0.12345916777849197,
+      0.1107829362154007,
+      0.1003185361623764,
+      0.1176886186003685,
+      0.09605075418949127,
+      0.12465865164995193,
+      0.09835167974233627,
+      0.1037013903260231,
+      0.11466304212808609,
+      0.11380521208047867,
+      0.13064700365066528,
+      0.13941724598407745,
+      0.08339516073465347,
+      0.11133090406656265,
+      0.10470746457576752,
+      0.11295027285814285,
+      0.11331796646118164,
+      0.09551217406988144,
+      0.11493296921253204,
+      0.09893112629652023,
+      0.11825107038021088,
+      0.12228565663099289,
+      0.10353125631809235,
+      0.10824307054281235,
+      0.10025204718112946,
+      0.07499944418668747,
+      0.09401531517505646,
+      0.0934530571103096,
+      0.11390485614538193,
+      0.10727037489414215,
+      0.09588678926229477,
+      0.0825532078742981,
+      0.10723954439163208,
+      0.1088128611445427,
+      0.11156749725341797,
+      0.09598823636770248,
+      0.08453374356031418,
+      0.10130847245454788,
+      0.09620478004217148,
+      0.11045535653829575,
+      0.10323013365268707,
+      0.13056471943855286,
+      0.08738001435995102,
+      0.11891673505306244,
+      0.09096094220876694,
+      0.08709857612848282,
+      0.09677024930715561,
+      0.08751770108938217,
+      0.10058047622442245,
+      0.09135355800390244,
+      0.1054212898015976,
+      0.09947032481431961,
+      0.10693687945604324,
+      0.0917934849858284,
+      0.10931998491287231,
+      0.09929491579532623,
+      0.11224542558193207,
+      0.11092674732208252,
+      0.09260962903499603,
+      0.12740318477153778,
+      0.11285065114498138,
+      0.10502947121858597,
+      0.07829917967319489,
+      0.1043354719877243,
+      0.10123976320028305,
+      0.08939259499311447,
+      0.11151234060525894,
+      0.13105222582817078,
+      0.10120860487222672,
+      0.06924744695425034,
+      0.10668231546878815,
+      0.09256572276353836,
+      0.08223498612642288,
+      0.09994925558567047,
+      0.08328042924404144,
+      0.0906481072306633
     ],
     "lr": [
       8.376068376068378e-06,
       20
     ],
     "eval_accuracy": [
+      0.01263157894736842,
+      0.2168421052631579,
+      0.3736842105263158,
+      0.4473684210526316,
+      0.4936842105263158,
+      0.5494736842105263,
+      0.5178947368421053,
+      0.52,
+      0.5873684210526315,
+      0.5136842105263157,
+      0.6073684210526316,
+      0.6021052631578947,
+      0.6431578947368422,
+      0.6621052631578948,
+      0.6168421052631579,
+      0.6536842105263158,
+      0.631578947368421,
+      0.6505263157894737,
+      0.6705263157894736,
+      0.6631578947368421
     ]
   },
+  "final_accuracy": 0.5342307692307692,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
       "K": null,
       "mode": "sft",
       "n_digits": 6,
+      "n_per_split": 100
     },
     "splits": {
       "add_S0": {
+        "full_accuracy": 0.86,
+        "digit_accuracy": 0.9771428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9768595041322314,
+            "count": 605
           },
           "SS": {
+            "accuracy": 0.9789473684210527,
+            "count": 95
           }
         }
       },
       "add_S1": {
+        "full_accuracy": 0.93,
+        "digit_accuracy": 0.99,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9803921568627451,
+            "count": 204
           },
           "SC": {
+            "accuracy": 0.9822485207100592,
+            "count": 169
           },
           "SS": {
+            "accuracy": 1.0,
+            "count": 31
           },
           "UC": {
+            "accuracy": 1.0,
+            "count": 296
           }
         }
       },
       "add_S2": {
+        "full_accuracy": 0.57,
+        "digit_accuracy": 0.9185714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9754601226993865,
+            "count": 163
           },
           "SC": {
+            "accuracy": 0.9461538461538461,
+            "count": 130
           },
           "SS": {
+            "accuracy": 0.9080459770114943,
+            "count": 87
           },
           "UC": {
+            "accuracy": 0.812807881773399,
+            "count": 203
           },
           "US": {
             "accuracy": 1.0,
+            "count": 117
           }
         }
       },
       "add_S3": {
+        "full_accuracy": 0.23,
+        "digit_accuracy": 0.8114285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9669421487603306,
+            "count": 121
           },
           "SC": {
+            "accuracy": 0.9504132231404959,
+            "count": 121
           },
           "SS": {
+            "accuracy": 0.9591836734693877,
+            "count": 49
           },
           "UC": {
+            "accuracy": 0.6505376344086021,
+            "count": 186
           },
           "US": {
+            "accuracy": 0.7533632286995515,
+            "count": 223
           }
         }
       },
       "add_S4": {
+        "full_accuracy": 0.33,
+        "digit_accuracy": 0.75,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
+            "count": 104
           },
           "SC": {
+            "accuracy": 0.9905660377358491,
+            "count": 106
           },
           "SS": {
             "accuracy": 1.0,
+            "count": 23
           },
           "UC": {
+            "accuracy": 0.6625,
+            "count": 160
           },
           "US": {
+            "accuracy": 0.6091205211726385,
+            "count": 307
           }
         }
       },
       "add_S5": {
+        "full_accuracy": 0.19,
         "digit_accuracy": 0.5971428571428572,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.34,
+            "count": 100
           },
           "US": {
+            "accuracy": 0.46,
+            "count": 400
           }
         }
       },
       "add_S6": {
+        "full_accuracy": 0.52,
+        "digit_accuracy": 0.6642857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.56,
+            "count": 100
           },
           "US": {
+            "accuracy": 0.618,
+            "count": 500
           }
         }
       },
       "add_random": {
+        "full_accuracy": 0.845,
+        "digit_accuracy": 0.9742857142857143,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9843400447427293,
+            "count": 447
           },
           "SC": {
+            "accuracy": 0.990625,
+            "count": 320
           },
           "SS": {
+            "accuracy": 0.9642857142857143,
+            "count": 56
           },
           "UC": {
+            "accuracy": 0.9584120982986768,
+            "count": 529
           },
           "US": {
+            "accuracy": 0.9583333333333334,
+            "count": 48
           }
         }
       },
       "add_C1": {
+        "full_accuracy": 0.97,
+        "digit_accuracy": 0.9957142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.994,
+            "count": 500
           },
           "SC": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UC": {
+            "accuracy": 1.0,
+            "count": 100
           }
         }
       },
       "add_C2": {
+        "full_accuracy": 0.8,
+        "digit_accuracy": 0.9714285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.985,
+            "count": 400
           },
           "SC": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.9102564102564102,
+            "count": 156
           },
           "US": {
+            "accuracy": 1.0,
+            "count": 44
           }
         }
       },
       "add_C3": {
+        "full_accuracy": 0.57,
+        "digit_accuracy": 0.9242857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.9966666666666667,
+            "count": 300
           },
           "SC": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.8040201005025126,
+            "count": 199
           },
           "US": {
+            "accuracy": 0.8712871287128713,
+            "count": 101
           }
         }
       },
       "add_C4": {
+        "full_accuracy": 0.49,
+        "digit_accuracy": 0.8914285714285715,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
+            "accuracy": 0.995,
+            "count": 200
           },
           "SC": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.8143939393939394,
+            "count": 264
           },
           "US": {
+            "accuracy": 0.8088235294117647,
+            "count": 136
           }
         }
       },
       "add_C5": {
+        "full_accuracy": 0.37,
+        "digit_accuracy": 0.8571428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.8096774193548387,
+            "count": 310
           },
           "US": {
+            "accuracy": 0.7842105263157895,
+            "count": 190
           }
         }
       },
       "add_C6": {
+        "full_accuracy": 0.42,
+        "digit_accuracy": 0.8442857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UC": {
+            "accuracy": 0.8405405405405405,
+            "count": 370
           },
           "US": {
+            "accuracy": 0.782608695652174,
+            "count": 230
           }
         }
       },
       "sub_M0": {
         "full_accuracy": 0.9,
         "digit_accuracy": 0.9857142857142858,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.983739837398374,
+            "count": 615
           },
           "ME": {
             "accuracy": 1.0,
+            "count": 85
           }
         }
       },
       "sub_M1": {
+        "full_accuracy": 0.94,
+        "digit_accuracy": 0.9914285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.9965753424657534,
+            "count": 292
           },
           "MB": {
             "accuracy": 0.9861111111111112,
+            "count": 144
           },
           "ME": {
+            "accuracy": 0.96,
+            "count": 25
           },
           "UB": {
+            "accuracy": 0.9916317991631799,
+            "count": 239
           }
         }
       },
       "sub_M2": {
+        "full_accuracy": 0.38,
+        "digit_accuracy": 0.8985714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.985781990521327,
+            "count": 211
           },
           "MB": {
+            "accuracy": 0.9739130434782609,
+            "count": 115
           },
           "ME": {
+            "accuracy": 0.9764705882352941,
+            "count": 85
           },
           "UB": {
+            "accuracy": 0.6519337016574586,
+            "count": 181
           },
           "UD": {
+            "accuracy": 1.0,
+            "count": 108
           }
         }
       },
       "sub_M3": {
+        "full_accuracy": 0.1,
+        "digit_accuracy": 0.8071428571428572,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 1.0,
+            "count": 179
           },
           "MB": {
+            "accuracy": 0.9805825242718447,
+            "count": 103
           },
           "ME": {
             "accuracy": 1.0,
+            "count": 56
           },
           "UB": {
+            "accuracy": 0.4899328859060403,
+            "count": 149
           },
           "UD": {
+            "accuracy": 0.7323943661971831,
+            "count": 213
           }
         }
       },
       "sub_M4": {
+        "full_accuracy": 0.06,
+        "digit_accuracy": 0.6885714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 200
           },
           "MB": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.36,
+            "count": 100
           },
           "UD": {
+            "accuracy": 0.4866666666666667,
+            "count": 300
           }
         }
       },
       "sub_M5": {
+        "full_accuracy": 0.06,
+        "digit_accuracy": 0.5571428571428572,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.49,
+            "count": 100
           },
           "UD": {
+            "accuracy": 0.3525,
+            "count": 400
           }
         }
       },
       "sub_random": {
+        "full_accuracy": 0.845,
+        "digit_accuracy": 0.9771428571428571,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.9866666666666667,
+            "count": 600
           },
           "MB": {
+            "accuracy": 0.9887640449438202,
+            "count": 267
           },
           "ME": {
             "accuracy": 1.0,
             "count": 53
           },
           "UB": {
+            "accuracy": 0.9521640091116174,
+            "count": 439
           },
           "UD": {
+            "accuracy": 1.0,
+            "count": 41
           }
         }
       },
       "sub_B3": {
+        "full_accuracy": 0.36,
+        "digit_accuracy": 0.8857142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
+            "accuracy": 0.9966666666666667,
+            "count": 300
           },
           "MB": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.7411167512690355,
+            "count": 197
           },
           "UD": {
+            "accuracy": 0.7281553398058253,
+            "count": 103
           }
         }
       },
       "sub_B4": {
+        "full_accuracy": 0.26,
+        "digit_accuracy": 0.84,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 200
           },
           "MB": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.7125506072874493,
+            "count": 247
           },
           "UD": {
+            "accuracy": 0.7320261437908496,
+            "count": 153
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.22,
+        "digit_accuracy": 0.7957142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
+            "count": 100
           },
           "UB": {
+            "accuracy": 0.7651006711409396,
+            "count": 298
           },
           "UD": {
+            "accuracy": 0.6386138613861386,
+            "count": 202
           }
         }
       }
     },
     "summary": {
+      "overall_accuracy": 0.5342307692307692,
+      "digit_accuracy": 0.8671428571428571,
+      "total_examples": 2600,
       "n_splits": 24
     }
   }

add_sub_baseline_50K_1L3H510d/model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2bfc5309917d4aa9bff45c52de66d373b6b27f50b2a5e8b3d16a116f52ff3678
 size 634642298

 version https://git-lfs.github.com/spec/v1
+oid sha256:52d5fc0719081cdcc41cebb47bfb62d7ccd33018f75f5738e461989b9f5e9219
 size 634642298

add_sub_baseline_50K_1L3H510d/train_config.json CHANGED Viewed

@@ -69,16 +69,20 @@
   "no_wandb": false,
   "n_params": 158584246,
   "run_name": "add_sub_baseline_50K_1L3H510d",
-  "git_commit": "17e935f460a7f9595b705c1d614101a6b0e520f7",
-  "timestamp": "2026-04-14T06:42:54.584652+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "sft",
-  "wandb_run_id": "ql0jdky0",
-  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/ql0jdky0",
-  "final_accuracy": 0.54125,
-  "sft_accuracy": 0.54125,
   "eval_method": "ArithmeticEvaluator"
 }

   "no_wandb": false,
   "n_params": 158584246,
   "run_name": "add_sub_baseline_50K_1L3H510d",
+  "git_commit": "1d5a160e16a5070d61b881494e832aa88149b15c",
+  "timestamp": "2026-04-15T03:54:44.956075+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
+  "train_dataset": "fixed_train/train_50K_seed42.pt",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "sft",
+  "wandb_run_id": "ougociky",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/ougociky",
+  "eval_final_dataset": "eval_sets/eval_add_sub_6d_N100_seed42.json",
+  "eval_epoch_dataset": "eval_sets/eval_add_sub_6d_N25_seed42.json",
+  "eval_hf_repo": "thoughtworks/arithmetic-sorl-data",
+  "final_accuracy": 0.5342307692307692,
+  "sft_accuracy": 0.5342307692307692,
   "eval_method": "ArithmeticEvaluator"
 }