Upload add_sub_baseline_25K_1L3H510d

add_sub_baseline_25K_1L3H510d/metrics.json

@@ -159,320 +159,320 @@
       7800
     ],
     "loss": [
-      8.74937915802002,
-      6.198915004730225,
-      4.366815090179443,
-      2.2759313583374023,
-      1.9526265859603882,
-      1.877150535583496,
-      1.8139151334762573,
-      1.822188377380371,
-      1.71466064453125,
-      1.590683937072754,
-      1.4826921224594116,
-      1.133190393447876,
-      0.8561835885047913,
-      0.7075616121292114,
-      0.6399551033973694,
-      0.6669952273368835,
-      0.6160100698471069,
-      0.5750561952590942,
-      0.5242418646812439,
-      0.5324519872665405,
-      0.49814528226852417,
-      0.4612176716327667,
-      0.5026556849479675,
-      0.47276973724365234,
-      0.43902602791786194,
-      0.446272611618042,
-      0.38574811816215515,
-      0.3711676001548767,
-      0.3969157338142395,
-      0.39037615060806274,
-      0.3659220337867737,
-      0.2901519238948822,
-      0.372765451669693,
-      0.33219239115715027,
-      0.34733083844184875,
-      0.31526169180870056,
-      0.33172616362571716,
-      0.3349834382534027,
-      0.3454124927520752,
-      0.32427385449409485,
-      0.29763302206993103,
-      0.2891975939273834,
-      0.3104265332221985,
-      0.2937973141670227,
-      0.29948028922080994,
-      0.26262804865837097,
-      0.26943641901016235,
-      0.2750188410282135,
-      0.2822719216346741,
-      0.2253115326166153,
-      0.2866693139076233,
-      0.29621443152427673,
-      0.2518032193183899,
-      0.25790685415267944,
-      0.33121928572654724,
-      0.2776740789413452,
-      0.240325927734375,
-      0.25502628087997437,
-      0.2516288459300995,
-      0.2600439190864563,
-      0.22953486442565918,
-      0.25047045946121216,
-      0.287687748670578,
-      0.24002419412136078,
-      0.20791330933570862,
-      0.24204131960868835,
-      0.27317410707473755,
-      0.24345578253269196,
-      0.21374334394931793,
-      0.21283268928527832,
-      0.21549828350543976,
-      0.19832827150821686,
-      0.21149209141731262,
-      0.1855495721101761,
-      0.19525133073329926,
-      0.23278847336769104,
-      0.2606830298900604,
-      0.20044934749603271,
-      0.181396022439003,
-      0.21328875422477722,
-      0.212825208902359,
-      0.21052296459674835,
-      0.19027332961559296,
-      0.2117026001214981,
-      0.22618059813976288,
-      0.22302715480327606,
-      0.17805133759975433,
-      0.18642711639404297,
-      0.2146042138338089,
-      0.19574260711669922,
-      0.16053321957588196,
-      0.21608540415763855,
-      0.20082899928092957,
-      0.23242345452308655,
-      0.17946086823940277,
-      0.2614651918411255,
-      0.1651460975408554,
-      0.1777617186307907,
-      0.15014827251434326,
-      0.20371000468730927,
-      0.1573261022567749,
-      0.19635023176670074,
-      0.18772493302822113,
-      0.1814812421798706,
-      0.18410103023052216,
-      0.20629878342151642,
-      0.1857832968235016,
-      0.17055557668209076,
-      0.16345703601837158,
-      0.16755636036396027,
-      0.17432273924350739,
-      0.15670685470104218,
-      0.15046896040439606,
-      0.14622943103313446,
-      0.1660774052143097,
-      0.16316993534564972,
-      0.16458259522914886,
-      0.16533777117729187,
-      0.18796680867671967,
-      0.16055859625339508,
-      0.16241776943206787,
-      0.15573906898498535,
-      0.18253065645694733,
-      0.15446387231349945,
-      0.16669152677059174,
-      0.16877718269824982,
-      0.15042708814144135,
-      0.14171156287193298,
-      0.1469859778881073,
-      0.16647039353847504,
-      0.1766190528869629,
-      0.15534673631191254,
-      0.15092872083187103,
-      0.16818329691886902,
-      0.14259760081768036,
-      0.16949117183685303,
-      0.13747766613960266,
-      0.16281220316886902,
-      0.1641203612089157,
-      0.15849816799163818,
-      0.14388667047023773,
-      0.14095112681388855,
-      0.14720116555690765,
-      0.15021368861198425,
-      0.15137387812137604,
-      0.15634380280971527,
-      0.14748618006706238,
-      0.1435571163892746,
-      0.13644962012767792,
-      0.16987796127796173,
-      0.15995793044567108,
-      0.17382332682609558,
-      0.157388374209404,
-      0.13605724275112152,
-      0.174538716673851,
-      0.15863211452960968
+      9.03099536895752,
+      6.222628116607666,
+      4.243917942047119,
+      2.320626974105835,
+      1.865071415901184,
+      1.8302096128463745,
+      1.8747438192367554,
+      1.8183810710906982,
+      1.750946283340454,
+      1.6808611154556274,
+      1.4773333072662354,
+      1.0558074712753296,
+      0.8097242712974548,
+      0.6758731603622437,
+      0.6198462247848511,
+      0.5841843485832214,
+      0.5899673104286194,
+      0.5845558047294617,
+      0.5196164846420288,
+      0.47981908917427063,
+      0.5051271319389343,
+      0.45223522186279297,
+      0.4028424322605133,
+      0.41621074080467224,
+      0.3799597918987274,
+      0.4078504741191864,
+      0.3867304027080536,
+      0.3984403610229492,
+      0.3459450602531433,
+      0.44437745213508606,
+      0.3283034861087799,
+      0.33987003564834595,
+      0.32265350222587585,
+      0.39120978116989136,
+      0.2970849573612213,
+      0.28256362676620483,
+      0.3255704343318939,
+      0.3375805914402008,
+      0.37903186678886414,
+      0.3113180696964264,
+      0.3064514696598053,
+      0.3122137188911438,
+      0.33857011795043945,
+      0.284681111574173,
+      0.2882470488548279,
+      0.27921387553215027,
+      0.25749853253364563,
+      0.28591188788414,
+      0.2735363841056824,
+      0.32027482986450195,
+      0.23003888130187988,
+      0.2517407238483429,
+      0.2710196077823639,
+      0.23125974833965302,
+      0.29058152437210083,
+      0.30273309350013733,
+      0.2546815276145935,
+      0.23205725848674774,
+      0.24823474884033203,
+      0.2757126986980438,
+      0.244472473859787,
+      0.2154902219772339,
+      0.22570113837718964,
+      0.2101653665304184,
+      0.26213961839675903,
+      0.2357279509305954,
+      0.20209209620952606,
+      0.22399720549583435,
+      0.24680034816265106,
+      0.2356012910604477,
+      0.21086706221103668,
+      0.23208554089069366,
+      0.21862395107746124,
+      0.23082730174064636,
+      0.18721143901348114,
+      0.20573721826076508,
+      0.22153320908546448,
+      0.23551729321479797,
+      0.19480657577514648,
+      0.21060045063495636,
+      0.17743727564811707,
+      0.18577153980731964,
+      0.2063155472278595,
+      0.2116478681564331,
+      0.2169320285320282,
+      0.21249254047870636,
+      0.20075924694538116,
+      0.20089474320411682,
+      0.16138426959514618,
+      0.1542191505432129,
+      0.1778700351715088,
+      0.17714276909828186,
+      0.1895652860403061,
+      0.20230571925640106,
+      0.19294473528862,
+      0.17365707457065582,
+      0.14464251697063446,
+      0.16429129242897034,
+      0.1803891807794571,
+      0.18245069682598114,
+      0.18170282244682312,
+      0.16513392329216003,
+      0.18308500945568085,
+      0.15528549253940582,
+      0.19069239497184753,
+      0.1680574268102646,
+      0.17276549339294434,
+      0.1893104612827301,
+      0.16881202161312103,
+      0.1330322027206421,
+      0.14430834352970123,
+      0.140597864985466,
+      0.18033921718597412,
+      0.14561453461647034,
+      0.15060050785541534,
+      0.14696337282657623,
+      0.1570371687412262,
+      0.13974200189113617,
+      0.15948486328125,
+      0.14828327298164368,
+      0.17222817242145538,
+      0.15832778811454773,
+      0.13328474760055542,
+      0.1352883279323578,
+      0.146428182721138,
+      0.14305901527404785,
+      0.14752517640590668,
+      0.14967180788516998,
+      0.16033518314361572,
+      0.15050823986530304,
+      0.12478389590978622,
+      0.13973259925842285,
+      0.16433657705783844,
+      0.15506653487682343,
+      0.10899337381124496,
+      0.13583232462406158,
+      0.1452769786119461,
+      0.1488191783428192,
+      0.16326965391635895,
+      0.13210436701774597,
+      0.1348404884338379,
+      0.15470661222934723,
+      0.0996326431632042,
+      0.14851771295070648,
+      0.14729616045951843,
+      0.13585257530212402,
+      0.16751345992088318,
+      0.12513908743858337,
+      0.12792132794857025,
+      0.14990487694740295,
+      0.14224524796009064,
+      0.14584632217884064,
+      0.1600850373506546,
+      0.12832213938236237,
+      0.12461556494235992,
+      0.14382870495319366
     ],
     "base_loss": [
-      8.74937915802002,
-      6.198915004730225,
-      4.366815090179443,
-      2.2759313583374023,
-      1.9526265859603882,
-      1.877150535583496,
-      1.8139151334762573,
-      1.822188377380371,
-      1.71466064453125,
-      1.590683937072754,
-      1.4826921224594116,
-      1.133190393447876,
-      0.8561835885047913,
-      0.7075616121292114,
-      0.6399551033973694,
-      0.6669952273368835,
-      0.6160100698471069,
-      0.5750561952590942,
-      0.5242418646812439,
-      0.5324519872665405,
-      0.49814528226852417,
-      0.4612176716327667,
-      0.5026556849479675,
-      0.47276973724365234,
-      0.43902602791786194,
-      0.446272611618042,
-      0.38574811816215515,
-      0.3711676001548767,
-      0.3969157338142395,
-      0.39037615060806274,
-      0.3659220337867737,
-      0.2901519238948822,
-      0.372765451669693,
-      0.33219239115715027,
-      0.34733083844184875,
-      0.31526169180870056,
-      0.33172616362571716,
-      0.3349834382534027,
-      0.3454124927520752,
-      0.32427385449409485,
-      0.29763302206993103,
-      0.2891975939273834,
-      0.3104265332221985,
-      0.2937973141670227,
-      0.29948028922080994,
-      0.26262804865837097,
-      0.26943641901016235,
-      0.2750188410282135,
-      0.2822719216346741,
-      0.2253115326166153,
-      0.2866693139076233,
-      0.29621443152427673,
-      0.2518032193183899,
-      0.25790685415267944,
-      0.33121928572654724,
-      0.2776740789413452,
-      0.240325927734375,
-      0.25502628087997437,
-      0.2516288459300995,
-      0.2600439190864563,
-      0.22953486442565918,
-      0.25047045946121216,
-      0.287687748670578,
-      0.24002419412136078,
-      0.20791330933570862,
-      0.24204131960868835,
-      0.27317410707473755,
-      0.24345578253269196,
-      0.21374334394931793,
-      0.21283268928527832,
-      0.21549828350543976,
-      0.19832827150821686,
-      0.21149209141731262,
-      0.1855495721101761,
-      0.19525133073329926,
-      0.23278847336769104,
-      0.2606830298900604,
-      0.20044934749603271,
-      0.181396022439003,
-      0.21328875422477722,
-      0.212825208902359,
-      0.21052296459674835,
-      0.19027332961559296,
-      0.2117026001214981,
-      0.22618059813976288,
-      0.22302715480327606,
-      0.17805133759975433,
-      0.18642711639404297,
-      0.2146042138338089,
-      0.19574260711669922,
-      0.16053321957588196,
-      0.21608540415763855,
-      0.20082899928092957,
-      0.23242345452308655,
-      0.17946086823940277,
-      0.2614651918411255,
-      0.1651460975408554,
-      0.1777617186307907,
-      0.15014827251434326,
-      0.20371000468730927,
-      0.1573261022567749,
-      0.19635023176670074,
-      0.18772493302822113,
-      0.1814812421798706,
-      0.18410103023052216,
-      0.20629878342151642,
-      0.1857832968235016,
-      0.17055557668209076,
-      0.16345703601837158,
-      0.16755636036396027,
-      0.17432273924350739,
-      0.15670685470104218,
-      0.15046896040439606,
-      0.14622943103313446,
-      0.1660774052143097,
-      0.16316993534564972,
-      0.16458259522914886,
-      0.16533777117729187,
-      0.18796680867671967,
-      0.16055859625339508,
-      0.16241776943206787,
-      0.15573906898498535,
-      0.18253065645694733,
-      0.15446387231349945,
-      0.16669152677059174,
-      0.16877718269824982,
-      0.15042708814144135,
-      0.14171156287193298,
-      0.1469859778881073,
-      0.16647039353847504,
-      0.1766190528869629,
-      0.15534673631191254,
-      0.15092872083187103,
-      0.16818329691886902,
-      0.14259760081768036,
-      0.16949117183685303,
-      0.13747766613960266,
-      0.16281220316886902,
-      0.1641203612089157,
-      0.15849816799163818,
-      0.14388667047023773,
-      0.14095112681388855,
-      0.14720116555690765,
-      0.15021368861198425,
-      0.15137387812137604,
-      0.15634380280971527,
-      0.14748618006706238,
-      0.1435571163892746,
-      0.13644962012767792,
-      0.16987796127796173,
-      0.15995793044567108,
-      0.17382332682609558,
-      0.157388374209404,
-      0.13605724275112152,
-      0.174538716673851,
-      0.15863211452960968
+      9.03099536895752,
+      6.222628116607666,
+      4.243917942047119,
+      2.320626974105835,
+      1.865071415901184,
+      1.8302096128463745,
+      1.8747438192367554,
+      1.8183810710906982,
+      1.750946283340454,
+      1.6808611154556274,
+      1.4773333072662354,
+      1.0558074712753296,
+      0.8097242712974548,
+      0.6758731603622437,
+      0.6198462247848511,
+      0.5841843485832214,
+      0.5899673104286194,
+      0.5845558047294617,
+      0.5196164846420288,
+      0.47981908917427063,
+      0.5051271319389343,
+      0.45223522186279297,
+      0.4028424322605133,
+      0.41621074080467224,
+      0.3799597918987274,
+      0.4078504741191864,
+      0.3867304027080536,
+      0.3984403610229492,
+      0.3459450602531433,
+      0.44437745213508606,
+      0.3283034861087799,
+      0.33987003564834595,
+      0.32265350222587585,
+      0.39120978116989136,
+      0.2970849573612213,
+      0.28256362676620483,
+      0.3255704343318939,
+      0.3375805914402008,
+      0.37903186678886414,
+      0.3113180696964264,
+      0.3064514696598053,
+      0.3122137188911438,
+      0.33857011795043945,
+      0.284681111574173,
+      0.2882470488548279,
+      0.27921387553215027,
+      0.25749853253364563,
+      0.28591188788414,
+      0.2735363841056824,
+      0.32027482986450195,
+      0.23003888130187988,
+      0.2517407238483429,
+      0.2710196077823639,
+      0.23125974833965302,
+      0.29058152437210083,
+      0.30273309350013733,
+      0.2546815276145935,
+      0.23205725848674774,
+      0.24823474884033203,
+      0.2757126986980438,
+      0.244472473859787,
+      0.2154902219772339,
+      0.22570113837718964,
+      0.2101653665304184,
+      0.26213961839675903,
+      0.2357279509305954,
+      0.20209209620952606,
+      0.22399720549583435,
+      0.24680034816265106,
+      0.2356012910604477,
+      0.21086706221103668,
+      0.23208554089069366,
+      0.21862395107746124,
+      0.23082730174064636,
+      0.18721143901348114,
+      0.20573721826076508,
+      0.22153320908546448,
+      0.23551729321479797,
+      0.19480657577514648,
+      0.21060045063495636,
+      0.17743727564811707,
+      0.18577153980731964,
+      0.2063155472278595,
+      0.2116478681564331,
+      0.2169320285320282,
+      0.21249254047870636,
+      0.20075924694538116,
+      0.20089474320411682,
+      0.16138426959514618,
+      0.1542191505432129,
+      0.1778700351715088,
+      0.17714276909828186,
+      0.1895652860403061,
+      0.20230571925640106,
+      0.19294473528862,
+      0.17365707457065582,
+      0.14464251697063446,
+      0.16429129242897034,
+      0.1803891807794571,
+      0.18245069682598114,
+      0.18170282244682312,
+      0.16513392329216003,
+      0.18308500945568085,
+      0.15528549253940582,
+      0.19069239497184753,
+      0.1680574268102646,
+      0.17276549339294434,
+      0.1893104612827301,
+      0.16881202161312103,
+      0.1330322027206421,
+      0.14430834352970123,
+      0.140597864985466,
+      0.18033921718597412,
+      0.14561453461647034,
+      0.15060050785541534,
+      0.14696337282657623,
+      0.1570371687412262,
+      0.13974200189113617,
+      0.15948486328125,
+      0.14828327298164368,
+      0.17222817242145538,
+      0.15832778811454773,
+      0.13328474760055542,
+      0.1352883279323578,
+      0.146428182721138,
+      0.14305901527404785,
+      0.14752517640590668,
+      0.14967180788516998,
+      0.16033518314361572,
+      0.15050823986530304,
+      0.12478389590978622,
+      0.13973259925842285,
+      0.16433657705783844,
+      0.15506653487682343,
+      0.10899337381124496,
+      0.13583232462406158,
+      0.1452769786119461,
+      0.1488191783428192,
+      0.16326965391635895,
+      0.13210436701774597,
+      0.1348404884338379,
+      0.15470661222934723,
+      0.0996326431632042,
+      0.14851771295070648,
+      0.14729616045951843,
+      0.13585257530212402,
+      0.16751345992088318,
+      0.12513908743858337,
+      0.12792132794857025,
+      0.14990487694740295,
+      0.14224524796009064,
+      0.14584632217884064,
+      0.1600850373506546,
+      0.12832213938236237,
+      0.12461556494235992,
+      0.14382870495319366
     ],
     "lr": [
       1.6752136752136756e-05,
@@ -677,595 +677,595 @@
       20
     ],
     "eval_accuracy": [
-      0.005555555555555556,
-      0.05333333333333334,
-      0.20555555555555555,
-      0.29333333333333333,
-      0.3055555555555556,
-      0.31777777777777777,
-      0.36777777777777776,
-      0.3988888888888889,
-      0.4388888888888889,
-      0.5322222222222223,
-      0.5355555555555556,
-      0.5366666666666666,
-      0.62,
-      0.5488888888888889,
-      0.5488888888888889,
-      0.5866666666666667,
-      0.5811111111111111,
-      0.5811111111111111,
-      0.5733333333333334,
-      0.5777777777777777
+      0.002105263157894737,
+      0.06736842105263158,
+      0.17894736842105263,
+      0.28526315789473683,
+      0.3336842105263158,
+      0.32842105263157895,
+      0.4421052631578947,
+      0.4168421052631579,
+      0.43894736842105264,
+      0.4305263157894737,
+      0.4431578947368421,
+      0.5505263157894736,
+      0.5305263157894737,
+      0.52,
+      0.5726315789473684,
+      0.5810526315789474,
+      0.5768421052631579,
+      0.5810526315789474,
+      0.5852631578947368,
+      0.5863157894736842
     ]
   },
-  "final_accuracy": 0.49541666666666667,
+  "final_accuracy": 0.46307692307692305,
   "sft_eval": {
     "config": {
       "ops": "add_sub",
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 50
+      "n_per_split": 100
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.74,
-        "digit_accuracy": 0.96,
-        "n_examples": 50,
+        "full_accuracy": 0.78,
+        "digit_accuracy": 0.9614285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9525423728813559,
-            "count": 295
+            "accuracy": 0.9636363636363636,
+            "count": 605
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 55
+            "accuracy": 0.9473684210526315,
+            "count": 95
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.74,
+        "full_accuracy": 0.76,
         "digit_accuracy": 0.9628571428571429,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9682539682539683,
-            "count": 126
+            "accuracy": 0.9607843137254902,
+            "count": 204
           },
           "SC": {
-            "accuracy": 0.9620253164556962,
-            "count": 79
+            "accuracy": 0.9704142011834319,
+            "count": 169
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 21
+            "accuracy": 0.9354838709677419,
+            "count": 31
           },
           "UC": {
-            "accuracy": 0.9516129032258065,
-            "count": 124
+            "accuracy": 0.9628378378378378,
+            "count": 296
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.6,
-        "digit_accuracy": 0.9342857142857143,
-        "n_examples": 50,
+        "full_accuracy": 0.5,
+        "digit_accuracy": 0.9114285714285715,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9733333333333334,
-            "count": 75
+            "accuracy": 0.9631901840490797,
+            "count": 163
           },
           "SC": {
-            "accuracy": 0.967741935483871,
-            "count": 62
+            "accuracy": 0.9461538461538461,
+            "count": 130
           },
           "SS": {
-            "accuracy": 0.9743589743589743,
-            "count": 39
+            "accuracy": 0.9195402298850575,
+            "count": 87
           },
           "UC": {
-            "accuracy": 0.8558558558558559,
-            "count": 111
+            "accuracy": 0.8078817733990148,
+            "count": 203
           },
           "US": {
-            "accuracy": 0.9682539682539683,
-            "count": 63
+            "accuracy": 0.9743589743589743,
+            "count": 117
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.38,
-        "digit_accuracy": 0.8428571428571429,
-        "n_examples": 50,
+        "full_accuracy": 0.14,
+        "digit_accuracy": 0.7757142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9833333333333333,
-            "count": 60
+            "accuracy": 0.9752066115702479,
+            "count": 121
           },
           "SC": {
-            "accuracy": 1.0,
-            "count": 57
+            "accuracy": 0.9256198347107438,
+            "count": 121
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 19
+            "accuracy": 0.9591836734693877,
+            "count": 49
           },
           "UC": {
-            "accuracy": 0.7115384615384616,
-            "count": 104
+            "accuracy": 0.6129032258064516,
+            "count": 186
           },
           "US": {
-            "accuracy": 0.7818181818181819,
-            "count": 110
+            "accuracy": 0.6816143497757847,
+            "count": 223
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.3,
-        "digit_accuracy": 0.7371428571428571,
-        "n_examples": 50,
+        "full_accuracy": 0.22,
+        "digit_accuracy": 0.7,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 48
+            "count": 104
           },
           "SC": {
-            "accuracy": 1.0,
-            "count": 52
+            "accuracy": 0.9905660377358491,
+            "count": 106
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 7
+            "count": 23
           },
           "UC": {
-            "accuracy": 0.6629213483146067,
-            "count": 89
+            "accuracy": 0.61875,
+            "count": 160
           },
           "US": {
-            "accuracy": 0.5974025974025974,
-            "count": 154
+            "accuracy": 0.5179153094462541,
+            "count": 307
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.24,
-        "digit_accuracy": 0.6028571428571429,
-        "n_examples": 50,
+        "full_accuracy": 0.21,
+        "digit_accuracy": 0.5942857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "SC": {
-            "accuracy": 1.0,
-            "count": 50
+            "accuracy": 0.97,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.52,
-            "count": 50
+            "accuracy": 0.43,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.425,
-            "count": 200
+            "accuracy": 0.44,
+            "count": 400
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.48,
-        "digit_accuracy": 0.6371428571428571,
-        "n_examples": 50,
+        "full_accuracy": 0.27,
+        "digit_accuracy": 0.5028571428571429,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.54,
-            "count": 50
+            "accuracy": 0.39,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.584,
-            "count": 250
+            "accuracy": 0.426,
+            "count": 500
           }
         }
       },
       "add_random": {
         "full_accuracy": 0.765,
-        "digit_accuracy": 0.9621428571428572,
+        "digit_accuracy": 0.96,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.988399071925754,
-            "count": 431
+            "accuracy": 0.9753914988814317,
+            "count": 447
           },
           "SC": {
-            "accuracy": 0.9936708860759493,
-            "count": 316
+            "accuracy": 0.984375,
+            "count": 320
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 39
+            "accuracy": 0.9642857142857143,
+            "count": 56
           },
           "UC": {
-            "accuracy": 0.9285714285714286,
-            "count": 560
+            "accuracy": 0.9376181474480151,
+            "count": 529
           },
           "US": {
-            "accuracy": 0.8888888888888888,
-            "count": 54
+            "accuracy": 0.8958333333333334,
+            "count": 48
           }
         }
       },
       "add_C1": {
-        "full_accuracy": 0.84,
+        "full_accuracy": 0.83,
         "digit_accuracy": 0.9742857142857143,
-        "n_examples": 50,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 0.976,
-            "count": 250
+            "count": 500
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
             "accuracy": 0.94,
-            "count": 50
+            "count": 100
           }
         }
       },
       "add_C2": {
-        "full_accuracy": 0.76,
-        "digit_accuracy": 0.96,
-        "n_examples": 50,
+        "full_accuracy": 0.69,
+        "digit_accuracy": 0.9485714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.975,
-            "count": 200
+            "accuracy": 0.97,
+            "count": 400
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.9156626506024096,
-            "count": 83
+            "accuracy": 0.8461538461538461,
+            "count": 156
           },
           "US": {
-            "accuracy": 0.8823529411764706,
-            "count": 17
+            "accuracy": 1.0,
+            "count": 44
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.38,
-        "digit_accuracy": 0.8857142857142857,
-        "n_examples": 50,
+        "full_accuracy": 0.5,
+        "digit_accuracy": 0.9085714285714286,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9866666666666667,
-            "count": 150
+            "accuracy": 0.9933333333333333,
+            "count": 300
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.78,
-            "count": 100
+            "accuracy": 0.7587939698492462,
+            "count": 199
           },
           "US": {
-            "accuracy": 0.68,
-            "count": 50
+            "accuracy": 0.8613861386138614,
+            "count": 101
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.5,
-        "digit_accuracy": 0.8971428571428571,
-        "n_examples": 50,
+        "full_accuracy": 0.47,
+        "digit_accuracy": 0.8842857142857142,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 0.985,
+            "count": 200
           },
           "SC": {
-            "accuracy": 1.0,
-            "count": 50
+            "accuracy": 0.99,
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.8333333333333334,
-            "count": 132
+            "accuracy": 0.8143939393939394,
+            "count": 264
           },
           "US": {
-            "accuracy": 0.8088235294117647,
-            "count": 68
+            "accuracy": 0.7941176470588235,
+            "count": 136
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.42,
-        "digit_accuracy": 0.8371428571428572,
-        "n_examples": 50,
+        "full_accuracy": 0.29,
+        "digit_accuracy": 0.8228571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.7808219178082192,
-            "count": 146
+            "accuracy": 0.7483870967741936,
+            "count": 310
           },
           "US": {
-            "accuracy": 0.7596153846153846,
-            "count": 104
+            "accuracy": 0.7578947368421053,
+            "count": 190
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.48,
-        "digit_accuracy": 0.86,
-        "n_examples": 50,
+        "full_accuracy": 0.33,
+        "digit_accuracy": 0.8,
+        "n_examples": 100,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UC": {
-            "accuracy": 0.8518518518518519,
-            "count": 189
+            "accuracy": 0.8081081081081081,
+            "count": 370
           },
           "US": {
-            "accuracy": 0.8108108108108109,
-            "count": 111
+            "accuracy": 0.7,
+            "count": 230
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.88,
-        "digit_accuracy": 0.9828571428571429,
-        "n_examples": 50,
+        "full_accuracy": 0.84,
+        "digit_accuracy": 0.9771428571428571,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9801980198019802,
-            "count": 303
+            "accuracy": 0.9739837398373984,
+            "count": 615
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 47
+            "count": 85
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.76,
-        "digit_accuracy": 0.96,
-        "n_examples": 50,
+        "full_accuracy": 0.83,
+        "digit_accuracy": 0.9728571428571429,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9858156028368794,
-            "count": 141
+            "accuracy": 0.9897260273972602,
+            "count": 292
           },
           "MB": {
-            "accuracy": 0.9583333333333334,
-            "count": 72
+            "accuracy": 0.9791666666666666,
+            "count": 144
           },
           "ME": {
-            "accuracy": 0.9444444444444444,
-            "count": 18
+            "accuracy": 0.96,
+            "count": 25
           },
           "UB": {
-            "accuracy": 0.9327731092436975,
-            "count": 119
+            "accuracy": 0.9497907949790795,
+            "count": 239
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.46,
-        "digit_accuracy": 0.9,
-        "n_examples": 50,
+        "full_accuracy": 0.3,
+        "digit_accuracy": 0.8842857142857142,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9732142857142857,
-            "count": 112
+            "accuracy": 0.985781990521327,
+            "count": 211
           },
           "MB": {
-            "accuracy": 0.9056603773584906,
-            "count": 53
+            "accuracy": 0.9565217391304348,
+            "count": 115
           },
           "ME": {
-            "accuracy": 0.9787234042553191,
-            "count": 47
+            "accuracy": 0.9764705882352941,
+            "count": 85
           },
           "UB": {
-            "accuracy": 0.7176470588235294,
-            "count": 85
+            "accuracy": 0.6243093922651933,
+            "count": 181
           },
           "UD": {
-            "accuracy": 0.9622641509433962,
-            "count": 53
+            "accuracy": 0.9722222222222222,
+            "count": 108
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.28,
-        "digit_accuracy": 0.84,
-        "n_examples": 50,
+        "full_accuracy": 0.11,
+        "digit_accuracy": 0.7928571428571428,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 97
+            "count": 179
           },
           "MB": {
-            "accuracy": 0.9803921568627451,
-            "count": 51
+            "accuracy": 0.941747572815534,
+            "count": 103
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 27
+            "count": 56
           },
           "UB": {
-            "accuracy": 0.6486486486486487,
-            "count": 74
+            "accuracy": 0.5100671140939598,
+            "count": 149
           },
           "UD": {
-            "accuracy": 0.7128712871287128,
-            "count": 101
+            "accuracy": 0.6901408450704225,
+            "count": 213
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.04,
-        "digit_accuracy": 0.6285714285714286,
-        "n_examples": 50,
+        "full_accuracy": 0.07,
+        "digit_accuracy": 0.6642857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 0.995,
+            "count": 200
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.34,
-            "count": 50
+            "accuracy": 0.46,
+            "count": 100
           },
           "UD": {
-            "accuracy": 0.36,
-            "count": 150
+            "accuracy": 0.4,
+            "count": 300
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.1,
-        "digit_accuracy": 0.5771428571428572,
-        "n_examples": 50,
+        "full_accuracy": 0.04,
+        "digit_accuracy": 0.5014285714285714,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.44,
-            "count": 50
+            "accuracy": 0.51,
+            "count": 100
           },
           "UD": {
-            "accuracy": 0.4,
-            "count": 200
+            "accuracy": 0.25,
+            "count": 400
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.75,
-        "digit_accuracy": 0.9578571428571429,
+        "full_accuracy": 0.775,
+        "digit_accuracy": 0.9664285714285714,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9912280701754386,
-            "count": 570
+            "accuracy": 0.98,
+            "count": 600
           },
           "MB": {
-            "accuracy": 0.9675090252707581,
-            "count": 277
+            "accuracy": 0.9737827715355806,
+            "count": 267
           },
           "ME": {
             "accuracy": 1.0,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.9087048832271762,
-            "count": 471
+            "accuracy": 0.9384965831435079,
+            "count": 439
           },
           "UD": {
-            "accuracy": 0.9310344827586207,
-            "count": 29
+            "accuracy": 0.975609756097561,
+            "count": 41
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.4,
-        "digit_accuracy": 0.8828571428571429,
-        "n_examples": 50,
+        "full_accuracy": 0.38,
+        "digit_accuracy": 0.8757142857142857,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 150
+            "accuracy": 0.9933333333333333,
+            "count": 300
           },
           "MB": {
-            "accuracy": 0.98,
-            "count": 50
+            "accuracy": 1.0,
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.7425742574257426,
-            "count": 101
+            "accuracy": 0.7208121827411168,
+            "count": 197
           },
           "UD": {
-            "accuracy": 0.7142857142857143,
-            "count": 49
+            "accuracy": 0.7087378640776699,
+            "count": 103
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.3,
-        "digit_accuracy": 0.8171428571428572,
-        "n_examples": 50,
+        "full_accuracy": 0.2,
+        "digit_accuracy": 0.8142857142857143,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 100
+            "accuracy": 0.99,
+            "count": 200
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.7107438016528925,
-            "count": 121
+            "accuracy": 0.6963562753036437,
+            "count": 247
           },
           "UD": {
-            "accuracy": 0.6329113924050633,
-            "count": 79
+            "accuracy": 0.6535947712418301,
+            "count": 153
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.28,
-        "digit_accuracy": 0.8,
-        "n_examples": 50,
+        "full_accuracy": 0.21,
+        "digit_accuracy": 0.77,
+        "n_examples": 100,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 50
+            "count": 100
           },
           "UB": {
-            "accuracy": 0.75,
-            "count": 152
+            "accuracy": 0.714765100671141,
+            "count": 298
           },
           "UD": {
-            "accuracy": 0.673469387755102,
-            "count": 98
+            "accuracy": 0.6237623762376238,
+            "count": 202
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.5473333333333333,
-      "digit_accuracy": 0.872,
-      "total_examples": 1500,
+      "overall_accuracy": 0.46307692307692305,
+      "digit_accuracy": 0.8403846153846154,
+      "total_examples": 2600,
       "n_splits": 24
     }
   }

add_sub_baseline_25K_1L3H510d/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c5b3d0a4f5926074c2ad67eaf23945b1b503b7915d0fbafb817b3c7222125cc7
+oid sha256:bc1136aa94b4c06fff5761cf5e96a59cb0caae50c49067b0a3d00ef1072485dd
 size 634642298

add_sub_baseline_25K_1L3H510d/train_config.json

@@ -69,16 +69,20 @@
   "no_wandb": false,
   "n_params": 158584246,
   "run_name": "add_sub_baseline_25K_1L3H510d",
-  "git_commit": "dc8dd776fb0c30a4c9073052dcc5e943e0fd80c6",
-  "timestamp": "2026-04-13T06:26:07.104526+00:00",
+  "git_commit": "f835493c19eb98267697007042c9d440cad2afbb",
+  "timestamp": "2026-04-15T12:26:18.246615+00:00",
   "tokenizer": "Qwen/Qwen3-0.6B",
   "dataset_repo": "thoughtworks/arithmetic-sorl-data",
   "dataset_config": "add_sub_6digit",
+  "train_dataset": "fixed_train/train_25K_seed42.pt",
   "model_repo": "thoughtworks/arithmetic-sorl",
   "trainer_version": "sft",
-  "wandb_run_id": "wpi2qkei",
-  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/wpi2qkei",
-  "final_accuracy": 0.49541666666666667,
-  "sft_accuracy": 0.49541666666666667,
+  "wandb_run_id": "sqn2003t",
+  "wandb_url": "https://wandb.ai/nlp_and_interpretability/sorl-arithmetic/runs/sqn2003t",
+  "eval_final_dataset": "eval_sets/eval_add_sub_6d_N100_seed42.json",
+  "eval_epoch_dataset": "eval_sets/eval_add_sub_6d_N25_seed42.json",
+  "eval_hf_repo": "thoughtworks/arithmetic-sorl-data",
+  "final_accuracy": 0.46307692307692305,
+  "sft_accuracy": 0.46307692307692305,
   "eval_method": "ArithmeticEvaluator"
 }